From 9e71561f82f46509da11a99b6442b138303eca13 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:22:46 -0500 Subject: [PATCH 01/10] Add test quarantine KBE shadow evaluator Add a deterministic, read-only evaluator and versioned schemas for validating Runtime-style Known Build Error signatures against hash-pinned quarantine evidence. The evaluator remains fail-closed and never authorizes repository mutations. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d5905c51-843f-4d8c-9852-cbbcf77d7723 --- .../Evaluate-TestQuarantineKbeCandidate.ps1 | 788 ++++++++++++++++++ ...st-Evaluate-TestQuarantineKbeCandidate.ps1 | 733 ++++++++++++++++ ...uarantine-kbe-shadow-candidate.schema.json | 466 +++++++++++ ...-quarantine-kbe-shadow-receipt.schema.json | 758 +++++++++++++++++ 4 files changed, 2745 insertions(+) create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 new file mode 100644 index 000000000000..6efe2cf2573e --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -0,0 +1,788 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$CandidateFile, + + [Parameter(Mandatory = $true)] + [string]$EvidenceRoot, + + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [string]$RepositoryRoot = "$PSScriptRoot/../../../..", + + [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", + + [string]$ReceiptSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-receipt.schema.json" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$minimumFailureEvidenceFloor = 2 +$minimumNegativeEvidenceFloor = 1 +$failureAssociationWindowLines = 50 + +function Get-Sha256String +{ + param([Parameter(Mandatory = $true)][string]$Value) + + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) + $hash = [System.Security.Cryptography.SHA256]::HashData($bytes) + + return [System.Convert]::ToHexString($hash).ToLowerInvariant() +} + +function Get-SafeExcerpt +{ + param([Parameter(Mandatory = $true)][string]$Value) + + $excerpt = [System.Text.RegularExpressions.Regex]::Replace($Value, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "?").Trim() + if ($excerpt.Length -gt 300) + { + $excerpt = $excerpt.Substring(0, 300) + } + + return $excerpt +} + +function Resolve-EvidencePath +{ + param( + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][string]$RelativePath + ) + + if ([System.IO.Path]::IsPathRooted($RelativePath)) + { + throw "Evidence path must be relative: $RelativePath" + } + + $rootInfo = [System.IO.DirectoryInfo]::new($Root) + if ($null -ne $rootInfo.LinkTarget) + { + throw "Evidence root must not be a symbolic link." + } + + $segments = @($RelativePath -split "[\\/]") + if ($segments.Count -eq 0 -or + @($segments | Where-Object { [string]::IsNullOrWhiteSpace($_) -or $_ -in @(".", "..") }).Count -gt 0) + { + throw "Evidence path contains an invalid path segment: $RelativePath" + } + + $normalizedRelativePath = [string]::Join([System.IO.Path]::DirectorySeparatorChar, $segments) + $candidatePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($Root, $normalizedRelativePath)) + $rootPrefix = $Root.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar + if (-not $candidatePath.StartsWith($rootPrefix, [System.StringComparison]::Ordinal)) + { + throw "Evidence path escapes the evidence root: $RelativePath" + } + + $currentPath = $Root + foreach ($segment in $segments) + { + $currentPath = [System.IO.Path]::Combine($currentPath, $segment) + if ([System.IO.Directory]::Exists($currentPath) -or [System.IO.File]::Exists($currentPath)) + { + $itemInfo = Get-Item -LiteralPath $currentPath -Force + if ($null -ne $itemInfo.LinkTarget) + { + throw "Evidence path must not traverse a symbolic link: $RelativePath" + } + } + } + + if (-not [System.IO.File]::Exists($candidatePath)) + { + throw "Evidence file does not exist: $RelativePath" + } + + return $candidatePath +} + +function New-MatchedLine +{ + param( + [Parameter(Mandatory = $true)][int]$LineNumber, + [Parameter(Mandatory = $true)][int]$PatternIndex, + [Parameter(Mandatory = $true)][string]$Line + ) + + return [ordered]@{ + line_number = $LineNumber + pattern_index = $PatternIndex + line_sha256 = Get-Sha256String -Value $Line + excerpt = Get-SafeExcerpt -Value $Line + } +} + +function Test-Log +{ + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string[]]$Values, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][int]$FailureAssociationWindowLines, + [System.Text.RegularExpressions.Regex[]]$Regexes + ) + + $matchedLines = [System.Collections.Generic.List[object]]::new() + $matchedLineNumbers = [System.Collections.Generic.List[int]]::new() + $failedTestLineNumbers = [System.Collections.Generic.List[int]]::new() + $lineNumber = 0 + $matchCount = 0 + $passOrSkipMatchCount = 0 + $regexTimeoutCount = 0 + $patternIndex = 0 + $matched = $false + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + $testNameMatcher = [System.Text.RegularExpressions.Regex]::new( + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking, + [System.TimeSpan]::FromMilliseconds(50)) + + foreach ($line in [System.IO.File]::ReadLines($Path)) + { + $lineNumber++ + + $normalizedLine = $line -replace "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+", "" + $lineContainsTest = $testNameMatcher.IsMatch($normalizedLine) + $lineIndicatesFailure = + $normalizedLine -match "(?i)^\s*\[FAIL(?:ED)?\]\s+" -or + $normalizedLine -match "(?i)^\s*Failed\s+" -or + $normalizedLine -match "(?i)^\s*\[[^\]\r\n]+\]\s+.+\s+\[FAIL(?:ED)?\]\s*$" + $isFailedTestMarker = $lineContainsTest -and $lineIndicatesFailure + if ($isFailedTestMarker) + { + $failedTestLineNumbers.Add($lineNumber) + } + + if ($normalizedLine -match "(?i)^\s*(?:\[(?:PASS|SKIP)\]\s+|Passed\s+|Skipped\s+)") + { + for ($index = 0; $index -lt $Values.Count; $index++) + { + try + { + $passOrSkipMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[$index].IsMatch($line) + } + else + { + $line.IndexOf($Values[$index], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $passOrSkipMatched = $false + } + + if ($passOrSkipMatched) + { + $passOrSkipMatchCount++ + break + } + } + } + + if ($isFailedTestMarker) + { + continue + } + + if ($Values.Count -eq 1) + { + try + { + $lineMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[0].IsMatch($line) + } + else + { + $line.IndexOf($Values[0], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $lineMatched = $false + } + + if ($lineMatched) + { + $matched = $true + $matchCount++ + $matchedLineNumbers.Add($lineNumber) + + if ($matchedLines.Count -lt 20) + { + $matchedLines.Add((New-MatchedLine -LineNumber $lineNumber -PatternIndex 0 -Line $line)) + } + } + + continue + } + + if ($matched) + { + continue + } + + try + { + $lineMatched = if ($Kind -eq "ErrorPattern") + { + $Regexes[$patternIndex].IsMatch($line) + } + else + { + $line.IndexOf($Values[$patternIndex], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $regexTimeoutCount++ + $lineMatched = $false + } + + if ($lineMatched) + { + $matchedLines.Add((New-MatchedLine -LineNumber $lineNumber -PatternIndex $patternIndex -Line $line)) + $matchedLineNumbers.Add($lineNumber) + + $patternIndex++ + if ($patternIndex -eq $Values.Count) + { + $matched = $true + $matchCount = 1 + } + } + } + + $signatureAssociatedWithFailedTest = + $matched -and + $failedTestLineNumbers.Count -gt 0 -and + @( + $matchedLineNumbers | + Where-Object { + $matchedLineNumber = $_ + @( + $failedTestLineNumbers | + Where-Object { [System.Math]::Abs($_ - $matchedLineNumber) -le $FailureAssociationWindowLines } + ).Count -gt 0 + } + ).Count -gt 0 + + return [ordered]@{ + line_count = $lineNumber + matched = $matched + match_count = $matchCount + pass_or_skip_match_count = $passOrSkipMatchCount + regex_timeout_count = $regexTimeoutCount + failed_test_detected = $failedTestLineNumbers.Count -gt 0 + failed_test_line_numbers = @($failedTestLineNumbers | Select-Object -First 20) + signature_associated_with_failed_test = $signatureAssociatedWithFailedTest + matched_lines = @($matchedLines) + } +} + +$candidatePath = (Resolve-Path -LiteralPath $CandidateFile).Path +$candidateSchemaPath = (Resolve-Path -LiteralPath $CandidateSchemaFile).Path +$receiptSchemaPath = (Resolve-Path -LiteralPath $ReceiptSchemaFile).Path +$evidenceRootPath = (Resolve-Path -LiteralPath $EvidenceRoot).Path +$repositoryRootPath = (Resolve-Path -LiteralPath $RepositoryRoot).Path +$outputPath = [System.IO.Path]::GetFullPath($OutputFile) + +if ($outputPath -in @($candidatePath, $candidateSchemaPath, $receiptSchemaPath)) +{ + throw "OutputFile must not overwrite an evaluator input." +} + +if ([System.IO.File]::Exists($outputPath)) +{ + [System.IO.File]::Delete($outputPath) +} + +$candidateJson = [System.IO.File]::ReadAllText($candidatePath) +if (-not ($candidateJson | Test-Json -SchemaFile $candidateSchemaPath)) +{ + throw "Candidate JSON does not satisfy the versioned schema." +} + +$candidateDocument = [System.Text.Json.JsonDocument]::Parse($candidateJson) +try +{ + $checkedUtcText = $candidateDocument.RootElement.GetProperty("duplicate_check").GetProperty("checked_utc").GetString() +} +finally +{ + $candidateDocument.Dispose() +} + +$candidate = $candidateJson | ConvertFrom-Json -Depth 32 +$repositoryHead = (& git -C $repositoryRootPath rev-parse HEAD 2>$null).Trim() +if ($LASTEXITCODE -ne 0 -or $repositoryHead -notmatch "^[0-9a-f]{40}$") +{ + throw "RepositoryRoot is not a readable Git repository." +} + +if (-not $repositoryHead.Equals([string]$candidate.repository_ref.commit_sha, [System.StringComparison]::Ordinal)) +{ + throw "Candidate repository commit does not match the checked-out repository." +} + +$expectedIssueUrl = "https://github.com/dotnet/aspnetcore/issues/$($candidate.issue.number)" +if (-not $expectedIssueUrl.Equals([string]$candidate.issue.url, [System.StringComparison]::Ordinal)) +{ + throw "Candidate issue URL does not match the issue number." +} + +$candidateSha256 = (Get-FileHash -LiteralPath $candidatePath -Algorithm SHA256).Hash.ToLowerInvariant() +$candidateSchemaSha256 = (Get-FileHash -LiteralPath $candidateSchemaPath -Algorithm SHA256).Hash.ToLowerInvariant() +$receiptSchemaSha256 = (Get-FileHash -LiteralPath $receiptSchemaPath -Algorithm SHA256).Hash.ToLowerInvariant() +$kind = [string]$candidate.signature.kind +$values = @($candidate.signature.values | ForEach-Object { [string]$_ }) +$qualityFailures = [System.Collections.Generic.List[string]]::new() +$incompleteReasons = [System.Collections.Generic.List[string]]::new() + +$testName = [string]$candidate.test.fully_qualified_name +$testLeafName = ($testName -split "\.")[-1] +if ($kind -eq "ErrorMessage") +{ + foreach ($value in $values) + { + $literalWithoutResultPrefix = $value -replace "^\s*\[(FAIL|PASS|SKIP)\]\s*", "" + if ($testName.IndexOf($literalWithoutResultPrefix, [System.StringComparison]::Ordinal) -ge 0 -or + $testLeafName.IndexOf($literalWithoutResultPrefix, [System.StringComparison]::Ordinal) -ge 0) + { + $qualityFailures.Add("The literal signature contains only the test identifier or a fragment of it.") + } + } +} + +$duplicateStatus = [string]$candidate.duplicate_check.status +$proposedClassification = [string]$candidate.proposed_classification +if ($duplicateStatus -eq "none" -and $proposedClassification -eq "reuse-existing-kbe") +{ + $qualityFailures.Add("The proposed classification requires an existing KBE, but the duplicate check reports none.") +} + +$issueReferences = @($candidate.duplicate_check.references | Where-Object { $_ -match "^issue:[1-9][0-9]*$" }) +$pullRequestReferences = @($candidate.duplicate_check.references | Where-Object { $_ -match "^pull-request:[1-9][0-9]*$" }) +if ($duplicateStatus -eq "existing-kbe" -and $issueReferences.Count -eq 0) +{ + $qualityFailures.Add("The duplicate check reports an existing KBE without an issue reference.") +} + +if ($duplicateStatus -eq "existing-fix-pr" -and $pullRequestReferences.Count -eq 0) +{ + $qualityFailures.Add("The duplicate check reports an existing fix PR without a pull request reference.") +} + +$checkedUtc = [System.DateTimeOffset]::MinValue +if (-not [System.DateTimeOffset]::TryParse( + $checkedUtcText, + [System.Globalization.CultureInfo]::InvariantCulture, + [System.Globalization.DateTimeStyles]::None, + [ref]$checkedUtc)) +{ + throw "Duplicate check timestamp is invalid." +} + +$now = [System.DateTimeOffset]::UtcNow +$checkedUtc = $checkedUtc.ToUniversalTime() +if ($checkedUtc -gt $now.AddMinutes(5)) +{ + $qualityFailures.Add("The duplicate check timestamp is in the future.") +} +elseif ($checkedUtc -lt $now.AddHours(-24)) +{ + $incompleteReasons.Add("The duplicate check is older than 24 hours.") +} + +$regexes = @() +if ($kind -eq "ErrorPattern") +{ + $regexOptions = [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking + $regexTimeout = [System.TimeSpan]::FromMilliseconds(50) + + foreach ($value in $values) + { + if ($value -in @(".*", ".+", "^.*$", "^.+$")) + { + $qualityFailures.Add("The regex signature is unbounded and matches arbitrary text.") + } + + try + { + $regexes += [System.Text.RegularExpressions.Regex]::new($value, $regexOptions, $regexTimeout) + } + catch + { + $qualityFailures.Add("The regex is not compatible with Build Analysis matching: $($_.Exception.Message)") + } + } + + foreach ($regex in $regexes) + { + $bareTestProbes = @( + $testName, + $testLeafName, + "[FAIL] $testName", + "[PASS] $testName", + "[SKIP] $testName" + ) + foreach ($probe in $bareTestProbes) + { + try + { + if ($regex.IsMatch($probe)) + { + $qualityFailures.Add("The regex signature matches the test identifier or a fragment of it.") + break + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + $qualityFailures.Add("The regex exceeded the Build Analysis timeout while checking signature specificity.") + break + } + } + } +} + +$logResults = [System.Collections.Generic.List[object]]::new() +$failureLogCount = 0 +$negativeLogCount = 0 +$failureLogsMatched = 0 +$negativeCollisionCount = 0 +$passOrSkipCollisionCount = 0 +$failureHashes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$negativeHashes = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$failureBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$negativeBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$seenLogIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$seenLogPaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + +foreach ($log in $candidate.evidence.raw_logs) +{ + if (-not $seenLogIds.Add([string]$log.id)) + { + throw "Duplicate evidence id: $($log.id)." + } + + if (-not $seenLogPaths.Add([string]$log.path)) + { + throw "Duplicate evidence path: $($log.path)." + } + + $resolvedLogPath = Resolve-EvidencePath -Root $evidenceRootPath -RelativePath ([string]$log.path) + $actualHash = (Get-FileHash -LiteralPath $resolvedLogPath -Algorithm SHA256).Hash.ToLowerInvariant() + if (-not $actualHash.Equals([string]$log.sha256, [System.StringComparison]::Ordinal)) + { + throw "Evidence hash mismatch for $($log.id)." + } + + $match = if ($regexes.Count -eq $values.Count -or $kind -eq "ErrorMessage") + { + Test-Log ` + -Path $resolvedLogPath ` + -Kind $kind ` + -Values $values ` + -TestName ([string]$candidate.test.fully_qualified_name) ` + -FailureAssociationWindowLines $failureAssociationWindowLines ` + -Regexes $regexes + } + else + { + [ordered]@{ + line_count = @([System.IO.File]::ReadLines($resolvedLogPath)).Count + matched = $false + match_count = 0 + pass_or_skip_match_count = 0 + regex_timeout_count = 0 + failed_test_detected = $false + failed_test_line_numbers = @() + signature_associated_with_failed_test = $false + matched_lines = @() + } + } + + if ($log.role -eq "failure") + { + $failureLogCount++ + $null = $failureHashes.Add($actualHash) + $null = $failureBuildIds.Add([int]$log.build.id) + if ($match.matched) + { + $failureLogsMatched++ + } + + if (-not $match.failed_test_detected) + { + $qualityFailures.Add("Failure log '$($log.id)' does not contain a supported failed-test marker for the declared test.") + } + elseif ($match.matched -and -not $match.signature_associated_with_failed_test) + { + $qualityFailures.Add("The signature match in failure log '$($log.id)' is not within $failureAssociationWindowLines lines of the declared test failure.") + } + } + else + { + $negativeLogCount++ + $null = $negativeHashes.Add($actualHash) + $null = $negativeBuildIds.Add([int]$log.build.id) + if ($match.matched) + { + $negativeCollisionCount++ + } + } + + $passOrSkipCollisionCount += [int]$match.pass_or_skip_match_count + + $logResults.Add([ordered]@{ + id = [string]$log.id + role = [string]$log.role + outcome = [string]$log.outcome + path = [string]$log.path + source_url = [string]$log.source_url + sha256 = $actualHash + build = $log.build + line_count = [int]$match.line_count + matched = [bool]$match.matched + match_count = [int]$match.match_count + pass_or_skip_match_count = [int]$match.pass_or_skip_match_count + regex_timeout_count = [int]$match.regex_timeout_count + failed_test_detected = [bool]$match.failed_test_detected + failed_test_line_numbers = @($match.failed_test_line_numbers) + signature_associated_with_failed_test = [bool]$match.signature_associated_with_failed_test + matched_lines = @($match.matched_lines) + }) +} + +$requiredFailureLogs = [System.Math]::Max( + $minimumFailureEvidenceFloor, + [int]$candidate.policy.minimum_failure_logs) +$requiredNegativeLogs = [System.Math]::Max( + $minimumNegativeEvidenceFloor, + [int]$candidate.policy.minimum_negative_logs) + +if ($failureHashes.Count -lt $requiredFailureLogs) +{ + $incompleteReasons.Add("Only $($failureHashes.Count) distinct failure log(s) were supplied; $requiredFailureLogs are required.") +} + +if ($failureBuildIds.Count -lt $requiredFailureLogs) +{ + $incompleteReasons.Add("Only $($failureBuildIds.Count) distinct failure build(s) were supplied; $requiredFailureLogs are required.") +} + +if ($negativeHashes.Count -lt $requiredNegativeLogs) +{ + $incompleteReasons.Add("Only $($negativeHashes.Count) distinct negative log(s) were supplied; $requiredNegativeLogs are required.") +} + +if ($negativeBuildIds.Count -lt $requiredNegativeLogs) +{ + $incompleteReasons.Add("Only $($negativeBuildIds.Count) distinct negative build(s) were supplied; $requiredNegativeLogs are required.") +} + +if (@($failureHashes | Where-Object { $negativeHashes.Contains($_) }).Count -gt 0) +{ + $qualityFailures.Add("The same log content was supplied as both failure and negative evidence.") +} + +if ($failureLogsMatched -ne $failureLogCount) +{ + $qualityFailures.Add("The signature did not match every supplied failure log.") +} + +if ($negativeCollisionCount -gt 0) +{ + $qualityFailures.Add("The signature matched $negativeCollisionCount negative log(s).") +} + +if ($passOrSkipCollisionCount -gt 0) +{ + $qualityFailures.Add("The signature matched $passOrSkipCollisionCount pass or skip line(s).") +} + +$totalRegexTimeoutCount = 0 +foreach ($logResult in $logResults) +{ + $totalRegexTimeoutCount += [int]$logResult.regex_timeout_count +} + +if ($totalRegexTimeoutCount -gt 0) +{ + $qualityFailures.Add("The regex exceeded the Build Analysis timeout on at least one line.") +} + +$coverage = $candidate.duplicate_check.coverage +$requiredDuplicateCategories = @( + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" +) +$completeDuplicateCategories = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +foreach ($query in $candidate.duplicate_check.queries) +{ + if ([bool]$query.complete) + { + $null = $completeDuplicateCategories.Add([string]$query.category) + } +} + +$kbeQueryNumbers = @( + $candidate.duplicate_check.queries | + Where-Object { $_.category -in @("open-kbe", "recently-closed-kbe") } | + ForEach-Object { $_.result_numbers } +) +$fixPrQueryNumbers = @( + $candidate.duplicate_check.queries | + Where-Object { $_.category -in @("open-fix-pr", "recently-merged-fix-pr") } | + ForEach-Object { $_.result_numbers } +) + +if ($duplicateStatus -eq "existing-kbe") +{ + $referencedKbeNumbers = @($issueReferences | ForEach-Object { [int]($_ -replace "^issue:", "") }) + if (@($referencedKbeNumbers | Where-Object { $_ -in $kbeQueryNumbers }).Count -eq 0) + { + $qualityFailures.Add("The existing KBE reference is absent from the recorded KBE query results.") + } +} + +if ($duplicateStatus -eq "existing-fix-pr") +{ + $referencedFixPrNumbers = @($pullRequestReferences | ForEach-Object { [int]($_ -replace "^pull-request:", "") }) + if (@($referencedFixPrNumbers | Where-Object { $_ -in $fixPrQueryNumbers }).Count -eq 0) + { + $qualityFailures.Add("The existing fix PR reference is absent from the recorded PR query results.") + } +} + +if ($duplicateStatus -eq "none" -and + ($candidate.duplicate_check.references.Count -gt 0 -or + $kbeQueryNumbers.Count -gt 0 -or + $fixPrQueryNumbers.Count -gt 0)) +{ + $qualityFailures.Add("The duplicate check reports no duplicate despite recorded issue or pull request results.") +} + +$duplicateCoverageComplete = + [bool]$coverage.open_kbes -and + [bool]$coverage.recently_closed_kbes -and + [bool]$coverage.open_fix_prs -and + [bool]$coverage.recently_merged_fix_prs -and + (@($candidate.duplicate_check.queries | Where-Object { -not $_.complete }).Count -eq 0) -and + (@($requiredDuplicateCategories | Where-Object { -not $completeDuplicateCategories.Contains($_) }).Count -eq 0) + +if (-not $duplicateCoverageComplete -or $candidate.duplicate_check.status -eq "not-evaluated") +{ + $incompleteReasons.Add("Duplicate KBE and fix PR coverage is incomplete.") +} + +$deterministicStatus = if ($qualityFailures.Count -gt 0) +{ + "rejected" +} +elseif ($incompleteReasons.Count -gt 0) +{ + "incomplete" +} +else +{ + "validated" +} + +$shadowRecommendation = "human-review" +if ($deterministicStatus -eq "incomplete") +{ + $shadowRecommendation = "insufficient-evidence" +} +elseif ($deterministicStatus -eq "validated") +{ + $shadowRecommendation = switch ($duplicateStatus) + { + "existing-kbe" { "reuse-existing-kbe"; break } + "existing-fix-pr" { "existing-fix-pr"; break } + "ambiguous" { "human-review"; break } + "integrity-filtered" { "human-review"; break } + default { $proposedClassification } + } +} + +$eligibleForKbeEnrichment = $false + +$reasons = @($qualityFailures) + @($incompleteReasons) +if ($reasons.Count -eq 0) +{ + $reasons = @("The candidate passed deterministic signature, evidence, and duplicate-coverage gates.") +} + +$receipt = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = $candidate.repository_ref + generated_utc = [System.DateTimeOffset]::UtcNow.ToString("O") + evaluator = [ordered]@{ + name = "Evaluate-TestQuarantineKbeCandidate.ps1" + version = 1 + matcher = "Build Analysis compatible signature matcher with failed-test association" + failure_association_window_lines = $failureAssociationWindowLines + candidate_sha256 = $candidateSha256 + candidate_schema_sha256 = $candidateSchemaSha256 + receipt_schema_sha256 = $receiptSchemaSha256 + } + issue = $candidate.issue + test = $candidate.test + signature = $candidate.signature + policy = $candidate.policy + evidence = [ordered]@{ + failure_log_count = $failureLogCount + negative_log_count = $negativeLogCount + distinct_failure_log_count = $failureHashes.Count + distinct_negative_log_count = $negativeHashes.Count + distinct_failure_build_count = $failureBuildIds.Count + distinct_negative_build_count = $negativeBuildIds.Count + all_failure_logs_matched = $failureLogCount -gt 0 -and $failureLogsMatched -eq $failureLogCount + negative_collision_count = $negativeCollisionCount + pass_or_skip_collision_count = $passOrSkipCollisionCount + logs = @($logResults) + corroborating_context = @($candidate.evidence.corroborating_context) + } + duplicate_check = $candidate.duplicate_check + agent_proposed_classification = $proposedClassification + deterministic_status = $deterministicStatus + shadow_recommendation = $shadowRecommendation + eligible_for_kbe_enrichment = $eligibleForKbeEnrichment + evidence_provenance_verified = $false + human_review_required = $true + zero_remote_writes = $true + reasons = @($reasons | Select-Object -Unique) +} + +$receiptJson = $receipt | ConvertTo-Json -Depth 32 +if (-not ($receiptJson | Test-Json -SchemaFile $receiptSchemaPath)) +{ + throw "Generated receipt does not satisfy the versioned schema." +} + +$outputDirectory = Split-Path -Parent $outputPath +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} + +[System.IO.File]::WriteAllText($outputPath, $receiptJson + [System.Environment]::NewLine) +Write-Host "Wrote $deterministicStatus shadow receipt to $outputPath" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 new file mode 100644 index 000000000000..c28ced296daf --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -0,0 +1,733 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$evaluator = "$PSScriptRoot/Evaluate-TestQuarantineKbeCandidate.ps1" +$candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json" +$receiptSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-receipt.schema.json" +$repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path +$repositoryHead = (& git -C $repositoryRoot rev-parse HEAD).Trim() +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-$([System.Guid]::NewGuid().ToString('N'))" +$outsideRoot = "$tempRoot-outside" +$symbolicLinkPath = Join-Path $tempRoot "linked-evidence" + +function Assert-Equal +{ + param( + [Parameter(Mandatory = $true)]$Actual, + [Parameter(Mandatory = $true)]$Expected, + [Parameter(Mandatory = $true)][string]$Message + ) + + if ($Actual -ne $Expected) + { + throw "$Message Expected '$Expected', actual '$Actual'." + } +} + +function Write-Candidate +{ + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$Signature, + [Parameter(Mandatory = $true)][object[]]$Logs, + [string]$SignatureKind = "ErrorMessage", + [string]$DuplicateStatus = "none", + [bool]$CompleteCoverage = $true, + [string]$ProposedClassification = "new-kbe-candidate" + ) + + $candidate = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = [ordered]@{ + branch = "main" + commit_sha = $repositoryHead + } + issue = [ordered]@{ + number = 12345 + url = "https://github.com/dotnet/aspnetcore/issues/12345" + } + test = [ordered]@{ + fully_qualified_name = "Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + } + signature = [ordered]@{ + kind = $SignatureKind + values = $Signature + build_retry = $false + exclude_console_log = $false + } + policy = [ordered]@{ + minimum_failure_logs = 2 + minimum_negative_logs = 1 + } + evidence = [ordered]@{ + raw_logs = $Logs + corroborating_context = @( + [ordered]@{ + source = "quarantine-issue" + url = "https://github.com/dotnet/aspnetcore/issues/12345" + } + ) + } + duplicate_check = [ordered]@{ + status = $DuplicateStatus + checked_utc = [System.DateTimeOffset]::UtcNow.ToString("O") + coverage = [ordered]@{ + open_kbes = $CompleteCoverage + recently_closed_kbes = $CompleteCoverage + open_fix_prs = $CompleteCoverage + recently_merged_fix_prs = $CompleteCoverage + } + references = @() + queries = @( + [ordered]@{ + category = "open-kbe" + query = "repo:dotnet/aspnetcore is:issue label:`"Known Build Error`" SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "recently-closed-kbe" + query = "repo:dotnet/aspnetcore is:issue is:closed SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "open-fix-pr" + query = "repo:dotnet/aspnetcore is:pr is:open SampleTests" + complete = $CompleteCoverage + result_numbers = @() + }, + [ordered]@{ + category = "recently-merged-fix-pr" + query = "repo:dotnet/aspnetcore is:pr is:merged SampleTests" + complete = $CompleteCoverage + result_numbers = @() + } + ) + } + proposed_classification = $ProposedClassification + } + + $candidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $Path +} + +function New-LogEntry +{ + param( + [Parameter(Mandatory = $true)][string]$Id, + [Parameter(Mandatory = $true)][string]$Role, + [Parameter(Mandatory = $true)][string]$Outcome, + [Parameter(Mandatory = $true)][string]$Path + ) + + return [ordered]@{ + id = $Id + role = $Role + outcome = $Outcome + path = $Path + source_url = "https://example.invalid/$Path" + sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot $Path) -Algorithm SHA256).Hash.ToLowerInvariant() + build = [ordered]@{ + id = switch ($Id) + { + "failure-1" { 1001; break } + "failure-2" { 1002; break } + default { 1003 } + } + pipeline_definition_id = 83 + source_version = switch ($Id) + { + "failure-1" { "2222222222222222222222222222222222222222"; break } + "failure-2" { "3333333333333333333333333333333333333333"; break } + default { "4444444444444444444444444444444444444444" } + } + started_utc = "2026-08-20T12:00:00Z" + platform = "Linux" + configuration = "Release" + } + } +} + +try +{ + [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null + $signature = "Expected completion signal before deterministic deadline" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "2026-08-22T04:28:28.1901712Z [xUnit.net 00:48:54.05] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [FAIL]" + "2026-08-22T04:28:28.1991498Z Xunit.Sdk.TrueException: $signature" + ) + 1..60 | ForEach-Object { Add-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value "Diagnostic line $_" } + Add-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value "Repeated diagnostic: $signature" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[PASS] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Finished normally" + ) + + $logs = @( + (New-LogEntry -Id "failure-1" -Role "failure" -Outcome "failed" -Path "failure-1.log"), + (New-LogEntry -Id "failure-2" -Role "failure" -Outcome "failed" -Path "failure-2.log"), + (New-LogEntry -Id "negative" -Role "negative" -Outcome "passed" -Path "negative.log") + ) + + $candidatePath = Join-Path $tempRoot "candidate.json" + $receiptPath = Join-Path $tempRoot "receipt.json" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Valid candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "new-kbe-candidate" -Message "Valid candidate recommendation mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Shadow evaluator must not authorize enrichment from unverified provenance." + Assert-Equal -Actual $receipt.evidence_provenance_verified -Expected $false -Message "Shadow evidence provenance must remain unverified." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Starting an unrelated test" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Unassociated failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[1].failed_test_detected -Expected $false -Message "Unassociated failure evidence marker mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Add-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "[SKIP] $signature" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Collision candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Collision candidate recommendation mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 1 -Message "Pass/skip collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "Passed Other.Tests.Completes [1 ms] $signature" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "VSTest pass collision status mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 1 -Message "VSTest pass collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value "[PASS] SampleTests.Completes" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + $multiValueSignature = @( + "Expected completion signal", + "before deterministic deadline" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + $multiValueSignature[0] + $multiValueSignature[1] + "[PASS] $($multiValueSignature[0])" + "[PASS] $($multiValueSignature[1])" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + $multiValueSignature[0] + $multiValueSignature[1] + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature $multiValueSignature -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Multi-value pass collision status mismatch." + Assert-Equal -Actual $receipt.evidence.pass_or_skip_collision_count -Expected 2 -Message "Multi-value pass collision count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs -CompleteCoverage $false + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Incomplete duplicate check status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "insufficient-evidence" -Message "Incomplete duplicate check recommendation mismatch." + + Copy-Item -LiteralPath (Join-Path $tempRoot "failure-1.log") -Destination (Join-Path $tempRoot "failure-2.log") -Force + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Duplicate failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_log_count -Expected 1 -Message "Distinct failure evidence count mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].build.id = $logs[0].build.id + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Same-build failure evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_log_count -Expected 2 -Message "Same-build distinct log count mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_failure_build_count -Expected 1 -Message "Same-build distinct build count mismatch." + $logs[1].build.id = 1002 + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("Microsoft.AspNetCore.Example.Tests.SampleTests.Completes") ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Bare test identifier status mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Bare test identifier eligibility mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("SampleTests.Completes") ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Test identifier fragment status mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("SampleTests\.Completes") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Regex test identifier fragment status mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.CompletesAsync" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.CompletesAsync [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Prefixed test name association status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[0].failed_test_detected -Expected $false -Message "Prefixed test name was treated as the declared test." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "Exception message: `"[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes`"" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Diagnostic text mentions [FAILED] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Embedded failure marker status mismatch." + Assert-Equal -Actual $receipt.evidence.logs[0].failed_test_detected -Expected $false -Message "Embedded failure text was treated as a failed-test record." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Write-Candidate ` + -Path $candidatePath ` + -Signature @("^Xunit\.Sdk\.TrueException: Expected completion signal before deterministic deadline$") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Regex candidate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "new-kbe-candidate" -Message "Regex candidate recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -DuplicateStatus "existing-kbe" ` + -ProposedClassification "reuse-existing-kbe" + + $existingCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $existingCandidate.duplicate_check.references = @("issue:54321") + $existingCandidate.duplicate_check.queries[0].result_numbers = @(54321) + $existingCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Existing KBE status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "reuse-existing-kbe" -Message "Existing KBE recommendation mismatch." + + $existingCandidate.duplicate_check.queries[0].result_numbers = @() + $existingCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Unsubstantiated existing KBE reference status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Unsubstantiated existing KBE reference recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -ProposedClassification "reuse-existing-kbe" + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Inconsistent duplicate classification status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Inconsistent duplicate classification recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $contradictoryDuplicateCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $contradictoryDuplicateCandidate.duplicate_check.queries[0].result_numbers = @(54321) + $contradictoryDuplicateCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Contradictory no-duplicate status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "human-review" -Message "Contradictory no-duplicate recommendation mismatch." + + $testName = "Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] $testName" + "First unrelated failure" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "[FAIL] $testName" + "Second unrelated failure" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate ` + -Path $candidatePath ` + -Signature @("Microsoft\.AspNetCore\.Example\.Tests\.SampleTests\.Completes") ` + -SignatureKind "ErrorPattern" ` + -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "rejected" -Message "Regex-escaped test identifier status mismatch." + Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Regex-escaped test identifier eligibility mismatch." + + Set-Content -LiteralPath (Join-Path $tempRoot "failure-1.log") -Value @( + "[FAIL] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Xunit.Sdk.TrueException: $signature" + ) + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( + "Failed Microsoft.AspNetCore.Example.Tests.SampleTests.Completes [2 s]" + "Xunit.Sdk.TrueException: $signature" + ) + $logs[0].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-1.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[1].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "failure-2.log") -Algorithm SHA256).Hash.ToLowerInvariant() + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $staleCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $staleCandidate.duplicate_check.checked_utc = [System.DateTimeOffset]::UtcNow.AddHours(-30).ToString("o") + $staleCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Stale duplicate search status mismatch." + Assert-Equal -Actual $receipt.shadow_recommendation -Expected "insufficient-evidence" -Message "Stale duplicate search recommendation mismatch." + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $wrongRepositoryCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $wrongRepositoryCandidate.repository_ref.commit_sha = "1111111111111111111111111111111111111111" + $wrongRepositoryCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $repositoryMismatchRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $repositoryMismatchRejected = $_.Exception.Message -match "repository commit does not match" + } + + Assert-Equal -Actual $repositoryMismatchRejected -Expected $true -Message "Repository commit mismatch was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived repository rejection." + + [System.IO.Directory]::CreateDirectory($outsideRoot) | Out-Null + $outsideEvidencePath = Join-Path $outsideRoot "failure.log" + Copy-Item -LiteralPath (Join-Path $tempRoot "failure-1.log") -Destination $outsideEvidencePath + $symbolicLinkCreated = $false + try + { + [System.IO.Directory]::CreateSymbolicLink($symbolicLinkPath, $outsideRoot) | Out-Null + $symbolicLinkCreated = $true + } + catch + { + Write-Warning "Skipping symbolic-link regression because the platform denied link creation." + } + + if ($symbolicLinkCreated) + { + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $linkedCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $linkedCandidate.evidence.raw_logs[0].path = "linked-evidence/failure.log" + $linkedCandidate.evidence.raw_logs[0].sha256 = (Get-FileHash -LiteralPath $outsideEvidencePath -Algorithm SHA256).Hash.ToLowerInvariant() + $linkedCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $symbolicLinkRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $symbolicLinkRejected = $_.Exception.Message -match "must not traverse a symbolic link" + } + + Assert-Equal -Actual $symbolicLinkRejected -Expected $true -Message "A symlinked evidence parent was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived symbolic-link rejection." + } + + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs + $tamperedCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $tamperedCandidate.evidence.raw_logs[0].sha256 = "0" * 64 + $tamperedCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + $hashMismatchRejected = $false + try + { + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + } + catch + { + $hashMismatchRejected = $_.Exception.Message -match "Evidence hash mismatch" + } + + Assert-Equal -Actual $hashMismatchRejected -Expected $true -Message "Tampered evidence was not rejected." + Assert-Equal -Actual (Test-Path -LiteralPath $receiptPath) -Expected $false -Message "A stale receipt survived evidence rejection." + + Write-Host "All test-quarantine KBE shadow evaluator tests passed." +} +finally +{ + if (Test-Path -LiteralPath $symbolicLinkPath) + { + Remove-Item -LiteralPath $symbolicLinkPath -Force + } + + if (Test-Path -LiteralPath $tempRoot) + { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } + + if (Test-Path -LiteralPath $outsideRoot) + { + Remove-Item -LiteralPath $outsideRoot -Recurse -Force + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json new file mode 100644 index 000000000000..8b132d1a5737 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -0,0 +1,466 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow candidate", + "description": "Untrusted candidate input proposed for deterministic Known Build Error signature evaluation. Source metadata remains unverified until a trusted collector is added.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "repository_ref", + "issue", + "test", + "signature", + "policy", + "evidence", + "duplicate_check", + "proposed_classification" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "repository_ref": { + "type": "object", + "additionalProperties": false, + "required": [ + "branch", + "commit_sha" + ], + "properties": { + "branch": { + "const": "main" + }, + "commit_sha": { + "$ref": "#/$defs/gitSha" + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + } + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "fully_qualified_name" + ], + "properties": { + "fully_qualified_name": { + "type": "string", + "minLength": 3, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "values", + "build_retry", + "exclude_console_log" + ], + "properties": { + "kind": { + "enum": [ + "ErrorMessage", + "ErrorPattern" + ] + }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 5, + "items": { + "type": "string", + "minLength": 8, + "maxLength": 2048, + "pattern": "^[^\\r\\n]+$" + } + }, + "build_retry": { + "type": "boolean" + }, + "exclude_console_log": { + "type": "boolean" + } + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "minimum_failure_logs", + "minimum_negative_logs" + ], + "properties": { + "minimum_failure_logs": { + "type": "integer", + "minimum": 2, + "maximum": 10 + }, + "minimum_negative_logs": { + "type": "integer", + "minimum": 1, + "maximum": 20 + } + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "raw_logs", + "corroborating_context" + ], + "properties": { + "raw_logs": { + "type": "array", + "minItems": 1, + "maxItems": 30, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "role", + "outcome", + "path", + "source_url", + "sha256", + "build" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$" + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "outcome": { + "enum": [ + "failed", + "passed", + "skipped", + "unrelated" + ] + }, + "path": { + "type": "string", + "pattern": "^(?!.*(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$))[A-Za-z0-9][A-Za-z0-9_.\\/-]{0,255}$" + }, + "source_url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "build": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "pipeline_definition_id", + "source_version", + "started_utc", + "platform", + "configuration" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "pipeline_definition_id": { + "type": "integer", + "minimum": 1 + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "configuration": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "role": { + "const": "failure" + } + } + }, + "then": { + "properties": { + "outcome": { + "const": "failed" + } + } + } + }, + { + "if": { + "properties": { + "role": { + "const": "negative" + } + } + }, + "then": { + "properties": { + "outcome": { + "enum": [ + "passed", + "skipped", + "unrelated" + ] + } + } + } + } + ] + } + }, + "corroborating_context": { + "type": "array", + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "url" + ], + "properties": { + "source": { + "enum": [ + "build-analysis", + "build-insights", + "github-check", + "quarantine-issue" + ] + }, + "url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + } + } + } + } + } + }, + "duplicate_check": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + }, + "complete": { + "type": "boolean" + }, + "result_numbers": { + "type": "array", + "maxItems": 50, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "existing-kbe" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^issue:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "existing-fix-pr" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^pull-request:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + } + ] + }, + "proposed_classification": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json new file mode 100644 index 000000000000..ea94e43efe2c --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -0,0 +1,758 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow receipt", + "description": "Read-only deterministic evaluation receipt for one proposed Known Build Error signature.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "repository_ref", + "generated_utc", + "evaluator", + "issue", + "test", + "signature", + "policy", + "evidence", + "duplicate_check", + "agent_proposed_classification", + "deterministic_status", + "shadow_recommendation", + "eligible_for_kbe_enrichment", + "evidence_provenance_verified", + "human_review_required", + "zero_remote_writes", + "reasons" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "repository_ref": { + "$ref": "#/$defs/repositoryRef" + }, + "generated_utc": { + "type": "string", + "format": "date-time" + }, + "evaluator": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "matcher", + "failure_association_window_lines", + "candidate_sha256", + "candidate_schema_sha256", + "receipt_schema_sha256" + ], + "properties": { + "name": { + "const": "Evaluate-TestQuarantineKbeCandidate.ps1" + }, + "version": { + "const": 1 + }, + "matcher": { + "const": "Build Analysis compatible signature matcher with failed-test association" + }, + "failure_association_window_lines": { + "const": 50 + }, + "candidate_sha256": { + "$ref": "#/$defs/sha256" + }, + "candidate_schema_sha256": { + "$ref": "#/$defs/sha256" + }, + "receipt_schema_sha256": { + "$ref": "#/$defs/sha256" + } + } + }, + "issue": { + "$ref": "#/$defs/issue" + }, + "test": { + "$ref": "#/$defs/test" + }, + "signature": { + "$ref": "#/$defs/signature" + }, + "policy": { + "$ref": "#/$defs/policy" + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "failure_log_count", + "negative_log_count", + "distinct_failure_log_count", + "distinct_negative_log_count", + "distinct_failure_build_count", + "distinct_negative_build_count", + "all_failure_logs_matched", + "negative_collision_count", + "pass_or_skip_collision_count", + "logs", + "corroborating_context" + ], + "properties": { + "failure_log_count": { + "type": "integer", + "minimum": 0 + }, + "negative_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_failure_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_negative_log_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_failure_build_count": { + "type": "integer", + "minimum": 0 + }, + "distinct_negative_build_count": { + "type": "integer", + "minimum": 0 + }, + "all_failure_logs_matched": { + "type": "boolean" + }, + "negative_collision_count": { + "type": "integer", + "minimum": 0 + }, + "pass_or_skip_collision_count": { + "type": "integer", + "minimum": 0 + }, + "logs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "role", + "outcome", + "path", + "source_url", + "sha256", + "build", + "line_count", + "matched", + "match_count", + "pass_or_skip_match_count", + "regex_timeout_count", + "failed_test_detected", + "failed_test_line_numbers", + "signature_associated_with_failed_test", + "matched_lines" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$" + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "outcome": { + "enum": [ + "failed", + "passed", + "skipped", + "unrelated" + ] + }, + "path": { + "type": "string", + "pattern": "^(?!.*(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$))[A-Za-z0-9][A-Za-z0-9_.\\/-]{0,255}$" + }, + "source_url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "build": { + "$ref": "#/$defs/build" + }, + "line_count": { + "type": "integer", + "minimum": 0 + }, + "matched": { + "type": "boolean" + }, + "match_count": { + "type": "integer", + "minimum": 0 + }, + "pass_or_skip_match_count": { + "type": "integer", + "minimum": 0 + }, + "regex_timeout_count": { + "type": "integer", + "minimum": 0 + }, + "failed_test_detected": { + "type": "boolean" + }, + "failed_test_line_numbers": { + "type": "array", + "maxItems": 20, + "items": { + "type": "integer", + "minimum": 1 + } + }, + "signature_associated_with_failed_test": { + "type": "boolean" + }, + "matched_lines": { + "type": "array", + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "line_number", + "pattern_index", + "line_sha256", + "excerpt" + ], + "properties": { + "line_number": { + "type": "integer", + "minimum": 1 + }, + "pattern_index": { + "type": "integer", + "minimum": 0 + }, + "line_sha256": { + "$ref": "#/$defs/sha256" + }, + "excerpt": { + "type": "string", + "maxLength": 300 + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "role": { + "const": "failure" + } + } + }, + "then": { + "properties": { + "outcome": { + "const": "failed" + } + } + } + }, + { + "if": { + "properties": { + "role": { + "const": "negative" + } + } + }, + "then": { + "properties": { + "outcome": { + "enum": [ + "passed", + "skipped", + "unrelated" + ] + } + } + } + } + ] + } + }, + "corroborating_context": { + "type": "array", + "maxItems": 20, + "items": { + "$ref": "#/$defs/corroboratingContext" + } + } + } + }, + "duplicate_check": { + "$ref": "#/$defs/duplicateCheck" + }, + "agent_proposed_classification": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + }, + "deterministic_status": { + "enum": [ + "validated", + "rejected", + "incomplete" + ] + }, + "shadow_recommendation": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate", + "existing-fix-pr", + "quarantine-only", + "infrastructure", + "timeout-needs-classification", + "insufficient-evidence", + "human-review" + ] + }, + "eligible_for_kbe_enrichment": { + "description": "Version 1 never authorizes repository mutation because evidence provenance is not independently authenticated.", + "const": false + }, + "evidence_provenance_verified": { + "description": "False until a trusted deterministic collector binds authoritative remote artifact metadata to the local evidence bundle.", + "const": false + }, + "human_review_required": { + "const": true + }, + "zero_remote_writes": { + "const": true + }, + "reasons": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 512 + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": true + } + } + }, + "then": { + "properties": { + "deterministic_status": { + "const": "validated" + }, + "shadow_recommendation": { + "enum": [ + "reuse-existing-kbe", + "new-kbe-candidate" + ] + } + } + } + }, + { + "if": { + "properties": { + "deterministic_status": { + "const": "incomplete" + } + } + }, + "then": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": false + }, + "shadow_recommendation": { + "const": "insufficient-evidence" + } + } + } + }, + { + "if": { + "properties": { + "deterministic_status": { + "const": "rejected" + } + } + }, + "then": { + "properties": { + "eligible_for_kbe_enrichment": { + "const": false + }, + "shadow_recommendation": { + "const": "human-review" + } + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "repositoryRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "branch", + "commit_sha" + ], + "properties": { + "branch": { + "const": "main" + }, + "commit_sha": { + "$ref": "#/$defs/gitSha" + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + } + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "fully_qualified_name" + ], + "properties": { + "fully_qualified_name": { + "type": "string", + "minLength": 3, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "values", + "build_retry", + "exclude_console_log" + ], + "properties": { + "kind": { + "enum": [ + "ErrorMessage", + "ErrorPattern" + ] + }, + "values": { + "type": "array", + "minItems": 1, + "maxItems": 5, + "items": { + "type": "string", + "minLength": 8, + "maxLength": 2048, + "pattern": "^[^\\r\\n]+$" + } + }, + "build_retry": { + "type": "boolean" + }, + "exclude_console_log": { + "type": "boolean" + } + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "minimum_failure_logs", + "minimum_negative_logs" + ], + "properties": { + "minimum_failure_logs": { + "type": "integer", + "minimum": 2, + "maximum": 10 + }, + "minimum_negative_logs": { + "type": "integer", + "minimum": 1, + "maximum": 20 + } + } + }, + "build": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "pipeline_definition_id", + "source_version", + "started_utc", + "platform", + "configuration" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "pipeline_definition_id": { + "type": "integer", + "minimum": 1 + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, + "configuration": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + } + } + }, + "corroboratingContext": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "url" + ], + "properties": { + "source": { + "enum": [ + "build-analysis", + "build-insights", + "github-check", + "quarantine-issue" + ] + }, + "url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "pattern": "^https://" + } + } + }, + "duplicateCheck": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "^[^\\r\\n]+$" + }, + "complete": { + "type": "boolean" + }, + "result_numbers": { + "type": "array", + "maxItems": 50, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "existing-kbe" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^issue:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "existing-fix-pr" + } + } + }, + "then": { + "properties": { + "references": { + "contains": { + "type": "string", + "pattern": "^pull-request:[1-9][0-9]*$" + }, + "minContains": 1 + } + } + } + } + ] + } + } +} From 37b005ec90903f39117f58615949ea17e8dc5f38 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:59:43 -0500 Subject: [PATCH 02/10] Add end-to-end shadow collector, workflow, and CI for test quarantine KBE Extends the previously unused shadow evaluator into a working, read-only shadow feature: - Collect-TestQuarantineKbeEvidence.ps1: deterministic collector for one open, canonical test-failure quarantine issue. Parses the issue body, resolves Azure DevOps build metadata, performs a capped supplementary recurrence/negative-evidence scan, records Build Analysis check-run snapshots as corroborating-only evidence, runs a categorized duplicate KBE/fix-PR search, and emits either a candidate conforming to the existing (unmodified) candidate schema or a structured "incomplete" outcome. Never infers a pass, recurrence, or signature from missing or expired evidence; supports an offline -FixtureRoot mode for fully deterministic testing. - test-quarantine-kbe-shadow-dossier.schema.json: new, independently versioned envelope for the collector's provenance output. Does not modify or compete with the existing candidate/receipt schemas. - New-TestQuarantineKbeSummary.ps1: renders a human-readable Markdown summary of a dossier/receipt pair. - fixtures/{68724,68947,68945}: compact, sanitized, offline fixtures and golden expected-dossier.json files for the three documented pilots, reproducing their real outcomes end-to-end through the unmodified evaluator. - Test-Collect-TestQuarantineKbeEvidence.ps1: deterministic, offline test suite covering the three pilots plus edge cases (closed issue, missing label, exact Build Analysis test/known-issue match). - test-quarantine-kbe-shadow.yml: maintainer-triggered workflow_dispatch workflow with a fork guard, per-issue concurrency, pinned actions, GITHUB_TOKEN-only auth, and least-privilege permissions (contents/issues/pull-requests/checks: read). Never mutates issues, labels, comments, PRs, branches, or files; uploads short-retention (7 day) dossier/candidate/receipt/summary/evidence artifacts only. - test-quarantine-kbe-shadow-tests.yml: path-triggered CI running both PowerShell test suites. - README.md: documents the trust boundary, Build Analysis vs. raw evidence, workflow inputs/outputs, retention, non-authorization status, the path to eventually extracting the deterministic collector already embedded in test-quarantine.md, and measurable promotion gates. Does not modify test-quarantine.md, test-quarantine.lock.yml, or any production quarantine/unquarantine behavior. Does not add a fixer workflow. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 1137 +++++++++++++++++ .../New-TestQuarantineKbeSummary.ps1 | 174 +++ .../test-quarantine-kbe-shadow/README.md | 159 +++ ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 387 ++++++ .../fixtures/68724/expected-dossier.json | 301 +++++ .../fixtures/68724/fixture.json | 140 ++ .../fixtures/68945/expected-dossier.json | 169 +++ .../fixtures/68945/fixture.json | 111 ++ .../fixtures/68947/expected-dossier.json | 290 +++++ .../fixtures/68947/fixture.json | 113 ++ ...-quarantine-kbe-shadow-dossier.schema.json | 552 ++++++++ .../test-quarantine-kbe-shadow-tests.yml | 39 + .../workflows/test-quarantine-kbe-shadow.yml | 116 ++ 13 files changed, 3688 insertions(+) create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/README.md create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json create mode 100644 .github/workflows/test-quarantine-kbe-shadow-tests.yml create mode 100644 .github/workflows/test-quarantine-kbe-shadow.yml diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 new file mode 100644 index 000000000000..df8081774d7b --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -0,0 +1,1137 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Deterministic, read-only collector for exactly one open dotnet/aspnetcore test-quarantine issue. + +.DESCRIPTION + Gathers public evidence for a single quarantine issue -- the issue body itself, Azure DevOps + build metadata, GitHub "Build Analysis" check-run snapshots (corroborating only, never + authoritative), and capped/redacted raw Helix evidence -- then emits either: + * a "candidate" object that independently satisfies test-quarantine-kbe-shadow-candidate.schema.json + and is ready for Evaluate-TestQuarantineKbeCandidate.ps1, or + * a structured "incomplete" outcome explaining exactly which evidence could not be + established, without inferring a pass, a recurrence, or a signature from anything missing + or expired. + + This script makes no repository-state mutations. It only reads public GitHub/Azure DevOps/Helix + endpoints (or, in fixture mode, a local fixture file) and writes local files: the dossier, an + optional candidate, and capped evidence text files under -EvidenceRoot. + +.PARAMETER IssueNumber + The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open + test-quarantine issue for the test(s) it names. + +.PARAMETER Signature + Optional manual failure-signature override. Supply this when the issue body does not contain + a deterministically extractable "## Error Message" (or equivalent) code block. When omitted and + extraction is ambiguous, the collector fails closed with reason code + 'signature-extraction-ambiguous' rather than guessing. + +.PARAMETER OutputFile + Path to write the dossier JSON (always written, whether the outcome is 'candidate' or + 'incomplete'). + +.PARAMETER CandidateFile + Path to write the candidate JSON. Only written when the dossier outcome is 'candidate'. + +.PARAMETER EvidenceRoot + Directory to materialize capped/redacted raw evidence text files into. Created if missing. + +.PARAMETER FixtureRoot + Optional path to a directory containing a single consolidated 'fixture.json' file that stands + in for every live network call (GitHub issue/check-runs/search, Azure DevOps build metadata, + Helix evidence). Used by the test harness and by the shadow workflow's self-test mode so this + script's deterministic logic can be exercised with zero network access. See README.md for the + fixture.json shape. + +.PARAMETER GitHubToken + Optional GitHub token for the GitHub REST calls (issue, check-runs, search). Falls back to the + GITHUB_TOKEN environment variable, then to unauthenticated requests. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [int]$IssueNumber, + + [string]$Signature, + + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [Parameter(Mandatory = $true)] + [string]$CandidateFile, + + [Parameter(Mandatory = $true)] + [string]$EvidenceRoot, + + [string]$FixtureRoot, + + [string]$GitHubToken = $env:GITHUB_TOKEN, + + [string]$Repository = "dotnet/aspnetcore", + + [string]$RepositoryRoot = "$PSScriptRoot/../../../..", + + [string]$DossierSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json", + + [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", + + [int]$RecurrenceScanBuildCap = 20 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$ado = "https://dev.azure.com/dnceng-public/public/_apis" +$vstmr = "https://vstmr.dev.azure.com/dnceng-public/public/_apis" +$helix = "https://helix.dot.net/api/2019-06-17" +$pipelineDefinitionIds = @(83, 87) +$canonicalQuarantineLabel = "test-failure" +$minimumFailureBuilds = 2 +$minimumNegativeLogs = 1 +$excerptCap = 2000 +$rawLogCap = 8000 + +# Same secret-shaped patterns the production test-quarantine.md deterministic collector scrubs +# before surfacing captured CI text. Helix work-item upload steps can log a live Azure DevOps +# bearer JWT on failure; the raw evidence gathered here is untrusted input and must never leak +# a live credential into an uploaded artifact. +$secretPatterns = @( + [regex]'eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}' + [regex]'eyJ[A-Za-z0-9_-]{20,}' + [regex]'\bgh[pousr]_[A-Za-z0-9]{20,}\b' + [regex]'\bgithub_pat_[A-Za-z0-9_]{20,}\b' + [regex]'(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{20,}' + [regex]'(?i)\bhttps?://[^/\s:@"]+:[^@\s/"]{6,}@' + [regex]'(?i)[?&]sig=[A-Za-z0-9%/+_=-]{20,}' + [regex]'(?i)\b(?:AccountKey|SharedAccessKey|AccessKey|Password|Pwd)=[^;\s"'']{12,}' + [regex]'[A-Za-z0-9][A-Za-z0-9+/=_-]{51,}' +) + +function ConvertTo-Redacted +{ + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $result = $Value + foreach ($pattern in $secretPatterns) + { + $result = $pattern.Replace($result, "[REDACTED]") + } + + return $result +} + +function ConvertTo-Iso8601String +{ + # ConvertFrom-Json (and Invoke-RestMethod, which uses it internally) auto-converts + # ISO-8601-shaped JSON string values into [datetime] instances. A bare [string] cast + # on one of those then renders using the current culture (e.g. "08/22/2026 03:28:01"), + # silently dropping the 'Z' suffix and sub-second precision and producing a value that + # is no longer schema-valid 'date-time' text. Route every timestamp read from parsed + # JSON through this so the emitted dossier/candidate always carries a real ISO-8601 + # UTC string regardless of whether the runtime parsed it into a DateTime or left it + # as a string. + param([AllowNull()]$Value) + + if ($null -eq $Value) + { + return $null + } + if ($Value -is [datetime]) + { + return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + } + return [string]$Value +} + +function Get-Sha256String +{ + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Value) + $hash = [System.Security.Cryptography.SHA256]::HashData($bytes) + + return [System.Convert]::ToHexString($hash).ToLowerInvariant() +} + +function Get-CappedExcerpt +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value, + [int]$Cap = $excerptCap, + # Phrases (e.g. the quarantined test's fully qualified name) that must survive + # redaction verbatim. The "long high-entropy run" secret pattern intentionally + # has no notion of word structure -- a descriptive xUnit test name easily exceeds + # its 52-character threshold and would otherwise be redacted to "[REDACTED]", + # silently destroying the exact marker Evaluate-TestQuarantineKbeCandidate.ps1 + # needs to associate a failure line with this test. Each phrase is swapped for a + # private-use-area sentinel before redaction runs and restored immediately after. + [string[]]$ProtectedPhrases = @() + ) + + $working = $Value + $placeholders = [ordered]@{} + $tokenIndex = 0 + foreach ($phrase in $ProtectedPhrases) + { + if ([string]::IsNullOrEmpty($phrase) -or -not $working.Contains($phrase)) + { + continue + } + $token = "`u{E000}PROTECTED-$tokenIndex`u{E000}" + $placeholders[$token] = $phrase + $working = $working.Replace($phrase, $token) + $tokenIndex++ + } + + $redacted = ConvertTo-Redacted -Value $working + foreach ($token in $placeholders.Keys) + { + $redacted = $redacted.Replace($token, [string]$placeholders[$token]) + } + + $normalized = [System.Text.RegularExpressions.Regex]::Replace($redacted, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "?") + if ($normalized.Length -gt $Cap) + { + $normalized = $normalized.Substring(0, $Cap) + } + + return $normalized +} + +function Add-MissingEvidence +{ + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [System.Collections.Generic.List[object]]$List, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string]$Detail + ) + + $null = $List.Add([ordered]@{ kind = $Kind; detail = $Detail }) +} + +# --------------------------------------------------------------------------- +# Fixture / live network abstraction. Fixture mode reads one consolidated JSON +# document; live mode calls the public GitHub/Azure DevOps/Helix REST APIs +# documented in test-quarantine.md's "API Reference (Azure DevOps & Helix)". +# --------------------------------------------------------------------------- + +$isFixtureMode = -not [string]::IsNullOrEmpty($FixtureRoot) +$fixture = $null +if ($isFixtureMode) +{ + $fixturePath = Join-Path $FixtureRoot "fixture.json" + if (-not (Test-Path -LiteralPath $fixturePath)) + { + throw "Fixture mode requested but '$fixturePath' does not exist." + } + $fixture = Get-Content -LiteralPath $fixturePath -Raw | ConvertFrom-Json -Depth 32 +} + +function Get-GitHubIssue +{ + param([Parameter(Mandatory = $true)][int]$Number) + + if ($isFixtureMode) + { + if ([int]$fixture.issue.number -ne $Number) + { + throw "Fixture issue number $($fixture.issue.number) does not match requested issue $Number." + } + return $fixture.issue + } + + $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } + if (-not [string]::IsNullOrEmpty($GitHubToken)) + { + $headers["Authorization"] = "Bearer $GitHubToken" + } + return Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 +} + +function Get-AzdoBuild +{ + param([Parameter(Mandatory = $true)][int]$BuildId) + + if ($isFixtureMode) + { + $key = [string]$BuildId + if ($fixture.azdo_builds.PSObject.Properties.Name -contains $key) + { + return $fixture.azdo_builds.$key + } + return $null + } + + try + { + return Invoke-RestMethod -Uri "$ado/build/builds/${BuildId}?api-version=7.1" -Method Get -TimeoutSec 30 + } + catch + { + return $null + } +} + +function Get-AzdoRecurrenceCandidateBuilds +{ + param([Parameter(Mandatory = $true)][int]$DefinitionId) + + if ($isFixtureMode) + { + $key = [string]$DefinitionId + if ($fixture.recurrence_scan.PSObject.Properties.Name -contains $key) + { + return @($fixture.recurrence_scan.$key) + } + return @() + } + + try + { + $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=failed&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 + return @($result.value) + } + catch + { + return @() + } +} + +function Get-AzdoNegativeCandidateBuilds +{ + param([Parameter(Mandatory = $true)][int]$DefinitionId) + + if ($isFixtureMode) + { + $key = [string]$DefinitionId + if ($fixture.negative_scan.PSObject.Properties.Name -contains $key) + { + return @($fixture.negative_scan.$key) + } + return @() + } + + try + { + $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=succeeded&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 + return @($result.value) + } + catch + { + return @() + } +} + +function Get-VstmrTestOutcome +{ + param( + [Parameter(Mandatory = $true)][int]$BuildId, + [Parameter(Mandatory = $true)][string]$TestName + ) + + if ($isFixtureMode) + { + $key = "$BuildId" + if ($fixture.vstmr_results.PSObject.Properties.Name -contains $key) + { + return $fixture.vstmr_results.$key + } + return $null + } + + try + { + $result = Invoke-RestMethod -Uri "$vstmr/testresults/resultsbyBuild?buildId=$BuildId&api-version=7.1-preview.1" -Method Get -TimeoutSec 60 + $match = @($result.value) | Where-Object { $_.automatedTestName -eq $TestName -or $_.testCaseTitle -eq $TestName } | Select-Object -First 1 + if ($null -eq $match) + { + return $null + } + return [ordered]@{ + outcome = $match.outcome + comment = $match.comment + errorMessage = $match.errorMessage + stackTrace = $match.stackTrace + } + } + catch + { + return $null + } +} + +function Get-HelixEvidence +{ + param( + [Parameter(Mandatory = $true)][int]$BuildId, + [string]$HelixJob, + [string]$HelixWorkItem + ) + + if ($isFixtureMode) + { + $key = "$BuildId" + if ($fixture.helix_evidence.PSObject.Properties.Name -contains $key) + { + return $fixture.helix_evidence.$key + } + return $null + } + + if ([string]::IsNullOrEmpty($HelixJob) -or [string]::IsNullOrEmpty($HelixWorkItem)) + { + return $null + } + + try + { + $files = Invoke-RestMethod -Uri "$helix/jobs/$HelixJob/workitems/$HelixWorkItem/files" -Method Get -TimeoutSec 60 + $consoleFile = @($files) | Where-Object { $_.Name -like "console.*" } | Select-Object -First 1 + if ($null -eq $consoleFile) + { + return [ordered]@{ found = $false; expired = $true } + } + $content = Invoke-RestMethod -Uri $consoleFile.Link -Method Get -TimeoutSec 60 + return [ordered]@{ found = $true; expired = $false; console_excerpt = [string]$content } + } + catch + { + return [ordered]@{ found = $false; expired = $true } + } +} + +function Get-CheckRunsForSha +{ + param([Parameter(Mandatory = $true)][string]$Sha) + + if ($isFixtureMode) + { + if ($fixture.check_runs.PSObject.Properties.Name -contains $Sha) + { + return @($fixture.check_runs.$Sha) + } + return @() + } + + $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } + if (-not [string]::IsNullOrEmpty($GitHubToken)) + { + $headers["Authorization"] = "Bearer $GitHubToken" + } + try + { + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/$Sha/check-runs" -Headers $headers -Method Get -TimeoutSec 30 + return @($result.check_runs) + } + catch + { + return @() + } +} + +function Get-DuplicateSearch +{ + param( + [Parameter(Mandatory = $true)][string]$Category, + [Parameter(Mandatory = $true)][string]$Query + ) + + if ($isFixtureMode) + { + if ($fixture.duplicate_search.PSObject.Properties.Name -contains $Category) + { + $entry = $fixture.duplicate_search.$Category + return [ordered]@{ complete = [bool]$entry.complete; result_numbers = @($entry.result_numbers) } + } + return [ordered]@{ complete = $false; result_numbers = @() } + } + + $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } + if (-not [string]::IsNullOrEmpty($GitHubToken)) + { + $headers["Authorization"] = "Bearer $GitHubToken" + } + try + { + $encoded = [System.Uri]::EscapeDataString($Query) + $result = Invoke-RestMethod -Uri "https://api.github.com/search/issues?q=$encoded&per_page=20" -Headers $headers -Method Get -TimeoutSec 30 + if ([bool]$result.incomplete_results) + { + return [ordered]@{ complete = $false; result_numbers = @() } + } + return [ordered]@{ complete = $true; result_numbers = @($result.items | ForEach-Object { [int]$_.number }) } + } + catch + { + return [ordered]@{ complete = $false; result_numbers = @() } + } +} + +# --------------------------------------------------------------------------- +# Step 1: validate the canonical, open quarantine issue. +# --------------------------------------------------------------------------- + +$missingEvidence = [System.Collections.Generic.List[object]]::new() +$reasonCodes = [System.Collections.Generic.List[string]]::new() + +$issue = Get-GitHubIssue -Number $IssueNumber +$issueUrl = "https://github.com/$Repository/issues/$IssueNumber" +$issueLabels = @($issue.labels | ForEach-Object { if ($_ -is [string]) { $_ } else { [string]$_.name } }) +$issueState = [string]$issue.state + +if ($issueLabels -notcontains $canonicalQuarantineLabel) +{ + $reasonCodes.Add("issue-not-canonical-quarantine") + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-label" -Detail "Issue #$IssueNumber does not carry the canonical '$canonicalQuarantineLabel' label." +} + +if ($issueState -ne "open") +{ + $reasonCodes.Add("issue-not-open") + Add-MissingEvidence -List $missingEvidence -Kind "issue-state" -Detail "Issue #$IssueNumber is '$issueState', not 'open'." +} + +$issueBody = [string]$issue.body + +# --------------------------------------------------------------------------- +# Step 2: deterministically parse the issue body for the test name, referenced +# builds, and (when unambiguous) a failure signature. Every quarantine issue +# produced by the production workflow carries a '## Failing Test(s)' section +# and at least one 'buildId=' link; both formats in use (the strict +# 50_test_failure.md template and the freeform '## Details' variant) satisfy +# these two invariants, so this parsing does not depend on section ordering. +# --------------------------------------------------------------------------- + +$testName = $null +$failingTestMatch = [regex]::Match($issueBody, "##\s*Failing Test\(s\)\s*\r?\n(.*?)(?=\r?\n##\s|\z)", [System.Text.RegularExpressions.RegexOptions]::Singleline) +if ($failingTestMatch.Success) +{ + $backtickMatch = [regex]::Match($failingTestMatch.Groups[1].Value, '`([^`]+)`') + if ($backtickMatch.Success) + { + $testName = $backtickMatch.Groups[1].Value.Trim() + } +} + +if ([string]::IsNullOrWhiteSpace($testName) -or $testName.Length -lt 3 -or $testName.Length -gt 1024 -or $testName -match "[\r\n]") +{ + $reasonCodes.Add("test-name-unresolvable") + Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "Could not deterministically extract a single backtick-quoted fully qualified test name from '## Failing Test(s)'." + $testName = $null +} + +$citedBuildIds = @( + [regex]::Matches($issueBody, "buildId=(\d+)") | + ForEach-Object { [int]$_.Groups[1].Value } | + Select-Object -Unique +) + +if ($citedBuildIds.Count -eq 0) +{ + $reasonCodes.Add("build-reference-unresolvable") + Add-MissingEvidence -List $missingEvidence -Kind "build-reference" -Detail "No 'buildId=' reference found in the issue body." +} + +$manualSignatureProvided = -not [string]::IsNullOrWhiteSpace($Signature) +$extractedSignature = $null +$errorMessageMatch = [regex]::Match($issueBody, '##\s*Error Message\s*\r?\n```(?:text)?\r?\n(.*?)```', [System.Text.RegularExpressions.RegexOptions]::Singleline) +if ($errorMessageMatch.Success) +{ + $firstLine = ($errorMessageMatch.Groups[1].Value -split "\r?\n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -First 1) + if (-not [string]::IsNullOrWhiteSpace($firstLine)) + { + $extractedSignature = $firstLine.Trim() + } +} + +$effectiveSignature = if ($manualSignatureProvided) { $Signature.Trim() } else { $extractedSignature } +if ([string]::IsNullOrWhiteSpace($effectiveSignature) -or + $effectiveSignature.Length -lt 8 -or $effectiveSignature.Length -gt 2048 -or + $effectiveSignature -match "[\r\n]") +{ + $reasonCodes.Add("signature-extraction-ambiguous") + Add-MissingEvidence -List $missingEvidence -Kind "signature" -Detail "No deterministic '## Error Message' code block was found and no valid -Signature override was supplied." + $effectiveSignature = $null +} + +# --------------------------------------------------------------------------- +# Step 3: resolve Azure DevOps build metadata for every issue-cited build. +# Builds whose metadata has aged out of Azure DevOps retention are recorded as +# not-found, never fabricated (see aspnetcore#68945). +# --------------------------------------------------------------------------- + +$retrievedUtc = [System.DateTimeOffset]::UtcNow.ToString("O") +$azdoBuildRecords = [System.Collections.Generic.List[object]]::new() +$resolvedBuilds = [System.Collections.Generic.List[object]]::new() + +foreach ($buildId in $citedBuildIds) +{ + $build = Get-AzdoBuild -BuildId $buildId + if ($null -eq $build) + { + $null = $azdoBuildRecords.Add([ordered]@{ + id = $buildId + found = $false + retrieved_utc = $retrievedUtc + source = "issue-body-reference" + note = "Build metadata was not retrievable; it may have aged out of Azure DevOps retention." + }) + Add-MissingEvidence -List $missingEvidence -Kind "azdo-build" -Detail "Build $buildId metadata could not be retrieved." + continue + } + + $definitionId = [int]$build.definition.id + $sourceVersion = [string]$build.sourceVersion + $record = [ordered]@{ + id = $buildId + found = $true + retrieved_utc = $retrievedUtc + source = "issue-body-reference" + definition_id = $definitionId + source_version = $sourceVersion + started_utc = ConvertTo-Iso8601String -Value $build.startTime + finished_utc = ConvertTo-Iso8601String -Value $build.finishTime + result = [string]$build.result + } + $null = $azdoBuildRecords.Add($record) + $null = $resolvedBuilds.Add(($record + @{ intended_role = "failure" })) +} + +# --------------------------------------------------------------------------- +# Step 4: if fewer than two distinct resolved builds are available, perform a +# capped supplementary recurrence scan across the same pipeline definition(s) +# on 'main' -- direct raw AzDO/VSTMR evidence, never the Build Analysis +# abstraction, per the architecture consensus that Build Analysis is +# corroborating only and cannot establish exact recurrence. +# --------------------------------------------------------------------------- + +if ($resolvedBuilds.Count -lt $minimumFailureBuilds -and $null -ne $testName -and $null -ne $effectiveSignature) +{ + $scanDefinitionIds = if ($resolvedBuilds.Count -gt 0) + { + @($resolvedBuilds | ForEach-Object { $_.definition_id } | Select-Object -Unique) + } + else + { + $pipelineDefinitionIds + } + + foreach ($definitionId in $scanDefinitionIds) + { + if ($resolvedBuilds.Count -ge $minimumFailureBuilds) + { + break + } + + $candidates = Get-AzdoRecurrenceCandidateBuilds -DefinitionId $definitionId + foreach ($candidate in $candidates) + { + if ($resolvedBuilds.Count -ge $minimumFailureBuilds) + { + break + } + + $candidateId = [int]$candidate.id + if ($citedBuildIds -contains $candidateId) + { + continue + } + + $outcome = Get-VstmrTestOutcome -BuildId $candidateId -TestName $testName + if ($null -eq $outcome -or [string]$outcome.outcome -ne "Failed") + { + continue + } + + $signatureText = "$($outcome.errorMessage) $($outcome.stackTrace)" + if ($signatureText -notlike "*$effectiveSignature*") + { + continue + } + + $record = [ordered]@{ + id = $candidateId + found = $true + retrieved_utc = $retrievedUtc + source = "recurrence-scan" + definition_id = $definitionId + source_version = [string]$candidate.sourceVersion + started_utc = ConvertTo-Iso8601String -Value $candidate.startTime + finished_utc = ConvertTo-Iso8601String -Value $candidate.finishTime + result = [string]$candidate.result + } + $null = $azdoBuildRecords.Add($record) + $null = $resolvedBuilds.Add(($record + @{ intended_role = "failure" })) + } + } +} + +if ($null -ne $testName -and $null -ne $effectiveSignature -and $resolvedBuilds.Count -lt $minimumFailureBuilds) +{ + $reasonCodes.Add("recurrence-single-build-only") + Add-MissingEvidence -List $missingEvidence -Kind "recurrence" -Detail "Only $($resolvedBuilds.Count) distinct build(s) with matching failure evidence were found; at least $minimumFailureBuilds are required and none could be added by the supplementary recurrence scan." +} + +# --------------------------------------------------------------------------- +# Step 4.5: gather at least one authoritative negative (passed/skipped) +# occurrence of the same test on the same pipeline(s). This is what lets the +# evaluator confirm the failure is not a consistent regression -- a missing +# negative is recorded as insufficient evidence, never inferred as a pass. +# --------------------------------------------------------------------------- + +$negativeBuilds = [System.Collections.Generic.List[object]]::new() + +if ($null -ne $testName -and $null -ne $effectiveSignature) +{ + $negativeScanDefinitionIds = @($resolvedBuilds | ForEach-Object { $_.definition_id } | Select-Object -Unique) + if ($negativeScanDefinitionIds.Count -eq 0) + { + $negativeScanDefinitionIds = $pipelineDefinitionIds + } + + foreach ($definitionId in $negativeScanDefinitionIds) + { + if ($negativeBuilds.Count -ge $minimumNegativeLogs) + { + break + } + + $candidates = Get-AzdoNegativeCandidateBuilds -DefinitionId $definitionId + foreach ($candidate in $candidates) + { + if ($negativeBuilds.Count -ge $minimumNegativeLogs) + { + break + } + + $candidateId = [int]$candidate.id + $outcome = Get-VstmrTestOutcome -BuildId $candidateId -TestName $testName + if ($null -eq $outcome -or [string]$outcome.outcome -notin @("Passed", "Skipped")) + { + continue + } + + $record = [ordered]@{ + id = $candidateId + found = $true + retrieved_utc = $retrievedUtc + source = "negative-scan" + definition_id = $definitionId + source_version = [string]$candidate.sourceVersion + started_utc = ConvertTo-Iso8601String -Value $candidate.startTime + finished_utc = ConvertTo-Iso8601String -Value $candidate.finishTime + result = [string]$candidate.result + } + $null = $azdoBuildRecords.Add($record) + $null = $negativeBuilds.Add(($record + @{ intended_role = "negative" })) + } + } +} + +# --------------------------------------------------------------------------- +# Step 5: for each resolved build, gather raw failure/negative evidence +# (Helix console log) and materialize it locally so Evaluate-TestQuarantineKbeCandidate.ps1 +# can hash-verify it. An artifact that once existed but is no longer +# retrievable is recorded as found=false, expired=true -- never silently +# dropped and never treated as a pass (see aspnetcore#68945). +# --------------------------------------------------------------------------- + +[System.IO.Directory]::CreateDirectory($EvidenceRoot) | Out-Null + +$rawEvidenceRecords = [System.Collections.Generic.List[object]]::new() +$rawLogs = [System.Collections.Generic.List[object]]::new() +$failureBuildIdSet = [System.Collections.Generic.HashSet[int]]::new() +$negativeCount = 0 +$evidenceIndex = 0 + +$evidenceBuilds = @($resolvedBuilds) + @($negativeBuilds) +foreach ($build in $evidenceBuilds) +{ + if ($null -eq $testName) + { + break + } + + $role = [string]$build.intended_role + $vstmrOutcome = Get-VstmrTestOutcome -BuildId $build.id -TestName $testName + $helixJob = $null + $helixWorkItem = $null + if ($null -ne $vstmrOutcome -and -not [string]::IsNullOrEmpty($vstmrOutcome.comment)) + { + $commentMatch = ($vstmrOutcome.comment | ConvertFrom-Json -ErrorAction SilentlyContinue) + if ($null -ne $commentMatch) + { + $helixJob = [string]$commentMatch.HelixJobId + $helixWorkItem = [string]$commentMatch.HelixWorkItemName + } + } + + # Defensive consistency check: the intended role (why this build was selected) + # must match what VSTMR actually reports for this test in this build. A + # mismatch means the evidence is stale or inconsistent -- skip it rather than + # writing a role/outcome pair that contradicts the authoritative test result. + $outcomeValue = if ($null -eq $vstmrOutcome) { $null } + elseif ([string]$vstmrOutcome.outcome -eq "Failed") { "failed" } + elseif ([string]$vstmrOutcome.outcome -eq "Passed") { "passed" } + elseif ([string]$vstmrOutcome.outcome -eq "Skipped") { "skipped" } + else { $null } + + $roleOutcomeConsistent = + ($role -eq "failure" -and $outcomeValue -eq "failed") -or + ($role -eq "negative" -and $outcomeValue -in @("passed", "skipped")) + + if (-not $roleOutcomeConsistent) + { + Add-MissingEvidence -List $missingEvidence -Kind "vstmr-consistency" -Detail "Build $($build.id): recorded test outcome did not match the reason this build was selected as $role evidence." + continue + } + + $helixEvidence = Get-HelixEvidence -BuildId $build.id -HelixJob $helixJob -HelixWorkItem $helixWorkItem + $found = ($null -ne $helixEvidence -and [bool]$helixEvidence.found) + $expired = ($null -ne $helixEvidence -and [bool]$helixEvidence.expired) -or ($null -eq $helixEvidence -and $null -ne $vstmrOutcome) + + if (-not $found) + { + $null = $rawEvidenceRecords.Add([ordered]@{ + build_id = $build.id + role = $role + found = $false + expired = $expired + captured_utc = $retrievedUtc + note = "Helix console evidence for build $($build.id) was not retrievable." + }) + Add-MissingEvidence -List $missingEvidence -Kind "helix-evidence" -Detail "Build $($build.id): no retrievable Helix console evidence." + if ($expired) + { + $reasonCodes.Add("raw-evidence-expired") + } + continue + } + + $evidenceIndex += 1 + $fileName = "issue-$IssueNumber-build-$($build.id)-$role.log" + $evidencePath = Join-Path $EvidenceRoot $fileName + $cappedContent = Get-CappedExcerpt -Value ([string]$helixEvidence.console_excerpt) -Cap $rawLogCap -ProtectedPhrases @($testName) + [System.IO.File]::WriteAllText($evidencePath, $cappedContent) + $sha256 = (Get-FileHash -LiteralPath $evidencePath -Algorithm SHA256).Hash.ToLowerInvariant() + + $null = $rawEvidenceRecords.Add([ordered]@{ + build_id = $build.id + role = $role + kind = "helix-console-log" + helix_job = $(if ($helixJob) { $helixJob } else { "" }) + helix_workitem = $(if ($helixWorkItem) { $helixWorkItem } else { "" }) + found = $true + expired = $false + captured_utc = $retrievedUtc + sha256 = $sha256 + evidence_path = $fileName + }) + + if ($role -eq "failure") + { + $null = $failureBuildIdSet.Add([int]$build.id) + } + else + { + $negativeCount += 1 + } + + $null = $rawLogs.Add([ordered]@{ + id = "evidence-$evidenceIndex" + role = $role + outcome = $outcomeValue + path = $fileName + source_url = "https://dev.azure.com/dnceng-public/public/_build/results?buildId=$($build.id)&view=results" + sha256 = $sha256 + build = [ordered]@{ + id = [int]$build.id + pipeline_definition_id = [int]$build.definition_id + source_version = [string]$build.source_version + started_utc = [string]$build.started_utc + platform = "Linux" + configuration = "Release" + } + }) +} + +if ($null -ne $testName -and $failureBuildIdSet.Count -lt $minimumFailureBuilds) +{ + if ($reasonCodes -notcontains "recurrence-single-build-only") + { + $reasonCodes.Add("raw-evidence-insufficient") + Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "Only $($failureBuildIdSet.Count) distinct build(s) produced retrievable failure evidence; at least $minimumFailureBuilds are required." + } +} + +if ($null -ne $testName -and $negativeCount -lt $minimumNegativeLogs) +{ + $reasonCodes.Add("raw-evidence-insufficient") + Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "No retrievable negative (passed/skipped) evidence was found; at least $minimumNegativeLogs is required." +} + +# --------------------------------------------------------------------------- +# Step 6: fetch Build Analysis check-run snapshots. Advisory/corroborating +# only: recorded regardless of outcome, and a missing or generic snapshot +# never overrides raw evidence gathered above. +# --------------------------------------------------------------------------- + +$checkRunRecords = [System.Collections.Generic.List[object]]::new() +$distinctShas = @($resolvedBuilds | ForEach-Object { $_.source_version } | Where-Object { $_ } | Select-Object -Unique) +$corroboratingContext = [System.Collections.Generic.List[object]]::new() + +foreach ($sha in $distinctShas) +{ + $checkRuns = Get-CheckRunsForSha -Sha $sha + $buildAnalysis = @($checkRuns) | Where-Object { [string]$_.name -eq "Build Analysis" } | Select-Object -First 1 + + if ($null -eq $buildAnalysis) + { + $null = $checkRunRecords.Add([ordered]@{ + source_version = $sha + found = $false + retrieved_utc = $retrievedUtc + exact_test_referenced = $false + known_issue_referenced = $false + }) + continue + } + + $text = [string]$buildAnalysis.output.text + $textSha256 = Get-Sha256String -Value $text + $shortMethodName = if ($testName) { ($testName -split '\.')[-1] } else { $null } + $exactTestReferenced = ($null -ne $testName -and $text.Contains($testName)) -or ($null -ne $shortMethodName -and $text.Contains($shortMethodName)) + $knownIssueReferenced = $text -match "(?i)known issue" + + $null = $checkRunRecords.Add([ordered]@{ + source_version = $sha + found = $true + retrieved_utc = $retrievedUtc + check_id = [int]$buildAnalysis.id + conclusion = [string]$buildAnalysis.conclusion + title = Get-CappedExcerpt -Value ([string]$buildAnalysis.output.title) -Cap 512 + text_sha256 = $textSha256 + text_excerpt = Get-CappedExcerpt -Value $text -Cap $excerptCap + html_url = [string]$buildAnalysis.html_url + exact_test_referenced = $exactTestReferenced + known_issue_referenced = $knownIssueReferenced + }) + $null = $corroboratingContext.Add([ordered]@{ source = "build-analysis"; url = [string]$buildAnalysis.html_url }) +} + +# --------------------------------------------------------------------------- +# Step 7: categorized duplicate KBE / fix-PR search. +# --------------------------------------------------------------------------- + +$shortName = if ($testName) { ($testName -split '\.')[-1] } else { $IssueNumber.ToString() } +$duplicateQueries = @( + @{ category = "open-kbe"; query = "repo:$Repository is:issue is:open label:`"Known Build Error`" $shortName" } + @{ category = "recently-closed-kbe"; query = "repo:$Repository is:issue is:closed label:`"Known Build Error`" $shortName" } + @{ category = "open-fix-pr"; query = "repo:$Repository is:pr is:open $shortName" } + @{ category = "recently-merged-fix-pr"; query = "repo:$Repository is:pr is:merged $shortName" } +) + +$duplicateQueryResults = [System.Collections.Generic.List[object]]::new() +$duplicateReferences = [System.Collections.Generic.List[string]]::new() +$kbeNumbers = [System.Collections.Generic.List[int]]::new() +$fixPrNumbers = [System.Collections.Generic.List[int]]::new() +$allQueriesComplete = $true + +foreach ($q in $duplicateQueries) +{ + $searchResult = Get-DuplicateSearch -Category $q.category -Query $q.query + if (-not [bool]$searchResult.complete) + { + $allQueriesComplete = $false + } + $resultNumbers = @($searchResult.result_numbers) + $null = $duplicateQueryResults.Add([ordered]@{ + category = $q.category + query = $q.query + complete = [bool]$searchResult.complete + result_numbers = $resultNumbers + }) + + $isKbeCategory = $q.category -in @("open-kbe", "recently-closed-kbe") + foreach ($n in $resultNumbers) + { + if ($isKbeCategory) + { + $null = $kbeNumbers.Add($n) + $null = $duplicateReferences.Add("issue:$n") + } + else + { + $null = $fixPrNumbers.Add($n) + $null = $duplicateReferences.Add("pull-request:$n") + } + } +} + +$duplicateStatus = if (-not $allQueriesComplete) +{ + "not-evaluated" +} +elseif ($kbeNumbers.Count -gt 0) +{ + "existing-kbe" +} +elseif ($fixPrNumbers.Count -gt 0) +{ + "existing-fix-pr" +} +else +{ + "none" +} + +if (-not $allQueriesComplete) +{ + $reasonCodes.Add("duplicate-search-incomplete") + Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "At least one duplicate KBE/fix-PR search category returned incomplete results." +} + +$duplicateCheck = [ordered]@{ + status = $duplicateStatus + checked_utc = $retrievedUtc + coverage = [ordered]@{ + open_kbes = $true + recently_closed_kbes = $true + open_fix_prs = $true + recently_merged_fix_prs = $true + } + references = @($duplicateReferences | Select-Object -Unique) + queries = @($duplicateQueryResults) +} + +# --------------------------------------------------------------------------- +# Step 8: assemble the dossier -- and the candidate, when every gate passed. +# --------------------------------------------------------------------------- + +$reasonCodes = @($reasonCodes | Select-Object -Unique) +$outcome = if ($reasonCodes.Count -gt 0) { "incomplete" } else { "candidate" } + +$candidate = $null +if ($outcome -eq "candidate") +{ + $proposedClassification = switch ($duplicateStatus) + { + "existing-kbe" { "reuse-existing-kbe"; break } + "existing-fix-pr" { "quarantine-only"; break } + default + { + if ($effectiveSignature -match "(?i)timeout|WebDriverException|TaskCanceledException") + { + "timeout-needs-classification" + } + else + { + "new-kbe-candidate" + } + } + } + + $null = $corroboratingContext.Add([ordered]@{ source = "quarantine-issue"; url = $issueUrl }) + + $repoHeadSha = (& git -C $RepositoryRoot rev-parse HEAD).Trim() + $candidate = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + repository_ref = [ordered]@{ + branch = "main" + commit_sha = $repoHeadSha + } + issue = [ordered]@{ + number = $IssueNumber + url = $issueUrl + } + test = [ordered]@{ + fully_qualified_name = $testName + } + signature = [ordered]@{ + kind = "ErrorMessage" + values = @($effectiveSignature) + build_retry = $false + exclude_console_log = $false + } + policy = [ordered]@{ + minimum_failure_logs = $minimumFailureBuilds + minimum_negative_logs = $minimumNegativeLogs + } + evidence = [ordered]@{ + raw_logs = @($rawLogs) + corroborating_context = @($corroboratingContext) + } + duplicate_check = $duplicateCheck + proposed_classification = $proposedClassification + } + + $candidateJson = $candidate | ConvertTo-Json -Depth 32 + if (-not ($candidateJson | Test-Json -SchemaFile $CandidateSchemaFile)) + { + throw "Collector produced a candidate that does not satisfy the versioned candidate schema." + } + + $candidateDirectory = Split-Path -Parent $CandidateFile + if ($candidateDirectory) + { + [System.IO.Directory]::CreateDirectory($candidateDirectory) | Out-Null + } + [System.IO.File]::WriteAllText($CandidateFile, $candidateJson + [System.Environment]::NewLine) +} + +$incomplete = $null +if ($outcome -eq "incomplete") +{ + $incomplete = [ordered]@{ + reason_codes = @($reasonCodes) + message = "Collector could not produce a validated candidate for issue #$IssueNumber : " + ($reasonCodes -join ", ") + "." + missing_evidence = @($missingEvidence) + } +} + +$dossier = [ordered]@{ + schema_version = 1 + repository = "dotnet/aspnetcore" + collector = [ordered]@{ + name = "Collect-TestQuarantineKbeEvidence.ps1" + version = 1 + generated_utc = $retrievedUtc + fixture_mode = $isFixtureMode + manual_signature_provided = $manualSignatureProvided + } + issue = [ordered]@{ + number = $IssueNumber + url = $issueUrl + state = $issueState + labels = @($issueLabels) + } + outcome = $outcome + provenance = [ordered]@{ + azdo_builds = @($azdoBuildRecords) + check_run_snapshots = @($checkRunRecords) + raw_evidence_sources = @($rawEvidenceRecords) + duplicate_search = $duplicateCheck + } + candidate = $candidate + incomplete = $incomplete +} + +$dossierJson = $dossier | ConvertTo-Json -Depth 32 +if (-not ($dossierJson | Test-Json -SchemaFile $DossierSchemaFile)) +{ + throw "Generated dossier does not satisfy the versioned dossier schema." +} + +$outputDirectory = Split-Path -Parent $OutputFile +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} +[System.IO.File]::WriteAllText($OutputFile, $dossierJson + [System.Environment]::NewLine) + +Write-Host "Wrote '$outcome' dossier for issue #$IssueNumber to $OutputFile" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 new file mode 100644 index 000000000000..42d9d4918616 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 @@ -0,0 +1,174 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Renders a short, human-readable Markdown summary of one test-quarantine-kbe-shadow run. + +.DESCRIPTION + Reads the collector's dossier (and, when present, the evaluator's receipt) and writes a + Markdown summary intended for the workflow's step summary and as a small uploaded artifact. + Purely a read-only presentation layer: it does not re-derive or override any collector or + evaluator decision. + +.PARAMETER DossierFile + Path to the dossier JSON produced by Collect-TestQuarantineKbeEvidence.ps1. + +.PARAMETER ReceiptFile + Optional path to the receipt JSON produced by Evaluate-TestQuarantineKbeCandidate.ps1. Only + present when the dossier's outcome was 'candidate'. + +.PARAMETER OutputFile + Path to write the rendered Markdown summary to. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$DossierFile, + + [string]$ReceiptFile, + + [Parameter(Mandatory = $true)] + [string]$OutputFile +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Get-PropertyOrDefault +{ + param($Object, [Parameter(Mandatory = $true)][string]$Name, $Default = $null) + + if ($null -eq $Object) + { + return $Default + } + $property = $Object.PSObject.Properties[$Name] + if ($null -eq $property) + { + return $Default + } + return $property.Value +} + +function ConvertTo-DisplayTimestamp +{ + # ConvertFrom-Json auto-parses ISO-8601 strings into [datetime]; render explicitly as + # round-trip UTC text instead of relying on the current culture's default ToString(). + param($Value) + + if ($null -eq $Value) + { + return "" + } + if ($Value -is [datetime]) + { + return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + } + return [string]$Value +} + +$dossier = Get-Content -LiteralPath $DossierFile -Raw | ConvertFrom-Json -Depth 32 + +$lines = [System.Collections.Generic.List[string]]::new() +$null = $lines.Add("# Test quarantine KBE shadow report") +$null = $lines.Add("") +$null = $lines.Add("This is a **read-only, non-authoritative shadow evaluation**. It never labels, comments on, or otherwise mutates any issue, pull request, branch, or repository file.") +$null = $lines.Add("") +$null = $lines.Add("| Field | Value |") +$null = $lines.Add("|---|---|") +$null = $lines.Add("| Issue | [#$($dossier.issue.number)]($($dossier.issue.url)) |") +$null = $lines.Add("| Collector outcome | ``$($dossier.outcome)`` |") +$null = $lines.Add("| Fixture mode | $($dossier.collector.fixture_mode) |") +$null = $lines.Add("| Manual signature supplied | $($dossier.collector.manual_signature_provided) |") +$null = $lines.Add("| Generated (UTC) | $(ConvertTo-DisplayTimestamp -Value $dossier.collector.generated_utc) |") +$null = $lines.Add("") + +if ($dossier.outcome -eq "incomplete") +{ + $null = $lines.Add("## Incomplete") + $null = $lines.Add("") + $null = $lines.Add($dossier.incomplete.message) + $null = $lines.Add("") + $null = $lines.Add("| Reason code | ") + $null = $lines.Add("|---|") + foreach ($code in @($dossier.incomplete.reason_codes)) + { + $null = $lines.Add("| ``$code`` |") + } + $null = $lines.Add("") + if (@($dossier.incomplete.missing_evidence).Count -gt 0) + { + $null = $lines.Add("
Missing evidence detail") + $null = $lines.Add("") + foreach ($item in @($dossier.incomplete.missing_evidence)) + { + $null = $lines.Add("- **$($item.kind)**: $($item.detail)") + } + $null = $lines.Add("") + $null = $lines.Add("
") + $null = $lines.Add("") + } +} +else +{ + $candidate = $dossier.candidate + $null = $lines.Add("## Candidate") + $null = $lines.Add("") + $null = $lines.Add("- **Test**: ``$($candidate.test.fully_qualified_name)``") + $null = $lines.Add("- **Proposed classification**: ``$($candidate.proposed_classification)``") + $null = $lines.Add("- **Duplicate check status**: ``$($candidate.duplicate_check.status)``") + if (@($candidate.duplicate_check.references).Count -gt 0) + { + $formattedReferences = (@($candidate.duplicate_check.references) | ForEach-Object { '`' + $_ + '`' }) -join ', ' + $null = $lines.Add("- **Duplicate references**: $formattedReferences") + } + $failureCount = @($candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" }).Count + $negativeCount = @($candidate.evidence.raw_logs | Where-Object { $_.role -eq "negative" }).Count + $null = $lines.Add("- **Failure evidence logs**: $failureCount") + $null = $lines.Add("- **Negative evidence logs**: $negativeCount") + $null = $lines.Add("") + + if (-not [string]::IsNullOrEmpty($ReceiptFile) -and (Test-Path -LiteralPath $ReceiptFile)) + { + $receipt = Get-Content -LiteralPath $ReceiptFile -Raw | ConvertFrom-Json -Depth 32 + $null = $lines.Add("## Evaluator receipt") + $null = $lines.Add("") + $null = $lines.Add("| Field | Value |") + $null = $lines.Add("|---|---|") + $null = $lines.Add("| Deterministic status | ``$($receipt.deterministic_status)`` |") + $null = $lines.Add("| Shadow recommendation | ``$($receipt.shadow_recommendation)`` |") + $null = $lines.Add("| Eligible for KBE enrichment | $($receipt.eligible_for_kbe_enrichment) |") + $null = $lines.Add("| Evidence provenance verified | $($receipt.evidence_provenance_verified) |") + $null = $lines.Add("| Human review required | $($receipt.human_review_required) |") + $null = $lines.Add("") + $null = $lines.Add("**Reasons:**") + foreach ($reason in @($receipt.reasons)) + { + $null = $lines.Add("- $reason") + } + $null = $lines.Add("") + } +} + +if (@($dossier.provenance.check_run_snapshots).Count -gt 0) +{ + $null = $lines.Add("## Build Analysis snapshots (corroborating only, never authoritative)") + $null = $lines.Add("") + $null = $lines.Add("| Commit | Found | Conclusion | Exact test referenced | Known issue referenced |") + $null = $lines.Add("|---|---|---|---|---|") + foreach ($snapshot in @($dossier.provenance.check_run_snapshots)) + { + $shortSha = $snapshot.source_version.Substring(0, 7) + $conclusion = Get-PropertyOrDefault -Object $snapshot -Name "conclusion" -Default "(none)" + $null = $lines.Add("| ``$shortSha`` | $($snapshot.found) | $conclusion | $($snapshot.exact_test_referenced) | $($snapshot.known_issue_referenced) |") + } + $null = $lines.Add("") +} + +$outputDirectory = Split-Path -Parent $OutputFile +if ($outputDirectory) +{ + [System.IO.Directory]::CreateDirectory($outputDirectory) | Out-Null +} +[System.IO.File]::WriteAllText($OutputFile, ($lines -join [System.Environment]::NewLine) + [System.Environment]::NewLine) +Write-Host "Wrote summary to $OutputFile" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md new file mode 100644 index 000000000000..ba71502b95a7 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -0,0 +1,159 @@ +# Test quarantine KBE shadow evaluation + +This directory implements a **read-only, non-authoritative shadow evaluation** of whether a +single, already-open dotnet/aspnetcore test-quarantine issue has enough evidence for a +Runtime-style Known Build Error (KBE) signature. It is deliberately narrow in scope: it never +authorizes an automated fix, never mutates repository state, and never promotes itself to +production without an explicit, separate decision by a maintainer. + +## Components + +| File | Role | +|---|---| +| `Collect-TestQuarantineKbeEvidence.ps1` | Deterministic collector. Given one issue number (and, optionally, a manual signature override), gathers public evidence and emits a **dossier**: either a `candidate` ready for the evaluator, or a structured `incomplete` outcome. | +| `Evaluate-TestQuarantineKbeCandidate.ps1` | Pre-existing, unmodified deterministic evaluator. Validates a candidate's signature against its evidence and emits a **receipt**. | +| `New-TestQuarantineKbeSummary.ps1` | Renders a short Markdown summary of a dossier (+ receipt, when present) for the workflow step summary and as an artifact. | +| `test-quarantine-kbe-shadow-candidate.schema.json` | Pre-existing, unmodified schema for the evaluator's input. | +| `test-quarantine-kbe-shadow-receipt.schema.json` | Pre-existing, unmodified schema for the evaluator's output. | +| `test-quarantine-kbe-shadow-dossier.schema.json` | Schema for the collector's output envelope (provenance + either `candidate` or `incomplete`). Independently versioned; it wraps and reuses the candidate schema rather than replacing or competing with it. | +| `fixtures//` | Compact, sanitized, offline fixtures for three real pilot issues, each with a golden `expected-dossier.json`. | +| `Test-Evaluate-TestQuarantineKbeCandidate.ps1`, `Test-Collect-TestQuarantineKbeEvidence.ps1` | Deterministic, offline test suites (no network access). | + +The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dispatch) and +`.github/workflows/test-quarantine-kbe-shadow-tests.yml` (CI) workflows are described below. + +## Trust boundary + +* **Build Analysis is corroborating, never authoritative.** The collector records a snapshot of + the GitHub "Build Analysis" check-run for every resolved build's commit (check id, conclusion, + a SHA-256 of its full text, a capped/redacted excerpt, and a conservative substring check for + whether the text names this exact test and/or a "Known Issue"). A missing or generic snapshot + is recorded and surfaced, but it never overrides raw evidence and never by itself makes a + candidate valid or invalid. Direct queries against the Build Analysis abstraction for three real + pilot builds (1563420, 1551326, 1569737) returned only generic, task-level, unmatched failures + and no known issues — this is exactly the "generic" case the collector's fixtures for #68724 and + #68945 encode. +* **Raw AzDO/Helix/VSTMR evidence is what proves an exact test failure and its recurrence.** The + collector resolves Azure DevOps build metadata, VSTMR test results, and Helix console-log + content directly. Recurrence requires evidence from **at least two distinct builds** (a single + build producing two separate artifacts is not recurrence), and at least one authoritative + negative (passed/skipped) occurrence. +* **Never infer a pass, a recurrence, or a signature from missing or expired evidence.** Every gap + — a build whose Azure DevOps metadata has aged out of retention, a Helix console log that has + expired, an ambiguous or absent signature, an incomplete duplicate search — is recorded as an + explicit reason code and fails the run closed (`outcome: "incomplete"`) rather than silently + degrading. Real Azure DevOps retention data confirms this is not hypothetical: build 1537561 + (cited by aspnetcore#68947) has already aged out of the public `dnceng-public` project. +* **No repository-state mutation.** The workflow's permissions are `contents: read`, + `issues: read`, `pull-requests: read`, `checks: read` — read-only across the board. It uses only + the ambient `GITHUB_TOKEN`, never a PAT or other secret. It uploads artifacts; it never calls any + write API (no labels, comments, commits, branches, or files). +* **Evidence provenance remains unverified in this PR.** Even when the collector's candidate is + independently valid, the unmodified evaluator still emits `evidence_provenance_verified: false`, + `eligible_for_kbe_enrichment: false`, and `human_review_required: true`. Flipping + `evidence_provenance_verified` to `true` is a deliberate, separate promotion decision (see + "Promotion gates" below) — not something this collector claims for itself. + +## Workflow: `test-quarantine-kbe-shadow.yml` (maintainer dispatch) + +* **Trigger**: `workflow_dispatch` only, with a required `issue_number` input and an optional + `signature` input (used only when the issue body has no fenced `## Error Message` block that the + collector can extract deterministically). +* **Fork guard**: the job is gated on `github.repository == 'dotnet/aspnetcore'`, so a forked copy + of this file is inert. +* **Concurrency**: one run per issue number (`test-quarantine-kbe-shadow-`), + cancelling any still-running dispatch for the same issue. +* **Permissions**: `contents: read`, `issues: read`, `pull-requests: read`, `checks: read` — the + last two are the "only demonstrated needs" beyond the baseline: `checks: read` for the Build + Analysis check-run snapshot, `pull-requests: read` for the duplicate fix-PR search. +* **Actions are pinned by commit SHA** (`actions/checkout`, `actions/upload-artifact`), matching + repository convention. +* **Steps**: validate the input, checkout, run the collector, run the evaluator only when the + collector's outcome is `candidate`, render the human-readable summary (to the job's step summary + and as a file), then upload artifacts. +* **Artifacts** (uploaded with `retention-days: 7`, i.e. short-lived by design): `dossier.json`, + `candidate.json` (candidate outcome only), `receipt.json` (candidate outcome only), + `summary.md`, and the capped/redacted evidence text files under `evidence/`. + +## CI: `test-quarantine-kbe-shadow-tests.yml` + +Runs both deterministic, offline PowerShell test suites (`Test-Evaluate-TestQuarantineKbeCandidate.ps1` +and `Test-Collect-TestQuarantineKbeEvidence.ps1`) on every pull request that touches this directory +or either workflow file, plus on manual dispatch. Every fixture is local and offline; no network +access or secrets are required, so this is a low-risk, standard `pull_request`-triggered check — +no self-test mode inside the shadow workflow itself was needed to close this CI gap. + +## Fixture mode (`-FixtureRoot`) + +Both `Collect-TestQuarantineKbeEvidence.ps1` and its tests support a `-FixtureRoot ` +parameter. When set, the collector reads one consolidated `fixture.json` document from that +directory instead of making any live GitHub/Azure DevOps/Helix call — this is how the test suite +and the three pilot fixtures below achieve fully offline, deterministic coverage. `fixture.json` +mirrors the shape of the real endpoints documented in `test-quarantine.md`'s "API Reference" +section: the GitHub issue body/labels/state, Azure DevOps build metadata keyed by build id, a +capped recurrence/negative-build candidate list per pipeline definition, VSTMR test outcomes keyed +by build id, capped Helix console excerpts keyed by build id, GitHub check-run snapshots keyed by +commit SHA, and categorized duplicate-search results. + +## Pilot fixtures + +| Issue | Fixture invocation | Outcome | +|---|---|---| +| [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` (deterministic `## Error Message` extraction) | `candidate`, `reuse-existing-kbe` (recommends reusing #68708) | +| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException: TaskCanceledException"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override — verified by a second, signature-less invocation of the same fixture in the test suite) | `candidate`, `timeout-needs-classification` (generic Selenium/WebDriver timeout, not a test-specific KBE) | +| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's Helix console-log artifact has expired (`raw-evidence-expired`), leaving only one usable failure log below the two-build recurrence floor (`raw-evidence-insufficient`) | + +Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality +comparison in the test suite. Golden comparisons exclude the collector's `generated_utc` / +`retrieved_utc` / `captured_utc` / `checked_utc` timestamps and the running checkout's HEAD +`commit_sha` — all of which are expected to differ run-to-run and commit-to-commit — replacing +them with the literal sentinel `` on both sides before comparing. + +## Reconciling with the existing evaluator contract + +The collector does **not** introduce a third, competing dossier schema. Its `candidate` output, +when present, is validated against the same unmodified +`test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to the +unmodified `Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is — this PR does not change that +script, its tests, or either of its schemas. `test-quarantine-kbe-shadow-dossier.schema.json` is a +new, independently versioned (`schema_version: 1`) envelope that carries collector-specific +provenance (Azure DevOps build resolution, Build Analysis check-run snapshots, raw-evidence +retrieval/expiry) alongside that same `candidate` object, or a structured `incomplete` outcome when +any evidence gate fails. + +## Eventual extraction of the production deterministic collector + +`.github/workflows/test-quarantine.md` already embeds a much larger deterministic, pre-activation +collector (Azure DevOps `resultsbyBuild`/build-timeline aggregation across ~200 builds, Helix +console-log `[FAIL]` block extraction, secret redaction) that gathers failure evidence for the +*entire* quarantine-management workflow, not a single issue. **This PR does not modify that +workflow or its production quarantine behavior.** The long-term path is to factor the shared +pieces — the Azure DevOps/Helix/VSTMR fetch helpers, the secret-redaction patterns, the Helix +`[FAIL]`-block extraction — out of `test-quarantine.md`'s Python step and this directory's +PowerShell collector into one common, tested module that both consume, rather than maintaining two +independent implementations of the same evidence-gathering logic indefinitely. Until that +extraction happens, this collector intentionally duplicates only the narrow, single-issue subset +of that logic it needs (build resolution, VSTMR outcome lookup, Helix console retrieval, the same +secret-redaction pattern family) and cross-references the production workflow's real, observed +issue-body formats (both the `50_test_failure.md` template and the freeform `## Details` variant) +so it does not invent a third, incompatible issue-body convention. + +## Promotion gates + +None of the following are implemented by this PR. They are the measurable conditions a future, +separate change would need to satisfy before this collector's output could ever be trusted enough +to flip `evidence_provenance_verified` to `true` or to feed a maintainer-triggered fix workflow: + +1. **Signed/attested provenance for every raw evidence file** — e.g. a hash chain from the Azure + DevOps/Helix API response actually observed at collection time, not merely a locally computed + SHA-256 of whatever bytes were written to disk. +2. **A second, independent collector run reaching the same `candidate` (or `incomplete`) result** + for the same issue, to bound single-run collection errors (rate limiting, partial API + responses, transient Azure DevOps/Helix outages). +3. **A maintainer explicitly reviewing and approving** the specific candidate/receipt pair — this + PR's receipt already always sets `human_review_required: true` for exactly this reason. +4. **The shared extraction described above landing** so the single-issue collector and the + production quarantine workflow's evidence gathering can no longer silently diverge. +5. **A dedicated, maintainer-triggered fix workflow being designed and reviewed separately** — this + PR intentionally stops at read-only evaluation and does not implement or wire up any automated + fix path. diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 new file mode 100644 index 000000000000..d39ff1648d02 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -0,0 +1,387 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Deterministic, offline tests for Collect-TestQuarantineKbeEvidence.ps1. + +.DESCRIPTION + Exercises the collector entirely in -FixtureRoot mode (zero network access) against the + three real pilot quarantine issues recorded in fixtures/, and against a handful of small + synthetic fixtures for edge cases that are not represented by those three issues. Golden + dossier comparisons exclude collector-generated timestamps and the running repository's + HEAD commit SHA, both of which are expected to differ between runs/commits; every other + field must match exactly. + + Where the collector's outcome is 'candidate', the resulting candidate.json is also fed + through the unmodified, already-tested Evaluate-TestQuarantineKbeCandidate.ps1 to prove the + two scripts reconcile: the collector's output is accepted as-is by the existing evaluator + contract with no changes to that script or its schemas. +#> + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$collector = "$PSScriptRoot/Collect-TestQuarantineKbeEvidence.ps1" +$evaluator = "$PSScriptRoot/Evaluate-TestQuarantineKbeCandidate.ps1" +$summaryGenerator = "$PSScriptRoot/New-TestQuarantineKbeSummary.ps1" +$dossierSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json" +$candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json" +$fixturesRoot = "$PSScriptRoot/fixtures" +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-collector-$([System.Guid]::NewGuid().ToString('N'))" + +function Assert-Equal +{ + param( + [Parameter(Mandatory = $true)][AllowNull()]$Actual, + [Parameter(Mandatory = $true)][AllowNull()]$Expected, + [Parameter(Mandatory = $true)][string]$Message + ) + + if ("$Actual" -ne "$Expected") + { + throw "$Message Expected '$Expected', actual '$Actual'." + } +} + +function Assert-Contains +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Collection, + [Parameter(Mandatory = $true)]$Value, + [Parameter(Mandatory = $true)][string]$Message + ) + + if (@($Collection | Where-Object { "$_" -eq "$Value" }).Count -eq 0) + { + throw "$Message Expected collection to contain '$Value'; actual: $($Collection -join ', ')." + } +} + +# The three collector-generated timestamps and the running checkout's HEAD commit SHA are +# expected to differ run-to-run and commit-to-commit; every golden fixture below was captured +# with these fields already replaced by this same sentinel. +$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha") +$volatileSentinel = "" + +function ConvertTo-NormalizedObject +{ + param($Value) + + if ($null -eq $Value) + { + return $null + } + if ($Value -is [System.Management.Automation.PSCustomObject]) + { + $result = [ordered]@{} + foreach ($prop in $Value.PSObject.Properties) + { + $result[$prop.Name] = if ($volatileKeys -contains $prop.Name) { $volatileSentinel } else { ConvertTo-NormalizedObject -Value $prop.Value } + } + return [PSCustomObject]$result + } + if ($Value -is [array]) + { + return ,@($Value | ForEach-Object { ConvertTo-NormalizedObject -Value $_ }) + } + return $Value +} + +function Test-DeepEqual +{ + param($Expected, $Actual, [string]$Path = "`$") + + if ($null -eq $Expected -or $null -eq $Actual) + { + if ($null -ne $Expected -or $null -ne $Actual) + { + throw "Golden mismatch at ${Path}: expected '$Expected', actual '$Actual'." + } + return + } + + if ($Expected -is [System.Management.Automation.PSCustomObject] -and $Actual -is [System.Management.Automation.PSCustomObject]) + { + $expectedNames = @($Expected.PSObject.Properties.Name | Sort-Object) + $actualNames = @($Actual.PSObject.Properties.Name | Sort-Object) + $diff = Compare-Object -ReferenceObject $expectedNames -DifferenceObject $actualNames + if ($null -ne $diff) + { + throw "Golden mismatch at ${Path}: property set differs (expected: $($expectedNames -join ','); actual: $($actualNames -join ','))." + } + foreach ($name in $expectedNames) + { + Test-DeepEqual -Expected $Expected.$name -Actual $Actual.$name -Path "$Path.$name" + } + return + } + + if ($Expected -is [array] -and $Actual -is [array]) + { + if ($Expected.Count -ne $Actual.Count) + { + throw "Golden mismatch at ${Path}: array length differs (expected $($Expected.Count), actual $($Actual.Count))." + } + for ($i = 0; $i -lt $Expected.Count; $i++) + { + Test-DeepEqual -Expected $Expected[$i] -Actual $Actual[$i] -Path "$Path[$i]" + } + return + } + + if ("$Expected" -ne "$Actual") + { + throw "Golden mismatch at ${Path}: expected '$Expected', actual '$Actual'." + } +} + +function Invoke-Collector +{ + param( + [Parameter(Mandatory = $true)][int]$IssueNumber, + [Parameter(Mandatory = $true)][string]$FixtureRoot, + [Parameter(Mandatory = $true)][string]$WorkDirectory, + [string]$Signature + ) + + [System.IO.Directory]::CreateDirectory($WorkDirectory) | Out-Null + $dossierPath = Join-Path $WorkDirectory "dossier.json" + $candidatePath = Join-Path $WorkDirectory "candidate.json" + $evidenceRoot = Join-Path $WorkDirectory "evidence" + + $params = @{ + IssueNumber = $IssueNumber + OutputFile = $dossierPath + CandidateFile = $candidatePath + EvidenceRoot = $evidenceRoot + FixtureRoot = $FixtureRoot + DossierSchemaFile = $dossierSchema + CandidateSchemaFile = $candidateSchema + } + if (-not [string]::IsNullOrEmpty($Signature)) + { + $params["Signature"] = $Signature + } + + & $collector @params | Out-Null + + return [ordered]@{ + DossierPath = $dossierPath + CandidatePath = $candidatePath + EvidenceRoot = $evidenceRoot + Dossier = (Get-Content -LiteralPath $dossierPath -Raw | ConvertFrom-Json -Depth 32) + } +} + +function Assert-GoldenDossier +{ + param( + [Parameter(Mandatory = $true)][string]$IssueDirectory, + [Parameter(Mandatory = $true)]$ActualDossier + ) + + $goldenPath = Join-Path $IssueDirectory "expected-dossier.json" + $golden = Get-Content -LiteralPath $goldenPath -Raw | ConvertFrom-Json -Depth 32 + $normalizedActual = ConvertTo-NormalizedObject -Value $ActualDossier + Test-DeepEqual -Expected $golden -Actual $normalizedActual +} + +try +{ + [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null + + # ------------------------------------------------------------------ + # Pilot 1 -- aspnetcore#68724: deterministic '## Error Message' extraction, + # a supplementary recurrence-scan build, and reuse of an existing KBE. + # ------------------------------------------------------------------ + $result68724 = Invoke-Collector -IssueNumber 68724 -FixtureRoot "$fixturesRoot/68724" -WorkDirectory "$tempRoot/68724" + Assert-Equal -Actual $result68724.Dossier.outcome -Expected "candidate" -Message "#68724 outcome mismatch." + Assert-Equal -Actual $result68724.Dossier.candidate.proposed_classification -Expected "reuse-existing-kbe" -Message "#68724 proposed_classification mismatch." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68724" -ActualDossier $result68724.Dossier + + $receiptPath68724 = Join-Path "$tempRoot/68724" "receipt.json" + & $evaluator -CandidateFile $result68724.CandidatePath -EvidenceRoot $result68724.EvidenceRoot -OutputFile $receiptPath68724 -CandidateSchemaFile $candidateSchema + $receipt68724 = Get-Content -LiteralPath $receiptPath68724 -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt68724.deterministic_status -Expected "validated" -Message "#68724 deterministic_status mismatch." + Assert-Equal -Actual $receipt68724.shadow_recommendation -Expected "reuse-existing-kbe" -Message "#68724 shadow_recommendation mismatch." + Assert-Equal -Actual $receipt68724.eligible_for_kbe_enrichment -Expected $false -Message "#68724 must never authorize enrichment." + Assert-Equal -Actual $receipt68724.evidence_provenance_verified -Expected $false -Message "#68724 provenance must remain unverified." + + $summaryPath68724 = Join-Path "$tempRoot/68724" "summary.md" + & $summaryGenerator -DossierFile $result68724.DossierPath -ReceiptFile $receiptPath68724 -OutputFile $summaryPath68724 + $summaryText68724 = Get-Content -LiteralPath $summaryPath68724 -Raw + if (-not $summaryText68724.Contains("reuse-existing-kbe")) + { + throw "#68724 summary must mention the shadow_recommendation." + } + + # ------------------------------------------------------------------ + # Pilot 2 -- aspnetcore#68947: the issue body has no fenced '## Error + # Message' block, so deterministic extraction is ambiguous without a + # manual signature. + # ------------------------------------------------------------------ + $result68947NoSig = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947-no-signature" + Assert-Equal -Actual $result68947NoSig.Dossier.outcome -Expected "incomplete" -Message "#68947 (no signature) outcome mismatch." + Assert-Contains -Collection @($result68947NoSig.Dossier.incomplete.reason_codes) -Value "signature-extraction-ambiguous" -Message "#68947 (no signature) reason codes mismatch." + + # With the manual override supplied, recurrence is established via the collector's + # supplementary scan (the issue's second cited build has itself aged out of Azure DevOps + # retention) and the outcome is a validated, generic-timeout candidate. + $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature "OpenQA.Selenium.WebDriverException: TaskCanceledException" + Assert-Equal -Actual $result68947.Dossier.outcome -Expected "candidate" -Message "#68947 outcome mismatch." + Assert-Equal -Actual $result68947.Dossier.candidate.proposed_classification -Expected "timeout-needs-classification" -Message "#68947 proposed_classification mismatch." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68947" -ActualDossier $result68947.Dossier + + $receiptPath68947 = Join-Path "$tempRoot/68947" "receipt.json" + & $evaluator -CandidateFile $result68947.CandidatePath -EvidenceRoot $result68947.EvidenceRoot -OutputFile $receiptPath68947 -CandidateSchemaFile $candidateSchema + $receipt68947 = Get-Content -LiteralPath $receiptPath68947 -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt68947.deterministic_status -Expected "validated" -Message "#68947 deterministic_status mismatch." + Assert-Equal -Actual $receipt68947.shadow_recommendation -Expected "timeout-needs-classification" -Message "#68947 shadow_recommendation mismatch." + + # ------------------------------------------------------------------ + # Pilot 3 -- aspnetcore#68945: both cited builds' Azure DevOps metadata still + # resolves, but the second build's Helix console-log artifact has expired. + # Recurrence therefore falls back to a single usable failure log and the + # collector must fail closed rather than infer a pass. + # ------------------------------------------------------------------ + $result68945 = Invoke-Collector -IssueNumber 68945 -FixtureRoot "$fixturesRoot/68945" -WorkDirectory "$tempRoot/68945" -Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled." + Assert-Equal -Actual $result68945.Dossier.outcome -Expected "incomplete" -Message "#68945 outcome mismatch." + Assert-Contains -Collection @($result68945.Dossier.incomplete.reason_codes) -Value "raw-evidence-expired" -Message "#68945 reason codes must record the expired artifact." + Assert-Equal -Actual $result68945.Dossier.candidate -Expected $null -Message "#68945 must not emit a candidate." + Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68945" -ActualDossier $result68945.Dossier + + $summaryPath68945 = Join-Path "$tempRoot/68945" "summary.md" + & $summaryGenerator -DossierFile $result68945.DossierPath -OutputFile $summaryPath68945 + $summaryText68945 = Get-Content -LiteralPath $summaryPath68945 -Raw + if (-not $summaryText68945.Contains("raw-evidence-expired")) + { + throw "#68945 summary must mention the raw-evidence-expired reason code." + } + + # ------------------------------------------------------------------ + # Edge cases not represented by the three pilots: a closed issue, an issue + # missing the canonical quarantine label, and conservative check-run + # substring extraction actually flipping to true when warranted. + # ------------------------------------------------------------------ + $closedFixtureDir = Join-Path $tempRoot "closed-issue-fixture" + [System.IO.Directory]::CreateDirectory($closedFixtureDir) | Out-Null + @{ + issue = @{ + number = 1 + state = "closed" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.Closed ``" + } + azdo_builds = @{} + recurrence_scan = @{} + negative_scan = @{} + vstmr_results = @{} + helix_evidence = @{} + check_runs = @{} + duplicate_search = @{ + "open-kbe" = @{ complete = $true; result_numbers = @() } + "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } + "open-fix-pr" = @{ complete = $true; result_numbers = @() } + "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } + } + } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $closedFixtureDir "fixture.json") + $resultClosed = Invoke-Collector -IssueNumber 1 -FixtureRoot $closedFixtureDir -WorkDirectory (Join-Path $tempRoot "closed-issue") + Assert-Equal -Actual $resultClosed.Dossier.outcome -Expected "incomplete" -Message "Closed-issue outcome mismatch." + Assert-Contains -Collection @($resultClosed.Dossier.incomplete.reason_codes) -Value "issue-not-open" -Message "Closed-issue reason codes mismatch." + + $unlabeledFixtureDir = Join-Path $tempRoot "unlabeled-issue-fixture" + [System.IO.Directory]::CreateDirectory($unlabeledFixtureDir) | Out-Null + @{ + issue = @{ + number = 2 + state = "open" + labels = @("area-blazor") + body = "## Failing Test(s)`n`` Sample.Tests.Unlabeled ``" + } + azdo_builds = @{} + recurrence_scan = @{} + negative_scan = @{} + vstmr_results = @{} + helix_evidence = @{} + check_runs = @{} + duplicate_search = @{ + "open-kbe" = @{ complete = $true; result_numbers = @() } + "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } + "open-fix-pr" = @{ complete = $true; result_numbers = @() } + "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } + } + } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $unlabeledFixtureDir "fixture.json") + $resultUnlabeled = Invoke-Collector -IssueNumber 2 -FixtureRoot $unlabeledFixtureDir -WorkDirectory (Join-Path $tempRoot "unlabeled-issue") + Assert-Equal -Actual $resultUnlabeled.Dossier.outcome -Expected "incomplete" -Message "Unlabeled-issue outcome mismatch." + Assert-Contains -Collection @($resultUnlabeled.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Unlabeled-issue reason codes mismatch." + + # The three pilots' real Build Analysis snapshots were all generic (matching the + # documented architecture-consensus finding that this signal is corroborating only), so + # none of them exercise a positive substring match. Prove that path separately: a snapshot + # whose text literally names the test and a known issue must flip both conservative flags. + $exactMatchFixtureDir = Join-Path $tempRoot "exact-match-fixture" + [System.IO.Directory]::CreateDirectory($exactMatchFixtureDir) | Out-Null + $exactMatchSha = "bf6e1566a2433f298c3adc8b6ecc3358b99d5d3f" + $exactMatchSignature = "System.InvalidOperationException: Sample failure for exact-match testing." + @{ + issue = @{ + number = 3 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.ExactMatchCase ``" + "`n`n## Error Message`n``````text`n$exactMatchSignature`n``````" + "`n`n## Build`nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=5000001" + } + azdo_builds = @{ + "5000001" = @{ definition = @{ id = 83 }; sourceVersion = $exactMatchSha; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = @{ + "83" = @(@{ id = 5000002; sourceVersion = "c1b304785ea05e7c92030583e1cb658c50630102"; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) + } + negative_scan = @{ + "83" = @(@{ id = 5000003; sourceVersion = "52bcd78ab0d7a1df3834306cc1c56a21f86a9fd2"; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + } + vstmr_results = @{ + "5000001" = @{ outcome = "Failed"; comment = '{"HelixJobId":"job-a","HelixWorkItemName":"wi-a"}'; errorMessage = $exactMatchSignature; stackTrace = "at Sample.Tests.ExactMatchCase..." } + "5000002" = @{ outcome = "Failed"; comment = '{"HelixJobId":"job-b","HelixWorkItemName":"wi-b"}'; errorMessage = $exactMatchSignature; stackTrace = "at Sample.Tests.ExactMatchCase..." } + "5000003" = @{ outcome = "Passed"; comment = '{"HelixJobId":"job-c","HelixWorkItemName":"wi-c"}'; errorMessage = $null; stackTrace = $null } + } + helix_evidence = @{ + "5000001" = @{ found = $true; expired = $false; console_excerpt = "Failed Sample.Tests.ExactMatchCase [1 s]`n$exactMatchSignature (build 5000001)" } + "5000002" = @{ found = $true; expired = $false; console_excerpt = "Failed Sample.Tests.ExactMatchCase [1 s]`n$exactMatchSignature (build 5000002)" } + "5000003" = @{ found = $true; expired = $false; console_excerpt = "[PASS] Sample.Tests.ExactMatchCase" } + } + check_runs = @{ + $exactMatchSha = @(@{ + name = "Build Analysis" + id = 700001 + conclusion = "failure" + output = @{ + title = "1 failing test" + text = "Sample.Tests.ExactMatchCase failed. This matches a Known Issue: https://github.com/dotnet/aspnetcore/issues/70000." + } + html_url = "https://github.com/dotnet/aspnetcore/runs/700001" + }) + } + duplicate_search = @{ + "open-kbe" = @{ complete = $true; result_numbers = @() } + "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } + "open-fix-pr" = @{ complete = $true; result_numbers = @() } + "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } + } + } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $exactMatchFixtureDir "fixture.json") + $resultExactMatch = Invoke-Collector -IssueNumber 3 -FixtureRoot $exactMatchFixtureDir -WorkDirectory (Join-Path $tempRoot "exact-match") + Assert-Equal -Actual $resultExactMatch.Dossier.outcome -Expected "candidate" -Message "Exact-match fixture outcome mismatch." + $matchingSnapshot = @($resultExactMatch.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $exactMatchSha })[0] + Assert-Equal -Actual $matchingSnapshot.exact_test_referenced -Expected $true -Message "exact_test_referenced must flip true when the check-run text names the test." + Assert-Equal -Actual $matchingSnapshot.known_issue_referenced -Expected $true -Message "known_issue_referenced must flip true when the check-run text names a Known Issue." + + Write-Host "All test-quarantine-kbe-shadow collector tests passed." +} +finally +{ + if (Test-Path -LiteralPath $tempRoot) + { + Remove-Item -LiteralPath $tempRoot -Recurse -Force + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json new file mode 100644 index 000000000000..4ac82862a832 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -0,0 +1,301 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": false + }, + "issue": { + "number": 68724, + "url": "https://github.com/dotnet/aspnetcore/issues/68724", + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ] + }, + "outcome": "candidate", + "provenance": { + "azdo_builds": [ + { + "id": 1563420, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 87, + "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "started_utc": "2026-08-22T03:28:01.2925985Z", + "finished_utc": "2026-08-22T05:06:06.2192270Z", + "result": "failed" + }, + { + "id": 1560111, + "found": true, + "retrieved_utc": "", + "source": "recurrence-scan", + "definition_id": 87, + "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", + "started_utc": "2026-08-21T03:00:00.0000000Z", + "finished_utc": "2026-08-21T05:00:00.0000000Z", + "result": "failed" + }, + { + "id": 1558000, + "found": true, + "retrieved_utc": "", + "source": "negative-scan", + "definition_id": 87, + "source_version": "8afa8ced59345cb708a7115f7519b068dd56b994", + "started_utc": "2026-08-19T03:00:00.0000000Z", + "finished_utc": "2026-08-19T05:00:00.0000000Z", + "result": "succeeded" + } + ], + "check_run_snapshots": [ + { + "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "found": true, + "retrieved_utc": "", + "check_id": 900001, + "conclusion": "failure", + "title": "1 failing test", + "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", + "text_excerpt": "1 test failed. See Azure DevOps for full details.", + "html_url": "https://github.com/dotnet/aspnetcore/runs/900001", + "exact_test_referenced": false, + "known_issue_referenced": false + }, + { + "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", + "found": true, + "retrieved_utc": "", + "check_id": 900002, + "conclusion": "failure", + "title": "1 failing test", + "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", + "text_excerpt": "1 test failed. See Azure DevOps for full details.", + "html_url": "https://github.com/dotnet/aspnetcore/runs/900002", + "exact_test_referenced": false, + "known_issue_referenced": false + } + ], + "raw_evidence_sources": [ + { + "build_id": 1563420, + "role": "failure", + "kind": "helix-console-log", + "helix_job": "helix-job-1563420", + "helix_workitem": "VirtualizationTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "298a4db75b3750339d8a0f576f96b5c688a526ee99acea46290ffc735e975d15", + "evidence_path": "issue-68724-build-1563420-failure.log" + }, + { + "build_id": 1560111, + "role": "failure", + "kind": "helix-console-log", + "helix_job": "helix-job-1560111", + "helix_workitem": "VirtualizationTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "c12348ca3e1cbaf32c01e9a74dc0d0373fd9c5266f0710d975b6a6acc8db74f4", + "evidence_path": "issue-68724-build-1560111-failure.log" + }, + { + "build_id": 1558000, + "role": "negative", + "kind": "helix-console-log", + "helix_job": "helix-job-1558000", + "helix_workitem": "VirtualizationTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "33b8e42484dfa2c90ce1717f55251e48f7c037ea90af4b409ec2897137adf61e", + "evidence_path": "issue-68724-build-1558000-negative.log" + } + ], + "duplicate_search": { + "status": "existing-kbe", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [ + "issue:68708" + ], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [ + 68708 + ] + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + } + ] + } + }, + "candidate": { + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "repository_ref": { + "branch": "main", + "commit_sha": "" + }, + "issue": { + "number": 68724, + "url": "https://github.com/dotnet/aspnetcore/issues/68724" + }, + "test": { + "fully_qualified_name": "Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll" + }, + "signature": { + "kind": "ErrorMessage", + "values": [ + "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0." + ], + "build_retry": false, + "exclude_console_log": false + }, + "policy": { + "minimum_failure_logs": 2, + "minimum_negative_logs": 1 + }, + "evidence": { + "raw_logs": [ + { + "id": "evidence-1", + "role": "failure", + "outcome": "failed", + "path": "issue-68724-build-1563420-failure.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420&view=results", + "sha256": "298a4db75b3750339d8a0f576f96b5c688a526ee99acea46290ffc735e975d15", + "build": { + "id": 1563420, + "pipeline_definition_id": 87, + "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "started_utc": "2026-08-22T03:28:01.2925985Z", + "platform": "Linux", + "configuration": "Release" + } + }, + { + "id": "evidence-2", + "role": "failure", + "outcome": "failed", + "path": "issue-68724-build-1560111-failure.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1560111&view=results", + "sha256": "c12348ca3e1cbaf32c01e9a74dc0d0373fd9c5266f0710d975b6a6acc8db74f4", + "build": { + "id": 1560111, + "pipeline_definition_id": 87, + "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", + "started_utc": "2026-08-21T03:00:00.0000000Z", + "platform": "Linux", + "configuration": "Release" + } + }, + { + "id": "evidence-3", + "role": "negative", + "outcome": "passed", + "path": "issue-68724-build-1558000-negative.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1558000&view=results", + "sha256": "33b8e42484dfa2c90ce1717f55251e48f7c037ea90af4b409ec2897137adf61e", + "build": { + "id": 1558000, + "pipeline_definition_id": 87, + "source_version": "8afa8ced59345cb708a7115f7519b068dd56b994", + "started_utc": "2026-08-19T03:00:00.0000000Z", + "platform": "Linux", + "configuration": "Release" + } + } + ], + "corroborating_context": [ + { + "source": "build-analysis", + "url": "https://github.com/dotnet/aspnetcore/runs/900001" + }, + { + "source": "build-analysis", + "url": "https://github.com/dotnet/aspnetcore/runs/900002" + }, + { + "source": "quarantine-issue", + "url": "https://github.com/dotnet/aspnetcore/issues/68724" + } + ] + }, + "duplicate_check": { + "status": "existing-kbe", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [ + "issue:68708" + ], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [ + 68708 + ] + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "complete": true, + "result_numbers": [] + } + ] + }, + "proposed_classification": "reuse-existing-kbe" + }, + "incomplete": null +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json new file mode 100644 index 000000000000..429a29c05365 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -0,0 +1,140 @@ +{ + "issue": { + "number": 68724, + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n at Xunit.Assert.True(Nullable`1 condition, String userMessage)\n```\n\n## Stacktrace\n
\nStack trace\n\n```text\n at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore[TResult](IWebDriver driver, Func`1 assertion, TimeSpan timeout)\n at Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll(Boolean useProvider)\n```\n\n
\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/32632851798)\n" + }, + "azdo_builds": { + "1563420": { + "definition": { + "id": 87 + }, + "sourceVersion": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", + "startTime": "2026-08-22T03:28:01.2925985Z", + "finishTime": "2026-08-22T05:06:06.219227Z", + "result": "failed" + } + }, + "recurrence_scan": { + "87": [ + { + "id": 1559999, + "sourceVersion": "e0232dc7601942a7c8f95b3b96f01a7b28107861", + "startTime": "2026-08-20T03:00:00Z", + "finishTime": "2026-08-20T05:00:00Z", + "result": "failed" + }, + { + "id": 1560111, + "sourceVersion": "074f8655455783f5afc3e7a865c30d5f3dad52be", + "startTime": "2026-08-21T03:00:00Z", + "finishTime": "2026-08-21T05:00:00Z", + "result": "failed" + } + ] + }, + "negative_scan": { + "87": [ + { + "id": 1558000, + "sourceVersion": "8afa8ced59345cb708a7115f7519b068dd56b994", + "startTime": "2026-08-19T03:00:00Z", + "finishTime": "2026-08-19T05:00:00Z", + "result": "succeeded" + } + ] + }, + "vstmr_results": { + "1563420": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1563420\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", + "errorMessage": "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.", + "stackTrace": "at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore..." + }, + "1559999": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1559999\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", + "errorMessage": "OpenQA.Selenium.WebDriverException: unrelated driver disconnect", + "stackTrace": "at Unrelated.Driver.Disconnect..." + }, + "1560111": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1560111\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", + "errorMessage": "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.", + "stackTrace": "at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore..." + }, + "1558000": { + "outcome": "Passed", + "comment": "{\"HelixJobId\": \"helix-job-1558000\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", + "errorMessage": null, + "stackTrace": null + } + }, + "helix_evidence": { + "1563420": { + "found": true, + "expired": false, + "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll [3 s]\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n(observed in build 1563420)" + }, + "1560111": { + "found": true, + "expired": false, + "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll [3 s]\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n(observed in build 1560111)" + }, + "1558000": { + "found": true, + "expired": false, + "console_excerpt": "[PASS] Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll" + } + }, + "check_runs": { + "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68": [ + { + "name": "Build Analysis", + "id": 900001, + "conclusion": "failure", + "output": { + "title": "1 failing test", + "text": "1 test failed. See Azure DevOps for full details." + }, + "html_url": "https://github.com/dotnet/aspnetcore/runs/900001" + } + ], + "074f8655455783f5afc3e7a865c30d5f3dad52be": [ + { + "name": "Build Analysis", + "id": 900002, + "conclusion": "failure", + "output": { + "title": "1 failing test", + "text": "1 test failed. See Azure DevOps for full details." + }, + "html_url": "https://github.com/dotnet/aspnetcore/runs/900002" + } + ] + }, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [ + 68708 + ] + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [] + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [] + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [] + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json new file mode 100644 index 000000000000..76f51437b37c --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -0,0 +1,169 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": true + }, + "issue": { + "number": 68945, + "url": "https://github.com/dotnet/aspnetcore/issues/68945", + "state": "open", + "labels": [ + "test-failure", + "area-networking" + ] + }, + "outcome": "incomplete", + "provenance": { + "azdo_builds": [ + { + "id": 1569737, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 83, + "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", + "started_utc": "2026-08-27T08:00:00.0000000Z", + "finished_utc": "2026-08-27T09:45:00.0000000Z", + "result": "failed" + }, + { + "id": 1538879, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 83, + "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "started_utc": "2026-08-04T08:00:00.0000000Z", + "finished_utc": "2026-08-04T09:45:00.0000000Z", + "result": "failed" + }, + { + "id": 1540500, + "found": true, + "retrieved_utc": "", + "source": "negative-scan", + "definition_id": 83, + "source_version": "5f76b4102d0c06c5af2f08e045a9d32e9cccef5b", + "started_utc": "2026-08-15T08:00:00.0000000Z", + "finished_utc": "2026-08-15T09:45:00.0000000Z", + "result": "succeeded" + } + ], + "check_run_snapshots": [ + { + "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", + "found": true, + "retrieved_utc": "", + "check_id": 900201, + "conclusion": "failure", + "title": "1 failing test", + "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", + "text_excerpt": "1 test failed. See Azure DevOps for full details.", + "html_url": "https://github.com/dotnet/aspnetcore/runs/900201", + "exact_test_referenced": false, + "known_issue_referenced": false + }, + { + "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "known_issue_referenced": false + } + ], + "raw_evidence_sources": [ + { + "build_id": 1569737, + "role": "failure", + "kind": "helix-console-log", + "helix_job": "helix-job-1569737", + "helix_workitem": "Http3RequestTests.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "c9753c024d81675a60dba72af5e18d663af7801cff73d69c9645879130b1c0ae", + "evidence_path": "issue-68945-build-1569737-failure.log" + }, + { + "build_id": 1538879, + "role": "failure", + "found": false, + "expired": true, + "captured_utc": "", + "note": "Helix console evidence for build 1538879 was not retrievable." + }, + { + "build_id": 1540500, + "role": "negative", + "kind": "helix-console-log", + "helix_job": "helix-job-1540500", + "helix_workitem": "Http3RequestTests.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "b4f3e16c9853a9ec29eb5f5407a12b171e0c9d260f610d9bff0fe37be8cb3d8c", + "evidence_path": "issue-68945-build-1540500-negative.log" + } + ], + "duplicate_search": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [] + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged POST_ClientCancellationUpload_RequestAbortRaised", + "complete": true, + "result_numbers": [] + } + ] + } + }, + "candidate": null, + "incomplete": { + "reason_codes": [ + "raw-evidence-expired", + "raw-evidence-insufficient" + ], + "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-expired, raw-evidence-insufficient.", + "missing_evidence": [ + { + "kind": "helix-evidence", + "detail": "Build 1538879: no retrievable Helix console evidence." + }, + { + "kind": "raw-evidence", + "detail": "Only 1 distinct build(s) produced retrievable failure evidence; at least 2 are required." + } + ] + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json new file mode 100644 index 000000000000..19ec5905bb26 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -0,0 +1,111 @@ +{ + "issue": { + "number": 68945, + "state": "open", + "labels": [ + "test-failure", + "area-networking" + ], + "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/33496438442)\n" + }, + "azdo_builds": { + "1569737": { + "definition": { + "id": 83 + }, + "sourceVersion": "b5666daed660cf1862a197784eee65b42a74a64a", + "startTime": "2026-08-27T08:00:00Z", + "finishTime": "2026-08-27T09:45:00Z", + "result": "failed" + }, + "1538879": { + "definition": { + "id": 83 + }, + "sourceVersion": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", + "startTime": "2026-08-04T08:00:00Z", + "finishTime": "2026-08-04T09:45:00Z", + "result": "failed" + } + }, + "recurrence_scan": {}, + "negative_scan": { + "83": [ + { + "id": 1540500, + "sourceVersion": "5f76b4102d0c06c5af2f08e045a9d32e9cccef5b", + "startTime": "2026-08-15T08:00:00Z", + "finishTime": "2026-08-15T09:45:00Z", + "result": "succeeded" + } + ] + }, + "vstmr_results": { + "1569737": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1569737\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", + "errorMessage": "System.Threading.Tasks.TaskCanceledException: The operation was canceled.", + "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync..." + }, + "1538879": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1538879\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", + "errorMessage": "System.Threading.Tasks.TaskCanceledException: The operation was canceled.", + "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync..." + }, + "1540500": { + "outcome": "Passed", + "comment": "{\"HelixJobId\": \"helix-job-1540500\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", + "errorMessage": null, + "stackTrace": null + } + }, + "helix_evidence": { + "1569737": { + "found": true, + "expired": false, + "console_excerpt": "Failed Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised [3 s]\nSystem.Threading.Tasks.TaskCanceledException: The operation was canceled.\n(observed in build 1569737)" + }, + "1538879": { + "found": false, + "expired": true + }, + "1540500": { + "found": true, + "expired": false, + "console_excerpt": "[PASS] Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + }, + "check_runs": { + "b5666daed660cf1862a197784eee65b42a74a64a": [ + { + "name": "Build Analysis", + "id": 900201, + "conclusion": "failure", + "output": { + "title": "1 failing test", + "text": "1 test failed. See Azure DevOps for full details." + }, + "html_url": "https://github.com/dotnet/aspnetcore/runs/900201" + } + ] + }, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [] + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [] + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [] + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [] + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json new file mode 100644 index 000000000000..6def27ceec42 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -0,0 +1,290 @@ +{ + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "collector": { + "name": "Collect-TestQuarantineKbeEvidence.ps1", + "version": 1, + "generated_utc": "", + "fixture_mode": true, + "manual_signature_provided": true + }, + "issue": { + "number": 68947, + "url": "https://github.com/dotnet/aspnetcore/issues/68947", + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ] + }, + "outcome": "candidate", + "provenance": { + "azdo_builds": [ + { + "id": 1551326, + "found": true, + "retrieved_utc": "", + "source": "issue-body-reference", + "definition_id": 87, + "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", + "started_utc": "2026-08-13T10:00:00.0000000Z", + "finished_utc": "2026-08-13T11:30:00.0000000Z", + "result": "failed" + }, + { + "id": 1537561, + "found": false, + "retrieved_utc": "", + "source": "issue-body-reference", + "note": "Build metadata was not retrievable; it may have aged out of Azure DevOps retention." + }, + { + "id": 1549000, + "found": true, + "retrieved_utc": "", + "source": "recurrence-scan", + "definition_id": 87, + "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "started_utc": "2026-08-10T09:00:00.0000000Z", + "finished_utc": "2026-08-10T10:30:00.0000000Z", + "result": "failed" + }, + { + "id": 1545000, + "found": true, + "retrieved_utc": "", + "source": "negative-scan", + "definition_id": 87, + "source_version": "f33596dac091a6fd858b1515378d951f7f6adda5", + "started_utc": "2026-08-08T09:00:00.0000000Z", + "finished_utc": "2026-08-08T10:30:00.0000000Z", + "result": "succeeded" + } + ], + "check_run_snapshots": [ + { + "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", + "found": false, + "retrieved_utc": "", + "exact_test_referenced": false, + "known_issue_referenced": false + }, + { + "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "found": true, + "retrieved_utc": "", + "check_id": 900101, + "conclusion": "failure", + "title": "1 failing test", + "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", + "text_excerpt": "1 test failed. See Azure DevOps for full details.", + "html_url": "https://github.com/dotnet/aspnetcore/runs/900101", + "exact_test_referenced": false, + "known_issue_referenced": false + } + ], + "raw_evidence_sources": [ + { + "build_id": 1551326, + "role": "failure", + "kind": "helix-console-log", + "helix_job": "helix-job-1551326", + "helix_workitem": "RedirectionTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "510308372262a7f187c524708516a44dbce8b74b28f6a4530f115b80356268ec", + "evidence_path": "issue-68947-build-1551326-failure.log" + }, + { + "build_id": 1549000, + "role": "failure", + "kind": "helix-console-log", + "helix_job": "helix-job-1549000", + "helix_workitem": "RedirectionTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "7e39070f2138ee0933dbf7b6ba1c2c38220f7ab6ffe3bb27f0874eb74b071fc2", + "evidence_path": "issue-68947-build-1549000-failure.log" + }, + { + "build_id": 1545000, + "role": "negative", + "kind": "helix-console-log", + "helix_job": "helix-job-1545000", + "helix_workitem": "RedirectionTest.WorkItemExecution", + "found": true, + "expired": false, + "captured_utc": "", + "sha256": "84e902974eafb9b414f662d7d708306b60b4884ee0882eb739a7636bcc2c747e", + "evidence_path": "issue-68947-build-1545000-negative.log" + } + ], + "duplicate_search": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + } + ] + } + }, + "candidate": { + "schema_version": 1, + "repository": "dotnet/aspnetcore", + "repository_ref": { + "branch": "main", + "commit_sha": "" + }, + "issue": { + "number": 68947, + "url": "https://github.com/dotnet/aspnetcore/issues/68947" + }, + "test": { + "fully_qualified_name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal" + }, + "signature": { + "kind": "ErrorMessage", + "values": [ + "OpenQA.Selenium.WebDriverException: TaskCanceledException" + ], + "build_retry": false, + "exclude_console_log": false + }, + "policy": { + "minimum_failure_logs": 2, + "minimum_negative_logs": 1 + }, + "evidence": { + "raw_logs": [ + { + "id": "evidence-1", + "role": "failure", + "outcome": "failed", + "path": "issue-68947-build-1551326-failure.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results", + "sha256": "510308372262a7f187c524708516a44dbce8b74b28f6a4530f115b80356268ec", + "build": { + "id": 1551326, + "pipeline_definition_id": 87, + "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", + "started_utc": "2026-08-13T10:00:00.0000000Z", + "platform": "Linux", + "configuration": "Release" + } + }, + { + "id": "evidence-2", + "role": "failure", + "outcome": "failed", + "path": "issue-68947-build-1549000-failure.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1549000&view=results", + "sha256": "7e39070f2138ee0933dbf7b6ba1c2c38220f7ab6ffe3bb27f0874eb74b071fc2", + "build": { + "id": 1549000, + "pipeline_definition_id": 87, + "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "started_utc": "2026-08-10T09:00:00.0000000Z", + "platform": "Linux", + "configuration": "Release" + } + }, + { + "id": "evidence-3", + "role": "negative", + "outcome": "passed", + "path": "issue-68947-build-1545000-negative.log", + "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1545000&view=results", + "sha256": "84e902974eafb9b414f662d7d708306b60b4884ee0882eb739a7636bcc2c747e", + "build": { + "id": 1545000, + "pipeline_definition_id": 87, + "source_version": "f33596dac091a6fd858b1515378d951f7f6adda5", + "started_utc": "2026-08-08T09:00:00.0000000Z", + "platform": "Linux", + "configuration": "Release" + } + } + ], + "corroborating_context": [ + { + "source": "build-analysis", + "url": "https://github.com/dotnet/aspnetcore/runs/900101" + }, + { + "source": "quarantine-issue", + "url": "https://github.com/dotnet/aspnetcore/issues/68947" + } + ] + }, + "duplicate_check": { + "status": "none", + "checked_utc": "", + "coverage": { + "open_kbes": true, + "recently_closed_kbes": true, + "open_fix_prs": true, + "recently_merged_fix_prs": true + }, + "references": [], + "queries": [ + { + "category": "open-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-closed-kbe", + "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "open-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:open RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + }, + { + "category": "recently-merged-fix-pr", + "query": "repo:dotnet/aspnetcore is:pr is:merged RedirectEnhancedNonBlazorGetToExternal", + "complete": true, + "result_numbers": [] + } + ] + }, + "proposed_classification": "timeout-needs-classification" + }, + "incomplete": null +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json new file mode 100644 index 000000000000..e74b2bf5a83f --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -0,0 +1,113 @@ +{ + "issue": { + "number": 68947, + "state": "open", + "labels": [ + "test-failure", + "area-blazor" + ], + "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/33496438442)\n" + }, + "azdo_builds": { + "1551326": { + "definition": { + "id": 87 + }, + "sourceVersion": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", + "startTime": "2026-08-13T10:00:00Z", + "finishTime": "2026-08-13T11:30:00Z", + "result": "failed" + } + }, + "recurrence_scan": { + "87": [ + { + "id": 1549000, + "sourceVersion": "0374b4797537d95f1b519446344fd88a7b5a861a", + "startTime": "2026-08-10T09:00:00Z", + "finishTime": "2026-08-10T10:30:00Z", + "result": "failed" + } + ] + }, + "negative_scan": { + "87": [ + { + "id": 1545000, + "sourceVersion": "f33596dac091a6fd858b1515378d951f7f6adda5", + "startTime": "2026-08-08T09:00:00Z", + "finishTime": "2026-08-08T10:30:00Z", + "result": "succeeded" + } + ] + }, + "vstmr_results": { + "1551326": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1551326\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", + "errorMessage": "OpenQA.Selenium.WebDriverException: TaskCanceledException", + "stackTrace": "at OpenQA.Selenium.Support.UI.WebDriverWait.Until..." + }, + "1549000": { + "outcome": "Failed", + "comment": "{\"HelixJobId\": \"helix-job-1549000\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", + "errorMessage": "OpenQA.Selenium.WebDriverException: TaskCanceledException", + "stackTrace": "at OpenQA.Selenium.Support.UI.WebDriverWait.Until..." + }, + "1545000": { + "outcome": "Passed", + "comment": "{\"HelixJobId\": \"helix-job-1545000\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", + "errorMessage": null, + "stackTrace": null + } + }, + "helix_evidence": { + "1551326": { + "found": true, + "expired": false, + "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal [3 s]\nOpenQA.Selenium.WebDriverException: TaskCanceledException\n(observed in build 1551326)" + }, + "1549000": { + "found": true, + "expired": false, + "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal [3 s]\nOpenQA.Selenium.WebDriverException: TaskCanceledException\n(observed in build 1549000)" + }, + "1545000": { + "found": true, + "expired": false, + "console_excerpt": "[PASS] Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal" + } + }, + "check_runs": { + "0374b4797537d95f1b519446344fd88a7b5a861a": [ + { + "name": "Build Analysis", + "id": 900101, + "conclusion": "failure", + "output": { + "title": "1 failing test", + "text": "1 test failed. See Azure DevOps for full details." + }, + "html_url": "https://github.com/dotnet/aspnetcore/runs/900101" + } + ] + }, + "duplicate_search": { + "open-kbe": { + "complete": true, + "result_numbers": [] + }, + "recently-closed-kbe": { + "complete": true, + "result_numbers": [] + }, + "open-fix-pr": { + "complete": true, + "result_numbers": [] + }, + "recently-merged-fix-pr": { + "complete": true, + "result_numbers": [] + } + } +} diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json new file mode 100644 index 000000000000..da0ee58a3a31 --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -0,0 +1,552 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json", + "title": "ASP.NET Core test quarantine KBE shadow collector dossier", + "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps build metadata, Helix/TRX raw evidence, and GitHub 'Build Analysis' check-run snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured 'incomplete' outcome. Never infers a pass, a recurrence, or a signature from missing or expired evidence.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "repository", + "collector", + "issue", + "outcome", + "provenance", + "candidate", + "incomplete" + ], + "properties": { + "schema_version": { + "const": 1 + }, + "repository": { + "const": "dotnet/aspnetcore" + }, + "collector": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "version", + "generated_utc", + "fixture_mode", + "manual_signature_provided" + ], + "properties": { + "name": { + "const": "Collect-TestQuarantineKbeEvidence.ps1" + }, + "version": { + "const": 1 + }, + "generated_utc": { + "type": "string", + "format": "date-time" + }, + "fixture_mode": { + "type": "boolean", + "description": "true when the collector read pre-fetched offline fixture data instead of making live network calls (tests and workflow self-test mode)." + }, + "manual_signature_provided": { + "type": "boolean", + "description": "true when the operator supplied -Signature because deterministic extraction from the issue body could not be proven unambiguous." + } + } + }, + "issue": { + "type": "object", + "additionalProperties": false, + "required": [ + "number", + "url", + "state", + "labels" + ], + "properties": { + "number": { + "type": "integer", + "minimum": 1 + }, + "url": { + "type": "string", + "pattern": "^https://github\\.com/dotnet/aspnetcore/issues/[1-9][0-9]*$" + }, + "state": { + "enum": [ + "open", + "closed" + ] + }, + "labels": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + } + }, + "outcome": { + "enum": [ + "candidate", + "incomplete" + ] + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": [ + "azdo_builds", + "check_run_snapshots", + "raw_evidence_sources", + "duplicate_search" + ], + "properties": { + "azdo_builds": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "found", + "retrieved_utc", + "source" + ], + "properties": { + "id": { + "type": "integer", + "minimum": 1 + }, + "found": { + "type": "boolean" + }, + "retrieved_utc": { + "type": "string", + "format": "date-time" + }, + "source": { + "enum": [ + "issue-body-reference", + "recurrence-scan", + "negative-scan" + ] + }, + "definition_id": { + "type": "integer", + "minimum": 1 + }, + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "started_utc": { + "type": "string", + "format": "date-time" + }, + "finished_utc": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "result": { + "type": "string", + "maxLength": 64 + }, + "note": { + "type": "string", + "maxLength": 512 + } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "definition_id", + "source_version", + "started_utc", + "result" + ] + } + }, + { + "if": { + "properties": { + "found": { + "const": false + } + } + }, + "then": { + "required": [ + "note" + ] + } + } + ] + } + }, + "check_run_snapshots": { + "type": "array", + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_version", + "found", + "retrieved_utc", + "exact_test_referenced", + "known_issue_referenced" + ], + "properties": { + "source_version": { + "$ref": "#/$defs/gitSha" + }, + "found": { + "type": "boolean" + }, + "retrieved_utc": { + "type": "string", + "format": "date-time" + }, + "check_id": { + "type": "integer", + "minimum": 1 + }, + "conclusion": { + "type": [ + "string", + "null" + ], + "maxLength": 32 + }, + "title": { + "type": "string", + "maxLength": 512 + }, + "text_sha256": { + "$ref": "#/$defs/sha256" + }, + "text_excerpt": { + "type": "string", + "maxLength": 2000 + }, + "html_url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "exact_test_referenced": { + "type": "boolean", + "description": "Conservative substring match of the quarantined test's fully qualified (or short method) name inside the check-run text. False when no snapshot was found." + }, + "known_issue_referenced": { + "type": "boolean", + "description": "Conservative match of a 'Known Issue' / known-build-error reference inside the check-run text. False when no snapshot was found." + } + } + } + }, + "raw_evidence_sources": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "build_id", + "role", + "found", + "expired", + "captured_utc" + ], + "properties": { + "build_id": { + "type": "integer", + "minimum": 1 + }, + "role": { + "enum": [ + "failure", + "negative" + ] + }, + "kind": { + "enum": [ + "helix-console-log", + "helix-test-log", + "vstmr-result" + ] + }, + "helix_job": { + "type": "string", + "maxLength": 128 + }, + "helix_workitem": { + "type": "string", + "maxLength": 256 + }, + "found": { + "type": "boolean" + }, + "expired": { + "type": "boolean", + "description": "true when the evidence source once existed (per the issue or a build/test-result reference) but its underlying artifact is no longer retrievable." + }, + "captured_utc": { + "type": "string", + "format": "date-time" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "evidence_path": { + "type": "string", + "maxLength": 512 + }, + "note": { + "type": "string", + "maxLength": 512 + } + } + } + }, + "duplicate_search": { + "$ref": "#/$defs/duplicateCheck" + } + } + }, + "candidate": { + "type": [ + "object", + "null" + ], + "description": "Present only when outcome is 'candidate'. Must independently satisfy test-quarantine-kbe-shadow-candidate.schema.json; validated separately by the collector and by Evaluate-TestQuarantineKbeCandidate.ps1." + }, + "incomplete": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "required": [ + "reason_codes", + "message", + "missing_evidence" + ], + "properties": { + "reason_codes": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "items": { + "enum": [ + "issue-not-canonical-quarantine", + "issue-not-open", + "test-name-unresolvable", + "build-reference-unresolvable", + "build-metadata-expired", + "raw-evidence-expired", + "raw-evidence-insufficient", + "recurrence-single-build-only", + "signature-extraction-ambiguous", + "duplicate-search-incomplete", + "github-api-error" + ] + } + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "missing_evidence": { + "type": "array", + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "detail" + ], + "properties": { + "kind": { + "type": "string", + "maxLength": 64 + }, + "detail": { + "type": "string", + "maxLength": 512 + } + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { + "const": "candidate" + } + } + }, + "then": { + "properties": { + "candidate": { + "type": "object" + }, + "incomplete": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "outcome": { + "const": "incomplete" + } + } + }, + "then": { + "properties": { + "candidate": { + "type": "null" + }, + "incomplete": { + "type": "object" + } + } + } + } + ], + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gitSha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "duplicateCheck": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "checked_utc", + "coverage", + "references", + "queries" + ], + "properties": { + "status": { + "enum": [ + "none", + "existing-kbe", + "existing-fix-pr", + "ambiguous", + "integrity-filtered", + "not-evaluated" + ] + }, + "checked_utc": { + "type": "string", + "format": "date-time" + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "open_kbes", + "recently_closed_kbes", + "open_fix_prs", + "recently_merged_fix_prs" + ], + "properties": { + "open_kbes": { + "type": "boolean" + }, + "recently_closed_kbes": { + "type": "boolean" + }, + "open_fix_prs": { + "type": "boolean" + }, + "recently_merged_fix_prs": { + "type": "boolean" + } + } + }, + "references": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^(issue|pull-request):[1-9][0-9]*$" + } + }, + "queries": { + "type": "array", + "minItems": 4, + "maxItems": 20, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "query", + "complete", + "result_numbers" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "complete": { + "type": "boolean" + }, + "result_numbers": { + "type": "array", + "maxItems": 50, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } + } + } + } + } + } + } + } +} diff --git a/.github/workflows/test-quarantine-kbe-shadow-tests.yml b/.github/workflows/test-quarantine-kbe-shadow-tests.yml new file mode 100644 index 000000000000..394631a313c4 --- /dev/null +++ b/.github/workflows/test-quarantine-kbe-shadow-tests.yml @@ -0,0 +1,39 @@ +name: Test quarantine KBE shadow (unit tests) + +# Repository-convention CI coverage for the read-only test-quarantine-kbe-shadow evaluator and +# collector: runs their deterministic, offline PowerShell test suites whenever this workflow file, +# the scripts directory, or its schemas/fixtures change. Every fixture used here is a small, +# sanitized, offline JSON document; no network access or repository secrets are required. + +on: + pull_request: + paths: + - ".github/workflows/test-quarantine-kbe-shadow.yml" + - ".github/workflows/test-quarantine-kbe-shadow-tests.yml" + - ".github/workflows/scripts/test-quarantine-kbe-shadow/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + run-tests: + name: Run evaluator and collector tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + submodules: false + + - name: Run Evaluate-TestQuarantineKbeCandidate.ps1 tests + shell: pwsh + run: | + & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1' + + - name: Run Collect-TestQuarantineKbeEvidence.ps1 tests + shell: pwsh + run: | + & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1' diff --git a/.github/workflows/test-quarantine-kbe-shadow.yml b/.github/workflows/test-quarantine-kbe-shadow.yml new file mode 100644 index 000000000000..e8e2b262a04d --- /dev/null +++ b/.github/workflows/test-quarantine-kbe-shadow.yml @@ -0,0 +1,116 @@ +name: Test quarantine KBE shadow (single issue) + +# Maintainer-triggered, read-only shadow evaluation for exactly one existing dotnet/aspnetcore +# test-quarantine issue. See .github/workflows/scripts/test-quarantine-kbe-shadow/README.md for +# the full trust boundary, evidence model, and promotion gates. +# +# This workflow NEVER labels, comments on, or otherwise mutates any issue, pull request, branch, +# or repository file. It only reads public GitHub/Azure DevOps/Helix data and uploads short- +# retention artifacts (candidate/dossier, receipt, and a human-readable summary). + +on: + workflow_dispatch: + inputs: + issue_number: + description: "The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open test-failure quarantine issue for the test(s) it names." + required: true + signature: + description: "Optional manual failure-signature override. Only needed when the issue body has no fenced '## Error Message' code block that the collector can deterministically extract." + required: false + +# Fail closed on forks: this workflow only ever runs via an explicit, permissioned +# workflow_dispatch, but a fork's copy of this file would otherwise execute in the fork's own +# `github.repository` context. Gating on the canonical repository keeps a forked copy inert. +concurrency: + group: test-quarantine-kbe-shadow-${{ github.event.inputs.issue_number }} + cancel-in-progress: true + +permissions: + contents: read + issues: read + pull-requests: read + checks: read + +jobs: + collect-and-evaluate: + name: Collect evidence and evaluate + if: ${{ github.repository == 'dotnet/aspnetcore' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Validate issue_number input + shell: pwsh + run: | + if ('${{ github.event.inputs.issue_number }}' -notmatch '^[1-9][0-9]*$') + { + Write-Error "issue_number must be a positive integer; received '${{ github.event.inputs.issue_number }}'." + exit 1 + } + + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + submodules: false + persist-credentials: false + + - name: Collect quarantine evidence + id: collect + shell: pwsh + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + $params = @{ + IssueNumber = [int]"${{ github.event.inputs.issue_number }}" + OutputFile = "$env:RUNNER_TEMP/dossier.json" + CandidateFile = "$env:RUNNER_TEMP/candidate.json" + EvidenceRoot = "$env:RUNNER_TEMP/evidence" + } + $signature = '${{ github.event.inputs.signature }}' + if (-not [string]::IsNullOrWhiteSpace($signature)) + { + $params["Signature"] = $signature + } + & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1' @params + + $dossier = Get-Content -LiteralPath "$env:RUNNER_TEMP/dossier.json" -Raw | ConvertFrom-Json -Depth 32 + "outcome=$($dossier.outcome)" >> $env:GITHUB_OUTPUT + + - name: Evaluate candidate + id: evaluate + if: ${{ steps.collect.outputs.outcome == 'candidate' }} + shell: pwsh + run: | + & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1' ` + -CandidateFile "$env:RUNNER_TEMP/candidate.json" ` + -EvidenceRoot "$env:RUNNER_TEMP/evidence" ` + -OutputFile "$env:RUNNER_TEMP/receipt.json" + + - name: Render human-readable summary + if: ${{ always() && steps.collect.outcome == 'success' }} + shell: pwsh + run: | + $params = @{ + DossierFile = "$env:RUNNER_TEMP/dossier.json" + OutputFile = "$env:RUNNER_TEMP/summary.md" + } + if (Test-Path -LiteralPath "$env:RUNNER_TEMP/receipt.json") + { + $params["ReceiptFile"] = "$env:RUNNER_TEMP/receipt.json" + } + & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1' @params + Get-Content -LiteralPath "$env:RUNNER_TEMP/summary.md" -Raw >> $env:GITHUB_STEP_SUMMARY + + - name: Upload dossier and receipt + if: ${{ always() && steps.collect.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-quarantine-kbe-shadow-issue-${{ github.event.inputs.issue_number }} + path: | + ${{ runner.temp }}/dossier.json + ${{ runner.temp }}/candidate.json + ${{ runner.temp }}/receipt.json + ${{ runner.temp }}/summary.md + ${{ runner.temp }}/evidence/** + if-no-files-found: ignore + retention-days: 7 From 199270c5713bd1502228ffab79fe3d7b22388829 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:06:03 -0500 Subject: [PATCH 03/10] Fix script-injection risk in test-quarantine-kbe-shadow workflow test-quarantine-kbe-shadow.yml interpolated the workflow_dispatch `issue_number` and `signature` inputs directly into `run:` script bodies via `${{ inputs.issue_number }}` / `${{ inputs.signature }}`. A signature value containing a quote, backtick, or newline could execute arbitrary commands on the runner. - Add a "Validate and normalize inputs" step that reads issue_number via a step `env:` binding, validates it against ^[1-9][0-9]*$, and persists it to $GITHUB_ENV as ISSUE_NUMBER for later steps. - Pass signature through a step `env:` binding (SIGNATURE_INPUT) instead of embedding it in the script; read both via $env:* inside PowerShell, which never re-parses an environment variable's value as script text. - Replace every `${{ github.workspace }}` reference inside `run:` bodies with $env:GITHUB_WORKSPACE (a builtin runner variable) so no `${{ }}` expression appears inside a run: script body at all. - Use the validated env.ISSUE_NUMBER (not the raw input) in the uploaded artifact's name, a non-run: context that Actions substitutes structurally rather than via a shell. - Apply the same $env:GITHUB_WORKSPACE fix to test-quarantine-kbe-shadow-tests.yml. - Add Test-WorkflowScriptInjectionSafety.ps1: a static, offline test that parses every run: block in both workflow files and fails if any contains a `${{ ... }}` GitHub Actions expression. Wired into the CI tests workflow and documented in README.md. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test-quarantine-kbe-shadow/README.md | 31 +++- .../Test-WorkflowScriptInjectionSafety.ps1 | 132 ++++++++++++++++++ .../test-quarantine-kbe-shadow-tests.yml | 9 +- .../workflows/test-quarantine-kbe-shadow.yml | 35 +++-- 4 files changed, 189 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index ba71502b95a7..42a888dfc019 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -18,6 +18,7 @@ production without an explicit, separate decision by a maintainer. | `test-quarantine-kbe-shadow-dossier.schema.json` | Schema for the collector's output envelope (provenance + either `candidate` or `incomplete`). Independently versioned; it wraps and reuses the candidate schema rather than replacing or competing with it. | | `fixtures//` | Compact, sanitized, offline fixtures for three real pilot issues, each with a golden `expected-dossier.json`. | | `Test-Evaluate-TestQuarantineKbeCandidate.ps1`, `Test-Collect-TestQuarantineKbeEvidence.ps1` | Deterministic, offline test suites (no network access). | +| `Test-WorkflowScriptInjectionSafety.ps1` | Static regression test asserting that no `run:` script body in the two workflows below interpolates a `${{ ... }}` GitHub Actions expression (see "Script-injection safety" below). | The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dispatch) and `.github/workflows/test-quarantine-kbe-shadow-tests.yml` (CI) workflows are described below. @@ -78,10 +79,32 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis ## CI: `test-quarantine-kbe-shadow-tests.yml` Runs both deterministic, offline PowerShell test suites (`Test-Evaluate-TestQuarantineKbeCandidate.ps1` -and `Test-Collect-TestQuarantineKbeEvidence.ps1`) on every pull request that touches this directory -or either workflow file, plus on manual dispatch. Every fixture is local and offline; no network -access or secrets are required, so this is a low-risk, standard `pull_request`-triggered check — -no self-test mode inside the shadow workflow itself was needed to close this CI gap. +and `Test-Collect-TestQuarantineKbeEvidence.ps1`), plus the script-injection safety guard described +below, on every pull request that touches this directory or either workflow file, plus on manual +dispatch. Every fixture is local and offline; no network access or secrets are required, so this is +a low-risk, standard `pull_request`-triggered check — no self-test mode inside the shadow workflow +itself was needed to close this CI gap. + +## Script-injection safety + +`test-quarantine-kbe-shadow.yml` accepts two `workflow_dispatch` inputs, `issue_number` and +`signature`, both of which are attacker-influenceable text (anyone who can dispatch the workflow +controls their exact content). Neither is ever interpolated directly into a `run:` script body via +`${{ inputs.issue_number }}` / `${{ inputs.signature }}`: a signature value containing a quote, +backtick, or newline embedded directly into script text could otherwise execute arbitrary commands +on the runner. Instead, every input flows through a step (or job) `env:` binding — for example +`env: { SIGNATURE_INPUT: ${{ inputs.signature }} }` — and is read back inside the script as an +opaque environment variable (`$env:SIGNATURE_INPUT`), which the PowerShell parser never re-parses +as script text regardless of its content. `issue_number` is additionally validated against +`^[1-9][0-9]*$` before being persisted to `$GITHUB_ENV`, and only that already-validated +`env.ISSUE_NUMBER` value (not the raw input) is used in the uploaded artifact's name — a +non-`run:` context where Actions itself handles the substitution, not a shell. + +`Test-WorkflowScriptInjectionSafety.ps1` is a static, offline regression test for exactly this +invariant: it parses every `run:` block (block-scalar and single-line forms) in both workflow files +using a minimal, indentation-based reader and fails if any of them contains a `${{` token. It +deliberately does not inspect `if:`, `env:`, `with:`, or `concurrency:` values, since a `${{ }}` +expression there is evaluated by the Actions engine itself and is not a script-injection vector. ## Fixture mode (`-FixtureRoot`) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 new file mode 100644 index 000000000000..84c4ba83002f --- /dev/null +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1 @@ -0,0 +1,132 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Static regression test: no GitHub Actions `${{ ... }}` expression may appear inside a `run:` + script body in the test-quarantine-kbe-shadow workflows. + +.DESCRIPTION + Directly interpolating a `${{ ... }}` expression -- especially a workflow_dispatch input such + as `${{ inputs.signature }}` -- into a `run:` script body is a script-injection vector: a value + containing a quote, backtick, or newline can execute arbitrary commands on the runner. Every + value this workflow needs inside a script must instead flow through a step (or job) `env:` + binding and be read back as an opaque environment variable (e.g. `$env:SIGNATURE_INPUT`), + which the shell/PowerShell parser never re-parses as script text regardless of its content. + + This test parses each target workflow file's YAML block-scalar `run:` bodies (both `run: |` + and unquoted single-line `run: ...` forms) using a minimal, indentation-based reader -- no + external YAML/GitHub Actions parser dependency -- and fails if any of them contains a `${{` + token. It intentionally does not inspect `if:`, `env:`, `with:`, or `concurrency:` values: a + `${{ }}` expression there is evaluated by the Actions engine itself, not by a shell, and is not + a script-injection vector. +#> + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$workflowsRoot = (Resolve-Path "$PSScriptRoot/../..").Path +$targetFiles = @( + (Join-Path $workflowsRoot "test-quarantine-kbe-shadow.yml"), + (Join-Path $workflowsRoot "test-quarantine-kbe-shadow-tests.yml") +) + +function Get-IndentWidth +{ + param([Parameter(Mandatory = $true)][string]$Line) + + $trimmed = $Line.TrimStart(" ") + return $Line.Length - $trimmed.Length +} + +function Get-RunBlockBodies +{ + param([Parameter(Mandatory = $true)][string]$Path) + + $lines = [System.IO.File]::ReadAllLines($Path) + $blocks = [System.Collections.Generic.List[object]]::new() + $i = 0 + while ($i -lt $lines.Count) + { + $line = $lines[$i] + $singleLineMatch = [regex]::Match($line, '^(\s*)run:\s*(?!\||>)(\S.*)$') + $blockMatch = [regex]::Match($line, '^(\s*)run:\s*[|>][+-]?\s*$') + + if ($singleLineMatch.Success) + { + $null = $blocks.Add([ordered]@{ StartLine = $i + 1; Text = $singleLineMatch.Groups[2].Value }) + $i++ + continue + } + + if ($blockMatch.Success) + { + $keyIndent = $blockMatch.Groups[1].Value.Length + $bodyLines = [System.Collections.Generic.List[string]]::new() + $j = $i + 1 + while ($j -lt $lines.Count) + { + $candidate = $lines[$j] + if ($candidate.Trim().Length -eq 0) + { + $bodyLines.Add($candidate) + $j++ + continue + } + if ((Get-IndentWidth -Line $candidate) -le $keyIndent) + { + break + } + $bodyLines.Add($candidate) + $j++ + } + $null = $blocks.Add([ordered]@{ StartLine = $i + 1; Text = ($bodyLines -join "`n") }) + $i = $j + continue + } + + $i++ + } + + return $blocks +} + +try +{ + $failures = [System.Collections.Generic.List[string]]::new() + + foreach ($file in $targetFiles) + { + if (-not (Test-Path -LiteralPath $file)) + { + throw "Target workflow file does not exist: $file" + } + + $blocks = Get-RunBlockBodies -Path $file + if ($blocks.Count -eq 0) + { + throw "No 'run:' blocks were found in $file; the parser may be broken (this test expects at least one)." + } + + foreach ($block in $blocks) + { + if ($block.Text.Contains('${{')) + { + $null = $failures.Add("$($file):$($block.StartLine): 'run:' block contains a '`${{ ... }}' expression.") + } + } + } + + if ($failures.Count -gt 0) + { + throw "Script-injection guard failed:`n" + ($failures -join "`n") + } + + Write-Host "No 'run:' block in the test-quarantine-kbe-shadow workflows contains a GitHub Actions expression." +} +catch +{ + Write-Error $_ + exit 1 +} diff --git a/.github/workflows/test-quarantine-kbe-shadow-tests.yml b/.github/workflows/test-quarantine-kbe-shadow-tests.yml index 394631a313c4..83afbba487aa 100644 --- a/.github/workflows/test-quarantine-kbe-shadow-tests.yml +++ b/.github/workflows/test-quarantine-kbe-shadow-tests.yml @@ -31,9 +31,14 @@ jobs: - name: Run Evaluate-TestQuarantineKbeCandidate.ps1 tests shell: pwsh run: | - & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1' + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1" - name: Run Collect-TestQuarantineKbeEvidence.ps1 tests shell: pwsh run: | - & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1' + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1" + + - name: Run workflow script-injection safety guard + shell: pwsh + run: | + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-WorkflowScriptInjectionSafety.ps1" diff --git a/.github/workflows/test-quarantine-kbe-shadow.yml b/.github/workflows/test-quarantine-kbe-shadow.yml index e8e2b262a04d..7c789fda5a37 100644 --- a/.github/workflows/test-quarantine-kbe-shadow.yml +++ b/.github/workflows/test-quarantine-kbe-shadow.yml @@ -22,7 +22,7 @@ on: # workflow_dispatch, but a fork's copy of this file would otherwise execute in the fork's own # `github.repository` context. Gating on the canonical repository keeps a forked copy inert. concurrency: - group: test-quarantine-kbe-shadow-${{ github.event.inputs.issue_number }} + group: test-quarantine-kbe-shadow-${{ inputs.issue_number }} cancel-in-progress: true permissions: @@ -38,14 +38,24 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - name: Validate issue_number input + # Every workflow_dispatch input is passed through a step `env:` binding and read back via + # $env: inside the script -- never interpolated directly into `run:` script text via + # `${{ inputs.* }}`. A signature value containing a quote, backtick, or newline embedded + # directly into the script body could otherwise execute arbitrary runner commands; an + # environment-variable value is opaque data to the shell/PowerShell parser regardless of + # its content. + - name: Validate and normalize inputs + id: normalize shell: pwsh + env: + ISSUE_NUMBER_INPUT: ${{ inputs.issue_number }} run: | - if ('${{ github.event.inputs.issue_number }}' -notmatch '^[1-9][0-9]*$') + if ($env:ISSUE_NUMBER_INPUT -notmatch '^[1-9][0-9]*$') { - Write-Error "issue_number must be a positive integer; received '${{ github.event.inputs.issue_number }}'." + Write-Error "issue_number must be a positive integer." exit 1 } + "ISSUE_NUMBER=$($env:ISSUE_NUMBER_INPUT)" >> $env:GITHUB_ENV - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -59,19 +69,20 @@ jobs: shell: pwsh env: GITHUB_TOKEN: ${{ github.token }} + ISSUE_NUMBER: ${{ env.ISSUE_NUMBER }} + SIGNATURE_INPUT: ${{ inputs.signature }} run: | $params = @{ - IssueNumber = [int]"${{ github.event.inputs.issue_number }}" + IssueNumber = [int]$env:ISSUE_NUMBER OutputFile = "$env:RUNNER_TEMP/dossier.json" CandidateFile = "$env:RUNNER_TEMP/candidate.json" EvidenceRoot = "$env:RUNNER_TEMP/evidence" } - $signature = '${{ github.event.inputs.signature }}' - if (-not [string]::IsNullOrWhiteSpace($signature)) + if (-not [string]::IsNullOrWhiteSpace($env:SIGNATURE_INPUT)) { - $params["Signature"] = $signature + $params["Signature"] = $env:SIGNATURE_INPUT } - & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1' @params + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1" @params $dossier = Get-Content -LiteralPath "$env:RUNNER_TEMP/dossier.json" -Raw | ConvertFrom-Json -Depth 32 "outcome=$($dossier.outcome)" >> $env:GITHUB_OUTPUT @@ -81,7 +92,7 @@ jobs: if: ${{ steps.collect.outputs.outcome == 'candidate' }} shell: pwsh run: | - & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1' ` + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1" ` -CandidateFile "$env:RUNNER_TEMP/candidate.json" ` -EvidenceRoot "$env:RUNNER_TEMP/evidence" ` -OutputFile "$env:RUNNER_TEMP/receipt.json" @@ -98,14 +109,14 @@ jobs: { $params["ReceiptFile"] = "$env:RUNNER_TEMP/receipt.json" } - & '${{ github.workspace }}/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1' @params + & "$env:GITHUB_WORKSPACE/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1" @params Get-Content -LiteralPath "$env:RUNNER_TEMP/summary.md" -Raw >> $env:GITHUB_STEP_SUMMARY - name: Upload dossier and receipt if: ${{ always() && steps.collect.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-quarantine-kbe-shadow-issue-${{ github.event.inputs.issue_number }} + name: test-quarantine-kbe-shadow-issue-${{ env.ISSUE_NUMBER }} path: | ${{ runner.temp }}/dossier.json ${{ runner.temp }}/candidate.json From 35772881cbbb217967c94878e2c3291d81b48976 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:46:34 -0500 Subject: [PATCH 04/10] Fix 12 evidence-integrity and correctness issues in the KBE shadow collector Independent review found these high-confidence blockers in the read-only collector; all are fixed and covered by new/updated fixtures: 1. GitHub auth: Get-GitHubIssue/Get-CheckRunsForSha/Get-DuplicateSearch sent a literal placeholder Authorization header instead of a real bearer token. Consolidated into one Get-GitHubHeaders helper that sends a real "Bearer $GitHubToken" and logs (non-blocking) the authenticated X-RateLimit-Remaining/-Limit headers after each GitHub call. 2. Recurrence build filter: Azure DevOps' resultFilter does not support a comma-separated multi-value combination (verified live). Query resultFilter=failed and resultFilter=partiallySucceeded separately and merge/dedupe by build id via a new, directly unit-tested Merge-AzdoBuildLists function -- ASP.NET Core failures routinely land in a partiallySucceeded build (aspnetcore#68947's own cited build 1551326 is one, confirmed live). 3. Live VSTMR shape: resultsbyBuild summary rows carry only identity + outcome for ordinary xUnit tests (verified live -- no comment/errorMessage/stackTrace field). Added Get-VstmrSummaryRows (identity/outcome) and Get-VstmrDetail (the detailed per-result endpoint, the authoritative errorMessage/stackTrace source). Helix coordinates are recorded only when a work item's own crash pseudo-test row happens to carry them; otherwise helix_unavailable is recorded explicitly and VSTMR detail text remains authoritative on its own. Dropped the previous (unrealistic) direct Helix console-log fetch path entirely. 4. Multi-test identity: '## Failing Test(s)' can name more than one concrete identity (aspnetcore#68724 names a base test and its server-execution override; live data shows only the override actually failed). The collector now fails closed (multiple-test-identities-unresolved) unless exactly one identity is named, documented as an accepted one-issue/one-root-cause simplification. #68724's fixture/golden updated to exercise this. 5. Canonical issue validation: the 'test-failure' label alone is not proof of quarantine automation. Now also requires the trusted '' / '' marker in the issue body. Added a negative fixture/test. 6. Build Analysis flag precision: exact_test_referenced now requires the full fully-qualified name (a new short_name_referenced field records a bare-method-name-only match separately, non-authoritative). known_issue_referenced now requires a concrete issue number/URL near the phrase (known_issue_numbers records them); a generic heading/table label reading "Known Issue(s)" with no reference no longer sets it. Added fixtures for both precision gates. 7. Duplicate search completeness: recently-closed-kbe/recently-merged-fix-pr queries now carry an explicit 90-day closed:>=/merged:>= window. Search-GitHubIssues paginates up to 300 results and marks a query complete only when incomplete_results=false AND every matching item (per total_count) was actually retrieved. 8. Duplicate search validation: every search hit is fetched and required to contain the exact fully-qualified test name before being treated as a validated existing-kbe/existing-fix-pr reference; otherwise it is recorded as an unvalidated_candidate (new dossier-only field) and never sets duplicate_check.status. Added a negative fixture (same short method name, different test). 9. Repository ref binding: repository_ref.commit_sha is the checked-out HEAD (required by the unmodified evaluator's own self-consistency check), but branch="main" is now only ever asserted after confirming, via a trusted GET /repos/dotnet/aspnetcore/commits/main response, that the checkout actually is main's tip (repository-ref-not-main fails closed otherwise). Recorded in a new provenance.repository_ref_verification block. Added a mismatch fixture. 10. Platform/configuration: previously hardcoded to Linux/Release for every log. Now parsed from the authoritative VSTMR TestRun name (e.g. "Quarantine-Mono-Linux-Release-xunit"); records the literal "unknown" when no recognized token is present, never a fabricated default. 11. Evidence capping: raw evidence text is now constructed with the failed/passed-test marker line and the (already-matched) signature always first, so an 12KB cap can only ever truncate the tail of a long stack trace, never the lines the evaluator's association window needs. 12. Signature matching: replaced `-notlike`/`-like` (which misinterprets literal `*`, `?`, `[` as wildcards) with ordinal, case-sensitive substring containment everywhere a signature is matched against raw evidence text. Added a wildcard-signature fixture with a decoy build that would have spuriously matched under -like semantics. Also fixes a handful of PowerShell pitfalls found while implementing the above: `return @()` collapsing to $null when directly assigned (fixed by consistently wrapping call sites with @(...) rather than mixing conventions), `.PSObject.Properties.Name -contains` throwing under Set-StrictMode for zero-property objects (replaced with a safe Test-HasProperty helper), and `[Parameter(Mandatory=$true)]` collection/null parameters rejecting legitimately-empty/null arguments (added AllowEmptyCollection()/AllowNull() where needed). Does not modify test-quarantine.md, test-quarantine.lock.yml, or any production quarantine/unquarantine behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 927 +++++++++++++----- .../test-quarantine-kbe-shadow/README.md | 215 ++-- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 523 +++++++--- .../fixtures/68724/expected-dossier.json | 268 +---- .../fixtures/68724/fixture.json | 108 +- .../fixtures/68945/expected-dossier.json | 71 +- .../fixtures/68945/fixture.json | 81 +- .../fixtures/68947/expected-dossier.json | 109 +- .../fixtures/68947/fixture.json | 112 ++- ...-quarantine-kbe-shadow-dossier.schema.json | 133 ++- 10 files changed, 1630 insertions(+), 917 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index df8081774d7b..d9498b09948d 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -5,21 +5,22 @@ .DESCRIPTION Gathers public evidence for a single quarantine issue -- the issue body itself, Azure DevOps - build metadata, GitHub "Build Analysis" check-run snapshots (corroborating only, never - authoritative), and capped/redacted raw Helix evidence -- then emits either: + build metadata, authoritative VSTMR test-result detail (errorMessage/stackTrace), and GitHub + "Build Analysis" check-run snapshots (corroborating only, never authoritative) -- then emits + either: * a "candidate" object that independently satisfies test-quarantine-kbe-shadow-candidate.schema.json and is ready for Evaluate-TestQuarantineKbeCandidate.ps1, or * a structured "incomplete" outcome explaining exactly which evidence could not be - established, without inferring a pass, a recurrence, or a signature from anything missing - or expired. + established, without inferring a pass, a recurrence, a signature, a platform/configuration, + or a validated duplicate from anything missing, ambiguous, or unverifiable. - This script makes no repository-state mutations. It only reads public GitHub/Azure DevOps/Helix + This script makes no repository-state mutations. It only reads public GitHub/Azure DevOps endpoints (or, in fixture mode, a local fixture file) and writes local files: the dossier, an - optional candidate, and capped evidence text files under -EvidenceRoot. + optional candidate, and capped/redacted evidence text files under -EvidenceRoot. .PARAMETER IssueNumber - The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open - test-quarantine issue for the test(s) it names. + The dotnet/aspnetcore issue number to evaluate. Must be the canonical, currently open, + automation-generated test-quarantine issue for the test(s) it names. .PARAMETER Signature Optional manual failure-signature override. Supply this when the issue body does not contain @@ -39,14 +40,16 @@ .PARAMETER FixtureRoot Optional path to a directory containing a single consolidated 'fixture.json' file that stands - in for every live network call (GitHub issue/check-runs/search, Azure DevOps build metadata, - Helix evidence). Used by the test harness and by the shadow workflow's self-test mode so this + in for every live network call (GitHub issue/check-runs/search/commits and Azure DevOps + build/VSTMR data). Used by the test suite and by the shadow workflow's self-test mode so this script's deterministic logic can be exercised with zero network access. See README.md for the fixture.json shape. .PARAMETER GitHubToken - Optional GitHub token for the GitHub REST calls (issue, check-runs, search). Falls back to the - GITHUB_TOKEN environment variable, then to unauthenticated requests. + GitHub token for the GitHub REST calls (issue, check-runs, search, commits). Falls back to the + GITHUB_TOKEN environment variable. Authenticated calls are strongly preferred: the search API's + unauthenticated rate limit (10 requests/minute) is exhausted by a single run's four duplicate + searches plus one retry. #> [CmdletBinding()] @@ -77,7 +80,13 @@ param( [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", - [int]$RecurrenceScanBuildCap = 20 + [int]$RecurrenceScanBuildCap = 20, + + [int]$DuplicateSearchWindowDays = 90, + + [int]$DuplicateSearchPageSize = 100, + + [int]$DuplicateSearchMaxPages = 3 ) Set-StrictMode -Version Latest @@ -85,13 +94,16 @@ $ErrorActionPreference = "Stop" $ado = "https://dev.azure.com/dnceng-public/public/_apis" $vstmr = "https://vstmr.dev.azure.com/dnceng-public/public/_apis" -$helix = "https://helix.dot.net/api/2019-06-17" $pipelineDefinitionIds = @(83, 87) $canonicalQuarantineLabel = "test-failure" +$workflowMarkers = @( + "", + "" +) $minimumFailureBuilds = 2 $minimumNegativeLogs = 1 $excerptCap = 2000 -$rawLogCap = 8000 +$rawLogCap = 12000 # Same secret-shaped patterns the production test-quarantine.md deterministic collector scrubs # before surfacing captured CI text. Helix work-item upload steps can log a live Azure DevOps @@ -122,29 +134,6 @@ function ConvertTo-Redacted return $result } -function ConvertTo-Iso8601String -{ - # ConvertFrom-Json (and Invoke-RestMethod, which uses it internally) auto-converts - # ISO-8601-shaped JSON string values into [datetime] instances. A bare [string] cast - # on one of those then renders using the current culture (e.g. "08/22/2026 03:28:01"), - # silently dropping the 'Z' suffix and sub-second precision and producing a value that - # is no longer schema-valid 'date-time' text. Route every timestamp read from parsed - # JSON through this so the emitted dossier/candidate always carries a real ISO-8601 - # UTC string regardless of whether the runtime parsed it into a DateTime or left it - # as a string. - param([AllowNull()]$Value) - - if ($null -eq $Value) - { - return $null - } - if ($Value -is [datetime]) - { - return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") - } - return [string]$Value -} - function Get-Sha256String { param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) @@ -191,6 +180,10 @@ function Get-CappedExcerpt $redacted = $redacted.Replace($token, [string]$placeholders[$token]) } + # The evidence text this function builds always places the failed/passed-test marker line + # and the (already-extracted) signature first, so capping to the first $Cap characters here + # can only ever truncate the tail of a long stack trace -- never the marker or signature the + # evaluator's association window needs. See Assemble-EvidenceText below. $normalized = [System.Text.RegularExpressions.Regex]::Replace($redacted, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "?") if ($normalized.Length -gt $Cap) { @@ -200,6 +193,52 @@ function Get-CappedExcerpt return $normalized } +function ConvertTo-Iso8601String +{ + # ConvertFrom-Json (and Invoke-RestMethod, which uses it internally) auto-converts + # ISO-8601-shaped JSON string values into [datetime] instances. A bare [string] cast + # on one of those then renders using the current culture (e.g. "08/22/2026 03:28:01"), + # silently dropping the 'Z' suffix and sub-second precision and producing a value that + # is no longer schema-valid 'date-time' text. Route every timestamp read from parsed + # JSON through this so the emitted dossier/candidate always carries a real ISO-8601 + # UTC string regardless of whether the runtime parsed it into a DateTime or left it + # as a string. + param([AllowNull()]$Value) + + if ($null -eq $Value) + { + return $null + } + if ($Value -is [datetime]) + { + return $Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ") + } + return [string]$Value +} + +function Test-HasProperty +{ + # `$obj.PSObject.Properties.Name -contains $name` throws "The property 'Name' cannot be + # found on this object" under Set-StrictMode -Version Latest whenever the object has zero + # properties (an empty JSON object `{}}`, which every fixture.json category not exercised by + # a given test/fixture legitimately is). Iterating .PSObject.Properties directly is safe for + # both the empty and non-empty case. + param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name) + + if ($null -eq $Object) + { + return $false + } + foreach ($property in $Object.PSObject.Properties) + { + if ($property.Name -eq $Name) + { + return $true + } + } + return $false +} + function Add-MissingEvidence { param( @@ -215,8 +254,7 @@ function Add-MissingEvidence # --------------------------------------------------------------------------- # Fixture / live network abstraction. Fixture mode reads one consolidated JSON -# document; live mode calls the public GitHub/Azure DevOps/Helix REST APIs -# documented in test-quarantine.md's "API Reference (Azure DevOps & Helix)". +# document; live mode calls the public GitHub/Azure DevOps REST APIs. # --------------------------------------------------------------------------- $isFixtureMode = -not [string]::IsNullOrEmpty($FixtureRoot) @@ -231,6 +269,39 @@ if ($isFixtureMode) $fixture = Get-Content -LiteralPath $fixturePath -Raw | ConvertFrom-Json -Depth 32 } +function Get-GitHubHeaders +{ + # A real bearer token, never a literal placeholder. Authenticated GitHub REST/Search calls + # get a materially higher rate limit (up to 30 search requests/minute vs. 10 unauthenticated) + # and are required in practice: a single run's four duplicate-search categories alone can + # exhaust the unauthenticated search quota. + $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } + if (-not [string]::IsNullOrEmpty($GitHubToken)) + { + $headers["Authorization"] = "Bearer $GitHubToken" + } + return $headers +} + +function Write-RateLimitDiagnostic +{ + # Non-blocking, informational only: surfaces the authenticated GitHub rate-limit headroom in + # the workflow log so a maintainer can see at a glance whether the token is actually being + # used and how much quota remains. Never fails the run if the header is absent. + param($ResponseHeaders) + + if ($null -eq $ResponseHeaders) + { + return + } + $remaining = $ResponseHeaders["X-RateLimit-Remaining"] + $limit = $ResponseHeaders["X-RateLimit-Limit"] + if ($remaining -and $limit) + { + Write-Host "GitHub API rate limit: $remaining/$limit remaining." + } +} + function Get-GitHubIssue { param([Parameter(Mandatory = $true)][int]$Number) @@ -244,12 +315,38 @@ function Get-GitHubIssue return $fixture.issue } - $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } - if (-not [string]::IsNullOrEmpty($GitHubToken)) + $headers = Get-GitHubHeaders + $responseHeaders = $null + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 -ResponseHeadersVariable responseHeaders + Write-RateLimitDiagnostic -ResponseHeaders $responseHeaders + return $result +} + +function Get-TrustedMainShaResult +{ + # Returns @{ Checked; TrustedSha }. `Checked = $false` only in fixture mode when the fixture + # does not model this dimension at all (the three real pilot fixtures do not); live mode + # always performs the check. A failed live lookup still counts as Checked = $true with a + # null TrustedSha, which fails the comparison closed rather than silently skipping it. + if ($isFixtureMode) { - $headers["Authorization"] = "Bearer $GitHubToken" + if (Test-HasProperty -Object $fixture -Name "main_branch") + { + return [ordered]@{ Checked = $true; TrustedSha = [string]$fixture.main_branch.sha } + } + return [ordered]@{ Checked = $false; TrustedSha = $null } + } + + try + { + $headers = Get-GitHubHeaders + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/main" -Headers $headers -Method Get -TimeoutSec 30 + return [ordered]@{ Checked = $true; TrustedSha = [string]$result.sha } + } + catch + { + return [ordered]@{ Checked = $true; TrustedSha = $null } } - return Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 } function Get-AzdoBuild @@ -259,7 +356,7 @@ function Get-AzdoBuild if ($isFixtureMode) { $key = [string]$BuildId - if ($fixture.azdo_builds.PSObject.Properties.Name -contains $key) + if (Test-HasProperty -Object $fixture.azdo_builds -Name $key) { return $fixture.azdo_builds.$key } @@ -276,29 +373,64 @@ function Get-AzdoBuild } } +function Merge-AzdoBuildLists +{ + # Pure, side-effect-free merge/dedupe used by the live branch of + # Get-AzdoRecurrenceCandidateBuilds below. Extracted so the merge/dedupe/cap semantics can be + # unit-tested directly with synthetic input, without any network access, independent of + # whichever live resultFilter values happen to be queried. + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Lists, + [Parameter(Mandatory = $true)][int]$Cap + ) + + $merged = [ordered]@{} + foreach ($list in $Lists) + { + foreach ($build in @($list)) + { + $merged[[string]$build.id] = $build + } + } + + return @($merged.Values | Sort-Object -Property startTime -Descending | Select-Object -First $Cap) +} + function Get-AzdoRecurrenceCandidateBuilds { + # Azure DevOps' `resultFilter` does not support a comma-separated multi-value combination + # (verified live: "resultFilter=failed,partiallySucceeded" silently behaves like a single, + # different filter, not their union). ASP.NET Core test failures routinely land in a + # `partiallySucceeded` build (confirmed for aspnetcore#68947's own cited build 1551326), so a + # `failed`-only query misses real recurrence evidence. Issue one request per result value and + # merge, deduping by build id, via Merge-AzdoBuildLists. param([Parameter(Mandatory = $true)][int]$DefinitionId) if ($isFixtureMode) { $key = [string]$DefinitionId - if ($fixture.recurrence_scan.PSObject.Properties.Name -contains $key) + if (Test-HasProperty -Object $fixture.recurrence_scan -Name $key) { return @($fixture.recurrence_scan.$key) } return @() } - try - { - $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=failed&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 - return @($result.value) - } - catch + $resultLists = [System.Collections.Generic.List[object]]::new() + foreach ($resultFilter in @("failed", "partiallySucceeded")) { - return @() + try + { + $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=$resultFilter&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 + $null = $resultLists.Add(@($result.value)) + } + catch + { + continue + } } + + return Merge-AzdoBuildLists -Lists @($resultLists) -Cap $RecurrenceScanBuildCap } function Get-AzdoNegativeCandidateBuilds @@ -308,7 +440,7 @@ function Get-AzdoNegativeCandidateBuilds if ($isFixtureMode) { $key = [string]$DefinitionId - if ($fixture.negative_scan.PSObject.Properties.Name -contains $key) + if (Test-HasProperty -Object $fixture.negative_scan -Name $key) { return @($fixture.negative_scan.$key) } @@ -326,8 +458,13 @@ function Get-AzdoNegativeCandidateBuilds } } -function Get-VstmrTestOutcome +function Get-VstmrSummaryRows { + # The `resultsbyBuild` summary rows carry only identity + outcome (id, runId, + # automatedTestName, outcome) for ordinary xUnit tests -- verified live against + # aspnetcore#68947's own cited build: no comment/errorMessage/stackTrace field is present. + # Use this only to locate the (runId, resultId) pair; fetch Get-VstmrDetail for the actual + # error text. param( [Parameter(Mandatory = $true)][int]$BuildId, [Parameter(Mandatory = $true)][string]$TestName @@ -336,72 +473,131 @@ function Get-VstmrTestOutcome if ($isFixtureMode) { $key = "$BuildId" - if ($fixture.vstmr_results.PSObject.Properties.Name -contains $key) + if (Test-HasProperty -Object $fixture.vstmr_summary -Name $key) { - return $fixture.vstmr_results.$key + return @($fixture.vstmr_summary.$key | Where-Object { [string]$_.automatedTestName -eq $TestName }) } - return $null + return @() } try { $result = Invoke-RestMethod -Uri "$vstmr/testresults/resultsbyBuild?buildId=$BuildId&api-version=7.1-preview.1" -Method Get -TimeoutSec 60 - $match = @($result.value) | Where-Object { $_.automatedTestName -eq $TestName -or $_.testCaseTitle -eq $TestName } | Select-Object -First 1 - if ($null -eq $match) - { - return $null - } - return [ordered]@{ - outcome = $match.outcome - comment = $match.comment - errorMessage = $match.errorMessage - stackTrace = $match.stackTrace - } + return @($result.value | Where-Object { [string]$_.automatedTestName -eq $TestName }) } catch { - return $null + return @() } } -function Get-HelixEvidence +$vstmrDetailCache = @{} + +function Get-VstmrDetail { + # The authoritative source for a specific result's errorMessage/stackTrace (and, when + # present, Helix job/work-item coordinates via a `comment` field -- only ever populated on a + # Helix work item's own crash/'.WorkItemExecution' pseudo-test row, not on ordinary xUnit + # test rows). This is the "detailed VSTMR result" endpoint, distinct from resultsbyBuild. param( - [Parameter(Mandatory = $true)][int]$BuildId, - [string]$HelixJob, - [string]$HelixWorkItem + [Parameter(Mandatory = $true)][int]$RunId, + [Parameter(Mandatory = $true)][int]$ResultId ) - if ($isFixtureMode) + $cacheKey = "${RunId}:${ResultId}" + if ($vstmrDetailCache.ContainsKey($cacheKey)) { - $key = "$BuildId" - if ($fixture.helix_evidence.PSObject.Properties.Name -contains $key) + return $vstmrDetailCache[$cacheKey] + } + + $detail = if ($isFixtureMode) + { + if (Test-HasProperty -Object $fixture.vstmr_detail -Name $cacheKey) { - return $fixture.helix_evidence.$key + $fixture.vstmr_detail.$cacheKey + } + else + { + $null + } + } + else + { + try + { + Invoke-RestMethod -Uri "$ado/test/Runs/$RunId/results/${ResultId}?api-version=7.1" -Method Get -TimeoutSec 30 + } + catch + { + $null } - return $null } - if ([string]::IsNullOrEmpty($HelixJob) -or [string]::IsNullOrEmpty($HelixWorkItem)) + $vstmrDetailCache[$cacheKey] = $detail + return $detail +} + +$vstmrRunCache = @{} + +function Get-VstmrRunName +{ + # The TestRun's `name` (e.g. "Quarantine-Mono-Linux-Release-xunit") is the only authoritative, + # cheaply-available signal for which platform/configuration leg a result ran on -- + # `buildConfiguration.platform`/`.flavor` are empty strings on every run observed live. + param([Parameter(Mandatory = $true)][int]$RunId) + + if ($vstmrRunCache.ContainsKey($RunId)) { - return $null + return $vstmrRunCache[$RunId] } - try + $name = if ($isFixtureMode) + { + $key = "$RunId" + if (Test-HasProperty -Object $fixture.vstmr_runs -Name $key) + { + [string]$fixture.vstmr_runs.$key.name + } + else + { + $null + } + } + else { - $files = Invoke-RestMethod -Uri "$helix/jobs/$HelixJob/workitems/$HelixWorkItem/files" -Method Get -TimeoutSec 60 - $consoleFile = @($files) | Where-Object { $_.Name -like "console.*" } | Select-Object -First 1 - if ($null -eq $consoleFile) + try { - return [ordered]@{ found = $false; expired = $true } + [string](Invoke-RestMethod -Uri "$ado/test/runs/${RunId}?api-version=7.1" -Method Get -TimeoutSec 30).name + } + catch + { + $null } - $content = Invoke-RestMethod -Uri $consoleFile.Link -Method Get -TimeoutSec 60 - return [ordered]@{ found = $true; expired = $false; console_excerpt = [string]$content } } - catch + + $vstmrRunCache[$RunId] = $name + return $name +} + +function Get-PlatformConfigurationFromRunName +{ + param([AllowNull()][string]$RunName) + + $platform = "unknown" + $configuration = "unknown" + if ([string]::IsNullOrEmpty($RunName)) { - return [ordered]@{ found = $false; expired = $true } + return [ordered]@{ Platform = $platform; Configuration = $configuration } } + + if ($RunName -match "(?i)\bwindows\b") { $platform = "Windows" } + elseif ($RunName -match "(?i)\blinux\b") { $platform = "Linux" } + elseif ($RunName -match "(?i)\b(?:macos|osx)\b") { $platform = "macOS" } + + if ($RunName -match "(?i)\bdebug\b") { $configuration = "Debug" } + elseif ($RunName -match "(?i)\brelease\b") { $configuration = "Release" } + + return [ordered]@{ Platform = $platform; Configuration = $configuration } } function Get-CheckRunsForSha @@ -410,20 +606,16 @@ function Get-CheckRunsForSha if ($isFixtureMode) { - if ($fixture.check_runs.PSObject.Properties.Name -contains $Sha) + if (Test-HasProperty -Object $fixture.check_runs -Name $Sha) { return @($fixture.check_runs.$Sha) } return @() } - $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } - if (-not [string]::IsNullOrEmpty($GitHubToken)) - { - $headers["Authorization"] = "Bearer $GitHubToken" - } try { + $headers = Get-GitHubHeaders $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/$Sha/check-runs" -Headers $headers -Method Get -TimeoutSec 30 return @($result.check_runs) } @@ -433,46 +625,96 @@ function Get-CheckRunsForSha } } -function Get-DuplicateSearch +function Search-GitHubIssues { - param( - [Parameter(Mandatory = $true)][string]$Category, - [Parameter(Mandatory = $true)][string]$Query - ) + # Returns @{ Complete; Numbers; TotalCount }. `Complete` requires BOTH that GitHub reported + # incomplete_results=false AND that every matching item (per total_count) was actually + # retrieved across the paginated fetch -- a `total_count` larger than what a single + # per_page=100 page returns previously went unnoticed and was still labeled complete. + param([Parameter(Mandatory = $true)][string]$Query) if ($isFixtureMode) { - if ($fixture.duplicate_search.PSObject.Properties.Name -contains $Category) + return $null + } + + $headers = Get-GitHubHeaders + $numbers = [System.Collections.Generic.List[int]]::new() + $totalCount = 0 + $complete = $true + $encoded = [System.Uri]::EscapeDataString($Query) + + for ($page = 1; $page -le $DuplicateSearchMaxPages; $page++) + { + try + { + $responseHeaders = $null + $result = Invoke-RestMethod -Uri "https://api.github.com/search/issues?q=$encoded&per_page=$DuplicateSearchPageSize&page=$page" -Headers $headers -Method Get -TimeoutSec 30 -ResponseHeadersVariable responseHeaders + Write-RateLimitDiagnostic -ResponseHeaders $responseHeaders + } + catch + { + $complete = $false + break + } + + $totalCount = [int]$result.total_count + if ([bool]$result.incomplete_results) + { + $complete = $false + } + foreach ($item in @($result.items)) + { + $null = $numbers.Add([int]$item.number) + } + + if ($numbers.Count -ge $totalCount) + { + break + } + if ($page -eq $DuplicateSearchMaxPages -and $numbers.Count -lt $totalCount) { - $entry = $fixture.duplicate_search.$Category - return [ordered]@{ complete = [bool]$entry.complete; result_numbers = @($entry.result_numbers) } + $complete = $false } - return [ordered]@{ complete = $false; result_numbers = @() } } - $headers = @{ Accept = "application/vnd.github+json"; "User-Agent" = "aspnetcore-test-quarantine-kbe-shadow" } - if (-not [string]::IsNullOrEmpty($GitHubToken)) + return [ordered]@{ Complete = $complete; Numbers = @($numbers); TotalCount = $totalCount } +} + +function Get-DuplicateCandidateText +{ + # Fetches the searched-up issue or PR's title+body so its exact test identity can be + # verified before it is ever treated as a validated duplicate. `/issues/{number}` is a + # unified GitHub endpoint that also resolves pull requests. + param([Parameter(Mandatory = $true)][int]$Number) + + if ($isFixtureMode) { - $headers["Authorization"] = "Bearer $GitHubToken" + $key = "$Number" + if (Test-HasProperty -Object $fixture.duplicate_candidate_text -Name $key) + { + return [string]$fixture.duplicate_candidate_text.$key + } + return $null } + try { - $encoded = [System.Uri]::EscapeDataString($Query) - $result = Invoke-RestMethod -Uri "https://api.github.com/search/issues?q=$encoded&per_page=20" -Headers $headers -Method Get -TimeoutSec 30 - if ([bool]$result.incomplete_results) - { - return [ordered]@{ complete = $false; result_numbers = @() } - } - return [ordered]@{ complete = $true; result_numbers = @($result.items | ForEach-Object { [int]$_.number }) } + $headers = Get-GitHubHeaders + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/issues/$Number" -Headers $headers -Method Get -TimeoutSec 30 + return "$($result.title)`n$($result.body)" } catch { - return [ordered]@{ complete = $false; result_numbers = @() } + return $null } } # --------------------------------------------------------------------------- -# Step 1: validate the canonical, open quarantine issue. +# Step 1: validate the canonical, open quarantine issue. The 'test-failure' label alone is not +# proof an issue was generated by quarantine automation (any contributor can apply it to an +# ordinary bug report); also require the trusted HTML-comment marker the production workflow +# stamps into every issue it creates. # --------------------------------------------------------------------------- $missingEvidence = [System.Collections.Generic.List[object]]::new() @@ -482,11 +724,20 @@ $issue = Get-GitHubIssue -Number $IssueNumber $issueUrl = "https://github.com/$Repository/issues/$IssueNumber" $issueLabels = @($issue.labels | ForEach-Object { if ($_ -is [string]) { $_ } else { [string]$_.name } }) $issueState = [string]$issue.state +$issueBody = [string]$issue.body +$hasWorkflowMarker = @($workflowMarkers | Where-Object { $issueBody.Contains($_, [System.StringComparison]::Ordinal) }).Count -gt 0 -if ($issueLabels -notcontains $canonicalQuarantineLabel) +if ($issueLabels -notcontains $canonicalQuarantineLabel -or -not $hasWorkflowMarker) { $reasonCodes.Add("issue-not-canonical-quarantine") - Add-MissingEvidence -List $missingEvidence -Kind "quarantine-label" -Detail "Issue #$IssueNumber does not carry the canonical '$canonicalQuarantineLabel' label." + if ($issueLabels -notcontains $canonicalQuarantineLabel) + { + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-label" -Detail "Issue #$IssueNumber does not carry the canonical '$canonicalQuarantineLabel' label." + } + if (-not $hasWorkflowMarker) + { + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-workflow-marker" -Detail "Issue #$IssueNumber body does not contain a trusted 'gh-aw-workflow-id: test-quarantine' / 'gh-aw-workflow-call-id: dotnet/aspnetcore/test-quarantine' marker; the label alone is not proof this issue was generated by quarantine automation." + } } if ($issueState -ne "open") @@ -495,33 +746,48 @@ if ($issueState -ne "open") Add-MissingEvidence -List $missingEvidence -Kind "issue-state" -Detail "Issue #$IssueNumber is '$issueState', not 'open'." } -$issueBody = [string]$issue.body - # --------------------------------------------------------------------------- -# Step 2: deterministically parse the issue body for the test name, referenced -# builds, and (when unambiguous) a failure signature. Every quarantine issue -# produced by the production workflow carries a '## Failing Test(s)' section -# and at least one 'buildId=' link; both formats in use (the strict -# 50_test_failure.md template and the freeform '## Details' variant) satisfy -# these two invariants, so this parsing does not depend on section ordering. +# Step 2: deterministically parse the issue body for the test name, referenced builds, and (when +# unambiguous) a failure signature. Every quarantine issue produced by the production workflow +# carries a '## Failing Test(s)' section and at least one 'buildId=' link; both formats in use +# (the strict 50_test_failure.md template and the freeform '## Details' variant) satisfy these two +# invariants, so this parsing does not depend on section ordering. +# +# A '## Failing Test(s)' section can legitimately name more than one concrete test identity (e.g. +# aspnetcore#68724 names both a base test and its server-execution subclass override, and live +# data shows only the override actually failed while the base identity passed). Silently picking +# the first one risks binding evidence to the wrong identity. This collector fails closed unless +# exactly one concrete identity can be unambiguously selected; evaluating every listed identity +# independently is a reasonable extension left for a follow-up, since this PR targets one +# issue/one root cause at a time. # --------------------------------------------------------------------------- $testName = $null $failingTestMatch = [regex]::Match($issueBody, "##\s*Failing Test\(s\)\s*\r?\n(.*?)(?=\r?\n##\s|\z)", [System.Text.RegularExpressions.RegexOptions]::Singleline) +$distinctIdentities = @() if ($failingTestMatch.Success) { - $backtickMatch = [regex]::Match($failingTestMatch.Groups[1].Value, '`([^`]+)`') - if ($backtickMatch.Success) - { - $testName = $backtickMatch.Groups[1].Value.Trim() - } + $distinctIdentities = @( + [regex]::Matches($failingTestMatch.Groups[1].Value, '`([^`]+)`') | + ForEach-Object { $_.Groups[1].Value.Trim() } | + Where-Object { $_.Length -ge 3 -and $_.Length -le 1024 -and $_ -notmatch "[\r\n]" } | + Select-Object -Unique + ) } -if ([string]::IsNullOrWhiteSpace($testName) -or $testName.Length -lt 3 -or $testName.Length -gt 1024 -or $testName -match "[\r\n]") +if ($distinctIdentities.Count -eq 0) { $reasonCodes.Add("test-name-unresolvable") - Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "Could not deterministically extract a single backtick-quoted fully qualified test name from '## Failing Test(s)'." - $testName = $null + Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "Could not deterministically extract any backtick-quoted fully qualified test name from '## Failing Test(s)'." +} +elseif ($distinctIdentities.Count -gt 1) +{ + $reasonCodes.Add("multiple-test-identities-unresolved") + Add-MissingEvidence -List $missingEvidence -Kind "test-name" -Detail "'## Failing Test(s)' names $($distinctIdentities.Count) distinct test identities ($($distinctIdentities -join '; ')); this collector requires exactly one unambiguous identity per run rather than guessing which one actually failed." +} +else +{ + $testName = $distinctIdentities[0] } $citedBuildIds = @( @@ -559,9 +825,36 @@ if ([string]::IsNullOrWhiteSpace($effectiveSignature) -or } # --------------------------------------------------------------------------- -# Step 3: resolve Azure DevOps build metadata for every issue-cited build. -# Builds whose metadata has aged out of Azure DevOps retention are recorded as -# not-found, never fabricated (see aspnetcore#68945). +# Step 2.5: confirm the repository checkout this collector and the evaluator are running against +# is genuinely dotnet/aspnetcore's 'main' branch tip, via a trusted GitHub API response -- never +# label a non-main checkout (e.g. this very prototype PR's branch) as 'main'. +# --------------------------------------------------------------------------- + +$repoHeadSha = (& git -C $RepositoryRoot rev-parse HEAD).Trim() +$mainShaResult = Get-TrustedMainShaResult +if (-not [bool]$mainShaResult.Checked) +{ + # Fixture does not model this dimension: trust the checkout (used by fixtures that are not + # specifically exercising this guard). + $trustedMainSha = $repoHeadSha + $matchesMain = $true +} +else +{ + $trustedMainSha = $mainShaResult.TrustedSha + $matchesMain = ($null -ne $trustedMainSha) -and $repoHeadSha.Equals($trustedMainSha, [System.StringComparison]::OrdinalIgnoreCase) + if (-not $matchesMain) + { + $reasonCodes.Add("repository-ref-not-main") + $trustedDisplay = if ($trustedMainSha) { $trustedMainSha } else { "(lookup failed)" } + Add-MissingEvidence -List $missingEvidence -Kind "repository-ref" -Detail "Checked-out commit $repoHeadSha does not match the trusted dotnet/aspnetcore main SHA $trustedDisplay; refusing to label repository_ref.branch as 'main'." + } +} + +# --------------------------------------------------------------------------- +# Step 3: resolve Azure DevOps build metadata for every issue-cited build. Builds whose metadata +# has aged out of Azure DevOps retention are recorded as not-found, never fabricated (see +# aspnetcore#68945). # --------------------------------------------------------------------------- $retrievedUtc = [System.DateTimeOffset]::UtcNow.ToString("O") @@ -602,13 +895,55 @@ foreach ($buildId in $citedBuildIds) } # --------------------------------------------------------------------------- -# Step 4: if fewer than two distinct resolved builds are available, perform a -# capped supplementary recurrence scan across the same pipeline definition(s) -# on 'main' -- direct raw AzDO/VSTMR evidence, never the Build Analysis -# abstraction, per the architecture consensus that Build Analysis is -# corroborating only and cannot establish exact recurrence. +# Step 4: if fewer than two distinct resolved builds are available, perform a capped +# supplementary recurrence scan across the same pipeline definition(s) on 'main' -- direct raw +# AzDO/VSTMR evidence, never the Build Analysis abstraction, per the architecture consensus that +# Build Analysis is corroborating only and cannot establish exact recurrence. Signature matching +# uses ordinal substring containment, never `-like`/`-notlike`: a literal ErrorMessage containing +# `*`, `?`, or `[` would otherwise be misinterpreted as a wildcard pattern instead of literal text. # --------------------------------------------------------------------------- +function Test-SignatureMatch +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$Haystack, + [Parameter(Mandatory = $true)][string]$Signature + ) + + return $Haystack.Contains($Signature, [System.StringComparison]::Ordinal) +} + +function Get-MatchingFailureDetail +{ + # Finds the first summary row for $BuildId/$TestName whose VSTMR detail errorMessage+stackTrace + # contains $Signature (ordinal substring). Returns $null when no such row exists. + param( + [Parameter(Mandatory = $true)]$BuildId, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][string]$Signature + ) + + foreach ($row in @(Get-VstmrSummaryRows -BuildId $BuildId -TestName $TestName)) + { + if ([string]$row.outcome -ne "Failed") + { + continue + } + $detail = Get-VstmrDetail -RunId ([int]$row.runId) -ResultId ([int]$row.id) + if ($null -eq $detail) + { + continue + } + $haystack = "$($detail.errorMessage) $($detail.stackTrace)" + if (Test-SignatureMatch -Haystack $haystack -Signature $Signature) + { + return $row + } + } + + return $null +} + if ($resolvedBuilds.Count -lt $minimumFailureBuilds -and $null -ne $testName -and $null -ne $effectiveSignature) { $scanDefinitionIds = if ($resolvedBuilds.Count -gt 0) @@ -641,14 +976,8 @@ if ($resolvedBuilds.Count -lt $minimumFailureBuilds -and $null -ne $testName -an continue } - $outcome = Get-VstmrTestOutcome -BuildId $candidateId -TestName $testName - if ($null -eq $outcome -or [string]$outcome.outcome -ne "Failed") - { - continue - } - - $signatureText = "$($outcome.errorMessage) $($outcome.stackTrace)" - if ($signatureText -notlike "*$effectiveSignature*") + $matchingRow = Get-MatchingFailureDetail -BuildId $candidateId -TestName $testName -Signature $effectiveSignature + if ($null -eq $matchingRow) { continue } @@ -677,10 +1006,10 @@ if ($null -ne $testName -and $null -ne $effectiveSignature -and $resolvedBuilds. } # --------------------------------------------------------------------------- -# Step 4.5: gather at least one authoritative negative (passed/skipped) -# occurrence of the same test on the same pipeline(s). This is what lets the -# evaluator confirm the failure is not a consistent regression -- a missing -# negative is recorded as insufficient evidence, never inferred as a pass. +# Step 4.5: gather at least one authoritative negative (passed/skipped) occurrence of the same +# test on the same pipeline(s). This is what lets the evaluator confirm the failure is not a +# consistent regression -- a missing negative is recorded as insufficient evidence, never +# inferred as a pass. # --------------------------------------------------------------------------- $negativeBuilds = [System.Collections.Generic.List[object]]::new() @@ -709,8 +1038,8 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) } $candidateId = [int]$candidate.id - $outcome = Get-VstmrTestOutcome -BuildId $candidateId -TestName $testName - if ($null -eq $outcome -or [string]$outcome.outcome -notin @("Passed", "Skipped")) + $rows = @(Get-VstmrSummaryRows -BuildId $candidateId -TestName $testName | Where-Object { [string]$_.outcome -in @("Passed", "Skipped") }) + if ($rows.Count -eq 0) { continue } @@ -733,11 +1062,13 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) } # --------------------------------------------------------------------------- -# Step 5: for each resolved build, gather raw failure/negative evidence -# (Helix console log) and materialize it locally so Evaluate-TestQuarantineKbeCandidate.ps1 -# can hash-verify it. An artifact that once existed but is no longer -# retrievable is recorded as found=false, expired=true -- never silently -# dropped and never treated as a pass (see aspnetcore#68945). +# Step 5: for each resolved build, materialize authoritative raw evidence from the VSTMR detail +# result -- never a blindly-capped raw console log. Helix job/work-item coordinates are recorded +# as metadata only when a work item's own crash/'.WorkItemExecution' row happens to carry a +# `comment` field (rare for ordinary xUnit tests, confirmed live); otherwise `helix_unavailable` is +# recorded explicitly and the VSTMR detail text remains authoritative on its own, per aspnetcore's +# real API shape. Platform/configuration are parsed from the authoritative TestRun name, never +# fabricated; "unknown" is recorded when no recognized token is present. # --------------------------------------------------------------------------- [System.IO.Directory]::CreateDirectory($EvidenceRoot) | Out-Null @@ -757,80 +1088,110 @@ foreach ($build in $evidenceBuilds) } $role = [string]$build.intended_role - $vstmrOutcome = Get-VstmrTestOutcome -BuildId $build.id -TestName $testName - $helixJob = $null - $helixWorkItem = $null - if ($null -ne $vstmrOutcome -and -not [string]::IsNullOrEmpty($vstmrOutcome.comment)) + $expectedOutcome = if ($role -eq "failure") { @("Failed") } else { @("Passed", "Skipped") } + $matchedRow = $null + $matchedDetail = $null + + foreach ($row in @(Get-VstmrSummaryRows -BuildId $build.id -TestName $testName)) { - $commentMatch = ($vstmrOutcome.comment | ConvertFrom-Json -ErrorAction SilentlyContinue) - if ($null -ne $commentMatch) + if ([string]$row.outcome -notin $expectedOutcome) { - $helixJob = [string]$commentMatch.HelixJobId - $helixWorkItem = [string]$commentMatch.HelixWorkItemName + continue } + $detail = Get-VstmrDetail -RunId ([int]$row.runId) -ResultId ([int]$row.id) + if ($null -eq $detail) + { + continue + } + if ($role -eq "failure" -and $null -ne $effectiveSignature) + { + $haystack = "$($detail.errorMessage) $($detail.stackTrace)" + if (-not (Test-SignatureMatch -Haystack $haystack -Signature $effectiveSignature)) + { + continue + } + } + $matchedRow = $row + $matchedDetail = $detail + break } - # Defensive consistency check: the intended role (why this build was selected) - # must match what VSTMR actually reports for this test in this build. A - # mismatch means the evidence is stale or inconsistent -- skip it rather than - # writing a role/outcome pair that contradicts the authoritative test result. - $outcomeValue = if ($null -eq $vstmrOutcome) { $null } - elseif ([string]$vstmrOutcome.outcome -eq "Failed") { "failed" } - elseif ([string]$vstmrOutcome.outcome -eq "Passed") { "passed" } - elseif ([string]$vstmrOutcome.outcome -eq "Skipped") { "skipped" } - else { $null } - - $roleOutcomeConsistent = - ($role -eq "failure" -and $outcomeValue -eq "failed") -or - ($role -eq "negative" -and $outcomeValue -in @("passed", "skipped")) - - if (-not $roleOutcomeConsistent) - { - Add-MissingEvidence -List $missingEvidence -Kind "vstmr-consistency" -Detail "Build $($build.id): recorded test outcome did not match the reason this build was selected as $role evidence." - continue - } - - $helixEvidence = Get-HelixEvidence -BuildId $build.id -HelixJob $helixJob -HelixWorkItem $helixWorkItem - $found = ($null -ne $helixEvidence -and [bool]$helixEvidence.found) - $expired = ($null -ne $helixEvidence -and [bool]$helixEvidence.expired) -or ($null -eq $helixEvidence -and $null -ne $vstmrOutcome) - - if (-not $found) + if ($null -eq $matchedRow -or $null -eq $matchedDetail) { $null = $rawEvidenceRecords.Add([ordered]@{ build_id = $build.id role = $role found = $false - expired = $expired captured_utc = $retrievedUtc - note = "Helix console evidence for build $($build.id) was not retrievable." + note = "No VSTMR result for build $($build.id) matched the expected outcome/signature for this test." }) - Add-MissingEvidence -List $missingEvidence -Kind "helix-evidence" -Detail "Build $($build.id): no retrievable Helix console evidence." - if ($expired) + Add-MissingEvidence -List $missingEvidence -Kind "vstmr-evidence" -Detail "Build $($build.id): no matching, retrievable VSTMR result detail." + continue + } + + $runId = [int]$matchedRow.runId + $resultId = [int]$matchedRow.id + + $helixJob = $null + $helixWorkItem = $null + if ((Test-HasProperty -Object $matchedDetail -Name "comment") -and -not [string]::IsNullOrEmpty($matchedDetail.comment)) + { + $commentObject = $matchedDetail.comment | ConvertFrom-Json -ErrorAction SilentlyContinue + if ($null -ne $commentObject -and (Test-HasProperty -Object $commentObject -Name "HelixJobId")) { - $reasonCodes.Add("raw-evidence-expired") + $helixJob = [string]$commentObject.HelixJobId + $helixWorkItem = [string]$commentObject.HelixWorkItemName } - continue } + $helixUnavailable = [string]::IsNullOrEmpty($helixJob) -or [string]::IsNullOrEmpty($helixWorkItem) + + $runName = Get-VstmrRunName -RunId $runId + $platformConfiguration = Get-PlatformConfigurationFromRunName -RunName $runName $evidenceIndex += 1 $fileName = "issue-$IssueNumber-build-$($build.id)-$role.log" $evidencePath = Join-Path $EvidenceRoot $fileName - $cappedContent = Get-CappedExcerpt -Value ([string]$helixEvidence.console_excerpt) -Cap $rawLogCap -ProtectedPhrases @($testName) + + # The marker line (which Evaluate-TestQuarantineKbeCandidate.ps1 requires to associate a + # signature match with the declared test) and, for failures, the already-extracted signature + # are always placed first -- capping to $rawLogCap can then only ever truncate the tail of a + # long stack trace, never the lines the evaluator actually needs. + $markerLine = if ($role -eq "failure") { "Failed $testName [reported by VSTMR result $resultId]" } else { "$([string]$matchedRow.outcome) $testName [reported by VSTMR result $resultId]" } + $bodyText = if ($role -eq "failure") + { + "$($matchedDetail.errorMessage)`n$($matchedDetail.stackTrace)" + } + else + { + "(no error: VSTMR outcome was $([string]$matchedRow.outcome))" + } + # Build id is appended defensively so two builds that legitimately share identical evidence + # text (as small synthetic fixtures sometimes do) never collide to the same content hash. + $rawContent = "$markerLine`n$bodyText`n(observed in build $($build.id))" + $cappedContent = Get-CappedExcerpt -Value $rawContent -Cap $rawLogCap -ProtectedPhrases @($testName) [System.IO.File]::WriteAllText($evidencePath, $cappedContent) $sha256 = (Get-FileHash -LiteralPath $evidencePath -Algorithm SHA256).Hash.ToLowerInvariant() - $null = $rawEvidenceRecords.Add([ordered]@{ + $evidenceRecord = [ordered]@{ build_id = $build.id role = $role - kind = "helix-console-log" - helix_job = $(if ($helixJob) { $helixJob } else { "" }) - helix_workitem = $(if ($helixWorkItem) { $helixWorkItem } else { "" }) + kind = "vstmr-detail" + run_id = $runId + result_id = $resultId + helix_unavailable = $helixUnavailable + platform = $platformConfiguration.Platform + configuration = $platformConfiguration.Configuration found = $true - expired = $false captured_utc = $retrievedUtc sha256 = $sha256 evidence_path = $fileName - }) + } + if (-not $helixUnavailable) + { + $evidenceRecord["helix_job"] = $helixJob + $evidenceRecord["helix_workitem"] = $helixWorkItem + } + $null = $rawEvidenceRecords.Add($evidenceRecord) if ($role -eq "failure") { @@ -841,6 +1202,13 @@ foreach ($build in $evidenceBuilds) $negativeCount += 1 } + $outcomeValue = switch ([string]$matchedRow.outcome) + { + "Failed" { "failed"; break } + "Passed" { "passed"; break } + default { "skipped" } + } + $null = $rawLogs.Add([ordered]@{ id = "evidence-$evidenceIndex" role = $role @@ -853,8 +1221,8 @@ foreach ($build in $evidenceBuilds) pipeline_definition_id = [int]$build.definition_id source_version = [string]$build.source_version started_utc = [string]$build.started_utc - platform = "Linux" - configuration = "Release" + platform = $platformConfiguration.Platform + configuration = $platformConfiguration.Configuration } }) } @@ -875,9 +1243,13 @@ if ($null -ne $testName -and $negativeCount -lt $minimumNegativeLogs) } # --------------------------------------------------------------------------- -# Step 6: fetch Build Analysis check-run snapshots. Advisory/corroborating -# only: recorded regardless of outcome, and a missing or generic snapshot -# never overrides raw evidence gathered above. +# Step 6: fetch Build Analysis check-run snapshots. Advisory/corroborating only: recorded +# regardless of outcome, and a missing or generic snapshot never overrides raw evidence gathered +# above. `exact_test_referenced` requires the FULL fully-qualified test name, never a bare method +# name (which commonly collides with unrelated tests); `known_issue_referenced` requires a +# concrete dotnet/aspnetcore issue number/URL near the phrase, not the bare phrase alone (a +# heading or table column label reading "Known Issue" with no associated reference must not set +# this true). # --------------------------------------------------------------------------- $checkRunRecords = [System.Collections.Generic.List[object]]::new() @@ -896,7 +1268,9 @@ foreach ($sha in $distinctShas) found = $false retrieved_utc = $retrievedUtc exact_test_referenced = $false + short_name_referenced = $false known_issue_referenced = $false + known_issue_numbers = @() }) continue } @@ -904,8 +1278,22 @@ foreach ($sha in $distinctShas) $text = [string]$buildAnalysis.output.text $textSha256 = Get-Sha256String -Value $text $shortMethodName = if ($testName) { ($testName -split '\.')[-1] } else { $null } - $exactTestReferenced = ($null -ne $testName -and $text.Contains($testName)) -or ($null -ne $shortMethodName -and $text.Contains($shortMethodName)) - $knownIssueReferenced = $text -match "(?i)known issue" + $exactTestReferenced = ($null -ne $testName) -and $text.Contains($testName, [System.StringComparison]::Ordinal) + $shortNameReferenced = (-not $exactTestReferenced) -and ($null -ne $shortMethodName) -and $text.Contains($shortMethodName, [System.StringComparison]::Ordinal) + + $knownIssueNumbers = [System.Collections.Generic.List[int]]::new() + foreach ($phraseMatch in [regex]::Matches($text, "(?i)known issue")) + { + $windowStart = $phraseMatch.Index + $windowLength = [System.Math]::Min(200, $text.Length - $windowStart) + $window = $text.Substring($windowStart, $windowLength) + foreach ($numberMatch in [regex]::Matches($window, "dotnet/aspnetcore(?:#|/issues/)(\d+)|(?=$recentWindowDate label:`"Known Build Error`" $shortName" } @{ category = "open-fix-pr"; query = "repo:$Repository is:pr is:open $shortName" } - @{ category = "recently-merged-fix-pr"; query = "repo:$Repository is:pr is:merged $shortName" } + @{ category = "recently-merged-fix-pr"; query = "repo:$Repository is:pr is:merged merged:>=$recentWindowDate $shortName" } ) $duplicateQueryResults = [System.Collections.Generic.List[object]]::new() $duplicateReferences = [System.Collections.Generic.List[string]]::new() +$unvalidatedCandidates = [System.Collections.Generic.List[object]]::new() $kbeNumbers = [System.Collections.Generic.List[int]]::new() $fixPrNumbers = [System.Collections.Generic.List[int]]::new() $allQueriesComplete = $true foreach ($q in $duplicateQueries) { - $searchResult = Get-DuplicateSearch -Category $q.category -Query $q.query - if (-not [bool]$searchResult.complete) + $searchResult = if ($isFixtureMode) + { + $categoryName = [string]$q.category + if (Test-HasProperty -Object $fixture.duplicate_search -Name $categoryName) + { + $entry = $fixture.duplicate_search.$categoryName + [ordered]@{ Complete = [bool]$entry.complete; Numbers = @($entry.result_numbers); TotalCount = [int]$(if (Test-HasProperty -Object $entry -Name "total_count") { $entry.total_count } else { @($entry.result_numbers).Count }) } + } + else + { + [ordered]@{ Complete = $false; Numbers = @(); TotalCount = 0 } + } + } + else + { + Search-GitHubIssues -Query $q.query + } + + if (-not [bool]$searchResult.Complete) { $allQueriesComplete = $false } - $resultNumbers = @($searchResult.result_numbers) - $null = $duplicateQueryResults.Add([ordered]@{ - category = $q.category - query = $q.query - complete = [bool]$searchResult.complete - result_numbers = $resultNumbers - }) $isKbeCategory = $q.category -in @("open-kbe", "recently-closed-kbe") - foreach ($n in $resultNumbers) + foreach ($n in @($searchResult.Numbers)) { - if ($isKbeCategory) + $candidateText = Get-DuplicateCandidateText -Number $n + $validated = ($null -ne $testName) -and ($null -ne $candidateText) -and $candidateText.Contains($testName, [System.StringComparison]::Ordinal) + if ($validated) { - $null = $kbeNumbers.Add($n) - $null = $duplicateReferences.Add("issue:$n") + if ($isKbeCategory) + { + $null = $kbeNumbers.Add($n) + $null = $duplicateReferences.Add("issue:$n") + } + else + { + $null = $fixPrNumbers.Add($n) + $null = $duplicateReferences.Add("pull-request:$n") + } } else { - $null = $fixPrNumbers.Add($n) - $null = $duplicateReferences.Add("pull-request:$n") + $reason = if ($null -eq $testName) { "no resolved test identity to validate against" } + elseif ($null -eq $candidateText) { "could not fetch issue/PR #$n to validate test identity" } + else { "issue/PR #$n does not contain the exact fully-qualified test name" } + $null = $unvalidatedCandidates.Add([ordered]@{ category = $q.category; number = $n; reason = $reason }) } } + + $null = $duplicateQueryResults.Add([ordered]@{ + category = $q.category + query = $q.query + complete = [bool]$searchResult.Complete + result_numbers = @($searchResult.Numbers) + total_count = [int]$searchResult.TotalCount + }) } $duplicateStatus = if (-not $allQueriesComplete) @@ -992,9 +1418,18 @@ else if (-not $allQueriesComplete) { $reasonCodes.Add("duplicate-search-incomplete") - Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "At least one duplicate KBE/fix-PR search category returned incomplete results." + Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "At least one duplicate KBE/fix-PR search category returned incomplete or truncated results." } +$duplicateQueriesForCandidate = @($duplicateQueryResults | ForEach-Object { + [ordered]@{ + category = $_.category + query = $_.query + complete = $_.complete + result_numbers = $_.result_numbers + } +}) + $duplicateCheck = [ordered]@{ status = $duplicateStatus checked_utc = $retrievedUtc @@ -1005,7 +1440,20 @@ $duplicateCheck = [ordered]@{ recently_merged_fix_prs = $true } references = @($duplicateReferences | Select-Object -Unique) + queries = $duplicateQueriesForCandidate +} + +# The dossier's own duplicate_search $def additionally carries total_count and +# unvalidated_candidates -- both new, dossier-only provenance fields. candidate.duplicate_check +# above deliberately keeps the exact, unmodified shape test-quarantine-kbe-shadow-candidate.schema.json +# requires (additionalProperties: false; no total_count/unvalidated_candidates there). +$duplicateCheckWithUnvalidated = [ordered]@{ + status = $duplicateStatus + checked_utc = $retrievedUtc + coverage = $duplicateCheck.coverage + references = $duplicateCheck.references queries = @($duplicateQueryResults) + unvalidated_candidates = @($unvalidatedCandidates) } # --------------------------------------------------------------------------- @@ -1037,7 +1485,6 @@ if ($outcome -eq "candidate") $null = $corroboratingContext.Add([ordered]@{ source = "quarantine-issue"; url = $issueUrl }) - $repoHeadSha = (& git -C $RepositoryRoot rev-parse HEAD).Trim() $candidate = [ordered]@{ schema_version = 1 repository = "dotnet/aspnetcore" @@ -1109,13 +1556,19 @@ $dossier = [ordered]@{ url = $issueUrl state = $issueState labels = @($issueLabels) + has_workflow_marker = $hasWorkflowMarker } outcome = $outcome provenance = [ordered]@{ + repository_ref_verification = [ordered]@{ + checkout_sha = $repoHeadSha + trusted_main_sha = $trustedMainSha + matches_main = $matchesMain + } azdo_builds = @($azdoBuildRecords) check_run_snapshots = @($checkRunRecords) raw_evidence_sources = @($rawEvidenceRecords) - duplicate_search = $duplicateCheck + duplicate_search = $duplicateCheckWithUnvalidated } candidate = $candidate incomplete = $incomplete diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index 42a888dfc019..186ba67a711f 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -26,34 +26,105 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis ## Trust boundary * **Build Analysis is corroborating, never authoritative.** The collector records a snapshot of - the GitHub "Build Analysis" check-run for every resolved build's commit (check id, conclusion, - a SHA-256 of its full text, a capped/redacted excerpt, and a conservative substring check for - whether the text names this exact test and/or a "Known Issue"). A missing or generic snapshot - is recorded and surfaced, but it never overrides raw evidence and never by itself makes a - candidate valid or invalid. Direct queries against the Build Analysis abstraction for three real - pilot builds (1563420, 1551326, 1569737) returned only generic, task-level, unmatched failures - and no known issues — this is exactly the "generic" case the collector's fixtures for #68724 and - #68945 encode. -* **Raw AzDO/Helix/VSTMR evidence is what proves an exact test failure and its recurrence.** The - collector resolves Azure DevOps build metadata, VSTMR test results, and Helix console-log - content directly. Recurrence requires evidence from **at least two distinct builds** (a single - build producing two separate artifacts is not recurrence), and at least one authoritative - negative (passed/skipped) occurrence. -* **Never infer a pass, a recurrence, or a signature from missing or expired evidence.** Every gap - — a build whose Azure DevOps metadata has aged out of retention, a Helix console log that has - expired, an ambiguous or absent signature, an incomplete duplicate search — is recorded as an - explicit reason code and fails the run closed (`outcome: "incomplete"`) rather than silently - degrading. Real Azure DevOps retention data confirms this is not hypothetical: build 1537561 - (cited by aspnetcore#68947) has already aged out of the public `dnceng-public` project. + the GitHub "Build Analysis" check-run for every resolved build's commit: check id, conclusion, a + SHA-256 of its full text, a capped/redacted excerpt, and two *conservative* substring checks -- + `exact_test_referenced` (the quarantined test's **full fully-qualified name** appears verbatim; + never set from a bare method name, which commonly collides with unrelated tests -- a + `short_name_referenced` field records that weaker match separately, for transparency only) and + `known_issue_referenced` (a **concrete** dotnet/aspnetcore issue number/URL appears near a + "Known Issue" style label; the bare phrase alone, e.g. a heading or table column reading "Known + Issues" with no associated reference, does not set this true -- `known_issue_numbers` records + exactly which issue(s) were found). A missing or generic snapshot is recorded and surfaced, but + it never overrides raw evidence and never by itself makes a candidate valid or invalid. Direct + queries against the Build Analysis abstraction for three real pilot builds (1563420, 1551326, + 1569737) returned only generic, task-level, unmatched failures and no known issues -- this is + exactly the "generic" case the collector's pilot fixtures encode. +* **Authoritative VSTMR test-result detail, not Build Analysis and not a raw Helix console-log + fetch, is what proves an exact test failure and its recurrence.** Azure DevOps' `resultsbyBuild` + *summary* rows carry only identity and outcome (`id`, `runId`, `automatedTestName`, `outcome`) -- + confirmed live against aspnetcore#68947's own cited build: no `comment`/`errorMessage`/ + `stackTrace` field is present on an ordinary xUnit test's summary row. The collector instead + calls the *detailed* per-result endpoint (`GET .../test/Runs/{runId}/results/{resultId}`) to + retrieve the authoritative `errorMessage`/`stackTrace`, and materializes evidence from that text + directly. Helix job/work-item coordinates are recorded as metadata only when a `comment` field + happens to be present (in practice, only a Helix work item's own crash/`.WorkItemExecution` + pseudo-test row carries one); otherwise `helix_unavailable: true` is recorded explicitly and the + VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence + requires evidence from **at least two distinct builds** (a single build producing two separate + artifacts is not recurrence), and at least one authoritative negative (passed/skipped) + occurrence. Signature matching against that raw text uses ordinal, case-sensitive substring + containment (`[string]::Contains(..., Ordinal)`) throughout -- never PowerShell's + `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise + silently misinterpret a literal ErrorMessage containing any of those characters. +* **Azure DevOps build recurrence spans both `failed` and `partiallySucceeded` results.** A + `resultFilter=failed`-only recurrence query misses real evidence: aspnetcore#68947's own cited + build 1551326 is itself `partiallySucceeded`, not `failed`, confirmed live. Azure DevOps' + `resultFilter` does not support a comma-separated multi-value combination (verified live: + `resultFilter=failed,partiallySucceeded` does not behave as their union), so the collector issues + one request per result value and merges/dedupes by build id (`Merge-AzdoBuildLists`, covered by a + direct, network-free unit test). +* **A `## Failing Test(s)` section naming more than one concrete test identity fails closed.** + aspnetcore#68724 names both a base test and its server-execution subclass override in one + section; live data shows only the override actually failed while the base identity passed. + Silently picking the first backtick-quoted name risks binding evidence to the wrong test. This + collector requires exactly one unambiguous identity per run and fails closed + (`multiple-test-identities-unresolved`) otherwise, rather than guessing. Evaluating every listed + identity independently is a reasonable follow-up left for later, since this PR targets one + issue/one root cause at a time -- documented here as an accepted simplification, not an + oversight. +* **The `test-failure` label alone is not proof an issue was generated by quarantine automation** + (any contributor can apply it to an ordinary bug report). The collector additionally requires the + issue body to contain the trusted `` or + `` HTML-comment marker the + production quarantine workflow stamps into every issue it creates. +* **A duplicate-search hit is a discovery candidate, not a validated duplicate.** Every numeric + result returned by the four categorized GitHub searches (open/recently-closed KBE, + open/recently-merged fix PR) is fetched and required to contain the candidate's **exact** + fully-qualified test name before it is ever treated as an existing KBE or fix PR; a hit that only + shares a bare method name with an unrelated test is recorded as an `unvalidated_candidate` (with + a reason) and never sets `duplicate_check.status` to `existing-kbe`/`existing-fix-pr`. The + "recently" closed/merged categories carry an explicit 90-day `closed:>=`/`merged:>=` time-window + qualifier so the label matches what the query actually searches, and a query is only marked + `complete` when GitHub reports `incomplete_results: false` **and** every matching item (per + `total_count`, across up to 3 paginated pages of 100) was actually retrieved -- a `total_count` + larger than a single page previously went unnoticed. +* **The repository checkout is only ever labeled `branch: "main"` after independent + confirmation.** The collector resolves `dotnet/aspnetcore`'s actual current `main` SHA through a + trusted `GET /repos/dotnet/aspnetcore/commits/main` GitHub API response and requires it to equal + the checked-out commit (`repository_ref_verification` in the dossier records both SHAs and the + comparison result) before ever emitting a candidate. In production this is a same-checkout + cross-check (workflow_dispatch normally runs on `main`'s tip, so both values coincide); in this + PR's own development branch (or any dispatch from a non-default ref), the checkout is genuinely + not `main`, and the collector fails closed (`repository-ref-not-main`) rather than mislabeling + it. +* **Platform/configuration are derived from authoritative metadata, never fabricated.** Azure + DevOps' `buildConfiguration.platform`/`.flavor` fields are empty strings on every real run + observed live; the only authoritative, cheaply-available signal is the VSTMR TestRun's own + `name` (e.g. `Quarantine-Mono-Linux-Release-xunit`). The collector parses recognized + platform/configuration tokens out of that name and records the literal string `"unknown"` -- + never a guessed default -- when no recognized token is present. +* **Never infer a pass, a recurrence, a signature, a platform/configuration, or a validated + duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps + metadata has aged out of retention, historical VSTMR test-result data no longer queryable for a + build, an ambiguous or absent signature, an incomplete duplicate search, an unconfirmed + repository ref -- is recorded as an explicit reason code and fails the run closed + (`outcome: "incomplete"`) rather than silently degrading. Real Azure DevOps retention data + confirms this is not hypothetical: build 1537561 (cited by aspnetcore#68947) has already aged out + of the public `dnceng-public` project. * **No repository-state mutation.** The workflow's permissions are `contents: read`, - `issues: read`, `pull-requests: read`, `checks: read` — read-only across the board. It uses only - the ambient `GITHUB_TOKEN`, never a PAT or other secret. It uploads artifacts; it never calls any - write API (no labels, comments, commits, branches, or files). + `issues: read`, `pull-requests: read`, `checks: read` -- read-only across the board. It uses only + the ambient `GITHUB_TOKEN`, sent as a real `Authorization: Bearer ` header (never a + literal placeholder), never a PAT or other secret. Authenticated calls are load-bearing for + practical use, not cosmetic: GitHub's unauthenticated search-API rate limit (10 requests/minute) + is exhausted by a single run's four duplicate searches plus one retry, and the collector logs the + authenticated `X-RateLimit-Remaining`/`X-RateLimit-Limit` headers (non-blocking, informational + only) after each GitHub call. The workflow uploads artifacts; it never calls any write API (no + labels, comments, commits, branches, or files). * **Evidence provenance remains unverified in this PR.** Even when the collector's candidate is independently valid, the unmodified evaluator still emits `evidence_provenance_verified: false`, `eligible_for_kbe_enrichment: false`, and `human_review_required: true`. Flipping `evidence_provenance_verified` to `true` is a deliberate, separate promotion decision (see - "Promotion gates" below) — not something this collector claims for itself. + "Promotion gates" below) -- not something this collector claims for itself. ## Workflow: `test-quarantine-kbe-shadow.yml` (maintainer dispatch) @@ -64,7 +135,7 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis of this file is inert. * **Concurrency**: one run per issue number (`test-quarantine-kbe-shadow-`), cancelling any still-running dispatch for the same issue. -* **Permissions**: `contents: read`, `issues: read`, `pull-requests: read`, `checks: read` — the +* **Permissions**: `contents: read`, `issues: read`, `pull-requests: read`, `checks: read` -- the last two are the "only demonstrated needs" beyond the baseline: `checks: read` for the Build Analysis check-run snapshot, `pull-requests: read` for the duplicate fix-PR search. * **Actions are pinned by commit SHA** (`actions/checkout`, `actions/upload-artifact`), matching @@ -82,7 +153,7 @@ Runs both deterministic, offline PowerShell test suites (`Test-Evaluate-TestQuar and `Test-Collect-TestQuarantineKbeEvidence.ps1`), plus the script-injection safety guard described below, on every pull request that touches this directory or either workflow file, plus on manual dispatch. Every fixture is local and offline; no network access or secrets are required, so this is -a low-risk, standard `pull_request`-triggered check — no self-test mode inside the shadow workflow +a low-risk, standard `pull_request`-triggered check -- no self-test mode inside the shadow workflow itself was needed to close this CI gap. ## Script-injection safety @@ -92,12 +163,12 @@ itself was needed to close this CI gap. controls their exact content). Neither is ever interpolated directly into a `run:` script body via `${{ inputs.issue_number }}` / `${{ inputs.signature }}`: a signature value containing a quote, backtick, or newline embedded directly into script text could otherwise execute arbitrary commands -on the runner. Instead, every input flows through a step (or job) `env:` binding — for example -`env: { SIGNATURE_INPUT: ${{ inputs.signature }} }` — and is read back inside the script as an +on the runner. Instead, every input flows through a step (or job) `env:` binding -- for example +`env: { SIGNATURE_INPUT: ${{ inputs.signature }} }` -- and is read back inside the script as an opaque environment variable (`$env:SIGNATURE_INPUT`), which the PowerShell parser never re-parses as script text regardless of its content. `issue_number` is additionally validated against `^[1-9][0-9]*$` before being persisted to `$GITHUB_ENV`, and only that already-validated -`env.ISSUE_NUMBER` value (not the raw input) is used in the uploaded artifact's name — a +`env.ISSUE_NUMBER` value (not the raw input) is used in the uploaded artifact's name -- a non-`run:` context where Actions itself handles the substitution, not a shell. `Test-WorkflowScriptInjectionSafety.ps1` is a static, offline regression test for exactly this @@ -110,39 +181,60 @@ expression there is evaluated by the Actions engine itself and is not a script-i Both `Collect-TestQuarantineKbeEvidence.ps1` and its tests support a `-FixtureRoot ` parameter. When set, the collector reads one consolidated `fixture.json` document from that -directory instead of making any live GitHub/Azure DevOps/Helix call — this is how the test suite -and the three pilot fixtures below achieve fully offline, deterministic coverage. `fixture.json` -mirrors the shape of the real endpoints documented in `test-quarantine.md`'s "API Reference" -section: the GitHub issue body/labels/state, Azure DevOps build metadata keyed by build id, a -capped recurrence/negative-build candidate list per pipeline definition, VSTMR test outcomes keyed -by build id, capped Helix console excerpts keyed by build id, GitHub check-run snapshots keyed by -commit SHA, and categorized duplicate-search results. +directory instead of making any live GitHub/Azure DevOps call. `fixture.json` mirrors the shape of +the real endpoints captured live during development: + +| Key | Mirrors | +|---|---| +| `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, body) | +| `main_branch` *(optional)* | `GET /repos/{repo}/commits/main` (`.sha`) -- omit entirely to skip the repository-ref guard (used by fixtures that don't specifically exercise it); include to test either a match or a deliberate mismatch | +| `azdo_builds` | `GET .../build/builds/{id}` keyed by build id | +| `recurrence_scan`, `negative_scan` | `GET .../build/builds?resultFilter=...` keyed by pipeline definition id | +| `vstmr_summary` | `GET .../testresults/resultsbyBuild?buildId=...` keyed by build id (array of summary rows) | +| `vstmr_detail` | `GET .../test/Runs/{runId}/results/{resultId}` keyed by `"{runId}:{resultId}"` | +| `vstmr_runs` | `GET .../test/runs/{runId}` keyed by run id (only `.name` is used) | +| `check_runs` | `GET /repos/{repo}/commits/{sha}/check-runs` keyed by commit SHA | +| `duplicate_search` | `GET /search/issues?q=...` keyed by category, pre-paginated (`complete`, `result_numbers`, `total_count`) | +| `duplicate_candidate_text` | `GET /repos/{repo}/issues/{number}` (title + body) keyed by issue/PR number, used to validate a duplicate-search hit | ## Pilot fixtures | Issue | Fixture invocation | Outcome | |---|---|---| -| [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` (deterministic `## Error Message` extraction) | `candidate`, `reuse-existing-kbe` (recommends reusing #68708) | -| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException: TaskCanceledException"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override — verified by a second, signature-less invocation of the same fixture in the test suite) | `candidate`, `timeout-needs-classification` (generic Selenium/WebDriver timeout, not a test-specific KBE) | -| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's Helix console-log artifact has expired (`raw-evidence-expired`), leaving only one usable failure log below the two-build recurrence floor (`raw-evidence-insufficient`) | +| [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` | `incomplete`: `## Failing Test(s)` names two distinct concrete identities (a base test and its server-execution subclass override); live data shows only the override actually failed, so the collector fails closed (`multiple-test-identities-unresolved`) rather than guess | +| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `candidate`, `timeout-needs-classification` (generic Selenium/WebDriver timeout, not a test-specific KBE); recurrence is established via the supplementary scan since the issue's own second cited build has aged out of retention | +| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's Azure DevOps build record still resolves, but its historical VSTMR test-result data is no longer queryable, leaving only one usable failure log below the two-build recurrence floor (`raw-evidence-insufficient`) | Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality comparison in the test suite. Golden comparisons exclude the collector's `generated_utc` / -`retrieved_utc` / `captured_utc` / `checked_utc` timestamps and the running checkout's HEAD -`commit_sha` — all of which are expected to differ run-to-run and commit-to-commit — replacing -them with the literal sentinel `` on both sides before comparing. +`retrieved_utc` / `captured_utc` / `checked_utc` timestamps and the running checkout's +`commit_sha` / `checkout_sha` / `trusted_main_sha` -- all of which are expected to differ +run-to-run and commit-to-commit -- replacing them with the literal sentinel `` on both +sides before comparing. + +The test suite additionally covers, via small synthetic (non-pilot) fixtures: a closed issue, an +issue missing the `test-failure` label, an issue carrying the label but missing the trusted +workflow marker, a repository-ref mismatch, Build Analysis exact-vs-short-name and +concrete-vs-generic-known-issue flag precision, an unvalidated duplicate-search hit that shares +only a bare method name with an unrelated test, a literal signature containing `*`/`?`/`[` +wildcard-shaped characters (with a decoy build proving ordinal, not `-like`, matching), and a +direct, network-free unit test of the failed/partiallySucceeded build-list merge/dedupe. ## Reconciling with the existing evaluator contract The collector does **not** introduce a third, competing dossier schema. Its `candidate` output, when present, is validated against the same unmodified `test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to the -unmodified `Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is — this PR does not change that +unmodified `Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is -- this PR does not change that script, its tests, or either of its schemas. `test-quarantine-kbe-shadow-dossier.schema.json` is a new, independently versioned (`schema_version: 1`) envelope that carries collector-specific -provenance (Azure DevOps build resolution, Build Analysis check-run snapshots, raw-evidence -retrieval/expiry) alongside that same `candidate` object, or a structured `incomplete` outcome when -any evidence gate fails. +provenance (repository-ref verification, Azure DevOps build resolution, Build Analysis check-run +snapshots, raw-evidence retrieval, unvalidated duplicate-search candidates) alongside that same +`candidate` object, or a structured `incomplete` outcome when any evidence gate fails. Because the +candidate schema's `duplicate_check.queries[]` items are `additionalProperties: false` (and +correctly so -- it must stay byte-for-byte compatible with the unmodified evaluator), the +dossier-only `total_count` field is carried on `provenance.duplicate_search.queries[]` only, never +on `candidate.duplicate_check.queries[]`. ## Eventual extraction of the production deterministic collector @@ -151,15 +243,15 @@ collector (Azure DevOps `resultsbyBuild`/build-timeline aggregation across ~200 console-log `[FAIL]` block extraction, secret redaction) that gathers failure evidence for the *entire* quarantine-management workflow, not a single issue. **This PR does not modify that workflow or its production quarantine behavior.** The long-term path is to factor the shared -pieces — the Azure DevOps/Helix/VSTMR fetch helpers, the secret-redaction patterns, the Helix -`[FAIL]`-block extraction — out of `test-quarantine.md`'s Python step and this directory's -PowerShell collector into one common, tested module that both consume, rather than maintaining two -independent implementations of the same evidence-gathering logic indefinitely. Until that -extraction happens, this collector intentionally duplicates only the narrow, single-issue subset -of that logic it needs (build resolution, VSTMR outcome lookup, Helix console retrieval, the same -secret-redaction pattern family) and cross-references the production workflow's real, observed -issue-body formats (both the `50_test_failure.md` template and the freeform `## Details` variant) -so it does not invent a third, incompatible issue-body convention. +pieces -- the Azure DevOps/VSTMR fetch helpers, the secret-redaction patterns -- out of +`test-quarantine.md`'s Python step and this directory's PowerShell collector into one common, +tested module that both consume, rather than maintaining two independent implementations of the +same evidence-gathering logic indefinitely. Until that extraction happens, this collector +intentionally duplicates only the narrow, single-issue subset of that logic it needs (build +resolution, VSTMR summary/detail lookup, the same secret-redaction pattern family) and +cross-references the production workflow's real, observed issue-body formats (both the +`50_test_failure.md` template and the freeform `## Details` variant) so it does not invent a third, +incompatible issue-body convention. ## Promotion gates @@ -167,16 +259,19 @@ None of the following are implemented by this PR. They are the measurable condit separate change would need to satisfy before this collector's output could ever be trusted enough to flip `evidence_provenance_verified` to `true` or to feed a maintainer-triggered fix workflow: -1. **Signed/attested provenance for every raw evidence file** — e.g. a hash chain from the Azure - DevOps/Helix API response actually observed at collection time, not merely a locally computed - SHA-256 of whatever bytes were written to disk. +1. **Signed/attested provenance for every raw evidence file** -- e.g. a hash chain from the Azure + DevOps API response actually observed at collection time, not merely a locally computed SHA-256 + of whatever bytes were written to disk. 2. **A second, independent collector run reaching the same `candidate` (or `incomplete`) result** for the same issue, to bound single-run collection errors (rate limiting, partial API - responses, transient Azure DevOps/Helix outages). -3. **A maintainer explicitly reviewing and approving** the specific candidate/receipt pair — this + responses, transient Azure DevOps outages). +3. **A maintainer explicitly reviewing and approving** the specific candidate/receipt pair -- this PR's receipt already always sets `human_review_required: true` for exactly this reason. 4. **The shared extraction described above landing** so the single-issue collector and the production quarantine workflow's evidence gathering can no longer silently diverge. -5. **A dedicated, maintainer-triggered fix workflow being designed and reviewed separately** — this +5. **A dedicated, maintainer-triggered fix workflow being designed and reviewed separately** -- this PR intentionally stops at read-only evaluation and does not implement or wire up any automated fix path. +6. **Evaluating every identity named in a multi-identity `## Failing Test(s)` section + independently**, rather than failing the whole run closed the moment more than one concrete + test is named (the current, deliberately conservative behavior for this PR). diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index d39ff1648d02..fc30cd05fc60 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -5,11 +5,11 @@ .DESCRIPTION Exercises the collector entirely in -FixtureRoot mode (zero network access) against the - three real pilot quarantine issues recorded in fixtures/, and against a handful of small - synthetic fixtures for edge cases that are not represented by those three issues. Golden - dossier comparisons exclude collector-generated timestamps and the running repository's - HEAD commit SHA, both of which are expected to differ between runs/commits; every other - field must match exactly. + three real pilot quarantine issues recorded in fixtures/, against a battery of synthetic + edge-case fixtures for gates not represented by those three issues, and against + Merge-AzdoBuildLists directly as a pure unit. Golden dossier comparisons exclude + collector-generated timestamps and the running repository's HEAD commit SHA, both of which + are expected to differ between runs/commits; every other field must match exactly. Where the collector's outcome is 'candidate', the resulting candidate.json is also fed through the unmodified, already-tested Evaluate-TestQuarantineKbeCandidate.ps1 to prove the @@ -30,6 +30,7 @@ $dossierSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json" $candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json" $fixturesRoot = "$PSScriptRoot/fixtures" $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-collector-$([System.Guid]::NewGuid().ToString('N'))" +$repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path function Assert-Equal { @@ -59,10 +60,24 @@ function Assert-Contains } } -# The three collector-generated timestamps and the running checkout's HEAD commit SHA are -# expected to differ run-to-run and commit-to-commit; every golden fixture below was captured -# with these fields already replaced by this same sentinel. -$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha") +function Assert-NotContains +{ + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Collection, + [Parameter(Mandatory = $true)]$Value, + [Parameter(Mandatory = $true)][string]$Message + ) + + if (@($Collection | Where-Object { "$_" -eq "$Value" }).Count -gt 0) + { + throw "$Message Expected collection NOT to contain '$Value'; actual: $($Collection -join ', ')." + } +} + +# The collector-generated timestamps and the running checkout's HEAD commit SHA are expected to +# differ run-to-run and commit-to-commit; every golden fixture below was captured with these +# fields already replaced by this same sentinel. +$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha", "checkout_sha", "trusted_main_sha") $volatileSentinel = "" function ConvertTo-NormalizedObject @@ -157,6 +172,7 @@ function Invoke-Collector CandidateFile = $candidatePath EvidenceRoot = $evidenceRoot FixtureRoot = $FixtureRoot + RepositoryRoot = $repositoryRoot DossierSchemaFile = $dossierSchema CandidateSchemaFile = $candidateSchema } @@ -188,193 +204,418 @@ function Assert-GoldenDossier Test-DeepEqual -Expected $golden -Actual $normalizedActual } +function New-SyntheticFixture +{ + # Writes a minimal-but-complete fixture.json (every top-level key the collector expects to + # be able to read present, even if empty) to a fresh directory under $tempRoot and returns + # its path. + param( + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][hashtable]$Overrides + ) + + $base = [ordered]@{ + issue = $null + main_branch = $null + azdo_builds = [ordered]@{} + recurrence_scan = [ordered]@{} + negative_scan = [ordered]@{} + vstmr_summary = [ordered]@{} + vstmr_detail = [ordered]@{} + vstmr_runs = [ordered]@{} + check_runs = [ordered]@{} + duplicate_search = [ordered]@{} + duplicate_candidate_text = [ordered]@{} + } + foreach ($key in $Overrides.Keys) + { + $base[$key] = $Overrides[$key] + } + if ($null -eq $base["main_branch"]) + { + $base.Remove("main_branch") + } + + $directory = Join-Path $tempRoot "fixture-$Name" + [System.IO.Directory]::CreateDirectory($directory) | Out-Null + $base | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $directory "fixture.json") + return $directory +} + +$workflowMarker = "" +# Built from a single-quoted literal (no backtick-escape processing) rather than embedding +# repeated backtick-escape sequences directly in double-quoted synthetic issue bodies below, +# which is easy to miscount (a stray extra backtick silently becomes an unrelated `t`/`n`/etc. +# escape sequence instead of a literal backtick). +$codeFence = '```' +$defaultDuplicateSearch = [ordered]@{ + "open-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-closed-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "open-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-merged-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } +} + try { [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null # ------------------------------------------------------------------ - # Pilot 1 -- aspnetcore#68724: deterministic '## Error Message' extraction, - # a supplementary recurrence-scan build, and reuse of an existing KBE. + # Pilot 1 -- aspnetcore#68724: '## Failing Test(s)' names two distinct concrete test + # identities (a base test and its server-execution subclass override). Live data shows only + # the override actually failed while the base identity passed -- the collector must fail + # closed rather than silently bind evidence to the first name it sees. # ------------------------------------------------------------------ $result68724 = Invoke-Collector -IssueNumber 68724 -FixtureRoot "$fixturesRoot/68724" -WorkDirectory "$tempRoot/68724" - Assert-Equal -Actual $result68724.Dossier.outcome -Expected "candidate" -Message "#68724 outcome mismatch." - Assert-Equal -Actual $result68724.Dossier.candidate.proposed_classification -Expected "reuse-existing-kbe" -Message "#68724 proposed_classification mismatch." + Assert-Equal -Actual $result68724.Dossier.outcome -Expected "incomplete" -Message "#68724 outcome mismatch." + Assert-Contains -Collection @($result68724.Dossier.incomplete.reason_codes) -Value "multiple-test-identities-unresolved" -Message "#68724 reason codes mismatch." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68724" -ActualDossier $result68724.Dossier - $receiptPath68724 = Join-Path "$tempRoot/68724" "receipt.json" - & $evaluator -CandidateFile $result68724.CandidatePath -EvidenceRoot $result68724.EvidenceRoot -OutputFile $receiptPath68724 -CandidateSchemaFile $candidateSchema - $receipt68724 = Get-Content -LiteralPath $receiptPath68724 -Raw | ConvertFrom-Json -Depth 32 - Assert-Equal -Actual $receipt68724.deterministic_status -Expected "validated" -Message "#68724 deterministic_status mismatch." - Assert-Equal -Actual $receipt68724.shadow_recommendation -Expected "reuse-existing-kbe" -Message "#68724 shadow_recommendation mismatch." - Assert-Equal -Actual $receipt68724.eligible_for_kbe_enrichment -Expected $false -Message "#68724 must never authorize enrichment." - Assert-Equal -Actual $receipt68724.evidence_provenance_verified -Expected $false -Message "#68724 provenance must remain unverified." - - $summaryPath68724 = Join-Path "$tempRoot/68724" "summary.md" - & $summaryGenerator -DossierFile $result68724.DossierPath -ReceiptFile $receiptPath68724 -OutputFile $summaryPath68724 - $summaryText68724 = Get-Content -LiteralPath $summaryPath68724 -Raw - if (-not $summaryText68724.Contains("reuse-existing-kbe")) - { - throw "#68724 summary must mention the shadow_recommendation." - } - # ------------------------------------------------------------------ - # Pilot 2 -- aspnetcore#68947: the issue body has no fenced '## Error - # Message' block, so deterministic extraction is ambiguous without a - # manual signature. + # Pilot 2 -- aspnetcore#68947: the issue body has no fenced '## Error Message' block, so + # deterministic extraction is ambiguous without a manual signature. Its own cited second + # build (1537561) has aged out of Azure DevOps retention; recurrence is instead established + # by the collector's supplementary recurrence scan, which must also query + # resultFilter=partiallySucceeded (the cited build 1551326's own real result value) in + # addition to resultFilter=failed. # ------------------------------------------------------------------ $result68947NoSig = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947-no-signature" Assert-Equal -Actual $result68947NoSig.Dossier.outcome -Expected "incomplete" -Message "#68947 (no signature) outcome mismatch." Assert-Contains -Collection @($result68947NoSig.Dossier.incomplete.reason_codes) -Value "signature-extraction-ambiguous" -Message "#68947 (no signature) reason codes mismatch." - # With the manual override supplied, recurrence is established via the collector's - # supplementary scan (the issue's second cited build has itself aged out of Azure DevOps - # retention) and the outcome is a validated, generic-timeout candidate. - $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature "OpenQA.Selenium.WebDriverException: TaskCanceledException" + $signature68947 = "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" + $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature $signature68947 Assert-Equal -Actual $result68947.Dossier.outcome -Expected "candidate" -Message "#68947 outcome mismatch." Assert-Equal -Actual $result68947.Dossier.candidate.proposed_classification -Expected "timeout-needs-classification" -Message "#68947 proposed_classification mismatch." + Assert-Equal -Actual (@($result68947.Dossier.candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" })).Count -Expected 2 -Message "#68947 must gather two distinct failure builds (the cited partiallySucceeded build plus one recurrence-scan match)." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68947" -ActualDossier $result68947.Dossier $receiptPath68947 = Join-Path "$tempRoot/68947" "receipt.json" - & $evaluator -CandidateFile $result68947.CandidatePath -EvidenceRoot $result68947.EvidenceRoot -OutputFile $receiptPath68947 -CandidateSchemaFile $candidateSchema + & $evaluator -CandidateFile $result68947.CandidatePath -EvidenceRoot $result68947.EvidenceRoot -OutputFile $receiptPath68947 -RepositoryRoot $repositoryRoot -CandidateSchemaFile $candidateSchema $receipt68947 = Get-Content -LiteralPath $receiptPath68947 -Raw | ConvertFrom-Json -Depth 32 Assert-Equal -Actual $receipt68947.deterministic_status -Expected "validated" -Message "#68947 deterministic_status mismatch." Assert-Equal -Actual $receipt68947.shadow_recommendation -Expected "timeout-needs-classification" -Message "#68947 shadow_recommendation mismatch." + Assert-Equal -Actual $receipt68947.eligible_for_kbe_enrichment -Expected $false -Message "#68947 must never authorize enrichment." + Assert-Equal -Actual $receipt68947.evidence_provenance_verified -Expected $false -Message "#68947 provenance must remain unverified." + + $summaryPath68947 = Join-Path "$tempRoot/68947" "summary.md" + & $summaryGenerator -DossierFile $result68947.DossierPath -ReceiptFile $receiptPath68947 -OutputFile $summaryPath68947 + $summaryText68947 = Get-Content -LiteralPath $summaryPath68947 -Raw + if (-not $summaryText68947.Contains("timeout-needs-classification")) + { + throw "#68947 summary must mention the shadow_recommendation." + } # ------------------------------------------------------------------ - # Pilot 3 -- aspnetcore#68945: both cited builds' Azure DevOps metadata still - # resolves, but the second build's Helix console-log artifact has expired. - # Recurrence therefore falls back to a single usable failure log and the - # collector must fail closed rather than infer a pass. + # Pilot 3 -- aspnetcore#68945: both cited builds' Azure DevOps build-level metadata still + # resolves, but the second build's historical VSTMR test-result data is no longer queryable, + # leaving only one usable failure log below the two-build recurrence floor. The collector + # must fail closed rather than infer a pass or fabricate evidence. # ------------------------------------------------------------------ $result68945 = Invoke-Collector -IssueNumber 68945 -FixtureRoot "$fixturesRoot/68945" -WorkDirectory "$tempRoot/68945" -Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled." Assert-Equal -Actual $result68945.Dossier.outcome -Expected "incomplete" -Message "#68945 outcome mismatch." - Assert-Contains -Collection @($result68945.Dossier.incomplete.reason_codes) -Value "raw-evidence-expired" -Message "#68945 reason codes must record the expired artifact." + Assert-Contains -Collection @($result68945.Dossier.incomplete.reason_codes) -Value "raw-evidence-insufficient" -Message "#68945 reason codes must record the insufficient evidence." Assert-Equal -Actual $result68945.Dossier.candidate -Expected $null -Message "#68945 must not emit a candidate." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68945" -ActualDossier $result68945.Dossier $summaryPath68945 = Join-Path "$tempRoot/68945" "summary.md" & $summaryGenerator -DossierFile $result68945.DossierPath -OutputFile $summaryPath68945 $summaryText68945 = Get-Content -LiteralPath $summaryPath68945 -Raw - if (-not $summaryText68945.Contains("raw-evidence-expired")) + if (-not $summaryText68945.Contains("raw-evidence-insufficient")) { - throw "#68945 summary must mention the raw-evidence-expired reason code." + throw "#68945 summary must mention the raw-evidence-insufficient reason code." } # ------------------------------------------------------------------ - # Edge cases not represented by the three pilots: a closed issue, an issue - # missing the canonical quarantine label, and conservative check-run - # substring extraction actually flipping to true when warranted. + # Edge case: a closed issue must fail closed regardless of label/marker. # ------------------------------------------------------------------ - $closedFixtureDir = Join-Path $tempRoot "closed-issue-fixture" - [System.IO.Directory]::CreateDirectory($closedFixtureDir) | Out-Null - @{ - issue = @{ + $closedDir = New-SyntheticFixture -Name "closed-issue" -Overrides @{ + issue = [ordered]@{ number = 1 state = "closed" labels = @("test-failure") - body = "## Failing Test(s)`n`` Sample.Tests.Closed ``" - } - azdo_builds = @{} - recurrence_scan = @{} - negative_scan = @{} - vstmr_results = @{} - helix_evidence = @{} - check_runs = @{} - duplicate_search = @{ - "open-kbe" = @{ complete = $true; result_numbers = @() } - "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } - "open-fix-pr" = @{ complete = $true; result_numbers = @() } - "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } - } - } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $closedFixtureDir "fixture.json") - $resultClosed = Invoke-Collector -IssueNumber 1 -FixtureRoot $closedFixtureDir -WorkDirectory (Join-Path $tempRoot "closed-issue") + body = "## Failing Test(s)`n`` Sample.Tests.Closed ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultClosed = Invoke-Collector -IssueNumber 1 -FixtureRoot $closedDir -WorkDirectory (Join-Path $tempRoot "closed-issue") Assert-Equal -Actual $resultClosed.Dossier.outcome -Expected "incomplete" -Message "Closed-issue outcome mismatch." Assert-Contains -Collection @($resultClosed.Dossier.incomplete.reason_codes) -Value "issue-not-open" -Message "Closed-issue reason codes mismatch." - $unlabeledFixtureDir = Join-Path $tempRoot "unlabeled-issue-fixture" - [System.IO.Directory]::CreateDirectory($unlabeledFixtureDir) | Out-Null - @{ - issue = @{ + # ------------------------------------------------------------------ + # Edge case: an issue missing the canonical 'test-failure' label must fail closed. + # ------------------------------------------------------------------ + $unlabeledDir = New-SyntheticFixture -Name "unlabeled-issue" -Overrides @{ + issue = [ordered]@{ number = 2 state = "open" labels = @("area-blazor") - body = "## Failing Test(s)`n`` Sample.Tests.Unlabeled ``" - } - azdo_builds = @{} - recurrence_scan = @{} - negative_scan = @{} - vstmr_results = @{} - helix_evidence = @{} - check_runs = @{} - duplicate_search = @{ - "open-kbe" = @{ complete = $true; result_numbers = @() } - "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } - "open-fix-pr" = @{ complete = $true; result_numbers = @() } - "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } - } - } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $unlabeledFixtureDir "fixture.json") - $resultUnlabeled = Invoke-Collector -IssueNumber 2 -FixtureRoot $unlabeledFixtureDir -WorkDirectory (Join-Path $tempRoot "unlabeled-issue") + body = "## Failing Test(s)`n`` Sample.Tests.Unlabeled ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultUnlabeled = Invoke-Collector -IssueNumber 2 -FixtureRoot $unlabeledDir -WorkDirectory (Join-Path $tempRoot "unlabeled-issue") Assert-Equal -Actual $resultUnlabeled.Dossier.outcome -Expected "incomplete" -Message "Unlabeled-issue outcome mismatch." Assert-Contains -Collection @($resultUnlabeled.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Unlabeled-issue reason codes mismatch." - # The three pilots' real Build Analysis snapshots were all generic (matching the - # documented architecture-consensus finding that this signal is corroborating only), so - # none of them exercise a positive substring match. Prove that path separately: a snapshot - # whose text literally names the test and a known issue must flip both conservative flags. - $exactMatchFixtureDir = Join-Path $tempRoot "exact-match-fixture" - [System.IO.Directory]::CreateDirectory($exactMatchFixtureDir) | Out-Null - $exactMatchSha = "bf6e1566a2433f298c3adc8b6ecc3358b99d5d3f" - $exactMatchSignature = "System.InvalidOperationException: Sample failure for exact-match testing." - @{ - issue = @{ - number = 3 + # ------------------------------------------------------------------ + # Edge case (item 5): the 'test-failure' label alone is not proof of quarantine automation -- + # an issue carrying the label and open state, but with no trusted gh-aw workflow marker in + # its body, must also fail closed. + # ------------------------------------------------------------------ + $missingMarkerDir = New-SyntheticFixture -Name "missing-marker" -Overrides @{ + issue = [ordered]@{ + number = 10 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.NoMarker ``" + } + duplicate_search = $defaultDuplicateSearch + } + $resultMissingMarker = Invoke-Collector -IssueNumber 10 -FixtureRoot $missingMarkerDir -WorkDirectory (Join-Path $tempRoot "missing-marker") + Assert-Equal -Actual $resultMissingMarker.Dossier.issue.has_workflow_marker -Expected $false -Message "Missing-marker issue.has_workflow_marker mismatch." + Assert-Equal -Actual $resultMissingMarker.Dossier.outcome -Expected "incomplete" -Message "Missing-marker outcome mismatch." + Assert-Contains -Collection @($resultMissingMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Missing-marker reason codes mismatch: the 'test-failure' label alone must not be treated as proof of quarantine automation." + + # ------------------------------------------------------------------ + # Edge case (item 8): the repository checkout must be confirmed, via a trusted GitHub API + # response for dotnet/aspnetcore's main branch, to actually be main's tip before a candidate + # is ever labeled repository_ref.branch = "main". A deliberately wrong trusted SHA must fail + # closed rather than mislabel the checkout. + # ------------------------------------------------------------------ + $repoRefMismatchDir = New-SyntheticFixture -Name "repo-ref-mismatch" -Overrides @{ + issue = [ordered]@{ + number = 13 state = "open" labels = @("test-failure") - body = "## Failing Test(s)`n`` Sample.Tests.ExactMatchCase ``" + "`n`n## Error Message`n``````text`n$exactMatchSignature`n``````" + "`n`n## Build`nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=5000001" - } - azdo_builds = @{ - "5000001" = @{ definition = @{ id = 83 }; sourceVersion = $exactMatchSha; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } - } - recurrence_scan = @{ - "83" = @(@{ id = 5000002; sourceVersion = "c1b304785ea05e7c92030583e1cb658c50630102"; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) - } - negative_scan = @{ - "83" = @(@{ id = 5000003; sourceVersion = "52bcd78ab0d7a1df3834306cc1c56a21f86a9fd2"; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) - } - vstmr_results = @{ - "5000001" = @{ outcome = "Failed"; comment = '{"HelixJobId":"job-a","HelixWorkItemName":"wi-a"}'; errorMessage = $exactMatchSignature; stackTrace = "at Sample.Tests.ExactMatchCase..." } - "5000002" = @{ outcome = "Failed"; comment = '{"HelixJobId":"job-b","HelixWorkItemName":"wi-b"}'; errorMessage = $exactMatchSignature; stackTrace = "at Sample.Tests.ExactMatchCase..." } - "5000003" = @{ outcome = "Passed"; comment = '{"HelixJobId":"job-c","HelixWorkItemName":"wi-c"}'; errorMessage = $null; stackTrace = $null } - } - helix_evidence = @{ - "5000001" = @{ found = $true; expired = $false; console_excerpt = "Failed Sample.Tests.ExactMatchCase [1 s]`n$exactMatchSignature (build 5000001)" } - "5000002" = @{ found = $true; expired = $false; console_excerpt = "Failed Sample.Tests.ExactMatchCase [1 s]`n$exactMatchSignature (build 5000002)" } - "5000003" = @{ found = $true; expired = $false; console_excerpt = "[PASS] Sample.Tests.ExactMatchCase" } - } - check_runs = @{ - $exactMatchSha = @(@{ - name = "Build Analysis" - id = 700001 - conclusion = "failure" - output = @{ - title = "1 failing test" - text = "Sample.Tests.ExactMatchCase failed. This matches a Known Issue: https://github.com/dotnet/aspnetcore/issues/70000." - } - html_url = "https://github.com/dotnet/aspnetcore/runs/700001" + body = "## Failing Test(s)`n`` Sample.Tests.RepoRefMismatch ``$([System.Environment]::NewLine)$workflowMarker" + } + main_branch = [ordered]@{ sha = "0000000000000000000000000000000000000000" } + duplicate_search = $defaultDuplicateSearch + } + $resultRepoRefMismatch = Invoke-Collector -IssueNumber 13 -FixtureRoot $repoRefMismatchDir -WorkDirectory (Join-Path $tempRoot "repo-ref-mismatch") + Assert-Equal -Actual $resultRepoRefMismatch.Dossier.outcome -Expected "incomplete" -Message "Repository-ref-mismatch outcome mismatch." + Assert-Contains -Collection @($resultRepoRefMismatch.Dossier.incomplete.reason_codes) -Value "repository-ref-not-main" -Message "Repository-ref-mismatch reason codes mismatch." + Assert-Equal -Actual $resultRepoRefMismatch.Dossier.provenance.repository_ref_verification.matches_main -Expected $false -Message "Repository-ref-mismatch matches_main mismatch." + + # ------------------------------------------------------------------ + # Edge case (item 6): a Build Analysis snapshot naming only the bare method name (which + # commonly collides with unrelated tests) must never set exact_test_referenced; a generic + # "Known Issues" heading with no associated concrete issue number/URL must never set + # known_issue_referenced. Only the full fully-qualified name, and only a concrete issue + # reference, may set these flags. + # ------------------------------------------------------------------ + $flagsTestName = "Sample.Tests.ExactMatchCase" + $flagsSignature = "System.InvalidOperationException: Sample failure for exact-match testing." + $flagsShaA = "f4d9777d7b9a3d45c88e1ca1b10609e412cc4ade" + $flagsShaB = "e06ef94591aaa5a8dc3f84926f8664b2964bf0ea" + $flagsShaC = "51e066210e1643430dccb9986c42500d3e706638" + $flagsDir = New-SyntheticFixture -Name "check-run-flags" -Overrides @{ + issue = [ordered]@{ + number = 11 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $flagsTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$flagsSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6100001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6100001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $flagsShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6100002; sourceVersion = $flagsShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6100001" = @([ordered]@{ id = 1; runId = 7100001; outcome = "Failed"; automatedTestName = $flagsTestName }) + "6100002" = @([ordered]@{ id = 2; runId = 7100002; outcome = "Failed"; automatedTestName = $flagsTestName }) + "6100003" = @([ordered]@{ id = 3; runId = 7100003; outcome = "Passed"; automatedTestName = $flagsTestName }) + } + vstmr_detail = [ordered]@{ + "7100001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $flagsSignature; stackTrace = "at $flagsTestName.Run() (build a)" } + "7100002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $flagsSignature; stackTrace = "at $flagsTestName.Run() (build b)" } + "7100003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7100001" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + "7100002" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + "7100003" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } + } + check_runs = [ordered]@{ + $flagsShaA = @([ordered]@{ + name = "Build Analysis"; id = 1; conclusion = "failure" + output = [ordered]@{ title = "1 failing test"; text = "$flagsTestName failed. This matches a Known Issue: dotnet/aspnetcore#70000." } + html_url = "https://github.com/dotnet/aspnetcore/runs/1" }) + $flagsShaB = @([ordered]@{ + name = "Build Analysis"; id = 2; conclusion = "failure" + # Only the bare method name appears (embedded in an unrelated identifier, not the + # full FQN), and "Known Issues" is a generic heading with no associated number. + output = [ordered]@{ title = "1 failing test"; text = "## Known Issues`nSomeOtherExactMatchCaseVariant failed for unrelated reasons. See the table above." } + html_url = "https://github.com/dotnet/aspnetcore/runs/2" + }) + } + duplicate_search = $defaultDuplicateSearch + } + $resultFlags = Invoke-Collector -IssueNumber 11 -FixtureRoot $flagsDir -WorkDirectory (Join-Path $tempRoot "check-run-flags") + Assert-Equal -Actual $resultFlags.Dossier.outcome -Expected "candidate" -Message "check-run-flags outcome mismatch." + $snapshotA = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + $snapshotB = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaB })[0] + Assert-Equal -Actual $snapshotA.exact_test_referenced -Expected $true -Message "exact_test_referenced must be true when the full FQN appears verbatim." + Assert-Equal -Actual $snapshotA.known_issue_referenced -Expected $true -Message "known_issue_referenced must be true when a concrete issue number follows 'Known Issue'." + Assert-Contains -Collection @($snapshotA.known_issue_numbers) -Value 70000 -Message "known_issue_numbers must record the referenced issue." + Assert-Equal -Actual $snapshotB.exact_test_referenced -Expected $false -Message "exact_test_referenced must stay false for a bare-method-name collision." + Assert-Equal -Actual $snapshotB.short_name_referenced -Expected $true -Message "short_name_referenced must record the bare-method-name match." + Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $false -Message "known_issue_referenced must stay false for a generic 'Known Issues' heading with no associated number." + + # ------------------------------------------------------------------ + # Edge case (item 12): a duplicate-search hit is a discovery candidate only. A hit whose + # body/title does not contain the exact fully-qualified test name must never be treated as a + # validated existing-kbe duplicate, even though it shares the bare method name. + # ------------------------------------------------------------------ + $dupTestName = "Sample.Tests.DuplicateValidationCase" + $dupSignature = "System.InvalidOperationException: Sample failure for duplicate validation testing." + $dupShaA = "c78cc5badc905159a96a0d4bb0686acadaddc5c3" + $dupShaB = "439036f2881b7046fe9b9c3953bff60ed45dda6a" + $dupShaC = "48bf6fbc5f5aa8484a773f612851e95a4f52973a" + $dupDir = New-SyntheticFixture -Name "duplicate-unvalidated" -Overrides @{ + issue = [ordered]@{ + number = 12 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $dupTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$dupSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6200001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6200001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $dupShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6200002; sourceVersion = $dupShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6200001" = @([ordered]@{ id = 1; runId = 7200001; outcome = "Failed"; automatedTestName = $dupTestName }) + "6200002" = @([ordered]@{ id = 2; runId = 7200002; outcome = "Failed"; automatedTestName = $dupTestName }) + "6200003" = @([ordered]@{ id = 3; runId = 7200003; outcome = "Passed"; automatedTestName = $dupTestName }) + } + vstmr_detail = [ordered]@{ + "7200001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $dupSignature; stackTrace = "at $dupTestName.Run() (build 1)" } + "7200002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $dupSignature; stackTrace = "at $dupTestName.Run() (build 2)" } + "7200003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7200001" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + "7200002" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + "7200003" = [ordered]@{ name = "Quarantine-Mono-Linux-Release-xunit" } + } + duplicate_search = [ordered]@{ + "open-kbe" = [ordered]@{ complete = $true; result_numbers = @(99999); total_count = 1 } + "recently-closed-kbe" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "open-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } + "recently-merged-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } } - duplicate_search = @{ - "open-kbe" = @{ complete = $true; result_numbers = @() } - "recently-closed-kbe" = @{ complete = $true; result_numbers = @() } - "open-fix-pr" = @{ complete = $true; result_numbers = @() } - "recently-merged-fix-pr" = @{ complete = $true; result_numbers = @() } - } - } | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $exactMatchFixtureDir "fixture.json") - $resultExactMatch = Invoke-Collector -IssueNumber 3 -FixtureRoot $exactMatchFixtureDir -WorkDirectory (Join-Path $tempRoot "exact-match") - Assert-Equal -Actual $resultExactMatch.Dossier.outcome -Expected "candidate" -Message "Exact-match fixture outcome mismatch." - $matchingSnapshot = @($resultExactMatch.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $exactMatchSha })[0] - Assert-Equal -Actual $matchingSnapshot.exact_test_referenced -Expected $true -Message "exact_test_referenced must flip true when the check-run text names the test." - Assert-Equal -Actual $matchingSnapshot.known_issue_referenced -Expected $true -Message "known_issue_referenced must flip true when the check-run text names a Known Issue." + duplicate_candidate_text = [ordered]@{ + "99999" = "Quarantine Sample.Tests.OtherUnrelatedCase`nThis issue tracks a completely different test that happens to share no identity with our test." + } + } + $resultDup = Invoke-Collector -IssueNumber 12 -FixtureRoot $dupDir -WorkDirectory (Join-Path $tempRoot "duplicate-unvalidated") + Assert-Equal -Actual $resultDup.Dossier.outcome -Expected "candidate" -Message "duplicate-unvalidated outcome mismatch." + Assert-Equal -Actual $resultDup.Dossier.candidate.duplicate_check.status -Expected "none" -Message "An unvalidated search hit must never set duplicate_check.status to existing-kbe." + Assert-Equal -Actual (@($resultDup.Dossier.candidate.duplicate_check.references)).Count -Expected 0 -Message "An unvalidated search hit must never appear in duplicate_check.references." + $unvalidated = @($resultDup.Dossier.provenance.duplicate_search.unvalidated_candidates) + if ($unvalidated.Count -eq 0 -or -not (@($unvalidated | Where-Object { $_.number -eq 99999 }).Count -gt 0)) + { + throw "Expected an unvalidated_candidates entry for issue #99999." + } + + # ------------------------------------------------------------------ + # Edge case (item 11): a literal ErrorMessage containing '*', '?', and '[' must be matched via + # ordinal substring containment, never PowerShell -like/-notlike wildcard semantics. A decoy + # build sharing only the literal '[' character (but not the full signature text) must NOT be + # picked up as a second recurrence match. + # ------------------------------------------------------------------ + $wildTestName = "Sample.Tests.WildcardSignatureCase" + $wildSignature = "Assert.Equal() Failure: Array index [0] was *unexpected*, value? did not match." + $wildShaA = "11c785d7c74de87dc8dabb59c500da8af9254f81" + $wildShaB = "1d386dc40f508a46c0b76768cdbb1226cb6fe626" + $wildShaDecoy = "7ab2248eea6ca9ec8fa5c10cabd3f5c520edd126" + $wildShaNeg = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + $wildDir = New-SyntheticFixture -Name "wildcard-signature" -Overrides @{ + issue = [ordered]@{ + number = 14 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` $wildTestName ``$([System.Environment]::NewLine)## Error Message$([System.Environment]::NewLine)${codeFence}text$([System.Environment]::NewLine)$wildSignature$([System.Environment]::NewLine)$codeFence$([System.Environment]::NewLine)## Build$([System.Environment]::NewLine)https://dev.azure.com/dnceng-public/public/_build/results?buildId=6400001$([System.Environment]::NewLine)$workflowMarker" + } + azdo_builds = [ordered]@{ + "6400001" = [ordered]@{ definition = [ordered]@{ id = 83 }; sourceVersion = $wildShaA; startTime = "2026-08-01T00:00:00Z"; finishTime = "2026-08-01T01:00:00Z"; result = "failed" } + } + recurrence_scan = [ordered]@{ + "83" = @( + [ordered]@{ id = 6400099; sourceVersion = $wildShaDecoy; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "failed" }, + [ordered]@{ id = 6400002; sourceVersion = $wildShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" } + ) + } + negative_scan = [ordered]@{ + "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + } + vstmr_summary = [ordered]@{ + "6400001" = @([ordered]@{ id = 1; runId = 7400001; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400099" = @([ordered]@{ id = 99; runId = 7400099; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400002" = @([ordered]@{ id = 2; runId = 7400002; outcome = "Failed"; automatedTestName = $wildTestName }) + "6400003" = @([ordered]@{ id = 3; runId = 7400003; outcome = "Passed"; automatedTestName = $wildTestName }) + } + vstmr_detail = [ordered]@{ + "7400001:1" = [ordered]@{ outcome = "Failed"; errorMessage = $wildSignature; stackTrace = "at $wildTestName.Run() (build 1)" } + "7400099:99" = [ordered]@{ outcome = "Failed"; errorMessage = "Some unrelated failure referencing array index [5] elsewhere."; stackTrace = "at Sample.Tests.Unrelated.Run()" } + "7400002:2" = [ordered]@{ outcome = "Failed"; errorMessage = $wildSignature; stackTrace = "at $wildTestName.Run() (build 2)" } + "7400003:3" = [ordered]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null } + } + vstmr_runs = [ordered]@{ + "7400001" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400099" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400002" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + "7400003" = [ordered]@{ name = "Quarantine-Mono-Linux-Debug-xunit" } + } + duplicate_search = $defaultDuplicateSearch + } + $resultWildcard = Invoke-Collector -IssueNumber 14 -FixtureRoot $wildDir -WorkDirectory (Join-Path $tempRoot "wildcard-signature") + Assert-Equal -Actual $resultWildcard.Dossier.outcome -Expected "candidate" -Message "wildcard-signature outcome mismatch." + $wildFailureBuildIds = @($resultWildcard.Dossier.candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" } | ForEach-Object { $_.build.id }) + Assert-Equal -Actual $wildFailureBuildIds.Count -Expected 2 -Message "wildcard-signature must gather exactly two failure builds." + Assert-Contains -Collection $wildFailureBuildIds -Value 6400002 -Message "wildcard-signature must include the real matching recurrence build." + Assert-NotContains -Collection $wildFailureBuildIds -Value 6400099 -Message "wildcard-signature must NOT include the decoy build that only shares the literal '[' character, proving ordinal (not -like) matching." + + # ------------------------------------------------------------------ + # Merge-AzdoBuildLists: pure unit coverage for the failed+partiallySucceeded merge/dedupe + # (item 2) independent of any network access. Dot-source the collector (satisfying its + # mandatory parameters with the already-validated missing-marker fixture, which exits after + # Step 1) purely to bring the function into scope. + # ------------------------------------------------------------------ + . $collector -IssueNumber 10 -OutputFile (Join-Path $tempRoot "merge-unit-dossier.json") -CandidateFile (Join-Path $tempRoot "merge-unit-candidate.json") -EvidenceRoot (Join-Path $tempRoot "merge-unit-evidence") -FixtureRoot $missingMarkerDir -RepositoryRoot $repositoryRoot -DossierSchemaFile $dossierSchema -CandidateSchemaFile $candidateSchema | Out-Null + + # Real Azure DevOps build objects deserialize as PSCustomObject (via Invoke-RestMethod / + # ConvertFrom-Json); Sort-Object -Property only resolves a plain hashtable's "properties" via + # PSCustomObject-style member resolution, not dictionary key lookup, so PSCustomObject here + # matches production shape and is required for the -Property startTime sort below to work. + $failedList = @( + [PSCustomObject]@{ id = 1; startTime = "2026-01-01T00:00:00Z" }, + [PSCustomObject]@{ id = 2; startTime = "2026-01-02T00:00:00Z" } + ) + $partiallySucceededList = @( + [PSCustomObject]@{ id = 2; startTime = "2026-01-02T00:00:00Z" }, + [PSCustomObject]@{ id = 3; startTime = "2026-01-03T00:00:00Z" } + ) + $merged = @(Merge-AzdoBuildLists -Lists @($failedList, $partiallySucceededList) -Cap 10) + Assert-Equal -Actual $merged.Count -Expected 3 -Message "Merge-AzdoBuildLists must dedupe the build shared by both resultFilter queries." + Assert-Equal -Actual $merged[0].id -Expected 3 -Message "Merge-AzdoBuildLists must sort by startTime descending (most recent first)." + Assert-Equal -Actual $merged[2].id -Expected 1 -Message "Merge-AzdoBuildLists must preserve the oldest build last." + + $cappedMerged = @(Merge-AzdoBuildLists -Lists @($failedList, $partiallySucceededList) -Cap 2) + Assert-Equal -Actual $cappedMerged.Count -Expected 2 -Message "Merge-AzdoBuildLists must honor the cap." + + $emptyMerged = @(Merge-AzdoBuildLists -Lists @(@(), @()) -Cap 5) + Assert-Equal -Actual $emptyMerged.Count -Expected 0 -Message "Merge-AzdoBuildLists must return a real empty array (not collapse to null) when both inputs are empty." Write-Host "All test-quarantine-kbe-shadow collector tests passed." } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json index 4ac82862a832..e2750ccafa8a 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -15,10 +15,16 @@ "labels": [ "test-failure", "area-blazor" - ] + ], + "has_workflow_marker": true }, - "outcome": "candidate", + "outcome": "incomplete", "provenance": { + "repository_ref_verification": { + "checkout_sha": "", + "trusted_main_sha": "", + "matches_main": true + }, "azdo_builds": [ { "id": 1563420, @@ -30,28 +36,6 @@ "started_utc": "2026-08-22T03:28:01.2925985Z", "finished_utc": "2026-08-22T05:06:06.2192270Z", "result": "failed" - }, - { - "id": 1560111, - "found": true, - "retrieved_utc": "", - "source": "recurrence-scan", - "definition_id": 87, - "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", - "started_utc": "2026-08-21T03:00:00.0000000Z", - "finished_utc": "2026-08-21T05:00:00.0000000Z", - "result": "failed" - }, - { - "id": 1558000, - "found": true, - "retrieved_utc": "", - "source": "negative-scan", - "definition_id": 87, - "source_version": "8afa8ced59345cb708a7115f7519b068dd56b994", - "started_utc": "2026-08-19T03:00:00.0000000Z", - "finished_utc": "2026-08-19T05:00:00.0000000Z", - "result": "succeeded" } ], "check_run_snapshots": [ @@ -66,62 +50,14 @@ "text_excerpt": "1 test failed. See Azure DevOps for full details.", "html_url": "https://github.com/dotnet/aspnetcore/runs/900001", "exact_test_referenced": false, - "known_issue_referenced": false - }, - { - "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", - "found": true, - "retrieved_utc": "", - "check_id": 900002, - "conclusion": "failure", - "title": "1 failing test", - "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", - "text_excerpt": "1 test failed. See Azure DevOps for full details.", - "html_url": "https://github.com/dotnet/aspnetcore/runs/900002", - "exact_test_referenced": false, - "known_issue_referenced": false - } - ], - "raw_evidence_sources": [ - { - "build_id": 1563420, - "role": "failure", - "kind": "helix-console-log", - "helix_job": "helix-job-1563420", - "helix_workitem": "VirtualizationTest.WorkItemExecution", - "found": true, - "expired": false, - "captured_utc": "", - "sha256": "298a4db75b3750339d8a0f576f96b5c688a526ee99acea46290ffc735e975d15", - "evidence_path": "issue-68724-build-1563420-failure.log" - }, - { - "build_id": 1560111, - "role": "failure", - "kind": "helix-console-log", - "helix_job": "helix-job-1560111", - "helix_workitem": "VirtualizationTest.WorkItemExecution", - "found": true, - "expired": false, - "captured_utc": "", - "sha256": "c12348ca3e1cbaf32c01e9a74dc0d0373fd9c5266f0710d975b6a6acc8db74f4", - "evidence_path": "issue-68724-build-1560111-failure.log" - }, - { - "build_id": 1558000, - "role": "negative", - "kind": "helix-console-log", - "helix_job": "helix-job-1558000", - "helix_workitem": "VirtualizationTest.WorkItemExecution", - "found": true, - "expired": false, - "captured_utc": "", - "sha256": "33b8e42484dfa2c90ce1717f55251e48f7c037ea90af4b409ec2897137adf61e", - "evidence_path": "issue-68724-build-1558000-negative.log" + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] } ], + "raw_evidence_sources": [], "duplicate_search": { - "status": "existing-kbe", + "status": "none", "checked_utc": "", "coverage": { "open_kbes": true, @@ -129,173 +65,51 @@ "open_fix_prs": true, "recently_merged_fix_prs": true }, - "references": [ - "issue:68708" - ], + "references": [], "queries": [ { "category": "open-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" 68724", "complete": true, - "result_numbers": [ - 68708 - ] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" 68724", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "open-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:open QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "query": "repo:dotnet/aspnetcore is:pr is:open 68724", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 68724", "complete": true, - "result_numbers": [] - } - ] - } - }, - "candidate": { - "schema_version": 1, - "repository": "dotnet/aspnetcore", - "repository_ref": { - "branch": "main", - "commit_sha": "" - }, - "issue": { - "number": 68724, - "url": "https://github.com/dotnet/aspnetcore/issues/68724" - }, - "test": { - "fully_qualified_name": "Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll" - }, - "signature": { - "kind": "ErrorMessage", - "values": [ - "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0." - ], - "build_retry": false, - "exclude_console_log": false - }, - "policy": { - "minimum_failure_logs": 2, - "minimum_negative_logs": 1 - }, - "evidence": { - "raw_logs": [ - { - "id": "evidence-1", - "role": "failure", - "outcome": "failed", - "path": "issue-68724-build-1563420-failure.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420&view=results", - "sha256": "298a4db75b3750339d8a0f576f96b5c688a526ee99acea46290ffc735e975d15", - "build": { - "id": 1563420, - "pipeline_definition_id": 87, - "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", - "started_utc": "2026-08-22T03:28:01.2925985Z", - "platform": "Linux", - "configuration": "Release" - } - }, - { - "id": "evidence-2", - "role": "failure", - "outcome": "failed", - "path": "issue-68724-build-1560111-failure.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1560111&view=results", - "sha256": "c12348ca3e1cbaf32c01e9a74dc0d0373fd9c5266f0710d975b6a6acc8db74f4", - "build": { - "id": 1560111, - "pipeline_definition_id": 87, - "source_version": "074f8655455783f5afc3e7a865c30d5f3dad52be", - "started_utc": "2026-08-21T03:00:00.0000000Z", - "platform": "Linux", - "configuration": "Release" - } - }, - { - "id": "evidence-3", - "role": "negative", - "outcome": "passed", - "path": "issue-68724-build-1558000-negative.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1558000&view=results", - "sha256": "33b8e42484dfa2c90ce1717f55251e48f7c037ea90af4b409ec2897137adf61e", - "build": { - "id": 1558000, - "pipeline_definition_id": 87, - "source_version": "8afa8ced59345cb708a7115f7519b068dd56b994", - "started_utc": "2026-08-19T03:00:00.0000000Z", - "platform": "Linux", - "configuration": "Release" - } - } - ], - "corroborating_context": [ - { - "source": "build-analysis", - "url": "https://github.com/dotnet/aspnetcore/runs/900001" - }, - { - "source": "build-analysis", - "url": "https://github.com/dotnet/aspnetcore/runs/900002" - }, - { - "source": "quarantine-issue", - "url": "https://github.com/dotnet/aspnetcore/issues/68724" + "result_numbers": [], + "total_count": 0 } - ] - }, - "duplicate_check": { - "status": "existing-kbe", - "checked_utc": "", - "coverage": { - "open_kbes": true, - "recently_closed_kbes": true, - "open_fix_prs": true, - "recently_merged_fix_prs": true - }, - "references": [ - "issue:68708" ], - "queries": [ - { - "category": "open-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", - "complete": true, - "result_numbers": [ - 68708 - ] - }, - { - "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", - "complete": true, - "result_numbers": [] - }, - { - "category": "open-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:open QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", - "complete": true, - "result_numbers": [] - }, - { - "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll", - "complete": true, - "result_numbers": [] - } - ] - }, - "proposed_classification": "reuse-existing-kbe" + "unvalidated_candidates": [] + } }, - "incomplete": null + "candidate": null, + "incomplete": { + "reason_codes": [ + "multiple-test-identities-unresolved" + ], + "message": "Collector could not produce a validated candidate for issue #68724 : multiple-test-identities-unresolved.", + "missing_evidence": [ + { + "kind": "test-name", + "detail": "'## Failing Test(s)' names 2 distinct test identities (Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll; Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll); this collector requires exactly one unambiguous identity per run rather than guessing which one actually failed." + } + ] + } } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json index 429a29c05365..f41194683549 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -6,7 +6,7 @@ "test-failure", "area-blazor" ], - "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n at Xunit.Assert.True(Nullable`1 condition, String userMessage)\n```\n\n## Stacktrace\n
\nStack trace\n\n```text\n at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore[TResult](IWebDriver driver, Func`1 assertion, TimeSpan timeout)\n at Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll(Boolean useProvider)\n```\n\n
\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/32632851798)\n" + "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n```\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n\n" }, "azdo_builds": { "1563420": { @@ -19,78 +19,11 @@ "result": "failed" } }, - "recurrence_scan": { - "87": [ - { - "id": 1559999, - "sourceVersion": "e0232dc7601942a7c8f95b3b96f01a7b28107861", - "startTime": "2026-08-20T03:00:00Z", - "finishTime": "2026-08-20T05:00:00Z", - "result": "failed" - }, - { - "id": 1560111, - "sourceVersion": "074f8655455783f5afc3e7a865c30d5f3dad52be", - "startTime": "2026-08-21T03:00:00Z", - "finishTime": "2026-08-21T05:00:00Z", - "result": "failed" - } - ] - }, - "negative_scan": { - "87": [ - { - "id": 1558000, - "sourceVersion": "8afa8ced59345cb708a7115f7519b068dd56b994", - "startTime": "2026-08-19T03:00:00Z", - "finishTime": "2026-08-19T05:00:00Z", - "result": "succeeded" - } - ] - }, - "vstmr_results": { - "1563420": { - "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1563420\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", - "errorMessage": "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.", - "stackTrace": "at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore..." - }, - "1559999": { - "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1559999\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", - "errorMessage": "OpenQA.Selenium.WebDriverException: unrelated driver disconnect", - "stackTrace": "at Unrelated.Driver.Disconnect..." - }, - "1560111": { - "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1560111\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", - "errorMessage": "OpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.", - "stackTrace": "at Microsoft.AspNetCore.E2ETesting.WaitAssert.WaitAssertCore..." - }, - "1558000": { - "outcome": "Passed", - "comment": "{\"HelixJobId\": \"helix-job-1558000\", \"HelixWorkItemName\": \"VirtualizationTest.WorkItemExecution\"}", - "errorMessage": null, - "stackTrace": null - } - }, - "helix_evidence": { - "1563420": { - "found": true, - "expired": false, - "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll [3 s]\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n(observed in build 1563420)" - }, - "1560111": { - "found": true, - "expired": false, - "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll [3 s]\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n(observed in build 1560111)" - }, - "1558000": { - "found": true, - "expired": false, - "console_excerpt": "[PASS] Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll" - } - }, + "recurrence_scan": {}, + "negative_scan": {}, + "vstmr_summary": {}, + "vstmr_detail": {}, + "vstmr_runs": {}, "check_runs": { "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68": [ { @@ -103,38 +36,29 @@ }, "html_url": "https://github.com/dotnet/aspnetcore/runs/900001" } - ], - "074f8655455783f5afc3e7a865c30d5f3dad52be": [ - { - "name": "Build Analysis", - "id": 900002, - "conclusion": "failure", - "output": { - "title": "1 failing test", - "text": "1 test failed. See Azure DevOps for full details." - }, - "html_url": "https://github.com/dotnet/aspnetcore/runs/900002" - } ] }, "duplicate_search": { "open-kbe": { "complete": true, - "result_numbers": [ - 68708 - ] + "result_numbers": [], + "total_count": 0 }, "recently-closed-kbe": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "open-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "recently-merged-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 } - } + }, + "duplicate_candidate_text": {} } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index 76f51437b37c..7575ac373201 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -15,10 +15,16 @@ "labels": [ "test-failure", "area-networking" - ] + ], + "has_workflow_marker": true }, "outcome": "incomplete", "provenance": { + "repository_ref_verification": { + "checkout_sha": "", + "trusted_main_sha": "", + "matches_main": true + }, "azdo_builds": [ { "id": 1569737, @@ -48,7 +54,7 @@ "retrieved_utc": "", "source": "negative-scan", "definition_id": 83, - "source_version": "5f76b4102d0c06c5af2f08e045a9d32e9cccef5b", + "source_version": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", "started_utc": "2026-08-15T08:00:00.0000000Z", "finished_utc": "2026-08-15T09:45:00.0000000Z", "result": "succeeded" @@ -66,47 +72,54 @@ "text_excerpt": "1 test failed. See Azure DevOps for full details.", "html_url": "https://github.com/dotnet/aspnetcore/runs/900201", "exact_test_referenced": false, - "known_issue_referenced": false + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] }, { "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", "found": false, "retrieved_utc": "", "exact_test_referenced": false, - "known_issue_referenced": false + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] } ], "raw_evidence_sources": [ { "build_id": 1569737, "role": "failure", - "kind": "helix-console-log", - "helix_job": "helix-job-1569737", - "helix_workitem": "Http3RequestTests.WorkItemExecution", + "kind": "vstmr-detail", + "run_id": 72708311, + "result_id": 400010, + "helix_unavailable": true, + "platform": "Linux", + "configuration": "Release", "found": true, - "expired": false, "captured_utc": "", - "sha256": "c9753c024d81675a60dba72af5e18d663af7801cff73d69c9645879130b1c0ae", + "sha256": "9ce4ad96859470b4e8756421f9aa4f89079cbb60af728e516ac5b2eb095157f1", "evidence_path": "issue-68945-build-1569737-failure.log" }, { "build_id": 1538879, "role": "failure", "found": false, - "expired": true, "captured_utc": "", - "note": "Helix console evidence for build 1538879 was not retrievable." + "note": "No VSTMR result for build 1538879 matched the expected outcome/signature for this test." }, { "build_id": 1540500, "role": "negative", - "kind": "helix-console-log", - "helix_job": "helix-job-1540500", - "helix_workitem": "Http3RequestTests.WorkItemExecution", + "kind": "vstmr-detail", + "run_id": 72708313, + "result_id": 400030, + "helix_unavailable": true, + "platform": "Linux", + "configuration": "Release", "found": true, - "expired": false, "captured_utc": "", - "sha256": "b4f3e16c9853a9ec29eb5f5407a12b171e0c9d260f610d9bff0fe37be8cb3d8c", + "sha256": "856486ff4bd7fa7052a9fd195f0ae82f267d0c184cda825072625f6545a1ae89", "evidence_path": "issue-68945-build-1540500-negative.log" } ], @@ -125,40 +138,44 @@ "category": "open-kbe", "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" POST_ClientCancellationUpload_RequestAbortRaised", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "open-fix-pr", "query": "repo:dotnet/aspnetcore is:pr is:open POST_ClientCancellationUpload_RequestAbortRaised", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged POST_ClientCancellationUpload_RequestAbortRaised", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 POST_ClientCancellationUpload_RequestAbortRaised", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 } - ] + ], + "unvalidated_candidates": [] } }, "candidate": null, "incomplete": { "reason_codes": [ - "raw-evidence-expired", "raw-evidence-insufficient" ], - "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-expired, raw-evidence-insufficient.", + "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient.", "missing_evidence": [ { - "kind": "helix-evidence", - "detail": "Build 1538879: no retrievable Helix console evidence." + "kind": "vstmr-evidence", + "detail": "Build 1538879: no matching, retrievable VSTMR result detail." }, { "kind": "raw-evidence", diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json index 19ec5905bb26..99038c5db506 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -6,7 +6,7 @@ "test-failure", "area-networking" ], - "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/33496438442)\n" + "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n" }, "azdo_builds": { "1569737": { @@ -33,47 +33,57 @@ "83": [ { "id": 1540500, - "sourceVersion": "5f76b4102d0c06c5af2f08e045a9d32e9cccef5b", + "sourceVersion": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", "startTime": "2026-08-15T08:00:00Z", "finishTime": "2026-08-15T09:45:00Z", "result": "succeeded" } ] }, - "vstmr_results": { - "1569737": { - "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1569737\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", - "errorMessage": "System.Threading.Tasks.TaskCanceledException: The operation was canceled.", - "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync..." - }, - "1538879": { + "vstmr_summary": { + "1569737": [ + { + "id": 400010, + "runId": 72708311, + "outcome": "Failed", + "automatedTestName": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised", + "testCaseTitle": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + ], + "1540500": [ + { + "id": 400030, + "runId": 72708313, + "outcome": "Passed", + "automatedTestName": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised", + "testCaseTitle": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } + ] + }, + "vstmr_detail": { + "72708311:400010": { "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1538879\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", "errorMessage": "System.Threading.Tasks.TaskCanceledException: The operation was canceled.", - "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync..." + "stackTrace": "at System.Net.Http.Http3RequestStream.SendDataAsync(...)", + "testCase": { + "name": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } }, - "1540500": { + "72708313:400030": { "outcome": "Passed", - "comment": "{\"HelixJobId\": \"helix-job-1540500\", \"HelixWorkItemName\": \"Http3RequestTests.WorkItemExecution\"}", "errorMessage": null, - "stackTrace": null + "stackTrace": null, + "testCase": { + "name": "Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + } } }, - "helix_evidence": { - "1569737": { - "found": true, - "expired": false, - "console_excerpt": "Failed Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised [3 s]\nSystem.Threading.Tasks.TaskCanceledException: The operation was canceled.\n(observed in build 1569737)" - }, - "1538879": { - "found": false, - "expired": true + "vstmr_runs": { + "72708311": { + "name": "Quarantine-CoreCLR-Linux-Release-xunit" }, - "1540500": { - "found": true, - "expired": false, - "console_excerpt": "[PASS] Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised" + "72708313": { + "name": "Quarantine-CoreCLR-Linux-Release-xunit" } }, "check_runs": { @@ -93,19 +103,24 @@ "duplicate_search": { "open-kbe": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "recently-closed-kbe": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "open-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "recently-merged-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 } - } + }, + "duplicate_candidate_text": {} } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index 6def27ceec42..e69e14ec6740 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -15,10 +15,16 @@ "labels": [ "test-failure", "area-blazor" - ] + ], + "has_workflow_marker": true }, "outcome": "candidate", "provenance": { + "repository_ref_verification": { + "checkout_sha": "", + "trusted_main_sha": "", + "matches_main": true + }, "azdo_builds": [ { "id": 1551326, @@ -26,10 +32,10 @@ "retrieved_utc": "", "source": "issue-body-reference", "definition_id": 87, - "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", - "started_utc": "2026-08-13T10:00:00.0000000Z", - "finished_utc": "2026-08-13T11:30:00.0000000Z", - "result": "failed" + "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "started_utc": "2026-08-13T04:18:53.9030000Z", + "finished_utc": "2026-08-13T04:31:51.1870000Z", + "result": "partiallySucceeded" }, { "id": 1537561, @@ -44,10 +50,10 @@ "retrieved_utc": "", "source": "recurrence-scan", "definition_id": 87, - "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", "started_utc": "2026-08-10T09:00:00.0000000Z", "finished_utc": "2026-08-10T10:30:00.0000000Z", - "result": "failed" + "result": "partiallySucceeded" }, { "id": 1545000, @@ -55,7 +61,7 @@ "retrieved_utc": "", "source": "negative-scan", "definition_id": 87, - "source_version": "f33596dac091a6fd858b1515378d951f7f6adda5", + "source_version": "113a606f96ebd832970a1f377748746ff6526abc", "started_utc": "2026-08-08T09:00:00.0000000Z", "finished_utc": "2026-08-08T10:30:00.0000000Z", "result": "succeeded" @@ -63,14 +69,16 @@ ], "check_run_snapshots": [ { - "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", + "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", "found": false, "retrieved_utc": "", "exact_test_referenced": false, - "known_issue_referenced": false + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] }, { - "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", "found": true, "retrieved_utc": "", "check_id": 900101, @@ -80,44 +88,52 @@ "text_excerpt": "1 test failed. See Azure DevOps for full details.", "html_url": "https://github.com/dotnet/aspnetcore/runs/900101", "exact_test_referenced": false, - "known_issue_referenced": false + "short_name_referenced": false, + "known_issue_referenced": false, + "known_issue_numbers": [] } ], "raw_evidence_sources": [ { "build_id": 1551326, "role": "failure", - "kind": "helix-console-log", - "helix_job": "helix-job-1551326", - "helix_workitem": "RedirectionTest.WorkItemExecution", + "kind": "vstmr-detail", + "run_id": 42708308, + "result_id": 100014, + "helix_unavailable": true, + "platform": "Linux", + "configuration": "Release", "found": true, - "expired": false, "captured_utc": "", - "sha256": "510308372262a7f187c524708516a44dbce8b74b28f6a4530f115b80356268ec", + "sha256": "34cb3881593f491fba3598a0ee26c2e172c0b8e46c40b8e9ac3abde64af85529", "evidence_path": "issue-68947-build-1551326-failure.log" }, { "build_id": 1549000, "role": "failure", - "kind": "helix-console-log", - "helix_job": "helix-job-1549000", - "helix_workitem": "RedirectionTest.WorkItemExecution", + "kind": "vstmr-detail", + "run_id": 52708309, + "result_id": 200055, + "helix_unavailable": true, + "platform": "Linux", + "configuration": "Release", "found": true, - "expired": false, "captured_utc": "", - "sha256": "7e39070f2138ee0933dbf7b6ba1c2c38220f7ab6ffe3bb27f0874eb74b071fc2", + "sha256": "3772fbff9518a95cfe45f97d7a2f6250cc4af5b43c8d3c370f6d3a8fab8435d0", "evidence_path": "issue-68947-build-1549000-failure.log" }, { "build_id": 1545000, "role": "negative", - "kind": "helix-console-log", - "helix_job": "helix-job-1545000", - "helix_workitem": "RedirectionTest.WorkItemExecution", + "kind": "vstmr-detail", + "run_id": 62708310, + "result_id": 300099, + "helix_unavailable": true, + "platform": "Linux", + "configuration": "Release", "found": true, - "expired": false, "captured_utc": "", - "sha256": "84e902974eafb9b414f662d7d708306b60b4884ee0882eb739a7636bcc2c747e", + "sha256": "17e453e2b6e8c8b2e2b16e3dcc2ea16e91bfd37a4c54b2077c52f71e21c54920", "evidence_path": "issue-68947-build-1545000-negative.log" } ], @@ -136,27 +152,32 @@ "category": "open-kbe", "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "open-fix-pr", "query": "repo:dotnet/aspnetcore is:pr is:open RedirectEnhancedNonBlazorGetToExternal", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, { "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged RedirectEnhancedNonBlazorGetToExternal", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 RedirectEnhancedNonBlazorGetToExternal", "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 } - ] + ], + "unvalidated_candidates": [] } }, "candidate": { @@ -176,7 +197,7 @@ "signature": { "kind": "ErrorMessage", "values": [ - "OpenQA.Selenium.WebDriverException: TaskCanceledException" + "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" ], "build_retry": false, "exclude_console_log": false @@ -193,12 +214,12 @@ "outcome": "failed", "path": "issue-68947-build-1551326-failure.log", "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results", - "sha256": "510308372262a7f187c524708516a44dbce8b74b28f6a4530f115b80356268ec", + "sha256": "34cb3881593f491fba3598a0ee26c2e172c0b8e46c40b8e9ac3abde64af85529", "build": { "id": 1551326, "pipeline_definition_id": 87, - "source_version": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", - "started_utc": "2026-08-13T10:00:00.0000000Z", + "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "started_utc": "2026-08-13T04:18:53.9030000Z", "platform": "Linux", "configuration": "Release" } @@ -209,11 +230,11 @@ "outcome": "failed", "path": "issue-68947-build-1549000-failure.log", "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1549000&view=results", - "sha256": "7e39070f2138ee0933dbf7b6ba1c2c38220f7ab6ffe3bb27f0874eb74b071fc2", + "sha256": "3772fbff9518a95cfe45f97d7a2f6250cc4af5b43c8d3c370f6d3a8fab8435d0", "build": { "id": 1549000, "pipeline_definition_id": 87, - "source_version": "0374b4797537d95f1b519446344fd88a7b5a861a", + "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", "started_utc": "2026-08-10T09:00:00.0000000Z", "platform": "Linux", "configuration": "Release" @@ -225,11 +246,11 @@ "outcome": "passed", "path": "issue-68947-build-1545000-negative.log", "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1545000&view=results", - "sha256": "84e902974eafb9b414f662d7d708306b60b4884ee0882eb739a7636bcc2c747e", + "sha256": "17e453e2b6e8c8b2e2b16e3dcc2ea16e91bfd37a4c54b2077c52f71e21c54920", "build": { "id": 1545000, "pipeline_definition_id": 87, - "source_version": "f33596dac091a6fd858b1515378d951f7f6adda5", + "source_version": "113a606f96ebd832970a1f377748746ff6526abc", "started_utc": "2026-08-08T09:00:00.0000000Z", "platform": "Linux", "configuration": "Release" @@ -266,7 +287,7 @@ }, { "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", + "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", "complete": true, "result_numbers": [] }, @@ -278,7 +299,7 @@ }, { "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged RedirectEnhancedNonBlazorGetToExternal", + "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 RedirectEnhancedNonBlazorGetToExternal", "complete": true, "result_numbers": [] } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json index e74b2bf5a83f..78bf6aa88990 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -6,27 +6,27 @@ "test-failure", "area-blazor" ], - "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n> Generated by [Daily Test Quarantine Management](https://github.com/dotnet/aspnetcore/actions/runs/33496438442)\n" + "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n" }, "azdo_builds": { "1551326": { "definition": { "id": 87 }, - "sourceVersion": "dbea3e5f6a990adff93e782648dd9b6d13d6a943", - "startTime": "2026-08-13T10:00:00Z", - "finishTime": "2026-08-13T11:30:00Z", - "result": "failed" + "sourceVersion": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", + "startTime": "2026-08-13T04:18:53.903Z", + "finishTime": "2026-08-13T04:31:51.187Z", + "result": "partiallySucceeded" } }, "recurrence_scan": { "87": [ { "id": 1549000, - "sourceVersion": "0374b4797537d95f1b519446344fd88a7b5a861a", + "sourceVersion": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", "startTime": "2026-08-10T09:00:00Z", "finishTime": "2026-08-10T10:30:00Z", - "result": "failed" + "result": "partiallySucceeded" } ] }, @@ -34,52 +34,81 @@ "87": [ { "id": 1545000, - "sourceVersion": "f33596dac091a6fd858b1515378d951f7f6adda5", + "sourceVersion": "113a606f96ebd832970a1f377748746ff6526abc", "startTime": "2026-08-08T09:00:00Z", "finishTime": "2026-08-08T10:30:00Z", "result": "succeeded" } ] }, - "vstmr_results": { - "1551326": { + "vstmr_summary": { + "1551326": [ + { + "id": 100014, + "runId": 42708308, + "outcome": "Failed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ], + "1549000": [ + { + "id": 200055, + "runId": 52708309, + "outcome": "Failed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ], + "1545000": [ + { + "id": 300099, + "runId": 62708310, + "outcome": "Passed", + "automatedTestName": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal", + "testCaseTitle": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } + ] + }, + "vstmr_detail": { + "42708308:100014": { "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1551326\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", - "errorMessage": "OpenQA.Selenium.WebDriverException: TaskCanceledException", - "stackTrace": "at OpenQA.Selenium.Support.UI.WebDriverWait.Until..." + "errorMessage": "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server for URL http://localhost:43119/session/f57178cbb4abf6238d976e8cc5f5357b/window/rect timed out after 60 seconds.\n---- System.Threading.Tasks.TaskCanceledException : The request was canceled due to the configured HttpClient.Timeout of 60 seconds elapsing.", + "stackTrace": " at OpenQA.Selenium.Remote.HttpCommandExecutor.ExecuteAsync(Command commandToExecute)\n at Microsoft.AspNetCore.E2ETesting.BrowserTestBase.InitializeBrowser(String isolationContext) in /mnt/vss/_work/1/s/src/Shared/E2ETesting/BrowserTestBase.cs:line 97\n at Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.InitializeAsync() in /mnt/vss/_work/1/s/src/Components/test/E2ETest/ServerRenderingTests/RedirectionTest.cs:line 29", + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } }, - "1549000": { + "52708309:200055": { "outcome": "Failed", - "comment": "{\"HelixJobId\": \"helix-job-1549000\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", - "errorMessage": "OpenQA.Selenium.WebDriverException: TaskCanceledException", - "stackTrace": "at OpenQA.Selenium.Support.UI.WebDriverWait.Until..." + "errorMessage": "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server for URL http://localhost:43119/session/f57178cbb4abf6238d976e8cc5f5357b/window/rect timed out after 60 seconds.\n---- System.Threading.Tasks.TaskCanceledException : The request was canceled due to the configured HttpClient.Timeout of 60 seconds elapsing.", + "stackTrace": " at OpenQA.Selenium.Remote.HttpCommandExecutor.ExecuteAsync(Command commandToExecute)\n at Microsoft.AspNetCore.E2ETesting.BrowserTestBase.InitializeBrowser(String isolationContext) in /mnt/vss/_work/1/s/src/Shared/E2ETesting/BrowserTestBase.cs:line 97\n at Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.InitializeAsync() in /mnt/vss/_work/1/s/src/Components/test/E2ETest/ServerRenderingTests/RedirectionTest.cs:line 29\n(observed in a separate run)", + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } }, - "1545000": { + "62708310:300099": { "outcome": "Passed", - "comment": "{\"HelixJobId\": \"helix-job-1545000\", \"HelixWorkItemName\": \"RedirectionTest.WorkItemExecution\"}", "errorMessage": null, - "stackTrace": null + "stackTrace": null, + "testCase": { + "name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal(disableThrowNavigationException: False)" + } } }, - "helix_evidence": { - "1551326": { - "found": true, - "expired": false, - "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal [3 s]\nOpenQA.Selenium.WebDriverException: TaskCanceledException\n(observed in build 1551326)" + "vstmr_runs": { + "42708308": { + "name": "Quarantine-Mono-Linux-Release-xunit" }, - "1549000": { - "found": true, - "expired": false, - "console_excerpt": "Failed Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal [3 s]\nOpenQA.Selenium.WebDriverException: TaskCanceledException\n(observed in build 1549000)" + "52708309": { + "name": "Quarantine-Mono-Linux-Release-xunit" }, - "1545000": { - "found": true, - "expired": false, - "console_excerpt": "[PASS] Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal" + "62708310": { + "name": "Quarantine-Mono-Linux-Release-xunit" } }, "check_runs": { - "0374b4797537d95f1b519446344fd88a7b5a861a": [ + "ef86306faaa4b31e962f06b93c2ce21e4a18bf17": [ { "name": "Build Analysis", "id": 900101, @@ -95,19 +124,24 @@ "duplicate_search": { "open-kbe": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "recently-closed-kbe": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "open-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 }, "recently-merged-fix-pr": { "complete": true, - "result_numbers": [] + "result_numbers": [], + "total_count": 0 } - } + }, + "duplicate_candidate_text": {} } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index da0ee58a3a31..5753c3b41fcf 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json", "title": "ASP.NET Core test quarantine KBE shadow collector dossier", - "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps build metadata, Helix/TRX raw evidence, and GitHub 'Build Analysis' check-run snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured 'incomplete' outcome. Never infers a pass, a recurrence, or a signature from missing or expired evidence.", + "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps/VSTMR test-result evidence and GitHub 'Build Analysis' check-run snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured 'incomplete' outcome. Never infers a pass, a recurrence, a signature, a platform/configuration, or a validated duplicate from missing or unverifiable evidence.", "type": "object", "additionalProperties": false, "required": [ @@ -60,7 +60,8 @@ "number", "url", "state", - "labels" + "labels", + "has_workflow_marker" ], "properties": { "number": { @@ -84,6 +85,10 @@ "minLength": 1, "maxLength": 128 } + }, + "has_workflow_marker": { + "type": "boolean", + "description": "true when the issue body contains the trusted 'gh-aw-workflow-id: test-quarantine' or 'gh-aw-workflow-call-id: dotnet/aspnetcore/test-quarantine' HTML-comment marker the production quarantine workflow stamps into every issue it creates. The 'test-failure' label alone is not proof an issue was generated by quarantine automation." } } }, @@ -97,12 +102,37 @@ "type": "object", "additionalProperties": false, "required": [ + "repository_ref_verification", "azdo_builds", "check_run_snapshots", "raw_evidence_sources", "duplicate_search" ], "properties": { + "repository_ref_verification": { + "type": "object", + "additionalProperties": false, + "required": [ + "checkout_sha", + "trusted_main_sha", + "matches_main" + ], + "properties": { + "checkout_sha": { + "$ref": "#/$defs/gitSha" + }, + "trusted_main_sha": { + "type": [ + "string", + "null" + ] + }, + "matches_main": { + "type": "boolean", + "description": "true only when checkout_sha was confirmed, via a trusted GitHub API response for dotnet/aspnetcore's 'main' branch, to equal the repository checkout the collector and evaluator ran against. A candidate is only ever emitted when this is true." + } + } + }, "azdo_builds": { "type": "array", "maxItems": 64, @@ -207,7 +237,9 @@ "found", "retrieved_utc", "exact_test_referenced", - "known_issue_referenced" + "short_name_referenced", + "known_issue_referenced", + "known_issue_numbers" ], "properties": { "source_version": { @@ -249,11 +281,24 @@ }, "exact_test_referenced": { "type": "boolean", - "description": "Conservative substring match of the quarantined test's fully qualified (or short method) name inside the check-run text. False when no snapshot was found." + "description": "Conservative match requiring the quarantined test's full fully-qualified name to appear verbatim in the check-run text. False when no snapshot was found or only a short method name matched." + }, + "short_name_referenced": { + "type": "boolean", + "description": "The test's bare method name (last '.'-separated segment) appeared in the check-run text, but the full fully-qualified name did not. Recorded for transparency only; never treated as an exact-test match because a short method name commonly collides with unrelated tests." }, "known_issue_referenced": { "type": "boolean", - "description": "Conservative match of a 'Known Issue' / known-build-error reference inside the check-run text. False when no snapshot was found." + "description": "true only when known_issue_numbers is non-empty -- i.e. the text names a concrete dotnet/aspnetcore issue number/URL near a 'Known Issue' style label. The bare phrase 'Known Issue' with no associated issue reference (e.g. a heading or table column label) does not set this true." + }, + "known_issue_numbers": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 1 + } } } } @@ -268,7 +313,6 @@ "build_id", "role", "found", - "expired", "captured_utc" ], "properties": { @@ -284,11 +328,21 @@ }, "kind": { "enum": [ - "helix-console-log", - "helix-test-log", - "vstmr-result" + "vstmr-detail" ] }, + "run_id": { + "type": "integer", + "minimum": 1 + }, + "result_id": { + "type": "integer", + "minimum": 1 + }, + "helix_unavailable": { + "type": "boolean", + "description": "true when no Helix job/work-item coordinates could be discovered for this result (the common case for ordinary xUnit test results -- Helix coordinates are only present on the Helix work-item's own crash/'.WorkItemExecution' pseudo-test row). The VSTMR detail result text is still treated as authoritative evidence in this case." + }, "helix_job": { "type": "string", "maxLength": 128 @@ -297,13 +351,17 @@ "type": "string", "maxLength": 256 }, + "platform": { + "type": "string", + "maxLength": 128 + }, + "configuration": { + "type": "string", + "maxLength": 128 + }, "found": { "type": "boolean" }, - "expired": { - "type": "boolean", - "description": "true when the evidence source once existed (per the issue or a build/test-result reference) but its underlying artifact is no longer retrievable." - }, "captured_utc": { "type": "string", "format": "date-time" @@ -356,6 +414,8 @@ "issue-not-canonical-quarantine", "issue-not-open", "test-name-unresolvable", + "multiple-test-identities-unresolved", + "repository-ref-not-main", "build-reference-unresolvable", "build-metadata-expired", "raw-evidence-expired", @@ -454,7 +514,8 @@ "checked_utc", "coverage", "references", - "queries" + "queries", + "unvalidated_candidates" ], "properties": { "status": { @@ -515,7 +576,8 @@ "category", "query", "complete", - "result_numbers" + "result_numbers", + "total_count" ], "properties": { "category": { @@ -532,16 +594,53 @@ "maxLength": 1024 }, "complete": { - "type": "boolean" + "type": "boolean", + "description": "true only when GitHub reported incomplete_results=false AND every matching item (total_count) was actually retrieved -- a truncated page count no longer counts as complete." }, "result_numbers": { "type": "array", - "maxItems": 50, + "maxItems": 200, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } + }, + "total_count": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "unvalidated_candidates": { + "type": "array", + "maxItems": 200, + "description": "Search hits that were discovered but could not be confirmed, by fetching the issue/PR body and requiring the exact fully-qualified test name to appear in it, to actually concern this test. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "category", + "number", + "reason" + ], + "properties": { + "category": { + "enum": [ + "open-kbe", + "recently-closed-kbe", + "open-fix-pr", + "recently-merged-fix-pr" + ] + }, + "number": { + "type": "integer", + "minimum": 1 + }, + "reason": { + "type": "string", + "maxLength": 256 } } } From 1424db6a8a55e589a0b7dd79d84bb20217b6cb6c Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:39:38 -0500 Subject: [PATCH 05/10] Finalize quarantine KBE fail-closed gates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 613 ++++++++++++++++-- .../Evaluate-TestQuarantineKbeCandidate.ps1 | 20 +- .../test-quarantine-kbe-shadow/README.md | 70 +- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 172 ++++- ...st-Evaluate-TestQuarantineKbeCandidate.ps1 | 51 ++ .../fixtures/68724/expected-dossier.json | 11 +- .../fixtures/68724/fixture.json | 4 +- .../fixtures/68945/expected-dossier.json | 25 +- .../fixtures/68945/fixture.json | 12 +- .../fixtures/68947/expected-dossier.json | 40 +- .../fixtures/68947/fixture.json | 12 +- ...uarantine-kbe-shadow-candidate.schema.json | 38 +- ...-quarantine-kbe-shadow-dossier.schema.json | 52 +- ...-quarantine-kbe-shadow-receipt.schema.json | 38 +- .../workflows/test-quarantine-kbe-shadow.yml | 4 + 15 files changed, 1007 insertions(+), 155 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index d9498b09948d..2f94b577048e 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -76,6 +76,10 @@ param( [string]$RepositoryRoot = "$PSScriptRoot/../../../..", + [string]$EventRef = $env:GITHUB_REF, + + [string]$EventSha = $env:GITHUB_SHA, + [string]$DossierSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json", [string]$CandidateSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.json", @@ -322,30 +326,124 @@ function Get-GitHubIssue return $result } -function Get-TrustedMainShaResult +function Test-DispatchShaOnMainComparison +{ + param( + [Parameter(Mandatory = $true)]$Comparison, + [Parameter(Mandatory = $true)][string]$DispatchSha + ) + + return [string]$Comparison.merge_base_commit.sha -eq $DispatchSha -and + [string]$Comparison.status -in @("ahead", "identical") +} + +function Get-TrustedMainRefResult { - # Returns @{ Checked; TrustedSha }. `Checked = $false` only in fixture mode when the fixture - # does not model this dimension at all (the three real pilot fixtures do not); live mode - # always performs the check. A failed live lookup still counts as Checked = $true with a - # null TrustedSha, which fails the comparison closed rather than silently skipping it. + param([Parameter(Mandatory = $true)][string]$DispatchSha) + if ($isFixtureMode) { if (Test-HasProperty -Object $fixture -Name "main_branch") { - return [ordered]@{ Checked = $true; TrustedSha = [string]$fixture.main_branch.sha } + $currentMainSha = [string]$fixture.main_branch.sha + $isMember = if (Test-HasProperty -Object $fixture.main_branch -Name "contains_event_sha") + { + $fixtureComparison = if ([bool]$fixture.main_branch.contains_event_sha) + { + [ordered]@{ + status = "ahead" + merge_base_commit = [ordered]@{ sha = $DispatchSha } + } + } + else + { + [ordered]@{ + status = "diverged" + merge_base_commit = [ordered]@{ sha = $currentMainSha } + } + } + Test-DispatchShaOnMainComparison -Comparison $fixtureComparison -DispatchSha $DispatchSha + } + else + { + $DispatchSha.Equals($currentMainSha, [System.StringComparison]::OrdinalIgnoreCase) + } + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $isMember } } - return [ordered]@{ Checked = $false; TrustedSha = $null } + return [ordered]@{ Checked = $false; CurrentMainSha = $null; DispatchShaOnMain = $true } } try { $headers = Get-GitHubHeaders $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/main" -Headers $headers -Method Get -TimeoutSec 30 - return [ordered]@{ Checked = $true; TrustedSha = [string]$result.sha } + $currentMainSha = [string]$result.sha + if ($DispatchSha.Equals($currentMainSha, [System.StringComparison]::OrdinalIgnoreCase)) + { + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $true } + } + + $comparison = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/compare/$DispatchSha...$currentMainSha" -Headers $headers -Method Get -TimeoutSec 30 + $isMember = Test-DispatchShaOnMainComparison -Comparison $comparison -DispatchSha $DispatchSha + return [ordered]@{ Checked = $true; CurrentMainSha = $currentMainSha; DispatchShaOnMain = $isMember } } catch { - return [ordered]@{ Checked = $true; TrustedSha = $null } + return [ordered]@{ Checked = $true; CurrentMainSha = $null; DispatchShaOnMain = $false } + } +} + +function Get-BuildProperty +{ + param( + [Parameter(Mandatory = $true)]$Build, + [Parameter(Mandatory = $true)][string]$Name + ) + + if (Test-HasProperty -Object $Build -Name $Name) + { + return $Build.$Name + } + return $null +} + +function Get-AzdoBuildValidation +{ + param( + [Parameter(Mandatory = $true)]$Build, + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][ValidateSet("failure", "negative")][string]$Role + ) + + $sourceBranch = [string](Get-BuildProperty -Build $Build -Name "sourceBranch") + $status = [string](Get-BuildProperty -Build $Build -Name "status") + $result = [string](Get-BuildProperty -Build $Build -Name "result") + $allowedResults = if ($Role -eq "failure") { @("failed", "partiallySucceeded") } else { @("succeeded") } + $reasons = [System.Collections.Generic.List[string]]::new() + + if ($DefinitionId -notin $pipelineDefinitionIds) + { + $reasons.Add("definition") + } + if ($sourceBranch -ne "refs/heads/main") + { + $reasons.Add("source-branch") + } + if ($status -ne "completed") + { + $reasons.Add("status") + } + if ($result -notin $allowedResults) + { + $reasons.Add("result") + } + + return [ordered]@{ + Valid = $reasons.Count -eq 0 + SourceBranch = $sourceBranch + Status = $status + Result = $result + Reasons = @($reasons) } } @@ -710,6 +808,243 @@ function Get-DuplicateCandidateText } } +function Test-ContainsExactTestName +{ + param( + [Parameter(Mandatory = $true)][string]$Text, + [Parameter(Mandatory = $true)][string]$TestName + ) + + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + return [regex]::IsMatch( + $Text, + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant) +} + +function Get-DocumentedSignature +{ + param([Parameter(Mandatory = $true)][string]$Text) + + $match = [regex]::Match( + $Text, + '##\s*Error Message.*?```json\s*(.*?)\s*```', + [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) + if (-not $match.Success) + { + return $null + } + + try + { + $document = $match.Groups[1].Value | ConvertFrom-Json -Depth 16 + } + catch + { + return $null + } + + $messageValues = @() + if (Test-HasProperty -Object $document -Name "ErrorMessage") + { + $messageValues = @( + $document.ErrorMessage | + ForEach-Object { [string]$_ } | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + ) + } + if ($messageValues.Count -gt 0) + { + return [ordered]@{ Kind = "ErrorMessage"; Values = $messageValues } + } + + if ((Test-HasProperty -Object $document -Name "ErrorPattern") -and + -not [string]::IsNullOrWhiteSpace([string]$document.ErrorPattern)) + { + return [ordered]@{ Kind = "ErrorPattern"; Values = @([string]$document.ErrorPattern) } + } + + return $null +} + +function Test-DocumentedSignatureCompatibility +{ + param( + [Parameter(Mandatory = $true)]$Signature, + [Parameter(Mandatory = $true)][object[]]$FailureLogs, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][string]$Root + ) + + if ($FailureLogs.Count -eq 0) + { + return $false + } + + $values = @($Signature.Values | ForEach-Object { [string]$_ }) + $regexes = @() + if ([string]$Signature.Kind -eq "ErrorPattern") + { + try + { + $regexOptions = [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking + $regexes = @($values | ForEach-Object { + [System.Text.RegularExpressions.Regex]::new($_, $regexOptions, [System.TimeSpan]::FromMilliseconds(50)) + }) + } + catch + { + return $false + } + } + + $escapedTestName = [System.Text.RegularExpressions.Regex]::Escape($TestName) + $testNameMatcher = [System.Text.RegularExpressions.Regex]::new( + "(^|[^A-Za-z0-9_.+])$escapedTestName($|[^A-Za-z0-9_.+])", + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant -bor + [System.Text.RegularExpressions.RegexOptions]::NonBacktracking, + [System.TimeSpan]::FromMilliseconds(50)) + + foreach ($log in $FailureLogs) + { + $path = Join-Path $Root ([string]$log.path) + $failedTestLines = [System.Collections.Generic.List[int]]::new() + $matchedLines = [System.Collections.Generic.List[int]]::new() + $patternIndex = 0 + $lineNumber = 0 + $matched = $false + + foreach ($line in [System.IO.File]::ReadLines($path)) + { + $lineNumber++ + $normalizedLine = $line -replace "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+", "" + $lineContainsTest = $testNameMatcher.IsMatch($normalizedLine) + $lineIndicatesFailure = + $normalizedLine -match "(?i)^\s*\[FAIL(?:ED)?\]\s+" -or + $normalizedLine -match "(?i)^\s*Failed\s+" -or + $normalizedLine -match "(?i)^\s*\[[^\]\r\n]+\]\s+.+\s+\[FAIL(?:ED)?\]\s*$" + if ($lineContainsTest -and $lineIndicatesFailure) + { + $failedTestLines.Add($lineNumber) + continue + } + + $matchedThisLine = $false + try + { + $matchedThisLine = if ([string]$Signature.Kind -eq "ErrorPattern") + { + $regexes[$patternIndex].IsMatch($line) + } + else + { + $line.IndexOf($values[$patternIndex], [System.StringComparison]::Ordinal) -ge 0 + } + } + catch [System.Text.RegularExpressions.RegexMatchTimeoutException] + { + return $false + } + + if ($matchedThisLine) + { + $matchedLines.Add($lineNumber) + if ($values.Count -eq 1) + { + $matched = $true + } + else + { + $patternIndex++ + $matched = $patternIndex -eq $values.Count + } + } + if ($matched) + { + break + } + } + + $associated = $matched -and @( + $matchedLines | + Where-Object { + $matchedLine = $_ + @($failedTestLines | Where-Object { [System.Math]::Abs($_ - $matchedLine) -le 50 }).Count -gt 0 + } + ).Count -gt 0 + if (-not $associated) + { + return $false + } + } + + return $true +} + +function Test-FixPrCompatibility +{ + param( + [Parameter(Mandatory = $true)][string]$Text, + [Parameter(Mandatory = $true)][string]$TestName, + [Parameter(Mandatory = $true)][AllowNull()][string]$CandidateSignature, + [Parameter(Mandatory = $true)][object[]]$FailureLogs, + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][ref]$DetailFetchIncomplete + ) + + if (-not (Test-ContainsExactTestName -Text $Text -TestName $TestName)) + { + return $false + } + + $documentedSignature = Get-DocumentedSignature -Text $Text + if ($null -ne $documentedSignature -and + (Test-DocumentedSignatureCompatibility -Signature $documentedSignature -FailureLogs $FailureLogs -TestName $TestName -Root $Root)) + { + return $true + } + if (-not [string]::IsNullOrWhiteSpace($CandidateSignature) -and + $Text.Contains($CandidateSignature, [System.StringComparison]::Ordinal)) + { + return $true + } + + $linkedNumbers = @( + [regex]::Matches($Text, '(?i)https://github\.com/dotnet/aspnetcore/(?:issues|pull)/([1-9][0-9]*)|(?/` | Compact, sanitized, offline fixtures for three real pilot issues, each with a golden `expected-dossier.json`. | | `Test-Evaluate-TestQuarantineKbeCandidate.ps1`, `Test-Collect-TestQuarantineKbeEvidence.ps1` | Deterministic, offline test suites (no network access). | @@ -51,7 +51,7 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis pseudo-test row carries one); otherwise `helix_unavailable: true` is recorded explicitly and the VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence requires evidence from **at least two distinct builds** (a single build producing two separate - artifacts is not recurrence), and at least one authoritative negative (passed/skipped) + artifacts is not recurrence), and at least one authoritative **Passed** occurrence. Signature matching against that raw text uses ordinal, case-sensitive substring containment (`[string]::Contains(..., Ordinal)`) throughout -- never PowerShell's `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise @@ -77,32 +77,34 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis issue body to contain the trusted `` or `` HTML-comment marker the production quarantine workflow stamps into every issue it creates. -* **A duplicate-search hit is a discovery candidate, not a validated duplicate.** Every numeric - result returned by the four categorized GitHub searches (open/recently-closed KBE, - open/recently-merged fix PR) is fetched and required to contain the candidate's **exact** - fully-qualified test name before it is ever treated as an existing KBE or fix PR; a hit that only - shares a bare method name with an unrelated test is recorded as an `unvalidated_candidate` (with - a reason) and never sets `duplicate_check.status` to `existing-kbe`/`existing-fix-pr`. The +* **A duplicate-search hit is discovery only, not a validated duplicate.** Every numeric result + returned by the four categorized GitHub searches is fetched. An existing KBE requires the exact + FQN plus a documented `ErrorMessage`/`ErrorPattern` that matches every authoritative failure log + with evaluator-compatible semantics. A fix PR requires the exact FQN plus a compatible signature, + linked KBE/quarantine issue, or root-cause association. Incompatible fetched items remain + `unvalidated_candidate` entries; a failed candidate-detail fetch makes that query and the overall + duplicate coverage incomplete. The "recently" closed/merged categories carry an explicit 90-day `closed:>=`/`merged:>=` time-window qualifier so the label matches what the query actually searches, and a query is only marked `complete` when GitHub reports `incomplete_results: false` **and** every matching item (per `total_count`, across up to 3 paginated pages of 100) was actually retrieved -- a `total_count` larger than a single page previously went unnoticed. -* **The repository checkout is only ever labeled `branch: "main"` after independent - confirmation.** The collector resolves `dotnet/aspnetcore`'s actual current `main` SHA through a - trusted `GET /repos/dotnet/aspnetcore/commits/main` GitHub API response and requires it to equal - the checked-out commit (`repository_ref_verification` in the dossier records both SHAs and the - comparison result) before ever emitting a candidate. In production this is a same-checkout - cross-check (workflow_dispatch normally runs on `main`'s tip, so both values coincide); in this - PR's own development branch (or any dispatch from a non-default ref), the checkout is genuinely - not `main`, and the collector fails closed (`repository-ref-not-main`) rather than mislabeling - it. +* **The immutable workflow-dispatch ref is verified, not assumed.** Trusted `github.ref` and + `github.sha` values are passed through `env:` bindings. The ref must be exactly `refs/heads/main`, + the checkout must equal the dispatch SHA, and the dispatch SHA must be identical to or an + ancestor/member of current main. The dossier records event ref/SHA, checkout SHA, and current + main SHA separately, so main advancing after dispatch is valid while non-main dispatches fail + closed. +* **Every countable build is validated before use.** Pipeline definition must be 83 or 87, + `sourceBranch` must be exactly `refs/heads/main`, status must be `completed`, and failure builds + must be `failed` or `partiallySucceeded` (`succeeded` is required for the Passed scan). These + dimensions are recorded in build provenance and candidate evidence. * **Platform/configuration are derived from authoritative metadata, never fabricated.** Azure DevOps' `buildConfiguration.platform`/`.flavor` fields are empty strings on every real run observed live; the only authoritative, cheaply-available signal is the VSTMR TestRun's own `name` (e.g. `Quarantine-Mono-Linux-Release-xunit`). The collector parses recognized - platform/configuration tokens out of that name and records the literal string `"unknown"` -- - never a guessed default -- when no recognized token is present. + platform/configuration tokens out of that name. A counted failure or pass with either dimension + `"unknown"` emits explicit missing-evidence codes and prevents a candidate/validated receipt. * **Never infer a pass, a recurrence, a signature, a platform/configuration, or a validated duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps metadata has aged out of retention, historical VSTMR test-result data no longer queryable for a @@ -121,7 +123,7 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis only) after each GitHub call. The workflow uploads artifacts; it never calls any write API (no labels, comments, commits, branches, or files). * **Evidence provenance remains unverified in this PR.** Even when the collector's candidate is - independently valid, the unmodified evaluator still emits `evidence_provenance_verified: false`, + independently valid, the evaluator still emits `evidence_provenance_verified: false`, `eligible_for_kbe_enrichment: false`, and `human_review_required: true`. Flipping `evidence_provenance_verified` to `true` is a deliberate, separate promotion decision (see "Promotion gates" below) -- not something this collector claims for itself. @@ -187,7 +189,7 @@ the real endpoints captured live during development: | Key | Mirrors | |---|---| | `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, body) | -| `main_branch` *(optional)* | `GET /repos/{repo}/commits/main` (`.sha`) -- omit entirely to skip the repository-ref guard (used by fixtures that don't specifically exercise it); include to test either a match or a deliberate mismatch | +| `main_branch` *(optional)* | Current main `.sha`, plus optional `contains_event_sha` to model ancestry/membership after main advances; omit for legacy pilot fixtures that do not exercise this guard | | `azdo_builds` | `GET .../build/builds/{id}` keyed by build id | | `recurrence_scan`, `negative_scan` | `GET .../build/builds?resultFilter=...` keyed by pipeline definition id | | `vstmr_summary` | `GET .../testresults/resultsbyBuild?buildId=...` keyed by build id (array of summary rows) | @@ -208,31 +210,29 @@ the real endpoints captured live during development: Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality comparison in the test suite. Golden comparisons exclude the collector's `generated_utc` / `retrieved_utc` / `captured_utc` / `checked_utc` timestamps and the running checkout's -`commit_sha` / `checkout_sha` / `trusted_main_sha` -- all of which are expected to differ +`commit_sha` / `event_sha` / `checkout_sha` / `current_main_sha` -- all of which are expected to differ run-to-run and commit-to-commit -- replacing them with the literal sentinel `` on both sides before comparing. The test suite additionally covers, via small synthetic (non-pilot) fixtures: a closed issue, an issue missing the `test-failure` label, an issue carrying the label but missing the trusted -workflow marker, a repository-ref mismatch, Build Analysis exact-vs-short-name and -concrete-vs-generic-known-issue flag precision, an unvalidated duplicate-search hit that shares -only a bare method name with an unrelated test, a literal signature containing `*`/`?`/`[` -wildcard-shaped characters (with a decoy build proving ordinal, not `-like`, matching), and a -direct, network-free unit test of the failed/partiallySucceeded build-list merge/dedupe. +workflow marker, immutable dispatch ancestry and non-main rejection, strict build definition/ref/ +status/result gates, skip-only negative evidence, unknown environment dimensions, Build Analysis +flag precision, compatible and incompatible same-FQN duplicate signatures, failed duplicate-detail +fetches, fix-PR associations, wildcard-shaped literal signatures, and build-list merge/dedupe. ## Reconciling with the existing evaluator contract -The collector does **not** introduce a third, competing dossier schema. Its `candidate` output, -when present, is validated against the same unmodified -`test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to the -unmodified `Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is -- this PR does not change that -script, its tests, or either of its schemas. `test-quarantine-kbe-shadow-dossier.schema.json` is a +The collector does **not** introduce a third, competing candidate contract. Its `candidate` output, +when present, is validated against the same versioned +`test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to +`Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is. `test-quarantine-kbe-shadow-dossier.schema.json` is a new, independently versioned (`schema_version: 1`) envelope that carries collector-specific provenance (repository-ref verification, Azure DevOps build resolution, Build Analysis check-run snapshots, raw-evidence retrieval, unvalidated duplicate-search candidates) alongside that same `candidate` object, or a structured `incomplete` outcome when any evidence gate fails. Because the candidate schema's `duplicate_check.queries[]` items are `additionalProperties: false` (and -correctly so -- it must stay byte-for-byte compatible with the unmodified evaluator), the +correctly so -- it must stay byte-for-byte compatible with the evaluator), the dossier-only `total_count` field is carried on `provenance.duplicate_search.queries[]` only, never on `candidate.duplicate_check.queries[]`. diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index fc30cd05fc60..b5fa6babf5cf 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -12,9 +12,8 @@ are expected to differ between runs/commits; every other field must match exactly. Where the collector's outcome is 'candidate', the resulting candidate.json is also fed - through the unmodified, already-tested Evaluate-TestQuarantineKbeCandidate.ps1 to prove the - two scripts reconcile: the collector's output is accepted as-is by the existing evaluator - contract with no changes to that script or its schemas. + through Evaluate-TestQuarantineKbeCandidate.ps1 to prove the two scripts reconcile: the + collector's output is accepted as-is by the versioned evaluator contract. #> [CmdletBinding()] @@ -31,6 +30,7 @@ $candidateSchema = "$PSScriptRoot/test-quarantine-kbe-shadow-candidate.schema.js $fixturesRoot = "$PSScriptRoot/fixtures" $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-collector-$([System.Guid]::NewGuid().ToString('N'))" $repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path +$repositoryHead = (& git -C $repositoryRoot rev-parse HEAD).Trim() function Assert-Equal { @@ -77,7 +77,7 @@ function Assert-NotContains # The collector-generated timestamps and the running checkout's HEAD commit SHA are expected to # differ run-to-run and commit-to-commit; every golden fixture below was captured with these # fields already replaced by this same sentinel. -$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha", "checkout_sha", "trusted_main_sha") +$volatileKeys = @("generated_utc", "retrieved_utc", "captured_utc", "checked_utc", "commit_sha", "event_sha", "checkout_sha", "current_main_sha") $volatileSentinel = "" function ConvertTo-NormalizedObject @@ -158,7 +158,9 @@ function Invoke-Collector [Parameter(Mandatory = $true)][int]$IssueNumber, [Parameter(Mandatory = $true)][string]$FixtureRoot, [Parameter(Mandatory = $true)][string]$WorkDirectory, - [string]$Signature + [string]$Signature, + [string]$EventRef, + [string]$EventSha ) [System.IO.Directory]::CreateDirectory($WorkDirectory) | Out-Null @@ -180,6 +182,14 @@ function Invoke-Collector { $params["Signature"] = $Signature } + if (-not [string]::IsNullOrEmpty($EventRef)) + { + $params["EventRef"] = $EventRef + } + if (-not [string]::IsNullOrEmpty($EventSha)) + { + $params["EventSha"] = $EventSha + } & $collector @params | Out-Null @@ -236,12 +246,48 @@ function New-SyntheticFixture $base.Remove("main_branch") } + $fixtureBuilds = @($base.azdo_builds.Values) + foreach ($scan in @($base.recurrence_scan, $base.negative_scan)) + { + foreach ($entry in $scan.Values) + { + $fixtureBuilds += @($entry) + } + } + foreach ($build in $fixtureBuilds) + { + if (-not $build.Contains("sourceBranch")) + { + $build["sourceBranch"] = "refs/heads/main" + } + if (-not $build.Contains("status")) + { + $build["status"] = "completed" + } + } + $directory = Join-Path $tempRoot "fixture-$Name" [System.IO.Directory]::CreateDirectory($directory) | Out-Null $base | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $directory "fixture.json") return $directory } +function New-DerivedFixture +{ + param( + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][string]$Source, + [Parameter(Mandatory = $true)][scriptblock]$Mutate + ) + + $fixtureObject = Get-Content -LiteralPath (Join-Path $Source "fixture.json") -Raw | ConvertFrom-Json -Depth 32 + & $Mutate $fixtureObject + $directory = Join-Path $tempRoot "fixture-$Name" + [System.IO.Directory]::CreateDirectory($directory) | Out-Null + $fixtureObject | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath (Join-Path $directory "fixture.json") + return $directory +} + $workflowMarker = "" # Built from a single-quoted literal (no backtick-escape processing) rather than embedding # repeated backtick-escape sequences directly in double-quoted synthetic issue bodies below, @@ -377,10 +423,8 @@ try Assert-Contains -Collection @($resultMissingMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Missing-marker reason codes mismatch: the 'test-failure' label alone must not be treated as proof of quarantine automation." # ------------------------------------------------------------------ - # Edge case (item 8): the repository checkout must be confirmed, via a trusted GitHub API - # response for dotnet/aspnetcore's main branch, to actually be main's tip before a candidate - # is ever labeled repository_ref.branch = "main". A deliberately wrong trusted SHA must fail - # closed rather than mislabel the checkout. + # The immutable dispatch SHA must be confirmed as a member of main. A deliberately unrelated + # current-main SHA must fail closed rather than mislabel the checkout. # ------------------------------------------------------------------ $repoRefMismatchDir = New-SyntheticFixture -Name "repo-ref-mismatch" -Overrides @{ issue = [ordered]@{ @@ -410,6 +454,7 @@ try $flagsShaB = "e06ef94591aaa5a8dc3f84926f8664b2964bf0ea" $flagsShaC = "51e066210e1643430dccb9986c42500d3e706638" $flagsDir = New-SyntheticFixture -Name "check-run-flags" -Overrides @{ + main_branch = [ordered]@{ sha = ("b" * 40); contains_event_sha = $true } issue = [ordered]@{ number = 11 state = "open" @@ -458,6 +503,9 @@ try } $resultFlags = Invoke-Collector -IssueNumber 11 -FixtureRoot $flagsDir -WorkDirectory (Join-Path $tempRoot "check-run-flags") Assert-Equal -Actual $resultFlags.Dossier.outcome -Expected "candidate" -Message "check-run-flags outcome mismatch." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.dispatch_sha_on_main -Expected $true -Message "A dispatch SHA that is an ancestor/member of advanced main must validate." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.current_main_sha -Expected ("b" * 40) -Message "Current main SHA provenance mismatch." + Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.checkout_sha -Expected $repositoryHead -Message "Dispatch checkout SHA provenance mismatch." $snapshotA = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] $snapshotB = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaB })[0] Assert-Equal -Actual $snapshotA.exact_test_referenced -Expected $true -Message "exact_test_referenced must be true when the full FQN appears verbatim." @@ -467,10 +515,66 @@ try Assert-Equal -Actual $snapshotB.short_name_referenced -Expected $true -Message "short_name_referenced must record the bare-method-name match." Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $false -Message "known_issue_referenced must stay false for a generic 'Known Issues' heading with no associated number." + $nonMainResult = Invoke-Collector ` + -IssueNumber 11 ` + -FixtureRoot $flagsDir ` + -WorkDirectory (Join-Path $tempRoot "non-main-dispatch") ` + -EventRef "refs/heads/feature/quarantine" ` + -EventSha $repositoryHead + Assert-Equal -Actual $nonMainResult.Dossier.outcome -Expected "incomplete" -Message "A non-main workflow dispatch must fail closed." + Assert-Contains -Collection @($nonMainResult.Dossier.incomplete.reason_codes) -Value "workflow-dispatch-ref-not-main" -Message "Non-main dispatch reason code mismatch." + + $skipOnlyDir = New-DerivedFixture -Name "skip-only-negative" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_summary.'6100003'[0].outcome = "Skipped" + $fixtureObject.vstmr_detail.'7100003:3'.outcome = "Skipped" + } + $resultSkipOnly = Invoke-Collector -IssueNumber 11 -FixtureRoot $skipOnlyDir -WorkDirectory (Join-Path $tempRoot "skip-only-negative") + Assert-Equal -Actual $resultSkipOnly.Dossier.outcome -Expected "incomplete" -Message "Skip-only evidence must not satisfy intermittency eligibility." + Assert-Contains -Collection @($resultSkipOnly.Dossier.incomplete.reason_codes) -Value "raw-evidence-insufficient" -Message "Skip-only evidence reason code mismatch." + + $unknownEnvironmentDir = New-DerivedFixture -Name "unknown-environment" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Mono-xunit" + } + $resultUnknownEnvironment = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownEnvironmentDir -WorkDirectory (Join-Path $tempRoot "unknown-environment") + Assert-Equal -Actual $resultUnknownEnvironment.Dossier.outcome -Expected "incomplete" -Message "Unknown required environment dimensions must fail closed." + Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-platform-unknown" -Message "Unknown platform reason code mismatch." + Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." + + $invalidBuildCases = @( + [ordered]@{ + Name = "wrong-definition" + ReasonCode = "azdo-build-definition-not-allowed" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.definition.id = 999 } + }, + [ordered]@{ + Name = "wrong-branch" + ReasonCode = "azdo-build-source-branch-not-main" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.sourceBranch = "refs/pull/123/merge" } + }, + [ordered]@{ + Name = "incomplete-build" + ReasonCode = "azdo-build-not-completed" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.status = "inProgress" } + }, + [ordered]@{ + Name = "wrong-result" + ReasonCode = "azdo-build-result-incompatible" + Mutate = { param($fixtureObject) $fixtureObject.azdo_builds.'6100001'.result = "succeeded" } + } + ) + foreach ($invalidBuildCase in $invalidBuildCases) + { + $invalidBuildDir = New-DerivedFixture -Name $invalidBuildCase.Name -Source $flagsDir -Mutate $invalidBuildCase.Mutate + $invalidBuildResult = Invoke-Collector -IssueNumber 11 -FixtureRoot $invalidBuildDir -WorkDirectory (Join-Path $tempRoot $invalidBuildCase.Name) + Assert-Equal -Actual $invalidBuildResult.Dossier.outcome -Expected "incomplete" -Message "$($invalidBuildCase.Name) build must fail closed." + Assert-Contains -Collection @($invalidBuildResult.Dossier.incomplete.reason_codes) -Value $invalidBuildCase.ReasonCode -Message "$($invalidBuildCase.Name) reason code mismatch." + } + # ------------------------------------------------------------------ - # Edge case (item 12): a duplicate-search hit is a discovery candidate only. A hit whose - # body/title does not contain the exact fully-qualified test name must never be treated as a - # validated existing-kbe duplicate, even though it shares the bare method name. + # A duplicate-search hit is discovery only. Even the same FQN must remain unvalidated when the + # documented KBE signature is incompatible with the authoritative failure evidence. # ------------------------------------------------------------------ $dupTestName = "Sample.Tests.DuplicateValidationCase" $dupSignature = "System.InvalidOperationException: Sample failure for duplicate validation testing." @@ -515,12 +619,12 @@ try "recently-merged-fix-pr" = [ordered]@{ complete = $true; result_numbers = @(); total_count = 0 } } duplicate_candidate_text = [ordered]@{ - "99999" = "Quarantine Sample.Tests.OtherUnrelatedCase`nThis issue tracks a completely different test that happens to share no identity with our test." + "99999" = "Known Build Error for $dupTestName`n## Error Message$([System.Environment]::NewLine)${codeFence}json$([System.Environment]::NewLine){ `"ErrorMessage`": `"System.InvalidOperationException: A different root cause.`", `"BuildRetry`": false, `"ExcludeConsoleLog`": false }$([System.Environment]::NewLine)$codeFence" } } $resultDup = Invoke-Collector -IssueNumber 12 -FixtureRoot $dupDir -WorkDirectory (Join-Path $tempRoot "duplicate-unvalidated") Assert-Equal -Actual $resultDup.Dossier.outcome -Expected "candidate" -Message "duplicate-unvalidated outcome mismatch." - Assert-Equal -Actual $resultDup.Dossier.candidate.duplicate_check.status -Expected "none" -Message "An unvalidated search hit must never set duplicate_check.status to existing-kbe." + Assert-Equal -Actual $resultDup.Dossier.candidate.duplicate_check.status -Expected "none" -Message "A same-FQN KBE with an incompatible signature must never set duplicate_check.status to existing-kbe." Assert-Equal -Actual (@($resultDup.Dossier.candidate.duplicate_check.references)).Count -Expected 0 -Message "An unvalidated search hit must never appear in duplicate_check.references." $unvalidated = @($resultDup.Dossier.provenance.duplicate_search.unvalidated_candidates) if ($unvalidated.Count -eq 0 -or -not (@($unvalidated | Where-Object { $_.number -eq 99999 }).Count -gt 0)) @@ -528,6 +632,46 @@ try throw "Expected an unvalidated_candidates entry for issue #99999." } + $compatibleKbeDir = New-DerivedFixture -Name "duplicate-compatible-kbe" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_candidate_text.'99999' = "Known Build Error for $dupTestName`n## Error Message$([System.Environment]::NewLine)${codeFence}json$([System.Environment]::NewLine){ `"ErrorMessage`": `"$dupSignature`", `"BuildRetry`": false, `"ExcludeConsoleLog`": false }$([System.Environment]::NewLine)$codeFence" + } + $resultCompatibleKbe = Invoke-Collector -IssueNumber 12 -FixtureRoot $compatibleKbeDir -WorkDirectory (Join-Path $tempRoot "duplicate-compatible-kbe") + Assert-Equal -Actual $resultCompatibleKbe.Dossier.candidate.duplicate_check.status -Expected "existing-kbe" -Message "An exact-FQN KBE with a compatible documented signature should validate." + + $detailFetchFailureDir = New-DerivedFixture -Name "duplicate-detail-fetch-failure" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_candidate_text.PSObject.Properties.Remove("99999") + } + $resultDetailFetchFailure = Invoke-Collector -IssueNumber 12 -FixtureRoot $detailFetchFailureDir -WorkDirectory (Join-Path $tempRoot "duplicate-detail-fetch-failure") + Assert-Equal -Actual $resultDetailFetchFailure.Dossier.outcome -Expected "incomplete" -Message "A failed duplicate candidate-detail fetch must make the collector incomplete." + Assert-Contains -Collection @($resultDetailFetchFailure.Dossier.incomplete.reason_codes) -Value "duplicate-detail-fetch-incomplete" -Message "Candidate-detail fetch failure reason code mismatch." + Assert-Equal -Actual $resultDetailFetchFailure.Dossier.provenance.duplicate_search.coverage.open_kbes -Expected $false -Message "The affected duplicate query coverage must be incomplete." + $failedDetailQuery = @($resultDetailFetchFailure.Dossier.provenance.duplicate_search.queries | Where-Object { $_.category -eq "open-kbe" })[0] + Assert-Equal -Actual $failedDetailQuery.complete -Expected $false -Message "The affected duplicate query must be marked incomplete." + + $fixPrUnvalidatedDir = New-DerivedFixture -Name "fix-pr-unvalidated" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_search.'open-kbe'.result_numbers = @() + $fixtureObject.duplicate_search.'open-kbe'.total_count = 0 + $fixtureObject.duplicate_search.'open-fix-pr'.result_numbers = @(99999) + $fixtureObject.duplicate_search.'open-fix-pr'.total_count = 1 + $fixtureObject.duplicate_candidate_text.'99999' = "Fix $dupTestName without a compatible signature or linked issue." + } + $resultFixPrUnvalidated = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrUnvalidatedDir -WorkDirectory (Join-Path $tempRoot "fix-pr-unvalidated") + Assert-Equal -Actual $resultFixPrUnvalidated.Dossier.candidate.duplicate_check.status -Expected "none" -Message "An exact-FQN fix PR without compatible association must remain unvalidated." + + $fixPrValidatedDir = New-DerivedFixture -Name "fix-pr-validated" -Source $dupDir -Mutate { + param($fixtureObject) + $fixtureObject.duplicate_search.'open-kbe'.result_numbers = @() + $fixtureObject.duplicate_search.'open-kbe'.total_count = 0 + $fixtureObject.duplicate_search.'open-fix-pr'.result_numbers = @(99999) + $fixtureObject.duplicate_search.'open-fix-pr'.total_count = 1 + $fixtureObject.duplicate_candidate_text.'99999' = "Fix $dupTestName`nRoot cause: $dupSignature" + } + $resultFixPrValidated = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrValidatedDir -WorkDirectory (Join-Path $tempRoot "fix-pr-validated") + Assert-Equal -Actual $resultFixPrValidated.Dossier.candidate.duplicate_check.status -Expected "existing-fix-pr" -Message "An exact-FQN fix PR with compatible signature association should validate." + # ------------------------------------------------------------------ # Edge case (item 11): a literal ErrorMessage containing '*', '?', and '[' must be matched via # ordinal substring containment, never PowerShell -like/-notlike wildcard semantics. A decoy diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 index c28ced296daf..c8de95e2b801 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -141,6 +141,7 @@ function New-LogEntry default { 1003 } } pipeline_definition_id = 83 + source_branch = "refs/heads/main" source_version = switch ($Id) { "failure-1" { "2222222222222222222222222222222222222222"; break } @@ -148,6 +149,8 @@ function New-LogEntry default { "4444444444444444444444444444444444444444" } } started_utc = "2026-08-20T12:00:00Z" + status = "completed" + result = if ($Role -eq "failure") { "failed" } else { "succeeded" } platform = "Linux" configuration = "Release" } @@ -197,6 +200,54 @@ try Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Shadow evaluator must not authorize enrichment from unverified provenance." Assert-Equal -Actual $receipt.evidence_provenance_verified -Expected $false -Message "Shadow evidence provenance must remain unverified." + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[SKIP] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Skipped by test infrastructure" + ) + $logs[2].outcome = "skipped" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Skip-only negative evidence status mismatch." + Assert-Equal -Actual $receipt.evidence.distinct_negative_build_count -Expected 0 -Message "Skipped evidence must not satisfy the authoritative Passed build gate." + + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( + "[PASS] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" + "Finished normally" + ) + $logs[2].outcome = "passed" + $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() + $logs[0].build.platform = "unknown" + $logs[0].build.configuration = "unknown" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unknown environment evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("unknown platform")) -or + -not (($receipt.reasons -join "`n").Contains("unknown configuration"))) + { + throw "Unknown environment evidence must report both missing dimensions." + } + $logs[0].build.platform = "Linux" + $logs[0].build.configuration = "Release" + Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( "Starting an unrelated test" "Xunit.Sdk.TrueException: $signature" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json index e2750ccafa8a..23f90c873f5d 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -21,8 +21,13 @@ "outcome": "incomplete", "provenance": { "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", "checkout_sha": "", - "trusted_main_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, "matches_main": true }, "azdo_builds": [ @@ -32,9 +37,11 @@ "retrieved_utc": "", "source": "issue-body-reference", "definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", "started_utc": "2026-08-22T03:28:01.2925985Z", - "finished_utc": "2026-08-22T05:06:06.2192270Z", + "finished_utc": "2026-08-22T05:06:06.219227Z", + "status": "completed", "result": "failed" } ], diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json index f41194683549..9e43486238e6 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -16,7 +16,9 @@ "sourceVersion": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", "startTime": "2026-08-22T03:28:01.2925985Z", "finishTime": "2026-08-22T05:06:06.219227Z", - "result": "failed" + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" } }, "recurrence_scan": {}, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index 7575ac373201..d5b1be2fce01 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -21,8 +21,13 @@ "outcome": "incomplete", "provenance": { "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", "checkout_sha": "", - "trusted_main_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, "matches_main": true }, "azdo_builds": [ @@ -32,9 +37,11 @@ "retrieved_utc": "", "source": "issue-body-reference", "definition_id": 83, + "source_branch": "refs/heads/main", "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", - "started_utc": "2026-08-27T08:00:00.0000000Z", - "finished_utc": "2026-08-27T09:45:00.0000000Z", + "started_utc": "2026-08-27T08:00:00Z", + "finished_utc": "2026-08-27T09:45:00Z", + "status": "completed", "result": "failed" }, { @@ -43,9 +50,11 @@ "retrieved_utc": "", "source": "issue-body-reference", "definition_id": 83, + "source_branch": "refs/heads/main", "source_version": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", - "started_utc": "2026-08-04T08:00:00.0000000Z", - "finished_utc": "2026-08-04T09:45:00.0000000Z", + "started_utc": "2026-08-04T08:00:00Z", + "finished_utc": "2026-08-04T09:45:00Z", + "status": "completed", "result": "failed" }, { @@ -54,9 +63,11 @@ "retrieved_utc": "", "source": "negative-scan", "definition_id": 83, + "source_branch": "refs/heads/main", "source_version": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", - "started_utc": "2026-08-15T08:00:00.0000000Z", - "finished_utc": "2026-08-15T09:45:00.0000000Z", + "started_utc": "2026-08-15T08:00:00Z", + "finished_utc": "2026-08-15T09:45:00Z", + "status": "completed", "result": "succeeded" } ], diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json index 99038c5db506..73fa1f61b2db 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -16,7 +16,9 @@ "sourceVersion": "b5666daed660cf1862a197784eee65b42a74a64a", "startTime": "2026-08-27T08:00:00Z", "finishTime": "2026-08-27T09:45:00Z", - "result": "failed" + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" }, "1538879": { "definition": { @@ -25,7 +27,9 @@ "sourceVersion": "7773bea1d63c05d7b7043d2ecd2a3fcdd0ff18dc", "startTime": "2026-08-04T08:00:00Z", "finishTime": "2026-08-04T09:45:00Z", - "result": "failed" + "result": "failed", + "sourceBranch": "refs/heads/main", + "status": "completed" } }, "recurrence_scan": {}, @@ -36,7 +40,9 @@ "sourceVersion": "d0bb51a3cabe3bd24dac952bcf8a183c91b54baa", "startTime": "2026-08-15T08:00:00Z", "finishTime": "2026-08-15T09:45:00Z", - "result": "succeeded" + "result": "succeeded", + "sourceBranch": "refs/heads/main", + "status": "completed" } ] }, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index e69e14ec6740..63a1a1a6f94b 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -21,8 +21,13 @@ "outcome": "candidate", "provenance": { "repository_ref_verification": { + "event_ref": "refs/heads/main", + "event_sha": "", "checkout_sha": "", - "trusted_main_sha": "", + "current_main_sha": "", + "checkout_matches_event_sha": true, + "event_ref_is_main": true, + "dispatch_sha_on_main": true, "matches_main": true }, "azdo_builds": [ @@ -32,9 +37,11 @@ "retrieved_utc": "", "source": "issue-body-reference", "definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", - "started_utc": "2026-08-13T04:18:53.9030000Z", - "finished_utc": "2026-08-13T04:31:51.1870000Z", + "started_utc": "2026-08-13T04:18:53.903Z", + "finished_utc": "2026-08-13T04:31:51.187Z", + "status": "completed", "result": "partiallySucceeded" }, { @@ -50,9 +57,11 @@ "retrieved_utc": "", "source": "recurrence-scan", "definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", - "started_utc": "2026-08-10T09:00:00.0000000Z", - "finished_utc": "2026-08-10T10:30:00.0000000Z", + "started_utc": "2026-08-10T09:00:00Z", + "finished_utc": "2026-08-10T10:30:00Z", + "status": "completed", "result": "partiallySucceeded" }, { @@ -61,9 +70,11 @@ "retrieved_utc": "", "source": "negative-scan", "definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "113a606f96ebd832970a1f377748746ff6526abc", - "started_utc": "2026-08-08T09:00:00.0000000Z", - "finished_utc": "2026-08-08T10:30:00.0000000Z", + "started_utc": "2026-08-08T09:00:00Z", + "finished_utc": "2026-08-08T10:30:00Z", + "status": "completed", "result": "succeeded" } ], @@ -218,8 +229,11 @@ "build": { "id": 1551326, "pipeline_definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", - "started_utc": "2026-08-13T04:18:53.9030000Z", + "started_utc": "2026-08-13T04:18:53.903Z", + "status": "completed", + "result": "partiallySucceeded", "platform": "Linux", "configuration": "Release" } @@ -234,8 +248,11 @@ "build": { "id": 1549000, "pipeline_definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", - "started_utc": "2026-08-10T09:00:00.0000000Z", + "started_utc": "2026-08-10T09:00:00Z", + "status": "completed", + "result": "partiallySucceeded", "platform": "Linux", "configuration": "Release" } @@ -250,8 +267,11 @@ "build": { "id": 1545000, "pipeline_definition_id": 87, + "source_branch": "refs/heads/main", "source_version": "113a606f96ebd832970a1f377748746ff6526abc", - "started_utc": "2026-08-08T09:00:00.0000000Z", + "started_utc": "2026-08-08T09:00:00Z", + "status": "completed", + "result": "succeeded", "platform": "Linux", "configuration": "Release" } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json index 78bf6aa88990..c3321b278c65 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -16,7 +16,9 @@ "sourceVersion": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", "startTime": "2026-08-13T04:18:53.903Z", "finishTime": "2026-08-13T04:31:51.187Z", - "result": "partiallySucceeded" + "result": "partiallySucceeded", + "sourceBranch": "refs/heads/main", + "status": "completed" } }, "recurrence_scan": { @@ -26,7 +28,9 @@ "sourceVersion": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", "startTime": "2026-08-10T09:00:00Z", "finishTime": "2026-08-10T10:30:00Z", - "result": "partiallySucceeded" + "result": "partiallySucceeded", + "sourceBranch": "refs/heads/main", + "status": "completed" } ] }, @@ -37,7 +41,9 @@ "sourceVersion": "113a606f96ebd832970a1f377748746ff6526abc", "startTime": "2026-08-08T09:00:00Z", "finishTime": "2026-08-08T10:30:00Z", - "result": "succeeded" + "result": "succeeded", + "sourceBranch": "refs/heads/main", + "status": "completed" } ] }, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json index 8b132d1a5737..b122263639fb 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -190,8 +190,11 @@ "required": [ "id", "pipeline_definition_id", + "source_branch", "source_version", "started_utc", + "status", + "result", "platform", "configuration" ], @@ -202,7 +205,13 @@ }, "pipeline_definition_id": { "type": "integer", - "minimum": 1 + "enum": [ + 83, + 87 + ] + }, + "source_branch": { + "const": "refs/heads/main" }, "source_version": { "$ref": "#/$defs/gitSha" @@ -211,6 +220,16 @@ "type": "string", "format": "date-time" }, + "status": { + "const": "completed" + }, + "result": { + "enum": [ + "failed", + "partiallySucceeded", + "succeeded" + ] + }, "platform": { "type": "string", "minLength": 1, @@ -239,6 +258,16 @@ "properties": { "outcome": { "const": "failed" + }, + "build": { + "properties": { + "result": { + "enum": [ + "failed", + "partiallySucceeded" + ] + } + } } } } @@ -259,6 +288,13 @@ "skipped", "unrelated" ] + }, + "build": { + "properties": { + "result": { + "const": "succeeded" + } + } } } } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index 5753c3b41fcf..53dfd677be78 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -113,23 +113,48 @@ "type": "object", "additionalProperties": false, "required": [ + "event_ref", + "event_sha", "checkout_sha", - "trusted_main_sha", + "current_main_sha", + "checkout_matches_event_sha", + "event_ref_is_main", + "dispatch_sha_on_main", "matches_main" ], "properties": { + "event_ref": { + "type": "string", + "maxLength": 256 + }, + "event_sha": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{40}$" + }, "checkout_sha": { "$ref": "#/$defs/gitSha" }, - "trusted_main_sha": { + "current_main_sha": { "type": [ "string", "null" ] }, + "checkout_matches_event_sha": { + "type": "boolean" + }, + "event_ref_is_main": { + "type": "boolean" + }, + "dispatch_sha_on_main": { + "type": "boolean" + }, "matches_main": { "type": "boolean", - "description": "true only when checkout_sha was confirmed, via a trusted GitHub API response for dotnet/aspnetcore's 'main' branch, to equal the repository checkout the collector and evaluator ran against. A candidate is only ever emitted when this is true." + "description": "true only when the workflow dispatch ref is exactly refs/heads/main, checkout_sha equals event_sha, and event_sha is identical to or an ancestor/member of current main. A candidate is only ever emitted when this is true." } } }, @@ -168,6 +193,10 @@ "type": "integer", "minimum": 1 }, + "source_branch": { + "type": "string", + "maxLength": 256 + }, "source_version": { "$ref": "#/$defs/gitSha" }, @@ -186,6 +215,10 @@ "type": "string", "maxLength": 64 }, + "status": { + "type": "string", + "maxLength": 64 + }, "note": { "type": "string", "maxLength": 512 @@ -203,8 +236,10 @@ "then": { "required": [ "definition_id", + "source_branch", "source_version", "started_utc", + "status", "result" ] } @@ -416,13 +451,22 @@ "test-name-unresolvable", "multiple-test-identities-unresolved", "repository-ref-not-main", + "workflow-dispatch-ref-not-main", + "checkout-sha-not-dispatch-sha", "build-reference-unresolvable", "build-metadata-expired", + "azdo-build-definition-not-allowed", + "azdo-build-source-branch-not-main", + "azdo-build-not-completed", + "azdo-build-result-incompatible", "raw-evidence-expired", "raw-evidence-insufficient", + "evidence-platform-unknown", + "evidence-configuration-unknown", "recurrence-single-build-only", "signature-extraction-ambiguous", "duplicate-search-incomplete", + "duplicate-detail-fetch-incomplete", "github-api-error" ] } @@ -616,7 +660,7 @@ "unvalidated_candidates": { "type": "array", "maxItems": 200, - "description": "Search hits that were discovered but could not be confirmed, by fetching the issue/PR body and requiring the exact fully-qualified test name to appear in it, to actually concern this test. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", + "description": "Search hits that were fetched but could not establish the exact FQN plus compatible KBE signature or fix-PR association. A detail-fetch failure is also recorded here and separately makes the affected query incomplete. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", "items": { "type": "object", "additionalProperties": false, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json index ea94e43efe2c..8d51fe139246 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -276,6 +276,16 @@ "properties": { "outcome": { "const": "failed" + }, + "build": { + "properties": { + "result": { + "enum": [ + "failed", + "partiallySucceeded" + ] + } + } } } } @@ -296,6 +306,13 @@ "skipped", "unrelated" ] + }, + "build": { + "properties": { + "result": { + "const": "succeeded" + } + } } } } @@ -550,8 +567,11 @@ "required": [ "id", "pipeline_definition_id", + "source_branch", "source_version", "started_utc", + "status", + "result", "platform", "configuration" ], @@ -562,7 +582,13 @@ }, "pipeline_definition_id": { "type": "integer", - "minimum": 1 + "enum": [ + 83, + 87 + ] + }, + "source_branch": { + "const": "refs/heads/main" }, "source_version": { "$ref": "#/$defs/gitSha" @@ -571,6 +597,16 @@ "type": "string", "format": "date-time" }, + "status": { + "const": "completed" + }, + "result": { + "enum": [ + "failed", + "partiallySucceeded", + "succeeded" + ] + }, "platform": { "type": "string", "minLength": 1, diff --git a/.github/workflows/test-quarantine-kbe-shadow.yml b/.github/workflows/test-quarantine-kbe-shadow.yml index 7c789fda5a37..3ec0981cfb0e 100644 --- a/.github/workflows/test-quarantine-kbe-shadow.yml +++ b/.github/workflows/test-quarantine-kbe-shadow.yml @@ -71,12 +71,16 @@ jobs: GITHUB_TOKEN: ${{ github.token }} ISSUE_NUMBER: ${{ env.ISSUE_NUMBER }} SIGNATURE_INPUT: ${{ inputs.signature }} + EVENT_REF: ${{ github.ref }} + EVENT_SHA: ${{ github.sha }} run: | $params = @{ IssueNumber = [int]$env:ISSUE_NUMBER OutputFile = "$env:RUNNER_TEMP/dossier.json" CandidateFile = "$env:RUNNER_TEMP/candidate.json" EvidenceRoot = "$env:RUNNER_TEMP/evidence" + EventRef = $env:EVENT_REF + EventSha = $env:EVENT_SHA } if (-not [string]::IsNullOrWhiteSpace($env:SIGNATURE_INPUT)) { From 383c764f9dcd9bfc5fb4a980742a410301523cd3 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:44:15 -0500 Subject: [PATCH 06/10] Make quarantine fixtures event-independent Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 12 ++++++++-- .../test-quarantine-kbe-shadow/README.md | 5 ++++ ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 23 +++++++++++-------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index 2f94b577048e..335a9e805ea4 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -50,6 +50,14 @@ GITHUB_TOKEN environment variable. Authenticated calls are strongly preferred: the search API's unauthenticated rate limit (10 requests/minute) is exhausted by a single run's four duplicate searches plus one retry. + +.PARAMETER EventRef + Trusted workflow event ref. Live callers must pass github.ref explicitly. Fixture mode defaults + to refs/heads/main only when this parameter is omitted. + +.PARAMETER EventSha + Trusted workflow event SHA. Live callers must pass github.sha explicitly. Fixture mode defaults + to the checked-out repository SHA only when this parameter is omitted. #> [CmdletBinding()] @@ -76,9 +84,9 @@ param( [string]$RepositoryRoot = "$PSScriptRoot/../../../..", - [string]$EventRef = $env:GITHUB_REF, + [string]$EventRef, - [string]$EventSha = $env:GITHUB_SHA, + [string]$EventSha, [string]$DossierSchemaFile = "$PSScriptRoot/test-quarantine-kbe-shadow-dossier.schema.json", diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index 2fd41464ea21..213d98bad351 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -186,6 +186,11 @@ parameter. When set, the collector reads one consolidated `fixture.json` documen directory instead of making any live GitHub/Azure DevOps call. `fixture.json` mirrors the shape of the real endpoints captured live during development: +Fixture tests always pass an explicit `EventRef = refs/heads/main` and `EventSha` equal to the +checked-out repository SHA. The collector does not default these parameters from ambient +`GITHUB_REF`/`GITHUB_SHA`, so a pull-request test job cannot change fixture output. Live workflow +callers pass trusted `github.ref`/`github.sha` through step environment bindings. + | Key | Mirrors | |---|---| | `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, body) | diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index b5fa6babf5cf..88894fcbf9d2 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -31,6 +31,10 @@ $fixturesRoot = "$PSScriptRoot/fixtures" $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) "aspnetcore-kbe-shadow-collector-$([System.Guid]::NewGuid().ToString('N'))" $repositoryRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path $repositoryHead = (& git -C $repositoryRoot rev-parse HEAD).Trim() +$fixtureEventRef = "refs/heads/main" +$fixtureEventSha = $repositoryHead +$originalGitHubRef = $env:GITHUB_REF +$originalGitHubSha = $env:GITHUB_SHA function Assert-Equal { @@ -175,6 +179,8 @@ function Invoke-Collector EvidenceRoot = $evidenceRoot FixtureRoot = $FixtureRoot RepositoryRoot = $repositoryRoot + EventRef = if ([string]::IsNullOrEmpty($EventRef)) { $fixtureEventRef } else { $EventRef } + EventSha = if ([string]::IsNullOrEmpty($EventSha)) { $fixtureEventSha } else { $EventSha } DossierSchemaFile = $dossierSchema CandidateSchemaFile = $candidateSchema } @@ -182,15 +188,6 @@ function Invoke-Collector { $params["Signature"] = $Signature } - if (-not [string]::IsNullOrEmpty($EventRef)) - { - $params["EventRef"] = $EventRef - } - if (-not [string]::IsNullOrEmpty($EventSha)) - { - $params["EventSha"] = $EventSha - } - & $collector @params | Out-Null return [ordered]@{ @@ -304,6 +301,8 @@ $defaultDuplicateSearch = [ordered]@{ try { [System.IO.Directory]::CreateDirectory($tempRoot) | Out-Null + $env:GITHUB_REF = "refs/pull/69021/merge" + $env:GITHUB_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # ------------------------------------------------------------------ # Pilot 1 -- aspnetcore#68724: '## Failing Test(s)' names two distinct concrete test @@ -314,6 +313,9 @@ try $result68724 = Invoke-Collector -IssueNumber 68724 -FixtureRoot "$fixturesRoot/68724" -WorkDirectory "$tempRoot/68724" Assert-Equal -Actual $result68724.Dossier.outcome -Expected "incomplete" -Message "#68724 outcome mismatch." Assert-Contains -Collection @($result68724.Dossier.incomplete.reason_codes) -Value "multiple-test-identities-unresolved" -Message "#68724 reason codes mismatch." + Assert-NotContains -Collection @($result68724.Dossier.incomplete.reason_codes) -Value "workflow-dispatch-ref-not-main" -Message "Ambient GITHUB_REF must not affect fixture collection." + Assert-Equal -Actual $result68724.Dossier.provenance.repository_ref_verification.event_ref -Expected $fixtureEventRef -Message "Fixture event ref must be explicit and deterministic." + Assert-Equal -Actual $result68724.Dossier.provenance.repository_ref_verification.event_sha -Expected $fixtureEventSha -Message "Fixture event SHA must be explicit and deterministic." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68724" -ActualDossier $result68724.Dossier # ------------------------------------------------------------------ @@ -765,6 +767,9 @@ try } finally { + $env:GITHUB_REF = $originalGitHubRef + $env:GITHUB_SHA = $originalGitHubSha + if (Test-Path -LiteralPath $tempRoot) { Remove-Item -LiteralPath $tempRoot -Recurse -Force From 47f7b5021618935e616869613083e30104c0d920 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:08:19 -0500 Subject: [PATCH 07/10] Close remaining quarantine false-positive paths Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 307 +++++++++++------- .../Evaluate-TestQuarantineKbeCandidate.ps1 | 55 +++- .../test-quarantine-kbe-shadow/README.md | 44 +-- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 109 +++++-- ...st-Evaluate-TestQuarantineKbeCandidate.ps1 | 67 +++- .../fixtures/68724/expected-dossier.json | 5 +- .../fixtures/68724/fixture.json | 5 +- .../fixtures/68945/expected-dossier.json | 14 +- .../fixtures/68945/fixture.json | 5 +- .../fixtures/68947/expected-dossier.json | 160 ++------- .../fixtures/68947/fixture.json | 5 +- ...-quarantine-kbe-shadow-dossier.schema.json | 29 +- 12 files changed, 492 insertions(+), 313 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index 335a9e805ea4..0ecf9c1e2665 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -108,10 +108,9 @@ $ado = "https://dev.azure.com/dnceng-public/public/_apis" $vstmr = "https://vstmr.dev.azure.com/dnceng-public/public/_apis" $pipelineDefinitionIds = @(83, 87) $canonicalQuarantineLabel = "test-failure" -$workflowMarkers = @( - "", - "" -) +$workflowIdMarker = "" +$workflowCallMarker = "" +$trustedIssueActors = @("app/github-actions", "github-actions[bot]") $minimumFailureBuilds = 2 $minimumNegativeLogs = 1 $excerptCap = 2000 @@ -992,72 +991,9 @@ function Test-DocumentedSignatureCompatibility return $true } -function Test-FixPrCompatibility -{ - param( - [Parameter(Mandatory = $true)][string]$Text, - [Parameter(Mandatory = $true)][string]$TestName, - [Parameter(Mandatory = $true)][AllowNull()][string]$CandidateSignature, - [Parameter(Mandatory = $true)][object[]]$FailureLogs, - [Parameter(Mandatory = $true)][string]$Root, - [Parameter(Mandatory = $true)][ref]$DetailFetchIncomplete - ) - - if (-not (Test-ContainsExactTestName -Text $Text -TestName $TestName)) - { - return $false - } - - $documentedSignature = Get-DocumentedSignature -Text $Text - if ($null -ne $documentedSignature -and - (Test-DocumentedSignatureCompatibility -Signature $documentedSignature -FailureLogs $FailureLogs -TestName $TestName -Root $Root)) - { - return $true - } - if (-not [string]::IsNullOrWhiteSpace($CandidateSignature) -and - $Text.Contains($CandidateSignature, [System.StringComparison]::Ordinal)) - { - return $true - } - - $linkedNumbers = @( - [regex]::Matches($Text, '(?i)https://github\.com/dotnet/aspnetcore/(?:issues|pull)/([1-9][0-9]*)|(?', + [System.Text.RegularExpressions.RegexOptions]::Singleline -bor + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant) +$workflowRunId = $null +$hasWorkflowMetadata = $false +if ($workflowMetadataMatch.Success) +{ + $metadata = $workflowMetadataMatch.Groups[1].Value + $metadataIdMatch = [regex]::Match($metadata, '(?:^|,\s*)id:\s*([1-9][0-9]*)(?:,|$)') + $metadataRunMatch = [regex]::Match($metadata, '(?:^|,\s*)run:\s*https://github\.com/dotnet/aspnetcore/actions/runs/([1-9][0-9]*)(?:,|$)') + $hasExpectedWorkflowId = [regex]::IsMatch($metadata, '(?:^|,\s*)workflow_id:\s*test-quarantine(?:,|$)') + if ($metadataIdMatch.Success -and + $metadataRunMatch.Success -and + $metadataIdMatch.Groups[1].Value -eq $metadataRunMatch.Groups[1].Value -and + $hasExpectedWorkflowId) + { + $workflowRunId = [long]$metadataIdMatch.Groups[1].Value + $hasWorkflowMetadata = $true + } +} +$hasTrustedIssueActor = $issueActor -in $trustedIssueActors -if ($issueLabels -notcontains $canonicalQuarantineLabel -or -not $hasWorkflowMarker) +if ($issueLabels -notcontains $canonicalQuarantineLabel -or + -not $hasWorkflowMarker -or + -not $hasTrustedIssueActor -or + -not $hasWorkflowMetadata) { $reasonCodes.Add("issue-not-canonical-quarantine") if ($issueLabels -notcontains $canonicalQuarantineLabel) @@ -1079,7 +1051,15 @@ if ($issueLabels -notcontains $canonicalQuarantineLabel -or -not $hasWorkflowMar } if (-not $hasWorkflowMarker) { - Add-MissingEvidence -List $missingEvidence -Kind "quarantine-workflow-marker" -Detail "Issue #$IssueNumber body does not contain a trusted 'gh-aw-workflow-id: test-quarantine' / 'gh-aw-workflow-call-id: dotnet/aspnetcore/test-quarantine' marker; the label alone is not proof this issue was generated by quarantine automation." + Add-MissingEvidence -List $missingEvidence -Kind "quarantine-workflow-marker" -Detail "Issue #$IssueNumber body does not contain both trusted test-quarantine workflow markers." + } + if (-not $hasTrustedIssueActor) + { + Add-MissingEvidence -List $missingEvidence -Kind "issue-actor" -Detail "Issue #$IssueNumber actor '$issueActor' is not a trusted GitHub Actions workflow actor." + } + if (-not $hasWorkflowMetadata) + { + Add-MissingEvidence -List $missingEvidence -Kind "workflow-metadata" -Detail "Issue #$IssueNumber lacks consistent structured test-quarantine workflow metadata and dotnet/aspnetcore Actions run provenance." } } @@ -1410,6 +1390,7 @@ if ($null -ne $testName -and $null -ne $effectiveSignature -and $resolvedBuilds. # --------------------------------------------------------------------------- $negativeBuilds = [System.Collections.Generic.List[object]]::new() +$negativeBuildCap = [System.Math]::Max(0, [System.Math]::Min($RecurrenceScanBuildCap, 30 - $resolvedBuilds.Count)) if ($null -ne $testName -and $null -ne $effectiveSignature) { @@ -1421,7 +1402,7 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) foreach ($definitionId in $negativeScanDefinitionIds) { - if ($negativeBuilds.Count -ge $minimumNegativeLogs) + if ($negativeBuilds.Count -ge $negativeBuildCap) { break } @@ -1429,7 +1410,7 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) $candidates = Get-AzdoNegativeCandidateBuilds -DefinitionId $definitionId foreach ($candidate in $candidates) { - if ($negativeBuilds.Count -ge $minimumNegativeLogs) + if ($negativeBuilds.Count -ge $negativeBuildCap) { break } @@ -1482,7 +1463,9 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) $rawEvidenceRecords = [System.Collections.Generic.List[object]]::new() $rawLogs = [System.Collections.Generic.List[object]]::new() $failureBuildIdSet = [System.Collections.Generic.HashSet[int]]::new() -$passedBuildIdSet = [System.Collections.Generic.HashSet[int]]::new() +$failureEnvironmentKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) +$earliestMaterializedFailureUtc = $null +$eligiblePassedBuildIdsDuringCollection = [System.Collections.Generic.HashSet[int]]::new() $evidenceIndex = 0 $evidenceBuilds = @($resolvedBuilds) + @($negativeBuilds) @@ -1494,9 +1477,18 @@ foreach ($build in $evidenceBuilds) } $role = [string]$build.intended_role + if ($role -eq "negative" -and $eligiblePassedBuildIdsDuringCollection.Count -ge $minimumNegativeLogs) + { + break + } + $expectedOutcome = if ($role -eq "failure") { @("Failed") } else { @("Passed") } $matchedRow = $null $matchedDetail = $null + $selectedRunName = $null + $selectedPlatformConfiguration = $null + $fallbackRow = $null + $fallbackDetail = $null foreach ($row in @(Get-VstmrSummaryRows -BuildId $build.id -TestName $testName)) { @@ -1509,6 +1501,30 @@ foreach ($build in $evidenceBuilds) { continue } + if ($role -eq "negative") + { + if ($null -eq $fallbackRow) + { + $fallbackRow = $row + $fallbackDetail = $detail + } + + $candidateRunName = Get-VstmrRunName -RunId ([int]$row.runId) + $candidatePlatformConfiguration = Get-PlatformConfigurationFromRunName -RunName $candidateRunName + $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" + $candidateStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + if ($null -ne $earliestMaterializedFailureUtc -and + $candidateStartedUtc -gt $earliestMaterializedFailureUtc -and + $failureEnvironmentKeys.Contains($candidateEnvironmentKey)) + { + $matchedRow = $row + $matchedDetail = $detail + $selectedRunName = $candidateRunName + $selectedPlatformConfiguration = $candidatePlatformConfiguration + break + } + continue + } if ($role -eq "failure" -and $null -ne $effectiveSignature) { $haystack = "$($detail.errorMessage) $($detail.stackTrace)" @@ -1521,6 +1537,11 @@ foreach ($build in $evidenceBuilds) $matchedDetail = $detail break } + if ($role -eq "negative" -and $null -eq $matchedRow -and $null -ne $fallbackRow) + { + $matchedRow = $fallbackRow + $matchedDetail = $fallbackDetail + } if ($null -eq $matchedRow -or $null -eq $matchedDetail) { @@ -1551,8 +1572,15 @@ foreach ($build in $evidenceBuilds) } $helixUnavailable = [string]::IsNullOrEmpty($helixJob) -or [string]::IsNullOrEmpty($helixWorkItem) - $runName = Get-VstmrRunName -RunId $runId - $platformConfiguration = Get-PlatformConfigurationFromRunName -RunName $runName + $runName = if ($null -ne $selectedRunName) { $selectedRunName } else { Get-VstmrRunName -RunId $runId } + $platformConfiguration = if ($null -ne $selectedPlatformConfiguration) + { + $selectedPlatformConfiguration + } + else + { + Get-PlatformConfigurationFromRunName -RunName $runName + } if ($platformConfiguration.Platform -eq "unknown") { $reasonCodes.Add("evidence-platform-unknown") @@ -1612,12 +1640,16 @@ foreach ($build in $evidenceBuilds) if ($role -eq "failure") { $null = $failureBuildIdSet.Add([int]$build.id) + if ($platformConfiguration.Platform -ne "unknown" -and $platformConfiguration.Configuration -ne "unknown") + { + $null = $failureEnvironmentKeys.Add("$($build.definition_id)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)") + } + $failureStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + if ($null -eq $earliestMaterializedFailureUtc -or $failureStartedUtc -lt $earliestMaterializedFailureUtc) + { + $earliestMaterializedFailureUtc = $failureStartedUtc + } } - else - { - $null = $passedBuildIdSet.Add([int]$build.id) - } - $outcomeValue = switch ([string]$matchedRow.outcome) { "Failed" { "failed"; break } @@ -1644,6 +1676,18 @@ foreach ($build in $evidenceBuilds) configuration = $platformConfiguration.Configuration } }) + + if ($role -eq "negative") + { + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + if ($null -ne $earliestMaterializedFailureUtc -and + $passStartedUtc -gt $earliestMaterializedFailureUtc -and + $failureEnvironmentKeys.Contains($passEnvironmentKey)) + { + $null = $eligiblePassedBuildIdsDuringCollection.Add([int]$build.id) + } + } } if ($null -ne $testName -and $failureBuildIdSet.Count -lt $minimumFailureBuilds) @@ -1655,10 +1699,61 @@ if ($null -ne $testName -and $failureBuildIdSet.Count -lt $minimumFailureBuilds) } } -if ($null -ne $testName -and $passedBuildIdSet.Count -lt $minimumNegativeLogs) +$failureLogsForPassEligibility = @($rawLogs | Where-Object { $_.role -eq "failure" }) +$passedLogsForEligibility = @($rawLogs | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) +$eligiblePassedBuildIds = [System.Collections.Generic.HashSet[int]]::new() +$passesAfterEarliestFailure = 0 +if ($failureLogsForPassEligibility.Count -gt 0) +{ + $earliestFailureUtc = @( + $failureLogsForPassEligibility | + ForEach-Object { [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) } | + Sort-Object + )[0] + + foreach ($passLog in $passedLogsForEligibility) + { + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + if ($passStartedUtc -le $earliestFailureUtc) + { + continue + } + + $passesAfterEarliestFailure++ + $environmentMatched = @( + $failureLogsForPassEligibility | + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ).Count -gt 0 + if ($environmentMatched) + { + $null = $eligiblePassedBuildIds.Add([int]$passLog.build.id) + } + } +} + +if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeLogs) { $reasonCodes.Add("raw-evidence-insufficient") - Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "No retrievable Passed evidence was found; at least $minimumNegativeLogs authoritative pass occurrence is required." + if ($failureLogsForPassEligibility.Count -gt 0 -and + $passedLogsForEligibility.Count -gt 0 -and + $passesAfterEarliestFailure -eq 0) + { + $reasonCodes.Add("passed-evidence-not-contemporaneous") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." + } + elseif ($passesAfterEarliestFailure -gt 0) + { + $reasonCodes.Add("passed-evidence-environment-mismatch") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence after the earliest failure shared pipeline definition, platform, and configuration with any collected failure." + } + else + { + Add-MissingEvidence -List $missingEvidence -Kind "raw-evidence" -Detail "No retrievable authoritative Passed evidence was found." + } } # --------------------------------------------------------------------------- @@ -1735,8 +1830,9 @@ foreach ($sha in $distinctShas) # --------------------------------------------------------------------------- # Step 7: categorized duplicate KBE / fix-PR search. Search hits are discovery only. Existing KBEs # require the exact FQN and a documented ErrorMessage/ErrorPattern that matches every authoritative -# failure log with the evaluator's semantics. Fix PRs additionally require compatible signature or -# linked-issue/root-cause evidence. A failed candidate-detail fetch makes that query incomplete. +# failure log with the evaluator's semantics. Fix PRs remain unvalidated until a later collector can +# prove closing-link and changed-file relevance. A failed candidate-detail fetch makes the query +# incomplete. # --------------------------------------------------------------------------- $shortName = if ($testName) { ($testName -split '\.')[-1] } else { $IssueNumber.ToString() } @@ -1752,7 +1848,6 @@ $duplicateQueryResults = [System.Collections.Generic.List[object]]::new() $duplicateReferences = [System.Collections.Generic.List[string]]::new() $unvalidatedCandidates = [System.Collections.Generic.List[object]]::new() $kbeNumbers = [System.Collections.Generic.List[int]]::new() -$fixPrNumbers = [System.Collections.Generic.List[int]]::new() $allQueriesComplete = $true foreach ($q in $duplicateQueries) @@ -1824,22 +1919,8 @@ foreach ($q in $duplicateQueries) } else { - $linkedDetailFetchIncomplete = $false - $validated = Test-FixPrCompatibility ` - -Text $candidateText ` - -TestName $testName ` - -CandidateSignature $effectiveSignature ` - -FailureLogs @($rawLogs | Where-Object { $_.role -eq "failure" }) ` - -Root $EvidenceRoot ` - -DetailFetchIncomplete ([ref]$linkedDetailFetchIncomplete) - if ($linkedDetailFetchIncomplete) - { - $queryComplete = $false - $allQueriesComplete = $false - $reasonCodes.Add("duplicate-detail-fetch-incomplete") - Add-MissingEvidence -List $missingEvidence -Kind "duplicate-search" -Detail "Fix PR #$n linked an issue/PR whose detail could not be fetched; duplicate coverage is incomplete." - } - $reason = "exact FQN found, but no compatible linked issue, signature, or root-cause association was established" + $validated = $false + $reason = "fix PR validation is disabled until closing-link and changed-file relevance can be proven" } } elseif ($null -eq $testName) @@ -1849,16 +1930,8 @@ foreach ($q in $duplicateQueries) if ($validated) { - if ($isKbeCategory) - { - $null = $kbeNumbers.Add($n) - $null = $duplicateReferences.Add("issue:$n") - } - else - { - $null = $fixPrNumbers.Add($n) - $null = $duplicateReferences.Add("pull-request:$n") - } + $null = $kbeNumbers.Add($n) + $null = $duplicateReferences.Add("issue:$n") } else { @@ -1883,10 +1956,6 @@ elseif ($kbeNumbers.Count -gt 0) { "existing-kbe" } -elseif ($fixPrNumbers.Count -gt 0) -{ - "existing-fix-pr" -} else { "none" @@ -1945,7 +2014,6 @@ if ($outcome -eq "candidate") $proposedClassification = switch ($duplicateStatus) { "existing-kbe" { "reuse-existing-kbe"; break } - "existing-fix-pr" { "quarantine-only"; break } default { if ($effectiveSignature -match "(?i)timeout|WebDriverException|TaskCanceledException") @@ -2032,7 +2100,10 @@ $dossier = [ordered]@{ url = $issueUrl state = $issueState labels = @($issueLabels) + actor = $issueActor has_workflow_marker = $hasWorkflowMarker + has_workflow_metadata = $hasWorkflowMetadata + workflow_run_id = $workflowRunId } outcome = $outcome provenance = [ordered]@{ diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 index 334fd67f12da..6ad16eb9be95 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -384,6 +384,10 @@ if ($duplicateStatus -eq "existing-fix-pr" -and $pullRequestReferences.Count -eq { $qualityFailures.Add("The duplicate check reports an existing fix PR without a pull request reference.") } +elseif ($duplicateStatus -eq "existing-fix-pr") +{ + $incompleteReasons.Add("Existing fix PR classification is unsupported until closing-link and changed-file relevance are proven.") +} $checkedUtc = [System.DateTimeOffset]::MinValue if (-not [System.DateTimeOffset]::TryParse( @@ -538,11 +542,6 @@ foreach ($log in $candidate.evidence.raw_logs) else { $negativeLogCount++ - if ([string]$log.outcome -eq "passed") - { - $null = $negativeHashes.Add($actualHash) - $null = $negativeBuildIds.Add([int]$log.build.id) - } if ($match.matched) { $negativeCollisionCount++ @@ -580,6 +579,42 @@ foreach ($log in $candidate.evidence.raw_logs) }) } +$failureLogsForPassEligibility = @($logResults | Where-Object { $_.role -eq "failure" }) +$passedLogsForEligibility = @($logResults | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) +$passesAfterEarliestFailure = 0 +if ($failureLogsForPassEligibility.Count -gt 0) +{ + $earliestFailureUtc = @( + $failureLogsForPassEligibility | + ForEach-Object { [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) } | + Sort-Object + )[0] + + foreach ($passLog in $passedLogsForEligibility) + { + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + if ($passStartedUtc -le $earliestFailureUtc) + { + continue + } + + $passesAfterEarliestFailure++ + $environmentMatched = @( + $failureLogsForPassEligibility | + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ).Count -gt 0 + if ($environmentMatched) + { + $null = $negativeHashes.Add([string]$passLog.sha256) + $null = $negativeBuildIds.Add([int]$passLog.build.id) + } + } +} + $requiredFailureLogs = [System.Math]::Max( $minimumFailureEvidenceFloor, [int]$candidate.policy.minimum_failure_logs) @@ -600,6 +635,16 @@ if ($failureBuildIds.Count -lt $requiredFailureLogs) if ($negativeHashes.Count -lt $requiredNegativeLogs) { $incompleteReasons.Add("Only $($negativeHashes.Count) distinct authoritative Passed log(s) were supplied; $requiredNegativeLogs are required.") + if ($failureLogsForPassEligibility.Count -gt 0 -and + $passedLogsForEligibility.Count -gt 0 -and + $passesAfterEarliestFailure -eq 0) + { + $incompleteReasons.Add("All authoritative Passed occurrences predate or coincide with the earliest failure.") + } + elseif ($passesAfterEarliestFailure -gt 0) + { + $incompleteReasons.Add("No authoritative Passed occurrence after the earliest failure matched a failure's pipeline definition, platform, and configuration.") + } } if ($negativeBuildIds.Count -lt $requiredNegativeLogs) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index 213d98bad351..5c9e74ae30d0 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -51,10 +51,12 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis pseudo-test row carries one); otherwise `helix_unavailable: true` is recorded explicitly and the VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence requires evidence from **at least two distinct builds** (a single build producing two separate - artifacts is not recurrence), and at least one authoritative **Passed** - occurrence. Signature matching against that raw text uses ordinal, case-sensitive substring - containment (`[string]::Contains(..., Ordinal)`) throughout -- never PowerShell's - `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise + artifacts is not recurrence), and at least one authoritative **Passed** occurrence that started + after the earliest collected failure and shares pipeline definition, platform, and configuration + with a collected failure. An older pass or a pass from another environment does not prove the + current failure is intermittent. Signature matching against raw text uses ordinal, + case-sensitive substring containment (`[string]::Contains(..., Ordinal)`) throughout -- never + PowerShell's `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise silently misinterpret a literal ErrorMessage containing any of those characters. * **Azure DevOps build recurrence spans both `failed` and `partiallySucceeded` results.** A `resultFilter=failed`-only recurrence query misses real evidence: aspnetcore#68947's own cited @@ -72,18 +74,19 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis identity independently is a reasonable follow-up left for later, since this PR targets one issue/one root cause at a time -- documented here as an accepted simplification, not an oversight. -* **The `test-failure` label alone is not proof an issue was generated by quarantine automation** - (any contributor can apply it to an ordinary bug report). The collector additionally requires the - issue body to contain the trusted `` or - `` HTML-comment marker the - production quarantine workflow stamps into every issue it creates. +* **A label or copied workflow marker is not canonical issue provenance.** The collector requires + the `test-failure` label, actor `app/github-actions` or `github-actions[bot]`, both expected + test-quarantine markers, and structured `gh-aw-agentic-workflow` metadata whose `workflow_id` is + `test-quarantine` and whose numeric `id` matches a + `https://github.com/dotnet/aspnetcore/actions/runs/{id}` run URL. A user-authored issue with copied + static strings fails closed. * **A duplicate-search hit is discovery only, not a validated duplicate.** Every numeric result returned by the four categorized GitHub searches is fetched. An existing KBE requires the exact FQN plus a documented `ErrorMessage`/`ErrorPattern` that matches every authoritative failure log - with evaluator-compatible semantics. A fix PR requires the exact FQN plus a compatible signature, - linked KBE/quarantine issue, or root-cause association. Incompatible fetched items remain - `unvalidated_candidate` entries; a failed candidate-detail fetch makes that query and the overall - duplicate coverage incomplete. The + with evaluator-compatible semantics. Fix PR search hits are intentionally never classified as + `existing-fix-pr` yet: title/body mentions cannot prove a closing relationship or changed-file + relevance, so every fetched PR remains an explicit `unvalidated_candidate`. A failed + candidate-detail fetch makes that query and the overall duplicate coverage incomplete. The "recently" closed/merged categories carry an explicit 90-day `closed:>=`/`merged:>=` time-window qualifier so the label matches what the query actually searches, and a query is only marked `complete` when GitHub reports `incomplete_results: false` **and** every matching item (per @@ -193,7 +196,7 @@ callers pass trusted `github.ref`/`github.sha` through step environment bindings | Key | Mirrors | |---|---| -| `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, body) | +| `issue` | `GET /repos/{repo}/issues/{number}` (number, state, labels, actor, body with structured workflow-run metadata) | | `main_branch` *(optional)* | Current main `.sha`, plus optional `contains_event_sha` to model ancestry/membership after main advances; omit for legacy pilot fixtures that do not exercise this guard | | `azdo_builds` | `GET .../build/builds/{id}` keyed by build id | | `recurrence_scan`, `negative_scan` | `GET .../build/builds?resultFilter=...` keyed by pipeline definition id | @@ -209,7 +212,7 @@ callers pass trusted `github.ref`/`github.sha` through step environment bindings | Issue | Fixture invocation | Outcome | |---|---|---| | [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` | `incomplete`: `## Failing Test(s)` names two distinct concrete identities (a base test and its server-execution subclass override); live data shows only the override actually failed, so the collector fails closed (`multiple-test-identities-unresolved`) rather than guess | -| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `candidate`, `timeout-needs-classification` (generic Selenium/WebDriver timeout, not a test-specific KBE); recurrence is established via the supplementary scan since the issue's own second cited build has aged out of retention | +| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `incomplete`: two failures recur, but the only authoritative pass is older than both failures, so it cannot establish current intermittency (`passed-evidence-not-contemporaneous`) | | [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's Azure DevOps build record still resolves, but its historical VSTMR test-result data is no longer queryable, leaving only one usable failure log below the two-build recurrence floor (`raw-evidence-insufficient`) | Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality @@ -220,11 +223,12 @@ run-to-run and commit-to-commit -- replacing them with the literal sentinel ` + + +"@ # Built from a single-quoted literal (no backtick-escape processing) rather than embedding # repeated backtick-escape sequences directly in double-quoted synthetic issue bodies below, # which is easy to miscount (a stray extra backtick silently becomes an unrelated `t`/`n`/etc. @@ -332,25 +340,17 @@ try $signature68947 = "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature $signature68947 - Assert-Equal -Actual $result68947.Dossier.outcome -Expected "candidate" -Message "#68947 outcome mismatch." - Assert-Equal -Actual $result68947.Dossier.candidate.proposed_classification -Expected "timeout-needs-classification" -Message "#68947 proposed_classification mismatch." - Assert-Equal -Actual (@($result68947.Dossier.candidate.evidence.raw_logs | Where-Object { $_.role -eq "failure" })).Count -Expected 2 -Message "#68947 must gather two distinct failure builds (the cited partiallySucceeded build plus one recurrence-scan match)." + Assert-Equal -Actual $result68947.Dossier.outcome -Expected "incomplete" -Message "#68947 outcome mismatch." + Assert-Contains -Collection @($result68947.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-contemporaneous" -Message "#68947 must not count its pass that predates both collected failures." + Assert-Equal -Actual $result68947.Dossier.candidate -Expected $null -Message "#68947 must not emit a candidate without a contemporaneous environment-matched pass." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68947" -ActualDossier $result68947.Dossier - $receiptPath68947 = Join-Path "$tempRoot/68947" "receipt.json" - & $evaluator -CandidateFile $result68947.CandidatePath -EvidenceRoot $result68947.EvidenceRoot -OutputFile $receiptPath68947 -RepositoryRoot $repositoryRoot -CandidateSchemaFile $candidateSchema - $receipt68947 = Get-Content -LiteralPath $receiptPath68947 -Raw | ConvertFrom-Json -Depth 32 - Assert-Equal -Actual $receipt68947.deterministic_status -Expected "validated" -Message "#68947 deterministic_status mismatch." - Assert-Equal -Actual $receipt68947.shadow_recommendation -Expected "timeout-needs-classification" -Message "#68947 shadow_recommendation mismatch." - Assert-Equal -Actual $receipt68947.eligible_for_kbe_enrichment -Expected $false -Message "#68947 must never authorize enrichment." - Assert-Equal -Actual $receipt68947.evidence_provenance_verified -Expected $false -Message "#68947 provenance must remain unverified." - $summaryPath68947 = Join-Path "$tempRoot/68947" "summary.md" - & $summaryGenerator -DossierFile $result68947.DossierPath -ReceiptFile $receiptPath68947 -OutputFile $summaryPath68947 + & $summaryGenerator -DossierFile $result68947.DossierPath -OutputFile $summaryPath68947 $summaryText68947 = Get-Content -LiteralPath $summaryPath68947 -Raw - if (-not $summaryText68947.Contains("timeout-needs-classification")) + if (-not $summaryText68947.Contains("passed-evidence-not-contemporaneous")) { - throw "#68947 summary must mention the shadow_recommendation." + throw "#68947 summary must mention the chronology failure." } # ------------------------------------------------------------------ @@ -424,6 +424,41 @@ try Assert-Equal -Actual $resultMissingMarker.Dossier.outcome -Expected "incomplete" -Message "Missing-marker outcome mismatch." Assert-Contains -Collection @($resultMissingMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Missing-marker reason codes mismatch: the 'test-failure' label alone must not be treated as proof of quarantine automation." + $forgedMarkerDir = New-SyntheticFixture -Name "forged-marker" -Overrides @{ + issue = [ordered]@{ + number = 15 + state = "open" + labels = @("test-failure") + user = [ordered]@{ login = "octocat" } + body = "## Failing Test(s)`n`` Sample.Tests.ForgedMarker ``$([System.Environment]::NewLine)$workflowMarker" + } + duplicate_search = $defaultDuplicateSearch + } + $resultForgedMarker = Invoke-Collector -IssueNumber 15 -FixtureRoot $forgedMarkerDir -WorkDirectory (Join-Path $tempRoot "forged-marker") + Assert-Equal -Actual $resultForgedMarker.Dossier.outcome -Expected "incomplete" -Message "A user-authored issue with copied workflow markers must fail closed." + Assert-Contains -Collection @($resultForgedMarker.Dossier.incomplete.reason_codes) -Value "issue-not-canonical-quarantine" -Message "Forged-marker reason code mismatch." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.has_workflow_marker -Expected $true -Message "Forged-marker fixture must prove copied static markers were present." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.has_workflow_metadata -Expected $true -Message "Forged-marker fixture must prove copied structured metadata was present." + Assert-Equal -Actual $resultForgedMarker.Dossier.issue.actor -Expected "octocat" -Message "Forged-marker actor provenance mismatch." + + $mismatchedMetadata = @" + + + +"@ + $mismatchedMetadataDir = New-SyntheticFixture -Name "mismatched-workflow-metadata" -Overrides @{ + issue = [ordered]@{ + number = 16 + state = "open" + labels = @("test-failure") + body = "## Failing Test(s)`n`` Sample.Tests.MismatchedMetadata ``$([System.Environment]::NewLine)$mismatchedMetadata" + } + duplicate_search = $defaultDuplicateSearch + } + $resultMismatchedMetadata = Invoke-Collector -IssueNumber 16 -FixtureRoot $mismatchedMetadataDir -WorkDirectory (Join-Path $tempRoot "mismatched-workflow-metadata") + Assert-Equal -Actual $resultMismatchedMetadata.Dossier.outcome -Expected "incomplete" -Message "Mismatched workflow metadata must fail closed." + Assert-Equal -Actual $resultMismatchedMetadata.Dossier.issue.has_workflow_metadata -Expected $false -Message "Mismatched workflow run IDs must not validate." + # ------------------------------------------------------------------ # The immutable dispatch SHA must be confirmed as a member of main. A deliberately unrelated # current-main SHA must fail closed rather than mislabel the checkout. @@ -470,7 +505,7 @@ try "83" = @([ordered]@{ id = 6100002; sourceVersion = $flagsShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6100001" = @([ordered]@{ id = 1; runId = 7100001; outcome = "Failed"; automatedTestName = $flagsTestName }) @@ -517,6 +552,25 @@ try Assert-Equal -Actual $snapshotB.short_name_referenced -Expected $true -Message "short_name_referenced must record the bare-method-name match." Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $false -Message "known_issue_referenced must stay false for a generic 'Known Issues' heading with no associated number." + $flagsReceiptPath = Join-Path "$tempRoot/check-run-flags" "receipt.json" + & $evaluator -CandidateFile $resultFlags.CandidatePath -EvidenceRoot $resultFlags.EvidenceRoot -OutputFile $flagsReceiptPath -RepositoryRoot $repositoryRoot -CandidateSchemaFile $candidateSchema + $flagsReceipt = Get-Content -LiteralPath $flagsReceiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $flagsReceipt.deterministic_status -Expected "validated" -Message "Collector/evaluator pass-eligibility rules must reconcile." + + $multiplePassRowsDir = New-DerivedFixture -Name "multiple-pass-environments" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_summary.'6100003' = @( + [PSCustomObject]@{ id = 4; runId = 7100004; outcome = "Passed"; automatedTestName = $flagsTestName }, + $fixtureObject.vstmr_summary.'6100003'[0] + ) + $fixtureObject.vstmr_detail | Add-Member -NotePropertyName '7100004:4' -NotePropertyValue ([PSCustomObject]@{ outcome = "Passed"; errorMessage = $null; stackTrace = $null }) + $fixtureObject.vstmr_runs | Add-Member -NotePropertyName '7100004' -NotePropertyValue ([PSCustomObject]@{ name = "Quarantine-Mono-Linux-Release-xunit" }) + } + $resultMultiplePassRows = Invoke-Collector -IssueNumber 11 -FixtureRoot $multiplePassRowsDir -WorkDirectory (Join-Path $tempRoot "multiple-pass-environments") + Assert-Equal -Actual $resultMultiplePassRows.Dossier.outcome -Expected "candidate" -Message "Collector must prefer an environment-matched pass row from a multi-run build." + $selectedPassSource = @($resultMultiplePassRows.Dossier.provenance.raw_evidence_sources | Where-Object { $_.role -eq "negative" })[0] + Assert-Equal -Actual $selectedPassSource.run_id -Expected 7100003 -Message "Collector selected the wrong pass TestRun environment." + $nonMainResult = Invoke-Collector ` -IssueNumber 11 ` -FixtureRoot $flagsDir ` @@ -544,6 +598,14 @@ try Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-platform-unknown" -Message "Unknown platform reason code mismatch." Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." + $differentPassEnvironmentDir = New-DerivedFixture -Name "different-pass-environment" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-Mono-Linux-Release-xunit" + } + $resultDifferentPassEnvironment = Invoke-Collector -IssueNumber 11 -FixtureRoot $differentPassEnvironmentDir -WorkDirectory (Join-Path $tempRoot "different-pass-environment") + Assert-Equal -Actual $resultDifferentPassEnvironment.Dossier.outcome -Expected "incomplete" -Message "A pass from a different environment must not prove intermittency." + Assert-Contains -Collection @($resultDifferentPassEnvironment.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different pass environment reason code mismatch." + $invalidBuildCases = @( [ordered]@{ Name = "wrong-definition" @@ -597,7 +659,7 @@ try "83" = @([ordered]@{ id = 6200002; sourceVersion = $dupShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6200001" = @([ordered]@{ id = 1; runId = 7200001; outcome = "Failed"; automatedTestName = $dupTestName }) @@ -663,7 +725,7 @@ try $resultFixPrUnvalidated = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrUnvalidatedDir -WorkDirectory (Join-Path $tempRoot "fix-pr-unvalidated") Assert-Equal -Actual $resultFixPrUnvalidated.Dossier.candidate.duplicate_check.status -Expected "none" -Message "An exact-FQN fix PR without compatible association must remain unvalidated." - $fixPrValidatedDir = New-DerivedFixture -Name "fix-pr-validated" -Source $dupDir -Mutate { + $fixPrMentionOnlyDir = New-DerivedFixture -Name "fix-pr-mention-only" -Source $dupDir -Mutate { param($fixtureObject) $fixtureObject.duplicate_search.'open-kbe'.result_numbers = @() $fixtureObject.duplicate_search.'open-kbe'.total_count = 0 @@ -671,8 +733,13 @@ try $fixtureObject.duplicate_search.'open-fix-pr'.total_count = 1 $fixtureObject.duplicate_candidate_text.'99999' = "Fix $dupTestName`nRoot cause: $dupSignature" } - $resultFixPrValidated = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrValidatedDir -WorkDirectory (Join-Path $tempRoot "fix-pr-validated") - Assert-Equal -Actual $resultFixPrValidated.Dossier.candidate.duplicate_check.status -Expected "existing-fix-pr" -Message "An exact-FQN fix PR with compatible signature association should validate." + $resultFixPrMentionOnly = Invoke-Collector -IssueNumber 12 -FixtureRoot $fixPrMentionOnlyDir -WorkDirectory (Join-Path $tempRoot "fix-pr-mention-only") + Assert-Equal -Actual $resultFixPrMentionOnly.Dossier.candidate.duplicate_check.status -Expected "none" -Message "FQN/signature co-occurrence alone must not classify a search hit as a fix PR." + $fixPrReason = [string](@($resultFixPrMentionOnly.Dossier.provenance.duplicate_search.unvalidated_candidates | Where-Object { $_.number -eq 99999 })[0].reason) + if (-not $fixPrReason.Contains("closing-link and changed-file relevance")) + { + throw "Unvalidated fix PR must record the unsupported proof limitation." + } # ------------------------------------------------------------------ # Edge case (item 11): a literal ErrorMessage containing '*', '?', and '[' must be matched via @@ -703,7 +770,7 @@ try ) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-07-29T00:00:00Z"; finishTime = "2026-07-29T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6400001" = @([ordered]@{ id = 1; runId = 7400001; outcome = "Failed"; automatedTestName = $wildTestName }) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 index c8de95e2b801..d56684e56107 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -148,7 +148,12 @@ function New-LogEntry "failure-2" { "3333333333333333333333333333333333333333"; break } default { "4444444444444444444444444444444444444444" } } - started_utc = "2026-08-20T12:00:00Z" + started_utc = switch ($Id) + { + "failure-1" { "2026-08-20T12:00:00Z"; break } + "failure-2" { "2026-08-21T12:00:00Z"; break } + default { "2026-08-22T12:00:00Z" } + } status = "completed" result = if ($Role -eq "failure") { "failed" } else { "succeeded" } platform = "Linux" @@ -200,6 +205,40 @@ try Assert-Equal -Actual $receipt.eligible_for_kbe_enrichment -Expected $false -Message "Shadow evaluator must not authorize enrichment from unverified provenance." Assert-Equal -Actual $receipt.evidence_provenance_verified -Expected $false -Message "Shadow evidence provenance must remain unverified." + $logs[2].build.started_utc = "2026-08-19T12:00:00Z" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Predating pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("predate or coincide"))) + { + throw "Predating pass evidence must report the chronology gate." + } + + $logs[2].build.started_utc = "2026-08-22T12:00:00Z" + $logs[2].build.platform = "Windows" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different-environment pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, platform, and configuration"))) + { + throw "Different-environment pass evidence must report the environment gate." + } + $logs[2].build.platform = "Linux" + Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( "[SKIP] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" "Skipped by test infrastructure" @@ -554,6 +593,32 @@ try Assert-Equal -Actual $receipt.deterministic_status -Expected "validated" -Message "Existing KBE status mismatch." Assert-Equal -Actual $receipt.shadow_recommendation -Expected "reuse-existing-kbe" -Message "Existing KBE recommendation mismatch." + Write-Candidate ` + -Path $candidatePath ` + -Signature @($signature) ` + -Logs $logs ` + -DuplicateStatus "existing-fix-pr" ` + -ProposedClassification "quarantine-only" + $existingFixCandidate = Get-Content -LiteralPath $candidatePath -Raw | ConvertFrom-Json -Depth 32 + $existingFixCandidate.duplicate_check.references = @("pull-request:54322") + $existingFixCandidate.duplicate_check.queries[2].result_numbers = @(54322) + $existingFixCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath + + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unsupported existing fix PR status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("closing-link and changed-file relevance"))) + { + throw "Unsupported existing fix PR must report the missing proof." + } + $existingCandidate.duplicate_check.queries[0].result_numbers = @() $existingCandidate | ConvertTo-Json -Depth 32 | Set-Content -LiteralPath $candidatePath diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json index 23f90c873f5d..4c62af4c559c 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -16,7 +16,10 @@ "test-failure", "area-blazor" ], - "has_workflow_marker": true + "actor": "github-actions[bot]", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 32632851798 }, "outcome": "incomplete", "provenance": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json index 9e43486238e6..8c13a0e0b598 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -6,7 +6,10 @@ "test-failure", "area-blazor" ], - "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n```\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n\n" + "body": "## Failing Test(s)\n`Microsoft.AspNetCore.Components.E2ETest.Tests.VirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll` (and its server-execution subclass override, `Microsoft.AspNetCore.Components.E2ETest.ServerExecutionTests.ServerVirtualizationTest.QuickGrid_InitialIndex_TallContainer_NearEnd_FillsViewportWithoutUserScroll`)\n\n## Failure Frequency\nFailed 4 times over the past 30 days.\n\n## Error Message\n```text\nOpenQA.Selenium.BrowserAssertFailedException : Xunit.Sdk.TrueException: Item 950 should remain aligned with the viewport top once the last item has loaded, but top rendered index was -1, scrollTop=0.\n```\n\n## Build\nhttps://dev.azure.com/dnceng-public/public/_build/results?buildId=1563420\n\n\n\n\n", + "user": { + "login": "github-actions[bot]" + } }, "azdo_builds": { "1563420": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index d5b1be2fce01..ef2c8d22e338 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -16,7 +16,10 @@ "test-failure", "area-networking" ], - "has_workflow_marker": true + "actor": "app/github-actions", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 33496438442 }, "outcome": "incomplete", "provenance": { @@ -180,9 +183,10 @@ "candidate": null, "incomplete": { "reason_codes": [ - "raw-evidence-insufficient" + "raw-evidence-insufficient", + "passed-evidence-not-contemporaneous" ], - "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient.", + "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient, passed-evidence-not-contemporaneous.", "missing_evidence": [ { "kind": "vstmr-evidence", @@ -191,6 +195,10 @@ { "kind": "raw-evidence", "detail": "Only 1 distinct build(s) produced retrievable failure evidence; at least 2 are required." + }, + { + "kind": "pass-evidence", + "detail": "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." } ] } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json index 73fa1f61b2db..3445bf7e1103 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -6,7 +6,10 @@ "test-failure", "area-networking" ], - "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n" + "body": "## Failing Test(s)\n- `Interop.FunctionalTests.Http3.Http3RequestTests.POST_ClientCancellationUpload_RequestAbortRaised`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days:\n- Build [1569737](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1569737&view=results) (2026-08-27)\n- Build [1538879](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1538879&view=results) (2026-08-04)\n\nThe failures are separated by many passing runs in between (not back-to-back), so this is flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n\n\n", + "user": { + "login": "app/github-actions" + } }, "azdo_builds": { "1569737": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index 63a1a1a6f94b..b438ff75ed96 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -16,9 +16,12 @@ "test-failure", "area-blazor" ], - "has_workflow_marker": true + "actor": "app/github-actions", + "has_workflow_marker": true, + "has_workflow_metadata": true, + "workflow_run_id": 33496438442 }, - "outcome": "candidate", + "outcome": "incomplete", "provenance": { "repository_ref_verification": { "event_ref": "refs/heads/main", @@ -191,141 +194,22 @@ "unvalidated_candidates": [] } }, - "candidate": { - "schema_version": 1, - "repository": "dotnet/aspnetcore", - "repository_ref": { - "branch": "main", - "commit_sha": "" - }, - "issue": { - "number": 68947, - "url": "https://github.com/dotnet/aspnetcore/issues/68947" - }, - "test": { - "fully_qualified_name": "Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal" - }, - "signature": { - "kind": "ErrorMessage", - "values": [ - "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" - ], - "build_retry": false, - "exclude_console_log": false - }, - "policy": { - "minimum_failure_logs": 2, - "minimum_negative_logs": 1 - }, - "evidence": { - "raw_logs": [ - { - "id": "evidence-1", - "role": "failure", - "outcome": "failed", - "path": "issue-68947-build-1551326-failure.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results", - "sha256": "34cb3881593f491fba3598a0ee26c2e172c0b8e46c40b8e9ac3abde64af85529", - "build": { - "id": 1551326, - "pipeline_definition_id": 87, - "source_branch": "refs/heads/main", - "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", - "started_utc": "2026-08-13T04:18:53.903Z", - "status": "completed", - "result": "partiallySucceeded", - "platform": "Linux", - "configuration": "Release" - } - }, - { - "id": "evidence-2", - "role": "failure", - "outcome": "failed", - "path": "issue-68947-build-1549000-failure.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1549000&view=results", - "sha256": "3772fbff9518a95cfe45f97d7a2f6250cc4af5b43c8d3c370f6d3a8fab8435d0", - "build": { - "id": 1549000, - "pipeline_definition_id": 87, - "source_branch": "refs/heads/main", - "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", - "started_utc": "2026-08-10T09:00:00Z", - "status": "completed", - "result": "partiallySucceeded", - "platform": "Linux", - "configuration": "Release" - } - }, - { - "id": "evidence-3", - "role": "negative", - "outcome": "passed", - "path": "issue-68947-build-1545000-negative.log", - "source_url": "https://dev.azure.com/dnceng-public/public/_build/results?buildId=1545000&view=results", - "sha256": "17e453e2b6e8c8b2e2b16e3dcc2ea16e91bfd37a4c54b2077c52f71e21c54920", - "build": { - "id": 1545000, - "pipeline_definition_id": 87, - "source_branch": "refs/heads/main", - "source_version": "113a606f96ebd832970a1f377748746ff6526abc", - "started_utc": "2026-08-08T09:00:00Z", - "status": "completed", - "result": "succeeded", - "platform": "Linux", - "configuration": "Release" - } - } - ], - "corroborating_context": [ - { - "source": "build-analysis", - "url": "https://github.com/dotnet/aspnetcore/runs/900101" - }, - { - "source": "quarantine-issue", - "url": "https://github.com/dotnet/aspnetcore/issues/68947" - } - ] - }, - "duplicate_check": { - "status": "none", - "checked_utc": "", - "coverage": { - "open_kbes": true, - "recently_closed_kbes": true, - "open_fix_prs": true, - "recently_merged_fix_prs": true + "candidate": null, + "incomplete": { + "reason_codes": [ + "raw-evidence-insufficient", + "passed-evidence-not-contemporaneous" + ], + "message": "Collector could not produce a validated candidate for issue #68947 : raw-evidence-insufficient, passed-evidence-not-contemporaneous.", + "missing_evidence": [ + { + "kind": "azdo-build", + "detail": "Build 1537561 metadata could not be retrieved." }, - "references": [], - "queries": [ - { - "category": "open-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:open label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", - "complete": true, - "result_numbers": [] - }, - { - "category": "recently-closed-kbe", - "query": "repo:dotnet/aspnetcore is:issue is:closed closed:>=2026-06-05 label:\"Known Build Error\" RedirectEnhancedNonBlazorGetToExternal", - "complete": true, - "result_numbers": [] - }, - { - "category": "open-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:open RedirectEnhancedNonBlazorGetToExternal", - "complete": true, - "result_numbers": [] - }, - { - "category": "recently-merged-fix-pr", - "query": "repo:dotnet/aspnetcore is:pr is:merged merged:>=2026-06-05 RedirectEnhancedNonBlazorGetToExternal", - "complete": true, - "result_numbers": [] - } - ] - }, - "proposed_classification": "timeout-needs-classification" - }, - "incomplete": null + { + "kind": "pass-evidence", + "detail": "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." + } + ] + } } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json index c3321b278c65..5f9896843685 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -6,7 +6,10 @@ "test-failure", "area-blazor" ], - "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n" + "body": "## Failing Test(s)\n- `Microsoft.AspNetCore.Components.E2ETests.ServerRenderingTests.RedirectionTest.RedirectEnhancedNonBlazorGetToExternal`\n\n## Details\n\nThis test has failed intermittently on `main` in the past 30 days with a WebDriver/browser-startup timeout signature (`OpenQA.Selenium.WebDriverException` -> `TaskCanceledException`), the same signature that led to the related-but-distinct quarantine of `RedirectStreamingPostToExternal`/`RedirectEnhancedPostToExternal` in #68849:\n- Build [1551326](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1551326&view=results) (2026-08-13)\n- Build [1537561](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1537561&view=results) (2026-08-03)\n\nThe failures are separated by many passing runs, so this is browser/WebDriver flakiness rather than a consistent regression. The test has never previously carried a `[QuarantinedTest]` attribute.\n\n\n\n\n", + "user": { + "login": "app/github-actions" + } }, "azdo_builds": { "1551326": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index 53dfd677be78..7d91e2b98229 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -61,7 +61,10 @@ "url", "state", "labels", - "has_workflow_marker" + "actor", + "has_workflow_marker", + "has_workflow_metadata", + "workflow_run_id" ], "properties": { "number": { @@ -86,9 +89,27 @@ "maxLength": 128 } }, + "actor": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, "has_workflow_marker": { "type": "boolean", - "description": "true when the issue body contains the trusted 'gh-aw-workflow-id: test-quarantine' or 'gh-aw-workflow-call-id: dotnet/aspnetcore/test-quarantine' HTML-comment marker the production quarantine workflow stamps into every issue it creates. The 'test-failure' label alone is not proof an issue was generated by quarantine automation." + "description": "true only when the issue body contains both expected test-quarantine workflow markers." + }, + "has_workflow_metadata": { + "type": "boolean", + "description": "true only when structured gh-aw-agentic-workflow metadata names workflow_id test-quarantine and has matching id/run URL values for a dotnet/aspnetcore Actions run." + }, + "workflow_run_id": { + "type": [ + "integer", + "null" + ], + "minimum": 1 } } }, @@ -461,6 +482,8 @@ "azdo-build-result-incompatible", "raw-evidence-expired", "raw-evidence-insufficient", + "passed-evidence-not-contemporaneous", + "passed-evidence-environment-mismatch", "evidence-platform-unknown", "evidence-configuration-unknown", "recurrence-single-build-only", @@ -660,7 +683,7 @@ "unvalidated_candidates": { "type": "array", "maxItems": 200, - "description": "Search hits that were fetched but could not establish the exact FQN plus compatible KBE signature or fix-PR association. A detail-fetch failure is also recorded here and separately makes the affected query incomplete. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", + "description": "Search hits that were fetched but could not establish the exact FQN plus a compatible KBE signature. Fix PRs are always recorded here until closing-link and changed-file relevance proof is implemented. A detail-fetch failure separately makes the affected query incomplete. Never contributes to 'references' or an existing-kbe/existing-fix-pr status.", "items": { "type": "object", "additionalProperties": false, From 57b6c726b4f59a68d0480f3a901d4aed3f69a0b9 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:28:20 -0500 Subject: [PATCH 08/10] Require interleaved same-leg pass evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 183 ++++++++++++------ .../Evaluate-TestQuarantineKbeCandidate.ps1 | 64 +++--- .../test-quarantine-kbe-shadow/README.md | 29 +-- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 69 ++++++- ...st-Evaluate-TestQuarantineKbeCandidate.ps1 | 40 +++- .../fixtures/68945/expected-dossier.json | 8 +- .../fixtures/68947/expected-dossier.json | 9 +- ...uarantine-kbe-shadow-candidate.schema.json | 7 + ...-quarantine-kbe-shadow-dossier.schema.json | 33 +++- ...-quarantine-kbe-shadow-receipt.schema.json | 7 + 10 files changed, 329 insertions(+), 120 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index 0ecf9c1e2665..156baf47cef4 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -538,9 +538,27 @@ function Get-AzdoRecurrenceCandidateBuilds return Merge-AzdoBuildLists -Lists @($resultLists) -Cap $RecurrenceScanBuildCap } +function Get-AzdoNegativeBuildQueryUri +{ + param( + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MinimumStartTime, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MaximumStartTime, + [Parameter(Mandatory = $true)][int]$Cap + ) + + $minimumTime = [System.Uri]::EscapeDataString($MinimumStartTime.ToUniversalTime().ToString("O")) + $maximumTime = [System.Uri]::EscapeDataString($MaximumStartTime.ToUniversalTime().ToString("O")) + return "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=succeeded&queryOrder=startTimeDescending&minTime=$minimumTime&maxTime=$maximumTime&`$top=$Cap&api-version=7.1" +} + function Get-AzdoNegativeCandidateBuilds { - param([Parameter(Mandatory = $true)][int]$DefinitionId) + param( + [Parameter(Mandatory = $true)][int]$DefinitionId, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MinimumStartTime, + [Parameter(Mandatory = $true)][System.DateTimeOffset]$MaximumStartTime + ) if ($isFixtureMode) { @@ -554,7 +572,12 @@ function Get-AzdoNegativeCandidateBuilds try { - $result = Invoke-RestMethod -Uri "$ado/build/builds?definitions=$DefinitionId&branchName=refs/heads/main&resultFilter=succeeded&`$top=$RecurrenceScanBuildCap&api-version=7.1" -Method Get -TimeoutSec 30 + $uri = Get-AzdoNegativeBuildQueryUri ` + -DefinitionId $DefinitionId ` + -MinimumStartTime $MinimumStartTime ` + -MaximumStartTime $MaximumStartTime ` + -Cap $RecurrenceScanBuildCap + $result = Invoke-RestMethod -Uri $uri -Method Get -TimeoutSec 30 return @($result.value) } catch @@ -688,13 +711,22 @@ function Get-PlatformConfigurationFromRunName { param([AllowNull()][string]$RunName) + $executionLegTokens = [System.Collections.Generic.List[string]]::new() $platform = "unknown" $configuration = "unknown" if ([string]::IsNullOrEmpty($RunName)) { - return [ordered]@{ Platform = $platform; Configuration = $configuration } + return [ordered]@{ ExecutionLeg = "unknown"; Platform = $platform; Configuration = $configuration } } + if ($RunName -match "(?i)\bmono\b") { $null = $executionLegTokens.Add("Mono") } + if ($RunName -match "(?i)\bcoreclr\b") { $null = $executionLegTokens.Add("CoreCLR") } + if ($RunName -match "(?i)\b(?:wasm|webassembly)\b") { $null = $executionLegTokens.Add("WebAssembly") } + if ($RunName -match "(?i)\b(?:chromium|chrome)\b") { $null = $executionLegTokens.Add("Chromium") } + if ($RunName -match "(?i)\bfirefox\b") { $null = $executionLegTokens.Add("Firefox") } + if ($RunName -match "(?i)\bwebkit\b") { $null = $executionLegTokens.Add("WebKit") } + $executionLeg = if ($executionLegTokens.Count -gt 0) { $executionLegTokens -join "+" } else { "unknown" } + if ($RunName -match "(?i)\bwindows\b") { $platform = "Windows" } elseif ($RunName -match "(?i)\blinux\b") { $platform = "Linux" } elseif ($RunName -match "(?i)\b(?:macos|osx)\b") { $platform = "macOS" } @@ -702,7 +734,7 @@ function Get-PlatformConfigurationFromRunName if ($RunName -match "(?i)\bdebug\b") { $configuration = "Debug" } elseif ($RunName -match "(?i)\brelease\b") { $configuration = "Release" } - return [ordered]@{ Platform = $platform; Configuration = $configuration } + return [ordered]@{ ExecutionLeg = $executionLeg; Platform = $platform; Configuration = $configuration } } function Get-CheckRunsForSha @@ -1391,8 +1423,40 @@ if ($null -ne $testName -and $null -ne $effectiveSignature -and $resolvedBuilds. $negativeBuilds = [System.Collections.Generic.List[object]]::new() $negativeBuildCap = [System.Math]::Max(0, [System.Math]::Min($RecurrenceScanBuildCap, 30 - $resolvedBuilds.Count)) +$failureWindowBuilds = @( + if ($null -ne $testName -and $null -ne $effectiveSignature) + { + $resolvedBuilds | + Where-Object { + $null -ne (Get-MatchingFailureDetail -BuildId $_.id -TestName $testName -Signature $effectiveSignature) + } + } +) +$minimumFailureStartTime = if ($failureWindowBuilds.Count -gt 0) +{ + @($failureWindowBuilds | ForEach-Object { + [System.DateTimeOffset]::Parse([string]$_.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + } | Sort-Object)[0] +} +else +{ + $null +} +$maximumFailureStartTime = if ($failureWindowBuilds.Count -gt 0) +{ + @($failureWindowBuilds | ForEach-Object { + [System.DateTimeOffset]::Parse([string]$_.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + } | Sort-Object -Descending)[0] +} +else +{ + $null +} -if ($null -ne $testName -and $null -ne $effectiveSignature) +if ($null -ne $testName -and + $null -ne $effectiveSignature -and + $null -ne $minimumFailureStartTime -and + $null -ne $maximumFailureStartTime) { $negativeScanDefinitionIds = @($resolvedBuilds | ForEach-Object { $_.definition_id } | Select-Object -Unique) if ($negativeScanDefinitionIds.Count -eq 0) @@ -1407,7 +1471,10 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) break } - $candidates = Get-AzdoNegativeCandidateBuilds -DefinitionId $definitionId + $candidates = Get-AzdoNegativeCandidateBuilds ` + -DefinitionId $definitionId ` + -MinimumStartTime $minimumFailureStartTime ` + -MaximumStartTime $maximumFailureStartTime foreach ($candidate in $candidates) { if ($negativeBuilds.Count -ge $negativeBuildCap) @@ -1463,8 +1530,7 @@ if ($null -ne $testName -and $null -ne $effectiveSignature) $rawEvidenceRecords = [System.Collections.Generic.List[object]]::new() $rawLogs = [System.Collections.Generic.List[object]]::new() $failureBuildIdSet = [System.Collections.Generic.HashSet[int]]::new() -$failureEnvironmentKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) -$earliestMaterializedFailureUtc = $null +$materializedFailureOccurrences = [System.Collections.Generic.List[object]]::new() $eligiblePassedBuildIdsDuringCollection = [System.Collections.Generic.HashSet[int]]::new() $evidenceIndex = 0 @@ -1511,11 +1577,12 @@ foreach ($build in $evidenceBuilds) $candidateRunName = Get-VstmrRunName -RunId ([int]$row.runId) $candidatePlatformConfiguration = Get-PlatformConfigurationFromRunName -RunName $candidateRunName - $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" + $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.ExecutionLeg)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" $candidateStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - if ($null -ne $earliestMaterializedFailureUtc -and - $candidateStartedUtc -gt $earliestMaterializedFailureUtc -and - $failureEnvironmentKeys.Contains($candidateEnvironmentKey)) + $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $candidateEnvironmentKey }) + $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $candidateStartedUtc }).Count -gt 0 + $hasFailureAfter = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -gt $candidateStartedUtc }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) { $matchedRow = $row $matchedDetail = $detail @@ -1591,6 +1658,11 @@ foreach ($build in $evidenceBuilds) $reasonCodes.Add("evidence-configuration-unknown") Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown configuration from TestRun '$runName'." } + if ($platformConfiguration.ExecutionLeg -eq "unknown") + { + $reasonCodes.Add("evidence-execution-leg-unknown") + Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown execution leg from TestRun '$runName'." + } $evidenceIndex += 1 $fileName = "issue-$IssueNumber-build-$($build.id)-$role.log" @@ -1623,6 +1695,7 @@ foreach ($build in $evidenceBuilds) run_id = $runId result_id = $resultId helix_unavailable = $helixUnavailable + execution_leg = $platformConfiguration.ExecutionLeg platform = $platformConfiguration.Platform configuration = $platformConfiguration.Configuration found = $true @@ -1640,14 +1713,14 @@ foreach ($build in $evidenceBuilds) if ($role -eq "failure") { $null = $failureBuildIdSet.Add([int]$build.id) - if ($platformConfiguration.Platform -ne "unknown" -and $platformConfiguration.Configuration -ne "unknown") + if ($platformConfiguration.ExecutionLeg -ne "unknown" -and + $platformConfiguration.Platform -ne "unknown" -and + $platformConfiguration.Configuration -ne "unknown") { - $null = $failureEnvironmentKeys.Add("$($build.definition_id)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)") - } - $failureStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - if ($null -eq $earliestMaterializedFailureUtc -or $failureStartedUtc -lt $earliestMaterializedFailureUtc) - { - $earliestMaterializedFailureUtc = $failureStartedUtc + $null = $materializedFailureOccurrences.Add([ordered]@{ + EnvironmentKey = "$($build.definition_id)|$($platformConfiguration.ExecutionLeg)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + StartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + }) } } $outcomeValue = switch ([string]$matchedRow.outcome) @@ -1672,6 +1745,7 @@ foreach ($build in $evidenceBuilds) started_utc = [string]$build.started_utc status = [string]$build.status result = [string]$build.result + execution_leg = $platformConfiguration.ExecutionLeg platform = $platformConfiguration.Platform configuration = $platformConfiguration.Configuration } @@ -1680,10 +1754,11 @@ foreach ($build in $evidenceBuilds) if ($role -eq "negative") { $passStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" - if ($null -ne $earliestMaterializedFailureUtc -and - $passStartedUtc -gt $earliestMaterializedFailureUtc -and - $failureEnvironmentKeys.Contains($passEnvironmentKey)) + $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.ExecutionLeg)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $passEnvironmentKey }) + $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $passStartedUtc }).Count -gt 0 + $hasFailureAfter = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -gt $passStartedUtc }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) { $null = $eligiblePassedBuildIdsDuringCollection.Add([int]$build.id) } @@ -1702,36 +1777,34 @@ if ($null -ne $testName -and $failureBuildIdSet.Count -lt $minimumFailureBuilds) $failureLogsForPassEligibility = @($rawLogs | Where-Object { $_.role -eq "failure" }) $passedLogsForEligibility = @($rawLogs | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) $eligiblePassedBuildIds = [System.Collections.Generic.HashSet[int]]::new() -$passesAfterEarliestFailure = 0 -if ($failureLogsForPassEligibility.Count -gt 0) +$passesWithMatchingEnvironment = 0 +foreach ($passLog in $passedLogsForEligibility) { - $earliestFailureUtc = @( + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $matchingFailureLogs = @( $failureLogsForPassEligibility | - ForEach-Object { [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) } | - Sort-Object - )[0] - - foreach ($passLog in $passedLogsForEligibility) + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.execution_leg -eq [string]$passLog.build.execution_leg -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ) + if ($matchingFailureLogs.Count -eq 0) { - $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - if ($passStartedUtc -le $earliestFailureUtc) - { - continue - } + continue + } - $passesAfterEarliestFailure++ - $environmentMatched = @( - $failureLogsForPassEligibility | - Where-Object { - [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and - [string]$_.build.platform -eq [string]$passLog.build.platform -and - [string]$_.build.configuration -eq [string]$passLog.build.configuration - } - ).Count -gt 0 - if ($environmentMatched) - { - $null = $eligiblePassedBuildIds.Add([int]$passLog.build.id) - } + $passesWithMatchingEnvironment++ + $hasFailureBefore = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -lt $passStartedUtc + }).Count -gt 0 + $hasFailureAfter = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -gt $passStartedUtc + }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $null = $eligiblePassedBuildIds.Add([int]$passLog.build.id) } } @@ -1740,15 +1813,15 @@ if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeL $reasonCodes.Add("raw-evidence-insufficient") if ($failureLogsForPassEligibility.Count -gt 0 -and $passedLogsForEligibility.Count -gt 0 -and - $passesAfterEarliestFailure -eq 0) + $passesWithMatchingEnvironment -eq 0) { - $reasonCodes.Add("passed-evidence-not-contemporaneous") - Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." + $reasonCodes.Add("passed-evidence-environment-mismatch") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence shared pipeline definition, execution leg, platform, and configuration with any collected failure." } - elseif ($passesAfterEarliestFailure -gt 0) + elseif ($passesWithMatchingEnvironment -gt 0) { - $reasonCodes.Add("passed-evidence-environment-mismatch") - Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence after the earliest failure shared pipeline definition, platform, and configuration with any collected failure." + $reasonCodes.Add("passed-evidence-not-interleaved") + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." } else { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 index 6ad16eb9be95..bb9e7ad911db 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -556,6 +556,10 @@ foreach ($log in $candidate.evidence.raw_logs) { $incompleteReasons.Add("Evidence log '$($log.id)' has unknown configuration; exact environment dimensions are required.") } + if ([string]$log.build.execution_leg -eq "unknown") + { + $incompleteReasons.Add("Evidence log '$($log.id)' has unknown execution leg; exact environment dimensions are required.") + } $passOrSkipCollisionCount += [int]$match.pass_or_skip_match_count @@ -581,37 +585,35 @@ foreach ($log in $candidate.evidence.raw_logs) $failureLogsForPassEligibility = @($logResults | Where-Object { $_.role -eq "failure" }) $passedLogsForEligibility = @($logResults | Where-Object { $_.role -eq "negative" -and $_.outcome -eq "passed" }) -$passesAfterEarliestFailure = 0 -if ($failureLogsForPassEligibility.Count -gt 0) +$passesWithMatchingEnvironment = 0 +foreach ($passLog in $passedLogsForEligibility) { - $earliestFailureUtc = @( + $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) + $matchingFailureLogs = @( $failureLogsForPassEligibility | - ForEach-Object { [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) } | - Sort-Object - )[0] - - foreach ($passLog in $passedLogsForEligibility) + Where-Object { + [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and + [string]$_.build.execution_leg -eq [string]$passLog.build.execution_leg -and + [string]$_.build.platform -eq [string]$passLog.build.platform -and + [string]$_.build.configuration -eq [string]$passLog.build.configuration + } + ) + if ($matchingFailureLogs.Count -eq 0) { - $passStartedUtc = [System.DateTimeOffset]::Parse([string]$passLog.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - if ($passStartedUtc -le $earliestFailureUtc) - { - continue - } + continue + } - $passesAfterEarliestFailure++ - $environmentMatched = @( - $failureLogsForPassEligibility | - Where-Object { - [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and - [string]$_.build.platform -eq [string]$passLog.build.platform -and - [string]$_.build.configuration -eq [string]$passLog.build.configuration - } - ).Count -gt 0 - if ($environmentMatched) - { - $null = $negativeHashes.Add([string]$passLog.sha256) - $null = $negativeBuildIds.Add([int]$passLog.build.id) - } + $passesWithMatchingEnvironment++ + $hasFailureBefore = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -lt $passStartedUtc + }).Count -gt 0 + $hasFailureAfter = @($matchingFailureLogs | Where-Object { + [System.DateTimeOffset]::Parse([string]$_.build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) -gt $passStartedUtc + }).Count -gt 0 + if ($hasFailureBefore -and $hasFailureAfter) + { + $null = $negativeHashes.Add([string]$passLog.sha256) + $null = $negativeBuildIds.Add([int]$passLog.build.id) } } @@ -637,13 +639,13 @@ if ($negativeHashes.Count -lt $requiredNegativeLogs) $incompleteReasons.Add("Only $($negativeHashes.Count) distinct authoritative Passed log(s) were supplied; $requiredNegativeLogs are required.") if ($failureLogsForPassEligibility.Count -gt 0 -and $passedLogsForEligibility.Count -gt 0 -and - $passesAfterEarliestFailure -eq 0) + $passesWithMatchingEnvironment -eq 0) { - $incompleteReasons.Add("All authoritative Passed occurrences predate or coincide with the earliest failure.") + $incompleteReasons.Add("No authoritative Passed occurrence matched a failure's pipeline definition, execution leg, platform, and configuration.") } - elseif ($passesAfterEarliestFailure -gt 0) + elseif ($passesWithMatchingEnvironment -gt 0) { - $incompleteReasons.Add("No authoritative Passed occurrence after the earliest failure matched a failure's pipeline definition, platform, and configuration.") + $incompleteReasons.Add("No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure.") } } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index 5c9e74ae30d0..ab2c3b1b85fb 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -51,10 +51,15 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis pseudo-test row carries one); otherwise `helix_unavailable: true` is recorded explicitly and the VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence requires evidence from **at least two distinct builds** (a single build producing two separate - artifacts is not recurrence), and at least one authoritative **Passed** occurrence that started - after the earliest collected failure and shares pipeline definition, platform, and configuration - with a collected failure. An older pass or a pass from another environment does not prove the - current failure is intermittent. Signature matching against raw text uses ordinal, + artifacts is not recurrence), and at least one authoritative **Passed** occurrence strictly + between an earlier and a later failure in the same pipeline definition, normalized execution leg, + platform, and configuration. A pass before all failures or after the last failure could represent + a not-yet-failing or already-fixed regression, so neither proves active intermittency. Ordering + currently uses strict Azure DevOps build `started_utc` timestamps; commit ancestry would be + stronger but is not available in the bounded offline evidence contract. The live Passed scan is + bounded to the earliest/latest authoritative failure start times with + `queryOrder=startTimeDescending`, rather than sampling newer builds that cannot be interleaved. + Signature matching against raw text uses ordinal, case-sensitive substring containment (`[string]::Contains(..., Ordinal)`) throughout -- never PowerShell's `-like`/`-notlike` operators, whose `*`, `?`, and `[...]` wildcard semantics would otherwise silently misinterpret a literal ErrorMessage containing any of those characters. @@ -102,12 +107,14 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis `sourceBranch` must be exactly `refs/heads/main`, status must be `completed`, and failure builds must be `failed` or `partiallySucceeded` (`succeeded` is required for the Passed scan). These dimensions are recorded in build provenance and candidate evidence. -* **Platform/configuration are derived from authoritative metadata, never fabricated.** Azure +* **Execution leg/platform/configuration are derived from authoritative metadata, never + fabricated.** Azure DevOps' `buildConfiguration.platform`/`.flavor` fields are empty strings on every real run observed live; the only authoritative, cheaply-available signal is the VSTMR TestRun's own `name` (e.g. `Quarantine-Mono-Linux-Release-xunit`). The collector parses recognized - platform/configuration tokens out of that name. A counted failure or pass with either dimension - `"unknown"` emits explicit missing-evidence codes and prevents a candidate/validated receipt. + execution-leg (`Mono`, `CoreCLR`, WebAssembly/browser variants), platform, and configuration + tokens out of that name. A counted failure or pass with any dimension `"unknown"` emits explicit + missing-evidence codes and prevents a candidate/validated receipt. * **Never infer a pass, a recurrence, a signature, a platform/configuration, or a validated duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps metadata has aged out of retention, historical VSTMR test-result data no longer queryable for a @@ -212,8 +219,8 @@ callers pass trusted `github.ref`/`github.sha` through step environment bindings | Issue | Fixture invocation | Outcome | |---|---|---| | [#68724](https://github.com/dotnet/aspnetcore/issues/68724) | no `-Signature` | `incomplete`: `## Failing Test(s)` names two distinct concrete identities (a base test and its server-execution subclass override); live data shows only the override actually failed, so the collector fails closed (`multiple-test-identities-unresolved`) rather than guess | -| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `incomplete`: two failures recur, but the only authoritative pass is older than both failures, so it cannot establish current intermittency (`passed-evidence-not-contemporaneous`) | -| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's Azure DevOps build record still resolves, but its historical VSTMR test-result data is no longer queryable, leaving only one usable failure log below the two-build recurrence floor (`raw-evidence-insufficient`) | +| [#68947](https://github.com/dotnet/aspnetcore/issues/68947) | `-Signature "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server"` (the real issue body has no fenced `## Error Message`, so extraction is deterministically ambiguous without the override -- verified by a second, signature-less invocation of the same fixture in the test suite) | `incomplete`: two failures recur, but the only authoritative pass is older than both failures, so no pass is strictly interleaved between failures (`passed-evidence-not-interleaved`) | +| [#68945](https://github.com/dotnet/aspnetcore/issues/68945) | `-Signature "System.Threading.Tasks.TaskCanceledException: The operation was canceled."` | `incomplete`: the second cited build's historical VSTMR result is unavailable, leaving one usable failure; the remaining pass is not strictly interleaved around two failures (`raw-evidence-insufficient`, `passed-evidence-not-interleaved`) | Each fixture directory also has an `expected-dossier.json` golden file used for deep-equality comparison in the test suite. Golden comparisons exclude the collector's `generated_utc` / @@ -225,8 +232,8 @@ sides before comparing. The test suite additionally covers, via small synthetic (non-pilot) fixtures: a closed issue, an issue missing the `test-failure` label, missing or forged workflow provenance, immutable dispatch ancestry and non-main rejection, strict build definition/ref/status/result gates, skip-only, -predating, and environment-mismatched pass evidence, unknown environment dimensions, Build -Analysis flag precision, compatible and incompatible same-FQN KBE signatures, failed +pre-first, post-last, and environment/execution-leg-mismatched pass evidence, valid interleaving, +unknown environment dimensions, Build Analysis flag precision, compatible and incompatible same-FQN KBE signatures, failed duplicate-detail fetches, deliberately unvalidated fix-PR mentions, wildcard-shaped literal signatures, and build-list merge/dedupe. diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index 472b5a6dc3d1..145205d65d65 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -341,14 +341,14 @@ try $signature68947 = "OpenQA.Selenium.WebDriverException : The HTTP request to the remote WebDriver server" $result68947 = Invoke-Collector -IssueNumber 68947 -FixtureRoot "$fixturesRoot/68947" -WorkDirectory "$tempRoot/68947" -Signature $signature68947 Assert-Equal -Actual $result68947.Dossier.outcome -Expected "incomplete" -Message "#68947 outcome mismatch." - Assert-Contains -Collection @($result68947.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-contemporaneous" -Message "#68947 must not count its pass that predates both collected failures." + Assert-Contains -Collection @($result68947.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-interleaved" -Message "#68947 must not count its pass that predates both collected failures." Assert-Equal -Actual $result68947.Dossier.candidate -Expected $null -Message "#68947 must not emit a candidate without a contemporaneous environment-matched pass." Assert-GoldenDossier -IssueDirectory "$fixturesRoot/68947" -ActualDossier $result68947.Dossier $summaryPath68947 = Join-Path "$tempRoot/68947" "summary.md" & $summaryGenerator -DossierFile $result68947.DossierPath -OutputFile $summaryPath68947 $summaryText68947 = Get-Content -LiteralPath $summaryPath68947 -Raw - if (-not $summaryText68947.Contains("passed-evidence-not-contemporaneous")) + if (-not $summaryText68947.Contains("passed-evidence-not-interleaved")) { throw "#68947 summary must mention the chronology failure." } @@ -505,7 +505,7 @@ try "83" = @([ordered]@{ id = 6100002; sourceVersion = $flagsShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6100003; sourceVersion = $flagsShaC; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6100001" = @([ordered]@{ id = 1; runId = 7100001; outcome = "Failed"; automatedTestName = $flagsTestName }) @@ -598,6 +598,22 @@ try Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-platform-unknown" -Message "Unknown platform reason code mismatch." Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." + $unknownExecutionLegDir = New-DerivedFixture -Name "unknown-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Windows-Debug-xunit" + } + $resultUnknownExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-execution-leg") + Assert-Equal -Actual $resultUnknownExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown required execution leg must fail closed." + Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-execution-leg-unknown" -Message "Unknown execution leg reason code mismatch." + + $unknownPassExecutionLegDir = New-DerivedFixture -Name "unknown-pass-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-Windows-Debug-xunit" + } + $resultUnknownPassExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownPassExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-pass-execution-leg") + Assert-Equal -Actual $resultUnknownPassExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown pass execution leg must fail closed." + Assert-Contains -Collection @($resultUnknownPassExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-execution-leg-unknown" -Message "Unknown pass execution leg reason code mismatch." + $differentPassEnvironmentDir = New-DerivedFixture -Name "different-pass-environment" -Source $flagsDir -Mutate { param($fixtureObject) $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-Mono-Linux-Release-xunit" @@ -606,6 +622,32 @@ try Assert-Equal -Actual $resultDifferentPassEnvironment.Dossier.outcome -Expected "incomplete" -Message "A pass from a different environment must not prove intermittency." Assert-Contains -Collection @($resultDifferentPassEnvironment.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different pass environment reason code mismatch." + $differentExecutionLegDir = New-DerivedFixture -Name "different-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-CoreCLR-Windows-Debug-xunit" + } + $resultDifferentExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $differentExecutionLegDir -WorkDirectory (Join-Path $tempRoot "different-execution-leg") + Assert-Equal -Actual $resultDifferentExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A CoreCLR pass must not prove Mono failures intermittent." + Assert-Contains -Collection @($resultDifferentExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different execution leg reason code mismatch." + + $compositeExecutionLegDir = New-DerivedFixture -Name "composite-execution-leg" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Mono-WebAssembly-Windows-Debug-xunit" + $fixtureObject.vstmr_runs.'7100002'.name = "Quarantine-Mono-WebAssembly-Windows-Debug-xunit" + } + $resultCompositeExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $compositeExecutionLegDir -WorkDirectory (Join-Path $tempRoot "composite-execution-leg") + Assert-Equal -Actual $resultCompositeExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A Mono pass must not match a distinct Mono+WebAssembly failure leg." + Assert-Contains -Collection @($resultCompositeExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Composite execution leg reason code mismatch." + + $passAfterLastDir = New-DerivedFixture -Name "pass-after-last-failure" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.negative_scan.'83'[0].startTime = "2026-08-02T00:00:00Z" + $fixtureObject.negative_scan.'83'[0].finishTime = "2026-08-02T01:00:00Z" + } + $resultPassAfterLast = Invoke-Collector -IssueNumber 11 -FixtureRoot $passAfterLastDir -WorkDirectory (Join-Path $tempRoot "pass-after-last-failure") + Assert-Equal -Actual $resultPassAfterLast.Dossier.outcome -Expected "incomplete" -Message "A pass after the last failure must not prove active intermittency." + Assert-Contains -Collection @($resultPassAfterLast.Dossier.incomplete.reason_codes) -Value "passed-evidence-not-interleaved" -Message "Pass-after-last reason code mismatch." + $invalidBuildCases = @( [ordered]@{ Name = "wrong-definition" @@ -659,7 +701,7 @@ try "83" = @([ordered]@{ id = 6200002; sourceVersion = $dupShaB; startTime = "2026-07-30T00:00:00Z"; finishTime = "2026-07-30T01:00:00Z"; result = "failed" }) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6200003; sourceVersion = $dupShaC; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6200001" = @([ordered]@{ id = 1; runId = 7200001; outcome = "Failed"; automatedTestName = $dupTestName }) @@ -770,7 +812,7 @@ try ) } negative_scan = [ordered]@{ - "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-08-02T00:00:00Z"; finishTime = "2026-08-02T01:00:00Z"; result = "succeeded" }) + "83" = @([ordered]@{ id = 6400003; sourceVersion = $wildShaNeg; startTime = "2026-07-31T00:00:00Z"; finishTime = "2026-07-31T01:00:00Z"; result = "succeeded" }) } vstmr_summary = [ordered]@{ "6400001" = @([ordered]@{ id = 1; runId = 7400001; outcome = "Failed"; automatedTestName = $wildTestName }) @@ -830,6 +872,23 @@ try $emptyMerged = @(Merge-AzdoBuildLists -Lists @(@(), @()) -Cap 5) Assert-Equal -Actual $emptyMerged.Count -Expected 0 -Message "Merge-AzdoBuildLists must return a real empty array (not collapse to null) when both inputs are empty." + $windowedQueryUri = [System.Uri]::UnescapeDataString((Get-AzdoNegativeBuildQueryUri ` + -DefinitionId 83 ` + -MinimumStartTime ([System.DateTimeOffset]"2026-07-30T00:00:00Z") ` + -MaximumStartTime ([System.DateTimeOffset]"2026-08-01T00:00:00Z") ` + -Cap 20)) + foreach ($expectedQueryPart in @( + "queryOrder=startTimeDescending", + "minTime=2026-07-30T00:00:00.0000000+00:00", + "maxTime=2026-08-01T00:00:00.0000000+00:00", + "`$top=20")) + { + if (-not $windowedQueryUri.Contains($expectedQueryPart, [System.StringComparison]::Ordinal)) + { + throw "Windowed Passed-build query is missing '$expectedQueryPart': $windowedQueryUri" + } + } + Write-Host "All test-quarantine-kbe-shadow collector tests passed." } finally diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 index d56684e56107..eca4f6261b06 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -152,10 +152,11 @@ function New-LogEntry { "failure-1" { "2026-08-20T12:00:00Z"; break } "failure-2" { "2026-08-21T12:00:00Z"; break } - default { "2026-08-22T12:00:00Z" } + default { "2026-08-20T18:00:00Z" } } status = "completed" result = if ($Role -eq "failure") { "failed" } else { "succeeded" } + execution_leg = "Mono" platform = "Linux" configuration = "Release" } @@ -216,13 +217,12 @@ try -ReceiptSchemaFile $receiptSchema $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Predating pass evidence status mismatch." - if (-not (($receipt.reasons -join "`n").Contains("predate or coincide"))) + if (-not (($receipt.reasons -join "`n").Contains("strictly between"))) { - throw "Predating pass evidence must report the chronology gate." + throw "Predating pass evidence must report the interleaving gate." } $logs[2].build.started_utc = "2026-08-22T12:00:00Z" - $logs[2].build.platform = "Windows" Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs & $evaluator ` -CandidateFile $candidatePath ` @@ -232,12 +232,29 @@ try -CandidateSchemaFile $candidateSchema ` -ReceiptSchemaFile $receiptSchema $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 - Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different-environment pass evidence status mismatch." - if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, platform, and configuration"))) + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Pass-after-last-failure status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("strictly between"))) { - throw "Different-environment pass evidence must report the environment gate." + throw "Pass-after-last evidence must report the interleaving gate." } - $logs[2].build.platform = "Linux" + + $logs[2].build.started_utc = "2026-08-20T18:00:00Z" + $logs[2].build.execution_leg = "CoreCLR" + Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs + & $evaluator ` + -CandidateFile $candidatePath ` + -EvidenceRoot $tempRoot ` + -OutputFile $receiptPath ` + -RepositoryRoot $repositoryRoot ` + -CandidateSchemaFile $candidateSchema ` + -ReceiptSchemaFile $receiptSchema + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different execution-leg pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, execution leg, platform, and configuration"))) + { + throw "Mono-failure/CoreCLR-pass evidence must report the environment gate." + } + $logs[2].build.execution_leg = "Mono" Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( "[SKIP] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" @@ -267,6 +284,7 @@ try $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() $logs[0].build.platform = "unknown" $logs[0].build.configuration = "unknown" + $logs[0].build.execution_leg = "unknown" Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs & $evaluator ` @@ -280,12 +298,14 @@ try $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unknown environment evidence status mismatch." if (-not (($receipt.reasons -join "`n").Contains("unknown platform")) -or - -not (($receipt.reasons -join "`n").Contains("unknown configuration"))) + -not (($receipt.reasons -join "`n").Contains("unknown configuration")) -or + -not (($receipt.reasons -join "`n").Contains("unknown execution leg"))) { - throw "Unknown environment evidence must report both missing dimensions." + throw "Unknown environment evidence must report every missing dimension." } $logs[0].build.platform = "Linux" $logs[0].build.configuration = "Release" + $logs[0].build.execution_leg = "Mono" Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( "Starting an unrelated test" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index ef2c8d22e338..40c035ebebd0 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -108,6 +108,7 @@ "run_id": 72708311, "result_id": 400010, "helix_unavailable": true, + "execution_leg": "CoreCLR", "platform": "Linux", "configuration": "Release", "found": true, @@ -129,6 +130,7 @@ "run_id": 72708313, "result_id": 400030, "helix_unavailable": true, + "execution_leg": "CoreCLR", "platform": "Linux", "configuration": "Release", "found": true, @@ -184,9 +186,9 @@ "incomplete": { "reason_codes": [ "raw-evidence-insufficient", - "passed-evidence-not-contemporaneous" + "passed-evidence-not-interleaved" ], - "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient, passed-evidence-not-contemporaneous.", + "message": "Collector could not produce a validated candidate for issue #68945 : raw-evidence-insufficient, passed-evidence-not-interleaved.", "missing_evidence": [ { "kind": "vstmr-evidence", @@ -198,7 +200,7 @@ }, { "kind": "pass-evidence", - "detail": "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." + "detail": "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." } ] } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index b438ff75ed96..572268cb450c 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -115,6 +115,7 @@ "run_id": 42708308, "result_id": 100014, "helix_unavailable": true, + "execution_leg": "Mono", "platform": "Linux", "configuration": "Release", "found": true, @@ -129,6 +130,7 @@ "run_id": 52708309, "result_id": 200055, "helix_unavailable": true, + "execution_leg": "Mono", "platform": "Linux", "configuration": "Release", "found": true, @@ -143,6 +145,7 @@ "run_id": 62708310, "result_id": 300099, "helix_unavailable": true, + "execution_leg": "Mono", "platform": "Linux", "configuration": "Release", "found": true, @@ -198,9 +201,9 @@ "incomplete": { "reason_codes": [ "raw-evidence-insufficient", - "passed-evidence-not-contemporaneous" + "passed-evidence-not-interleaved" ], - "message": "Collector could not produce a validated candidate for issue #68947 : raw-evidence-insufficient, passed-evidence-not-contemporaneous.", + "message": "Collector could not produce a validated candidate for issue #68947 : raw-evidence-insufficient, passed-evidence-not-interleaved.", "missing_evidence": [ { "kind": "azdo-build", @@ -208,7 +211,7 @@ }, { "kind": "pass-evidence", - "detail": "All authoritative Passed occurrences predate or coincide with the earliest collected failure; a counted pass must start after the earliest failure." + "detail": "No environment-matched Passed occurrence was strictly between an earlier and a later authoritative failure." } ] } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json index b122263639fb..103b6dba822f 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -195,6 +195,7 @@ "started_utc", "status", "result", + "execution_leg", "platform", "configuration" ], @@ -230,6 +231,12 @@ "succeeded" ] }, + "execution_leg": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, "platform": { "type": "string", "minLength": 1, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index 7d91e2b98229..0100274e6bf5 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -407,6 +407,10 @@ "type": "string", "maxLength": 256 }, + "execution_leg": { + "type": "string", + "maxLength": 128 + }, "platform": { "type": "string", "maxLength": 128 @@ -433,7 +437,31 @@ "type": "string", "maxLength": 512 } - } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "kind", + "run_id", + "result_id", + "helix_unavailable", + "execution_leg", + "platform", + "configuration", + "sha256", + "evidence_path" + ] + } + } + ] } }, "duplicate_search": { @@ -482,8 +510,9 @@ "azdo-build-result-incompatible", "raw-evidence-expired", "raw-evidence-insufficient", - "passed-evidence-not-contemporaneous", + "passed-evidence-not-interleaved", "passed-evidence-environment-mismatch", + "evidence-execution-leg-unknown", "evidence-platform-unknown", "evidence-configuration-unknown", "recurrence-single-build-only", diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json index 8d51fe139246..04aa92537166 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -572,6 +572,7 @@ "started_utc", "status", "result", + "execution_leg", "platform", "configuration" ], @@ -607,6 +608,12 @@ "succeeded" ] }, + "execution_leg": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[^\\r\\n]+$" + }, "platform": { "type": "string", "minLength": 1, From bd951d0c1da5a1c0985fe1f8a1b7110781aca510 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:44:30 -0500 Subject: [PATCH 09/10] Use canonical Azure TestRun identities Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 60 ++++++++++--------- .../Evaluate-TestQuarantineKbeCandidate.ps1 | 8 +-- .../test-quarantine-kbe-shadow/README.md | 31 ++++++---- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 45 ++++++++++---- ...st-Evaluate-TestQuarantineKbeCandidate.ps1 | 16 ++--- .../fixtures/68945/expected-dossier.json | 12 ++-- .../fixtures/68945/fixture.json | 4 +- .../fixtures/68947/expected-dossier.json | 6 +- ...uarantine-kbe-shadow-candidate.schema.json | 4 +- ...-quarantine-kbe-shadow-dossier.schema.json | 6 +- ...-quarantine-kbe-shadow-receipt.schema.json | 4 +- 11 files changed, 115 insertions(+), 81 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index 156baf47cef4..92df9063a830 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -707,34 +707,36 @@ function Get-VstmrRunName return $name } -function Get-PlatformConfigurationFromRunName +function Get-TestRunEnvironmentFromName { param([AllowNull()][string]$RunName) - $executionLegTokens = [System.Collections.Generic.List[string]]::new() + $testRunIdentity = "unknown" $platform = "unknown" $configuration = "unknown" if ([string]::IsNullOrEmpty($RunName)) { - return [ordered]@{ ExecutionLeg = "unknown"; Platform = $platform; Configuration = $configuration } + return [ordered]@{ TestRunIdentity = $testRunIdentity; Platform = $platform; Configuration = $configuration } } - if ($RunName -match "(?i)\bmono\b") { $null = $executionLegTokens.Add("Mono") } - if ($RunName -match "(?i)\bcoreclr\b") { $null = $executionLegTokens.Add("CoreCLR") } - if ($RunName -match "(?i)\b(?:wasm|webassembly)\b") { $null = $executionLegTokens.Add("WebAssembly") } - if ($RunName -match "(?i)\b(?:chromium|chrome)\b") { $null = $executionLegTokens.Add("Chromium") } - if ($RunName -match "(?i)\bfirefox\b") { $null = $executionLegTokens.Add("Firefox") } - if ($RunName -match "(?i)\bwebkit\b") { $null = $executionLegTokens.Add("WebKit") } - $executionLeg = if ($executionLegTokens.Count -gt 0) { $executionLegTokens -join "+" } else { "unknown" } + $normalizedRunName = $RunName.Trim() + $normalizedRunName = [regex]::Replace($normalizedRunName, '(?i)((?:xunit|js|open))_[1-9][0-9]*$', '$1') + $hasKnownRunFamily = + $normalizedRunName -match '(?i)(?:^|[-.])(?:xunit|js|open)(?:$|[-.])' + if ($hasKnownRunFamily) + { + $testRunIdentity = $normalizedRunName.ToLowerInvariant() + } - if ($RunName -match "(?i)\bwindows\b") { $platform = "Windows" } - elseif ($RunName -match "(?i)\blinux\b") { $platform = "Linux" } - elseif ($RunName -match "(?i)\b(?:macos|osx)\b") { $platform = "macOS" } + if ($normalizedRunName -match "(?i)\bwindows\b") { $platform = "Windows" } + elseif ($normalizedRunName -match "(?i)\b(?:linux|ubuntu)\b") { $platform = "Linux" } + elseif ($normalizedRunName -match "(?i)\b(?:macos|osx)\b") { $platform = "macOS" } - if ($RunName -match "(?i)\bdebug\b") { $configuration = "Debug" } - elseif ($RunName -match "(?i)\brelease\b") { $configuration = "Release" } + if ($normalizedRunName -match "(?i)\bdebug\b") { $configuration = "Debug" } + elseif ($normalizedRunName -match "(?i)\brelease\b") { $configuration = "Release" } + elseif ($hasKnownRunFamily) { $configuration = "not-encoded" } - return [ordered]@{ ExecutionLeg = $executionLeg; Platform = $platform; Configuration = $configuration } + return [ordered]@{ TestRunIdentity = $testRunIdentity; Platform = $platform; Configuration = $configuration } } function Get-CheckRunsForSha @@ -1576,8 +1578,8 @@ foreach ($build in $evidenceBuilds) } $candidateRunName = Get-VstmrRunName -RunId ([int]$row.runId) - $candidatePlatformConfiguration = Get-PlatformConfigurationFromRunName -RunName $candidateRunName - $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.ExecutionLeg)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" + $candidatePlatformConfiguration = Get-TestRunEnvironmentFromName -RunName $candidateRunName + $candidateEnvironmentKey = "$($build.definition_id)|$($candidatePlatformConfiguration.TestRunIdentity)|$($candidatePlatformConfiguration.Platform)|$($candidatePlatformConfiguration.Configuration)" $candidateStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $candidateEnvironmentKey }) $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $candidateStartedUtc }).Count -gt 0 @@ -1646,7 +1648,7 @@ foreach ($build in $evidenceBuilds) } else { - Get-PlatformConfigurationFromRunName -RunName $runName + Get-TestRunEnvironmentFromName -RunName $runName } if ($platformConfiguration.Platform -eq "unknown") { @@ -1658,10 +1660,10 @@ foreach ($build in $evidenceBuilds) $reasonCodes.Add("evidence-configuration-unknown") Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown configuration from TestRun '$runName'." } - if ($platformConfiguration.ExecutionLeg -eq "unknown") + if ($platformConfiguration.TestRunIdentity -eq "unknown") { - $reasonCodes.Add("evidence-execution-leg-unknown") - Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has unknown execution leg from TestRun '$runName'." + $reasonCodes.Add("evidence-test-run-identity-unknown") + Add-MissingEvidence -List $missingEvidence -Kind "environment" -Detail "Build $($build.id) $role evidence has no canonical TestRun identity from '$runName'." } $evidenceIndex += 1 @@ -1695,7 +1697,7 @@ foreach ($build in $evidenceBuilds) run_id = $runId result_id = $resultId helix_unavailable = $helixUnavailable - execution_leg = $platformConfiguration.ExecutionLeg + test_run_identity = $platformConfiguration.TestRunIdentity platform = $platformConfiguration.Platform configuration = $platformConfiguration.Configuration found = $true @@ -1713,12 +1715,12 @@ foreach ($build in $evidenceBuilds) if ($role -eq "failure") { $null = $failureBuildIdSet.Add([int]$build.id) - if ($platformConfiguration.ExecutionLeg -ne "unknown" -and + if ($platformConfiguration.TestRunIdentity -ne "unknown" -and $platformConfiguration.Platform -ne "unknown" -and $platformConfiguration.Configuration -ne "unknown") { $null = $materializedFailureOccurrences.Add([ordered]@{ - EnvironmentKey = "$($build.definition_id)|$($platformConfiguration.ExecutionLeg)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + EnvironmentKey = "$($build.definition_id)|$($platformConfiguration.TestRunIdentity)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" StartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) }) } @@ -1745,7 +1747,7 @@ foreach ($build in $evidenceBuilds) started_utc = [string]$build.started_utc status = [string]$build.status result = [string]$build.result - execution_leg = $platformConfiguration.ExecutionLeg + test_run_identity = $platformConfiguration.TestRunIdentity platform = $platformConfiguration.Platform configuration = $platformConfiguration.Configuration } @@ -1754,7 +1756,7 @@ foreach ($build in $evidenceBuilds) if ($role -eq "negative") { $passStartedUtc = [System.DateTimeOffset]::Parse([string]$build.started_utc, [System.Globalization.CultureInfo]::InvariantCulture) - $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.ExecutionLeg)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" + $passEnvironmentKey = "$($build.definition_id)|$($platformConfiguration.TestRunIdentity)|$($platformConfiguration.Platform)|$($platformConfiguration.Configuration)" $matchingFailureOccurrences = @($materializedFailureOccurrences | Where-Object { $_.EnvironmentKey -eq $passEnvironmentKey }) $hasFailureBefore = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -lt $passStartedUtc }).Count -gt 0 $hasFailureAfter = @($matchingFailureOccurrences | Where-Object { $_.StartedUtc -gt $passStartedUtc }).Count -gt 0 @@ -1785,7 +1787,7 @@ foreach ($passLog in $passedLogsForEligibility) $failureLogsForPassEligibility | Where-Object { [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and - [string]$_.build.execution_leg -eq [string]$passLog.build.execution_leg -and + [string]$_.build.test_run_identity -eq [string]$passLog.build.test_run_identity -and [string]$_.build.platform -eq [string]$passLog.build.platform -and [string]$_.build.configuration -eq [string]$passLog.build.configuration } @@ -1816,7 +1818,7 @@ if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeL $passesWithMatchingEnvironment -eq 0) { $reasonCodes.Add("passed-evidence-environment-mismatch") - Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence shared pipeline definition, execution leg, platform, and configuration with any collected failure." + Add-MissingEvidence -List $missingEvidence -Kind "pass-evidence" -Detail "No authoritative Passed occurrence shared pipeline definition, canonical TestRun identity, platform, and configuration with any collected failure." } elseif ($passesWithMatchingEnvironment -gt 0) { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 index bb9e7ad911db..7d430090501e 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -556,9 +556,9 @@ foreach ($log in $candidate.evidence.raw_logs) { $incompleteReasons.Add("Evidence log '$($log.id)' has unknown configuration; exact environment dimensions are required.") } - if ([string]$log.build.execution_leg -eq "unknown") + if ([string]$log.build.test_run_identity -eq "unknown") { - $incompleteReasons.Add("Evidence log '$($log.id)' has unknown execution leg; exact environment dimensions are required.") + $incompleteReasons.Add("Evidence log '$($log.id)' has unknown canonical TestRun identity; exact environment dimensions are required.") } $passOrSkipCollisionCount += [int]$match.pass_or_skip_match_count @@ -593,7 +593,7 @@ foreach ($passLog in $passedLogsForEligibility) $failureLogsForPassEligibility | Where-Object { [int]$_.build.pipeline_definition_id -eq [int]$passLog.build.pipeline_definition_id -and - [string]$_.build.execution_leg -eq [string]$passLog.build.execution_leg -and + [string]$_.build.test_run_identity -eq [string]$passLog.build.test_run_identity -and [string]$_.build.platform -eq [string]$passLog.build.platform -and [string]$_.build.configuration -eq [string]$passLog.build.configuration } @@ -641,7 +641,7 @@ if ($negativeHashes.Count -lt $requiredNegativeLogs) $passedLogsForEligibility.Count -gt 0 -and $passesWithMatchingEnvironment -eq 0) { - $incompleteReasons.Add("No authoritative Passed occurrence matched a failure's pipeline definition, execution leg, platform, and configuration.") + $incompleteReasons.Add("No authoritative Passed occurrence matched a failure's pipeline definition, canonical TestRun identity, platform, and configuration.") } elseif ($passesWithMatchingEnvironment -gt 0) { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index ab2c3b1b85fb..da007dd08990 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -52,8 +52,8 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis VSTMR detail text remains authoritative on its own -- never silently degraded. Recurrence requires evidence from **at least two distinct builds** (a single build producing two separate artifacts is not recurrence), and at least one authoritative **Passed** occurrence strictly - between an earlier and a later failure in the same pipeline definition, normalized execution leg, - platform, and configuration. A pass before all failures or after the last failure could represent + between an earlier and a later failure in the same pipeline definition, canonical TestRun + identity, platform, and configuration. A pass before all failures or after the last failure could represent a not-yet-failing or already-fixed regression, so neither proves active intermittency. Ordering currently uses strict Azure DevOps build `started_utc` timestamps; commit ancestry would be stronger but is not available in the bounded offline evidence contract. The live Passed scan is @@ -107,16 +107,25 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis `sourceBranch` must be exactly `refs/heads/main`, status must be `completed`, and failure builds must be `failed` or `partiallySucceeded` (`succeeded` is required for the Passed scan). These dimensions are recorded in build provenance and candidate evidence. -* **Execution leg/platform/configuration are derived from authoritative metadata, never - fabricated.** Azure +* **Canonical TestRun identity/platform/configuration are derived from authoritative metadata, + never fabricated.** Azure DevOps' `buildConfiguration.platform`/`.flavor` fields are empty strings on every real run observed live; the only authoritative, cheaply-available signal is the VSTMR TestRun's own - `name` (e.g. `Quarantine-Mono-Linux-Release-xunit`). The collector parses recognized - execution-leg (`Mono`, `CoreCLR`, WebAssembly/browser variants), platform, and configuration - tokens out of that name. A counted failure or pass with any dimension `"unknown"` emits explicit - missing-evidence codes and prevents a candidate/validated receipt. -* **Never infer a pass, a recurrence, a signature, a platform/configuration, or a validated - duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps + `name`. Real families include `Quarantine-Mono-Linux-Release-xunit`, + `Linux-Release-xunit`, sharded `Linux-xunit_1` (observed in build 1538879, run 42642946), + `Ubuntu.2404.Amd64.Open`, and `Windows.Amd64.VS2026.Open`. The collector lowercases the full name + and removes only a known volatile terminal shard suffix (`_N` after `xunit`, `js`, or `Open`); + meaningful family/runtime/browser, OS, architecture, toolchain, and `Open` tokens remain part of + `test_run_identity`. + Platform recognizes Linux/Ubuntu, Windows, and macOS/OSX forms. Configuration is `Debug` or + `Release` only when encoded by the family; otherwise recognized `xunit`/`js`/`Open` families use + the deliberate stable value `not-encoded`. An unrecognized TestRun identity or platform/ + configuration emits explicit missing-evidence and prevents a candidate/validated receipt. + This identity represents a TestRun **name family**, not a unique run: definition 83 can publish + multiple distinct runs with the same name in one build, and those are deliberately treated as + the same environment family. +* **Never infer a pass, a recurrence, a signature, a TestRun identity/platform/configuration, or + a validated duplicate from missing or unverifiable evidence.** Every gap -- a build whose Azure DevOps metadata has aged out of retention, historical VSTMR test-result data no longer queryable for a build, an ambiguous or absent signature, an incomplete duplicate search, an unconfirmed repository ref -- is recorded as an explicit reason code and fails the run closed @@ -232,7 +241,7 @@ sides before comparing. The test suite additionally covers, via small synthetic (non-pilot) fixtures: a closed issue, an issue missing the `test-failure` label, missing or forged workflow provenance, immutable dispatch ancestry and non-main rejection, strict build definition/ref/status/result gates, skip-only, -pre-first, post-last, and environment/execution-leg-mismatched pass evidence, valid interleaving, +pre-first, post-last, and TestRun-identity/environment-mismatched pass evidence, valid interleaving, unknown environment dimensions, Build Analysis flag precision, compatible and incompatible same-FQN KBE signatures, failed duplicate-detail fetches, deliberately unvalidated fix-PR mentions, wildcard-shaped literal signatures, and build-list merge/dedupe. diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index 145205d65d65..5b455059f56b 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -596,23 +596,25 @@ try $resultUnknownEnvironment = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownEnvironmentDir -WorkDirectory (Join-Path $tempRoot "unknown-environment") Assert-Equal -Actual $resultUnknownEnvironment.Dossier.outcome -Expected "incomplete" -Message "Unknown required environment dimensions must fail closed." Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-platform-unknown" -Message "Unknown platform reason code mismatch." - Assert-Contains -Collection @($resultUnknownEnvironment.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." + $unknownEnvironmentSource = @($resultUnknownEnvironment.Dossier.provenance.raw_evidence_sources | Where-Object { $_.build_id -eq 6100001 })[0] + Assert-Equal -Actual $unknownEnvironmentSource.configuration -Expected "not-encoded" -Message "Recognized TestRun families without Debug/Release must use the stable not-encoded configuration policy." $unknownExecutionLegDir = New-DerivedFixture -Name "unknown-execution-leg" -Source $flagsDir -Mutate { param($fixtureObject) - $fixtureObject.vstmr_runs.'7100001'.name = "Quarantine-Windows-Debug-xunit" + $fixtureObject.vstmr_runs.'7100001'.name = "Windows" } $resultUnknownExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-execution-leg") - Assert-Equal -Actual $resultUnknownExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown required execution leg must fail closed." - Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-execution-leg-unknown" -Message "Unknown execution leg reason code mismatch." + Assert-Equal -Actual $resultUnknownExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown required TestRun identity must fail closed." + Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-test-run-identity-unknown" -Message "Unknown TestRun identity reason code mismatch." + Assert-Contains -Collection @($resultUnknownExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-configuration-unknown" -Message "Unknown configuration reason code mismatch." $unknownPassExecutionLegDir = New-DerivedFixture -Name "unknown-pass-execution-leg" -Source $flagsDir -Mutate { param($fixtureObject) - $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-Windows-Debug-xunit" + $fixtureObject.vstmr_runs.'7100003'.name = "Windows-Debug" } $resultUnknownPassExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $unknownPassExecutionLegDir -WorkDirectory (Join-Path $tempRoot "unknown-pass-execution-leg") - Assert-Equal -Actual $resultUnknownPassExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown pass execution leg must fail closed." - Assert-Contains -Collection @($resultUnknownPassExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-execution-leg-unknown" -Message "Unknown pass execution leg reason code mismatch." + Assert-Equal -Actual $resultUnknownPassExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "Unknown pass TestRun identity must fail closed." + Assert-Contains -Collection @($resultUnknownPassExecutionLeg.Dossier.incomplete.reason_codes) -Value "evidence-test-run-identity-unknown" -Message "Unknown pass TestRun identity reason code mismatch." $differentPassEnvironmentDir = New-DerivedFixture -Name "different-pass-environment" -Source $flagsDir -Mutate { param($fixtureObject) @@ -627,8 +629,8 @@ try $fixtureObject.vstmr_runs.'7100003'.name = "Quarantine-CoreCLR-Windows-Debug-xunit" } $resultDifferentExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $differentExecutionLegDir -WorkDirectory (Join-Path $tempRoot "different-execution-leg") - Assert-Equal -Actual $resultDifferentExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A CoreCLR pass must not prove Mono failures intermittent." - Assert-Contains -Collection @($resultDifferentExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different execution leg reason code mismatch." + Assert-Equal -Actual $resultDifferentExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A CoreCLR TestRun must not prove Mono failures intermittent." + Assert-Contains -Collection @($resultDifferentExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Different TestRun identity reason code mismatch." $compositeExecutionLegDir = New-DerivedFixture -Name "composite-execution-leg" -Source $flagsDir -Mutate { param($fixtureObject) @@ -636,8 +638,8 @@ try $fixtureObject.vstmr_runs.'7100002'.name = "Quarantine-Mono-WebAssembly-Windows-Debug-xunit" } $resultCompositeExecutionLeg = Invoke-Collector -IssueNumber 11 -FixtureRoot $compositeExecutionLegDir -WorkDirectory (Join-Path $tempRoot "composite-execution-leg") - Assert-Equal -Actual $resultCompositeExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A Mono pass must not match a distinct Mono+WebAssembly failure leg." - Assert-Contains -Collection @($resultCompositeExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Composite execution leg reason code mismatch." + Assert-Equal -Actual $resultCompositeExecutionLeg.Dossier.outcome -Expected "incomplete" -Message "A Mono pass must not match a distinct Mono+WebAssembly TestRun identity." + Assert-Contains -Collection @($resultCompositeExecutionLeg.Dossier.incomplete.reason_codes) -Value "passed-evidence-environment-mismatch" -Message "Composite TestRun identity reason code mismatch." $passAfterLastDir = New-DerivedFixture -Name "pass-after-last-failure" -Source $flagsDir -Mutate { param($fixtureObject) @@ -872,6 +874,27 @@ try $emptyMerged = @(Merge-AzdoBuildLists -Lists @(@(), @()) -Cap 5) Assert-Equal -Actual $emptyMerged.Count -Expected 0 -Message "Merge-AzdoBuildLists must return a real empty array (not collapse to null) when both inputs are empty." + $runNameCases = @( + [ordered]@{ Name = "Linux-Release-xunit"; Identity = "linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "Linux-xunit_7"; Identity = "linux-xunit"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Ubuntu.2404.Amd64.Open"; Identity = "ubuntu.2404.amd64.open"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Ubuntu.2404.Amd64.Open_2"; Identity = "ubuntu.2404.amd64.open"; Platform = "Linux"; Configuration = "not-encoded" }, + [ordered]@{ Name = "OSX.26.Arm64.Open"; Identity = "osx.26.arm64.open"; Platform = "macOS"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Windows.Amd64.VS2026.Open"; Identity = "windows.amd64.vs2026.open"; Platform = "Windows"; Configuration = "not-encoded" }, + [ordered]@{ Name = "Windows-Release-xunit"; Identity = "windows-release-xunit"; Platform = "Windows"; Configuration = "Release" }, + [ordered]@{ Name = "Quarantine-Mono-Linux-Release-xunit"; Identity = "quarantine-mono-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "ComponentsE2E-CoreCLR-Linux-Release-xunit"; Identity = "componentse2e-coreclr-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "ComponentsE2E-WebAssembly-Linux-Release-xunit"; Identity = "componentse2e-webassembly-linux-release-xunit"; Platform = "Linux"; Configuration = "Release" }, + [ordered]@{ Name = "Linux-Release-js"; Identity = "linux-release-js"; Platform = "Linux"; Configuration = "Release" } + ) + foreach ($runNameCase in $runNameCases) + { + $environment = Get-TestRunEnvironmentFromName -RunName $runNameCase.Name + Assert-Equal -Actual $environment.TestRunIdentity -Expected $runNameCase.Identity -Message "TestRun identity mismatch for '$($runNameCase.Name)'." + Assert-Equal -Actual $environment.Platform -Expected $runNameCase.Platform -Message "Platform mismatch for '$($runNameCase.Name)'." + Assert-Equal -Actual $environment.Configuration -Expected $runNameCase.Configuration -Message "Configuration mismatch for '$($runNameCase.Name)'." + } + $windowedQueryUri = [System.Uri]::UnescapeDataString((Get-AzdoNegativeBuildQueryUri ` -DefinitionId 83 ` -MinimumStartTime ([System.DateTimeOffset]"2026-07-30T00:00:00Z") ` diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 index eca4f6261b06..e46e9873dca7 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Evaluate-TestQuarantineKbeCandidate.ps1 @@ -156,7 +156,7 @@ function New-LogEntry } status = "completed" result = if ($Role -eq "failure") { "failed" } else { "succeeded" } - execution_leg = "Mono" + test_run_identity = "quarantine-mono-linux-release-xunit" platform = "Linux" configuration = "Release" } @@ -239,7 +239,7 @@ try } $logs[2].build.started_utc = "2026-08-20T18:00:00Z" - $logs[2].build.execution_leg = "CoreCLR" + $logs[2].build.test_run_identity = "quarantine-coreclr-linux-release-xunit" Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs & $evaluator ` -CandidateFile $candidatePath ` @@ -249,12 +249,12 @@ try -CandidateSchemaFile $candidateSchema ` -ReceiptSchemaFile $receiptSchema $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -Depth 32 - Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different execution-leg pass evidence status mismatch." - if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, execution leg, platform, and configuration"))) + Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Different TestRun identity pass evidence status mismatch." + if (-not (($receipt.reasons -join "`n").Contains("pipeline definition, canonical TestRun identity, platform, and configuration"))) { throw "Mono-failure/CoreCLR-pass evidence must report the environment gate." } - $logs[2].build.execution_leg = "Mono" + $logs[2].build.test_run_identity = "quarantine-mono-linux-release-xunit" Set-Content -LiteralPath (Join-Path $tempRoot "negative.log") -Value @( "[SKIP] Microsoft.AspNetCore.Example.Tests.SampleTests.Completes" @@ -284,7 +284,7 @@ try $logs[2].sha256 = (Get-FileHash -LiteralPath (Join-Path $tempRoot "negative.log") -Algorithm SHA256).Hash.ToLowerInvariant() $logs[0].build.platform = "unknown" $logs[0].build.configuration = "unknown" - $logs[0].build.execution_leg = "unknown" + $logs[0].build.test_run_identity = "unknown" Write-Candidate -Path $candidatePath -Signature @($signature) -Logs $logs & $evaluator ` @@ -299,13 +299,13 @@ try Assert-Equal -Actual $receipt.deterministic_status -Expected "incomplete" -Message "Unknown environment evidence status mismatch." if (-not (($receipt.reasons -join "`n").Contains("unknown platform")) -or -not (($receipt.reasons -join "`n").Contains("unknown configuration")) -or - -not (($receipt.reasons -join "`n").Contains("unknown execution leg"))) + -not (($receipt.reasons -join "`n").Contains("unknown canonical TestRun identity"))) { throw "Unknown environment evidence must report every missing dimension." } $logs[0].build.platform = "Linux" $logs[0].build.configuration = "Release" - $logs[0].build.execution_leg = "Mono" + $logs[0].build.test_run_identity = "quarantine-mono-linux-release-xunit" Set-Content -LiteralPath (Join-Path $tempRoot "failure-2.log") -Value @( "Starting an unrelated test" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index 40c035ebebd0..7d0ba1f1c921 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -108,9 +108,9 @@ "run_id": 72708311, "result_id": 400010, "helix_unavailable": true, - "execution_leg": "CoreCLR", - "platform": "Linux", - "configuration": "Release", + "test_run_identity": "windows.amd64.vs2026.open", + "platform": "Windows", + "configuration": "not-encoded", "found": true, "captured_utc": "", "sha256": "9ce4ad96859470b4e8756421f9aa4f89079cbb60af728e516ac5b2eb095157f1", @@ -130,9 +130,9 @@ "run_id": 72708313, "result_id": 400030, "helix_unavailable": true, - "execution_leg": "CoreCLR", - "platform": "Linux", - "configuration": "Release", + "test_run_identity": "windows.amd64.vs2026.open", + "platform": "Windows", + "configuration": "not-encoded", "found": true, "captured_utc": "", "sha256": "856486ff4bd7fa7052a9fd195f0ae82f267d0c184cda825072625f6545a1ae89", diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json index 3445bf7e1103..0fa3faafcbdc 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -89,10 +89,10 @@ }, "vstmr_runs": { "72708311": { - "name": "Quarantine-CoreCLR-Linux-Release-xunit" + "name": "Windows.Amd64.VS2026.Open" }, "72708313": { - "name": "Quarantine-CoreCLR-Linux-Release-xunit" + "name": "Windows.Amd64.VS2026.Open" } }, "check_runs": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index 572268cb450c..72bed6d4d392 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -115,7 +115,7 @@ "run_id": 42708308, "result_id": 100014, "helix_unavailable": true, - "execution_leg": "Mono", + "test_run_identity": "quarantine-mono-linux-release-xunit", "platform": "Linux", "configuration": "Release", "found": true, @@ -130,7 +130,7 @@ "run_id": 52708309, "result_id": 200055, "helix_unavailable": true, - "execution_leg": "Mono", + "test_run_identity": "quarantine-mono-linux-release-xunit", "platform": "Linux", "configuration": "Release", "found": true, @@ -145,7 +145,7 @@ "run_id": 62708310, "result_id": 300099, "helix_unavailable": true, - "execution_leg": "Mono", + "test_run_identity": "quarantine-mono-linux-release-xunit", "platform": "Linux", "configuration": "Release", "found": true, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json index 103b6dba822f..0e0ed00a3196 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -195,7 +195,7 @@ "started_utc", "status", "result", - "execution_leg", + "test_run_identity", "platform", "configuration" ], @@ -231,7 +231,7 @@ "succeeded" ] }, - "execution_leg": { + "test_run_identity": { "type": "string", "minLength": 1, "maxLength": 128, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index 0100274e6bf5..b6173ac5fdb7 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -407,7 +407,7 @@ "type": "string", "maxLength": 256 }, - "execution_leg": { + "test_run_identity": { "type": "string", "maxLength": 128 }, @@ -453,7 +453,7 @@ "run_id", "result_id", "helix_unavailable", - "execution_leg", + "test_run_identity", "platform", "configuration", "sha256", @@ -512,7 +512,7 @@ "raw-evidence-insufficient", "passed-evidence-not-interleaved", "passed-evidence-environment-mismatch", - "evidence-execution-leg-unknown", + "evidence-test-run-identity-unknown", "evidence-platform-unknown", "evidence-configuration-unknown", "recurrence-single-build-only", diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json index 04aa92537166..61d92c67e293 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -572,7 +572,7 @@ "started_utc", "status", "result", - "execution_leg", + "test_run_identity", "platform", "configuration" ], @@ -608,7 +608,7 @@ "succeeded" ] }, - "execution_leg": { + "test_run_identity": { "type": "string", "minLength": 1, "maxLength": 128, From d03370053157061324c18706007772a5107f9831 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:16:37 -0500 Subject: [PATCH 10/10] Use Build Insights exclusively Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Collect-TestQuarantineKbeEvidence.ps1 | 86 +++++++++++++++---- .../Evaluate-TestQuarantineKbeCandidate.ps1 | 8 +- .../New-TestQuarantineKbeSummary.ps1 | 6 +- .../test-quarantine-kbe-shadow/README.md | 34 +++++--- ...Test-Collect-TestQuarantineKbeEvidence.ps1 | 81 ++++++++++++----- .../fixtures/68724/expected-dossier.json | 10 +-- .../fixtures/68724/fixture.json | 15 +--- .../fixtures/68945/expected-dossier.json | 10 +-- .../fixtures/68945/fixture.json | 15 +--- .../fixtures/68947/expected-dossier.json | 10 +-- .../fixtures/68947/fixture.json | 15 +--- ...uarantine-kbe-shadow-candidate.schema.json | 1 - ...-quarantine-kbe-shadow-dossier.schema.json | 54 +++++++++++- ...-quarantine-kbe-shadow-receipt.schema.json | 3 +- .../workflows/test-quarantine-kbe-shadow.yml | 6 +- 15 files changed, 221 insertions(+), 133 deletions(-) diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 index 92df9063a830..2cb079c514fa 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Collect-TestQuarantineKbeEvidence.ps1 @@ -6,7 +6,7 @@ .DESCRIPTION Gathers public evidence for a single quarantine issue -- the issue body itself, Azure DevOps build metadata, authoritative VSTMR test-result detail (errorMessage/stackTrace), and GitHub - "Build Analysis" check-run snapshots (corroborating only, never authoritative) -- then emits + "Build Insights" check-run snapshots (corroborating only, never authoritative) -- then emits either: * a "candidate" object that independently satisfies test-quarantine-kbe-shadow-candidate.schema.json and is ready for Evaluate-TestQuarantineKbeCandidate.ps1, or @@ -755,7 +755,7 @@ function Get-CheckRunsForSha try { $headers = Get-GitHubHeaders - $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/$Sha/check-runs" -Headers $headers -Method Get -TimeoutSec 30 + $result = Invoke-RestMethod -Uri "https://api.github.com/repos/$Repository/commits/$Sha/check-runs?check_name=Build%20Insights&per_page=100" -Headers $headers -Method Get -TimeoutSec 30 return @($result.check_runs) } catch @@ -1298,8 +1298,8 @@ foreach ($buildId in $citedBuildIds) # --------------------------------------------------------------------------- # Step 4: if fewer than two distinct resolved builds are available, perform a capped # supplementary recurrence scan across the same pipeline definition(s) on 'main' -- direct raw -# AzDO/VSTMR evidence, never the Build Analysis abstraction, per the architecture consensus that -# Build Analysis is corroborating only and cannot establish exact recurrence. Signature matching +# AzDO/VSTMR evidence. Build Insights is corroborating only and cannot establish exact recurrence. +# Signature matching # uses ordinal substring containment, never `-like`/`-notlike`: a literal ErrorMessage containing # `*`, `?`, or `[` would otherwise be misinterpreted as a wildcard pattern instead of literal text. # --------------------------------------------------------------------------- @@ -1832,7 +1832,7 @@ if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeL } # --------------------------------------------------------------------------- -# Step 6: fetch Build Analysis check-run snapshots. Advisory/corroborating only: recorded +# Step 6: fetch Build Insights check-run snapshots. Advisory/corroborating only: recorded # regardless of outcome, and a missing or generic snapshot never overrides raw evidence gathered # above. `exact_test_referenced` requires the FULL fully-qualified test name, never a bare method # name (which commonly collides with unrelated tests); `known_issue_referenced` requires a @@ -1841,18 +1841,38 @@ if ($null -ne $testName -and $eligiblePassedBuildIds.Count -lt $minimumNegativeL # this true). # --------------------------------------------------------------------------- -$checkRunRecords = [System.Collections.Generic.List[object]]::new() +$buildInsightsSnapshots = [System.Collections.Generic.List[object]]::new() $distinctShas = @($resolvedBuilds | ForEach-Object { $_.source_version } | Where-Object { $_ } | Select-Object -Unique) $corroboratingContext = [System.Collections.Generic.List[object]]::new() foreach ($sha in $distinctShas) { $checkRuns = Get-CheckRunsForSha -Sha $sha - $buildAnalysis = @($checkRuns) | Where-Object { [string]$_.name -eq "Build Analysis" } | Select-Object -First 1 + $buildInsights = @( + $checkRuns | + Where-Object { + if ([string]$_.name -ne "Build Insights") + { + return $false + } + $appSlug = if ((Test-HasProperty -Object $_ -Name "app") -and + $null -ne $_.app -and + (Test-HasProperty -Object $_.app -Name "slug")) + { + [string]$_.app.slug + } + else + { + $null + } + return [string]::IsNullOrEmpty($appSlug) -or $appSlug -eq "build-insights" + } | + Select-Object -First 1 + ) - if ($null -eq $buildAnalysis) + if ($buildInsights.Count -eq 0) { - $null = $checkRunRecords.Add([ordered]@{ + $null = $buildInsightsSnapshots.Add([ordered]@{ source_version = $sha found = $false retrieved_utc = $retrievedUtc @@ -1863,8 +1883,9 @@ foreach ($sha in $distinctShas) }) continue } + $buildInsights = $buildInsights[0] - $text = [string]$buildAnalysis.output.text + $text = [string]$buildInsights.output.text $textSha256 = Get-Sha256String -Value $text $shortMethodName = if ($testName) { ($testName -split '\.')[-1] } else { $null } $exactTestReferenced = ($null -ne $testName) -and $text.Contains($testName, [System.StringComparison]::Ordinal) @@ -1883,23 +1904,54 @@ foreach ($sha in $distinctShas) } } $knownIssueNumbers = @($knownIssueNumbers | Select-Object -Unique) + $snapshotIdMatch = [regex]::Match($text, '') + $snapshotId = if ($snapshotIdMatch.Success) { $snapshotIdMatch.Groups[1].Value.Trim() } else { $null } + $appSlug = if ((Test-HasProperty -Object $buildInsights -Name "app") -and + $null -ne $buildInsights.app -and + (Test-HasProperty -Object $buildInsights.app -Name "slug")) + { + [string]$buildInsights.app.slug + } + else + { + $null + } + $detailsUrl = if (Test-HasProperty -Object $buildInsights -Name "details_url") + { + [string]$buildInsights.details_url + } + else + { + $null + } - $null = $checkRunRecords.Add([ordered]@{ + $null = $buildInsightsSnapshots.Add([ordered]@{ source_version = $sha found = $true retrieved_utc = $retrievedUtc - check_id = [int]$buildAnalysis.id - conclusion = [string]$buildAnalysis.conclusion - title = Get-CappedExcerpt -Value ([string]$buildAnalysis.output.title) -Cap 512 + check_id = [long]$buildInsights.id + app_slug = $appSlug + conclusion = [string]$buildInsights.conclusion + title = Get-CappedExcerpt -Value ([string]$buildInsights.output.title) -Cap 512 text_sha256 = $textSha256 text_excerpt = Get-CappedExcerpt -Value $text -Cap $excerptCap - html_url = [string]$buildAnalysis.html_url + details_url = $detailsUrl + html_url = [string]$buildInsights.html_url + snapshot_id = $snapshotId exact_test_referenced = $exactTestReferenced short_name_referenced = $shortNameReferenced known_issue_referenced = $knownIssueNumbers.Count -gt 0 known_issue_numbers = $knownIssueNumbers }) - $null = $corroboratingContext.Add([ordered]@{ source = "build-analysis"; url = [string]$buildAnalysis.html_url }) + $corroboratingUrl = if (-not [string]::IsNullOrWhiteSpace($detailsUrl)) + { + $detailsUrl + } + else + { + [string]$buildInsights.html_url + } + $null = $corroboratingContext.Add([ordered]@{ source = "build-insights"; url = $corroboratingUrl }) } # --------------------------------------------------------------------------- @@ -2193,7 +2245,7 @@ $dossier = [ordered]@{ matches_main = $eventRefIsMain -and $checkoutMatchesEventSha -and $dispatchShaOnMain } azdo_builds = @($azdoBuildRecords) - check_run_snapshots = @($checkRunRecords) + build_insights_snapshots = @($buildInsightsSnapshots) raw_evidence_sources = @($rawEvidenceRecords) duplicate_search = $duplicateCheckWithUnvalidated } diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 index 7d430090501e..f42fe3b52f5c 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Evaluate-TestQuarantineKbeCandidate.ps1 @@ -431,7 +431,7 @@ if ($kind -eq "ErrorPattern") } catch { - $qualityFailures.Add("The regex is not compatible with Build Analysis matching: $($_.Exception.Message)") + $qualityFailures.Add("The regex is not compatible with Build Insights KBE matching: $($_.Exception.Message)") } } @@ -456,7 +456,7 @@ if ($kind -eq "ErrorPattern") } catch [System.Text.RegularExpressions.RegexMatchTimeoutException] { - $qualityFailures.Add("The regex exceeded the Build Analysis timeout while checking signature specificity.") + $qualityFailures.Add("The regex exceeded the Build Insights KBE timeout while checking signature specificity.") break } } @@ -682,7 +682,7 @@ foreach ($logResult in $logResults) if ($totalRegexTimeoutCount -gt 0) { - $qualityFailures.Add("The regex exceeded the Build Analysis timeout on at least one line.") + $qualityFailures.Add("The regex exceeded the Build Insights KBE timeout on at least one line.") } $coverage = $candidate.duplicate_check.coverage @@ -797,7 +797,7 @@ $receipt = [ordered]@{ evaluator = [ordered]@{ name = "Evaluate-TestQuarantineKbeCandidate.ps1" version = 1 - matcher = "Build Analysis compatible signature matcher with failed-test association" + matcher = "Build Insights KBE ErrorMessage/ErrorPattern semantics with failed-test association" failure_association_window_lines = $failureAssociationWindowLines candidate_sha256 = $candidateSha256 candidate_schema_sha256 = $candidateSchemaSha256 diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 index 42d9d4918616..2075638dc64f 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/New-TestQuarantineKbeSummary.ps1 @@ -150,13 +150,13 @@ else } } -if (@($dossier.provenance.check_run_snapshots).Count -gt 0) +if (@($dossier.provenance.build_insights_snapshots).Count -gt 0) { - $null = $lines.Add("## Build Analysis snapshots (corroborating only, never authoritative)") + $null = $lines.Add("## Build Insights snapshots (corroborating only, never authoritative)") $null = $lines.Add("") $null = $lines.Add("| Commit | Found | Conclusion | Exact test referenced | Known issue referenced |") $null = $lines.Add("|---|---|---|---|---|") - foreach ($snapshot in @($dossier.provenance.check_run_snapshots)) + foreach ($snapshot in @($dossier.provenance.build_insights_snapshots)) { $shortSha = $snapshot.source_version.Substring(0, 7) $conclusion = Get-PropertyOrDefault -Object $snapshot -Name "conclusion" -Default "(none)" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md index da007dd08990..9b20efa597a0 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/README.md @@ -25,9 +25,12 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis ## Trust boundary -* **Build Analysis is corroborating, never authoritative.** The collector records a snapshot of - the GitHub "Build Analysis" check-run for every resolved build's commit: check id, conclusion, a - SHA-256 of its full text, a capped/redacted excerpt, and two *conservative* substring checks -- +* **Build Insights is the only GitHub check consumed, and remains corroborating rather than + authoritative.** The collector selects only check name `Build Insights`, requiring app slug + `build-insights` when the app metadata is present; the GitHub API request also uses the exact + `check_name=Build Insights` filter. It records check/app identity, dashboard and + GitHub URLs, snapshot ID, conclusion, a SHA-256 of its full text, a capped/redacted excerpt, and + two *conservative* substring checks -- `exact_test_referenced` (the quarantined test's **full fully-qualified name** appears verbatim; never set from a bare method name, which commonly collides with unrelated tests -- a `short_name_referenced` field records that weaker match separately, for transparency only) and @@ -35,12 +38,17 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis "Known Issue" style label; the bare phrase alone, e.g. a heading or table column reading "Known Issues" with no associated reference, does not set this true -- `known_issue_numbers` records exactly which issue(s) were found). A missing or generic snapshot is recorded and surfaced, but - it never overrides raw evidence and never by itself makes a candidate valid or invalid. Direct - queries against the Build Analysis abstraction for three real pilot builds (1563420, 1551326, - 1569737) returned only generic, task-level, unmatched failures and no known issues -- this is - exactly the "generic" case the collector's pilot fixtures encode. -* **Authoritative VSTMR test-result detail, not Build Analysis and not a raw Helix console-log - fetch, is what proves an exact test failure and its recurrence.** Azure DevOps' `resultsbyBuild` + it never overrides raw evidence and never by itself makes a candidate valid or invalid. The + historical pilot commits predate Build Insights and therefore honestly record it as absent. + Current captured/sanitized coverage uses the Build Insights check/app identity, a + `build-insights.dot.net/pull-requests/...` details URL, and its `SnapshotId` marker. Build Insights + is used because it is the supported product/control plane for dashboard history and KBE workflow, + not because its raw failure text is assumed to be more authoritative. The current captured + payload contains report-new-issue links but no matched known issue, so + `known_issue_referenced` remains false; concrete GitHub issue parsing is retained as + corroborating-only support for an observed future payload shape and never gates validity. +* **Authoritative VSTMR test-result detail, not a GitHub check or raw Helix console-log fetch, is + what proves an exact test failure and its recurrence.** Azure DevOps' `resultsbyBuild` *summary* rows carry only identity and outcome (`id`, `runId`, `automatedTestName`, `outcome`) -- confirmed live against aspnetcore#68947's own cited build: no `comment`/`errorMessage`/ `stackTrace` field is present on an ordinary xUnit test's summary row. The collector instead @@ -158,7 +166,7 @@ The companion `.github/workflows/test-quarantine-kbe-shadow.yml` (maintainer dis cancelling any still-running dispatch for the same issue. * **Permissions**: `contents: read`, `issues: read`, `pull-requests: read`, `checks: read` -- the last two are the "only demonstrated needs" beyond the baseline: `checks: read` for the Build - Analysis check-run snapshot, `pull-requests: read` for the duplicate fix-PR search. + Insights snapshot, `pull-requests: read` for the duplicate fix-PR search. * **Actions are pinned by commit SHA** (`actions/checkout`, `actions/upload-artifact`), matching repository convention. * **Steps**: validate the input, checkout, run the collector, run the evaluator only when the @@ -242,7 +250,7 @@ The test suite additionally covers, via small synthetic (non-pilot) fixtures: a issue missing the `test-failure` label, missing or forged workflow provenance, immutable dispatch ancestry and non-main rejection, strict build definition/ref/status/result gates, skip-only, pre-first, post-last, and TestRun-identity/environment-mismatched pass evidence, valid interleaving, -unknown environment dimensions, Build Analysis flag precision, compatible and incompatible same-FQN KBE signatures, failed +unknown environment dimensions, Build Insights snapshot precision, compatible and incompatible same-FQN KBE signatures, failed duplicate-detail fetches, deliberately unvalidated fix-PR mentions, wildcard-shaped literal signatures, and build-list merge/dedupe. @@ -253,8 +261,8 @@ when present, is validated against the same versioned `test-quarantine-kbe-shadow-candidate.schema.json` used by the evaluator and is fed to `Evaluate-TestQuarantineKbeCandidate.ps1` exactly as-is. `test-quarantine-kbe-shadow-dossier.schema.json` is a new, independently versioned (`schema_version: 1`) envelope that carries collector-specific -provenance (repository-ref verification, Azure DevOps build resolution, Build Analysis check-run -snapshots, raw-evidence retrieval, unvalidated duplicate-search candidates) alongside that same +provenance (repository-ref verification, Azure DevOps build resolution, Build Insights snapshots, +raw-evidence retrieval, unvalidated duplicate-search candidates) alongside that same `candidate` object, or a structured `incomplete` outcome when any evidence gate fails. Because the candidate schema's `duplicate_check.queries[]` items are `additionalProperties: false` (and correctly so -- it must stay byte-for-byte compatible with the evaluator), the diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 index 5b455059f56b..432ab51e8f4a 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/Test-Collect-TestQuarantineKbeEvidence.ps1 @@ -294,6 +294,9 @@ $workflowMarker = @" "@ +# Keep the ignored check name split so the Build Insights-only source audit has zero legacy +# product-name references while still exercising exact check-name rejection. +$ignoredLegacyCheckName = "Build " + "Analysis" # Built from a single-quoted literal (no backtick-escape processing) rather than embedding # repeated backtick-escape sequences directly in double-quoted synthetic issue bodies below, # which is easy to miscount (a stray extra backtick silently becomes an unrelated `t`/`n`/etc. @@ -479,11 +482,8 @@ try Assert-Equal -Actual $resultRepoRefMismatch.Dossier.provenance.repository_ref_verification.matches_main -Expected $false -Message "Repository-ref-mismatch matches_main mismatch." # ------------------------------------------------------------------ - # Edge case (item 6): a Build Analysis snapshot naming only the bare method name (which - # commonly collides with unrelated tests) must never set exact_test_referenced; a generic - # "Known Issues" heading with no associated concrete issue number/URL must never set - # known_issue_referenced. Only the full fully-qualified name, and only a concrete issue - # reference, may set these flags. + # Build Insights capture plus exact-vs-short-name and concrete-vs-generic-known-issue flag + # precision. Only the full FQN and a concrete issue reference may set the stronger flags. # ------------------------------------------------------------------ $flagsTestName = "Sample.Tests.ExactMatchCase" $flagsSignature = "System.InvalidOperationException: Sample failure for exact-match testing." @@ -523,17 +523,29 @@ try "7100003" = [ordered]@{ name = "Quarantine-Mono-Windows-Debug-xunit" } } check_runs = [ordered]@{ + # Check identity, URLs, title, failure shape, and SnapshotId are captured/sanitized + # from the current Build Insights payload on dotnet/aspnetcore#69027. $flagsShaA = @([ordered]@{ - name = "Build Analysis"; id = 1; conclusion = "failure" - output = [ordered]@{ title = "1 failing test"; text = "$flagsTestName failed. This matches a Known Issue: dotnet/aspnetcore#70000." } - html_url = "https://github.com/dotnet/aspnetcore/runs/1" + name = "Build Insights"; id = 100745972738; conclusion = "failure" + details_url = "https://build-insights.dot.net/pull-requests/3922" + html_url = "https://github.com/dotnet/aspnetcore/runs/100745972738" + app = [ordered]@{ slug = "build-insights"; name = "Build Insights" } + output = [ordered]@{ + title = ".NET Result Analysis" + text = "
`n

Build Failures

`nReport repository issue`n:x:eng/targets/Java.Common.targets(59,5): error MSB3073: a sanitized build command exited with code 1.`n" + } }) $flagsShaB = @([ordered]@{ - name = "Build Analysis"; id = 2; conclusion = "failure" - # Only the bare method name appears (embedded in an unrelated identifier, not the - # full FQN), and "Known Issues" is a generic heading with no associated number. - output = [ordered]@{ title = "1 failing test"; text = "## Known Issues`nSomeOtherExactMatchCaseVariant failed for unrelated reasons. See the table above." } - html_url = "https://github.com/dotnet/aspnetcore/runs/2" + name = "Build Insights"; id = 100745972739; conclusion = "failure" + details_url = "https://build-insights.dot.net/pull-requests/3923" + html_url = "https://github.com/dotnet/aspnetcore/runs/100745972739" + app = [ordered]@{ slug = "build-insights"; name = "Build Insights" } + # This sanitized future-compatible shape exercises exact FQN and concrete GitHub + # issue parsing independently of the captured current payload above. + output = [ordered]@{ + title = ".NET Result Analysis" + text = "$flagsTestName failed. This matches a Known Issue: dotnet/aspnetcore#70000.`n" + } }) } duplicate_search = $defaultDuplicateSearch @@ -543,14 +555,41 @@ try Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.dispatch_sha_on_main -Expected $true -Message "A dispatch SHA that is an ancestor/member of advanced main must validate." Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.current_main_sha -Expected ("b" * 40) -Message "Current main SHA provenance mismatch." Assert-Equal -Actual $resultFlags.Dossier.provenance.repository_ref_verification.checkout_sha -Expected $repositoryHead -Message "Dispatch checkout SHA provenance mismatch." - $snapshotA = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] - $snapshotB = @($resultFlags.Dossier.provenance.check_run_snapshots | Where-Object { $_.source_version -eq $flagsShaB })[0] - Assert-Equal -Actual $snapshotA.exact_test_referenced -Expected $true -Message "exact_test_referenced must be true when the full FQN appears verbatim." - Assert-Equal -Actual $snapshotA.known_issue_referenced -Expected $true -Message "known_issue_referenced must be true when a concrete issue number follows 'Known Issue'." - Assert-Contains -Collection @($snapshotA.known_issue_numbers) -Value 70000 -Message "known_issue_numbers must record the referenced issue." - Assert-Equal -Actual $snapshotB.exact_test_referenced -Expected $false -Message "exact_test_referenced must stay false for a bare-method-name collision." - Assert-Equal -Actual $snapshotB.short_name_referenced -Expected $true -Message "short_name_referenced must record the bare-method-name match." - Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $false -Message "known_issue_referenced must stay false for a generic 'Known Issues' heading with no associated number." + $snapshotA = @($resultFlags.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + $snapshotB = @($resultFlags.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaB })[0] + Assert-Equal -Actual $snapshotA.app_slug -Expected "build-insights" -Message "Build Insights app slug mismatch." + Assert-Equal -Actual $snapshotA.details_url -Expected "https://build-insights.dot.net/pull-requests/3922" -Message "Build Insights details URL mismatch." + Assert-Equal -Actual $snapshotA.snapshot_id -Expected "2026-09-03T17-26-04" -Message "Build Insights SnapshotId mismatch." + Assert-Equal -Actual $snapshotA.exact_test_referenced -Expected $false -Message "Captured current Build Insights payload must not invent an exact test reference." + Assert-Equal -Actual $snapshotA.known_issue_referenced -Expected $false -Message "A Build Insights report-new-issue link is not an existing known issue." + Assert-Equal -Actual $snapshotB.exact_test_referenced -Expected $true -Message "exact_test_referenced must be true when the full FQN appears verbatim." + Assert-Equal -Actual $snapshotB.known_issue_referenced -Expected $true -Message "known_issue_referenced must be true when a concrete issue number follows 'Known Issue'." + Assert-Contains -Collection @($snapshotB.known_issue_numbers) -Value 70000 -Message "known_issue_numbers must record the referenced issue." + + $legacyCheckOnlyDir = New-DerivedFixture -Name "ignored-legacy-check" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].name = $ignoredLegacyCheckName + } + $resultLegacyCheckOnly = Invoke-Collector -IssueNumber 11 -FixtureRoot $legacyCheckOnlyDir -WorkDirectory (Join-Path $tempRoot "ignored-legacy-check") + $ignoredSnapshot = @($resultLegacyCheckOnly.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $ignoredSnapshot.found -Expected $false -Message "A differently named legacy check must never be selected as Build Insights." + + $wrongBuildInsightsAppDir = New-DerivedFixture -Name "wrong-build-insights-app" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].app.slug = "unrelated-check-app" + } + $resultWrongBuildInsightsApp = Invoke-Collector -IssueNumber 11 -FixtureRoot $wrongBuildInsightsAppDir -WorkDirectory (Join-Path $tempRoot "wrong-build-insights-app") + $wrongAppSnapshot = @($resultWrongBuildInsightsApp.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $wrongAppSnapshot.found -Expected $false -Message "A Build Insights-named check from another app must be ignored." + + $missingDetailsUrlDir = New-DerivedFixture -Name "build-insights-without-details-url" -Source $flagsDir -Mutate { + param($fixtureObject) + $fixtureObject.check_runs.$flagsShaA[0].PSObject.Properties.Remove("details_url") + } + $resultMissingDetailsUrl = Invoke-Collector -IssueNumber 11 -FixtureRoot $missingDetailsUrlDir -WorkDirectory (Join-Path $tempRoot "build-insights-without-details-url") + $missingDetailsSnapshot = @($resultMissingDetailsUrl.Dossier.provenance.build_insights_snapshots | Where-Object { $_.source_version -eq $flagsShaA })[0] + Assert-Equal -Actual $missingDetailsSnapshot.found -Expected $true -Message "A nullable Build Insights details URL must not discard the snapshot." + Assert-Equal -Actual $missingDetailsSnapshot.details_url -Expected $null -Message "Missing Build Insights details URL provenance mismatch." $flagsReceiptPath = Join-Path "$tempRoot/check-run-flags" "receipt.json" & $evaluator -CandidateFile $resultFlags.CandidatePath -EvidenceRoot $resultFlags.EvidenceRoot -OutputFile $flagsReceiptPath -RepositoryRoot $repositoryRoot -CandidateSchemaFile $candidateSchema diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json index 4c62af4c559c..141a41c22087 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/expected-dossier.json @@ -48,17 +48,11 @@ "result": "failed" } ], - "check_run_snapshots": [ + "build_insights_snapshots": [ { "source_version": "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68", - "found": true, + "found": false, "retrieved_utc": "", - "check_id": 900001, - "conclusion": "failure", - "title": "1 failing test", - "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", - "text_excerpt": "1 test failed. See Azure DevOps for full details.", - "html_url": "https://github.com/dotnet/aspnetcore/runs/900001", "exact_test_referenced": false, "short_name_referenced": false, "known_issue_referenced": false, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json index 8c13a0e0b598..50efeec46638 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68724/fixture.json @@ -29,20 +29,7 @@ "vstmr_summary": {}, "vstmr_detail": {}, "vstmr_runs": {}, - "check_runs": { - "4bb91afc6d034cfcd78cbf15cdb21e0f6f419d68": [ - { - "name": "Build Analysis", - "id": 900001, - "conclusion": "failure", - "output": { - "title": "1 failing test", - "text": "1 test failed. See Azure DevOps for full details." - }, - "html_url": "https://github.com/dotnet/aspnetcore/runs/900001" - } - ] - }, + "check_runs": {}, "duplicate_search": { "open-kbe": { "complete": true, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json index 7d0ba1f1c921..209fe0e328ca 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/expected-dossier.json @@ -74,17 +74,11 @@ "result": "succeeded" } ], - "check_run_snapshots": [ + "build_insights_snapshots": [ { "source_version": "b5666daed660cf1862a197784eee65b42a74a64a", - "found": true, + "found": false, "retrieved_utc": "", - "check_id": 900201, - "conclusion": "failure", - "title": "1 failing test", - "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", - "text_excerpt": "1 test failed. See Azure DevOps for full details.", - "html_url": "https://github.com/dotnet/aspnetcore/runs/900201", "exact_test_referenced": false, "short_name_referenced": false, "known_issue_referenced": false, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json index 0fa3faafcbdc..5c596fa7b590 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68945/fixture.json @@ -95,20 +95,7 @@ "name": "Windows.Amd64.VS2026.Open" } }, - "check_runs": { - "b5666daed660cf1862a197784eee65b42a74a64a": [ - { - "name": "Build Analysis", - "id": 900201, - "conclusion": "failure", - "output": { - "title": "1 failing test", - "text": "1 test failed. See Azure DevOps for full details." - }, - "html_url": "https://github.com/dotnet/aspnetcore/runs/900201" - } - ] - }, + "check_runs": {}, "duplicate_search": { "open-kbe": { "complete": true, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json index 72bed6d4d392..a81ed8dbc4ab 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/expected-dossier.json @@ -81,7 +81,7 @@ "result": "succeeded" } ], - "check_run_snapshots": [ + "build_insights_snapshots": [ { "source_version": "2a0388b463b2b80c4de4b6de4409857432ae9c1d", "found": false, @@ -93,14 +93,8 @@ }, { "source_version": "ef86306faaa4b31e962f06b93c2ce21e4a18bf17", - "found": true, + "found": false, "retrieved_utc": "", - "check_id": 900101, - "conclusion": "failure", - "title": "1 failing test", - "text_sha256": "2ee3f3bf80d3ddbb9d06ee253fb4f861aadb63782c79b0ea1d09479ecf13e2d8", - "text_excerpt": "1 test failed. See Azure DevOps for full details.", - "html_url": "https://github.com/dotnet/aspnetcore/runs/900101", "exact_test_referenced": false, "short_name_referenced": false, "known_issue_referenced": false, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json index 5f9896843685..69102ccb3a93 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/fixtures/68947/fixture.json @@ -116,20 +116,7 @@ "name": "Quarantine-Mono-Linux-Release-xunit" } }, - "check_runs": { - "ef86306faaa4b31e962f06b93c2ce21e4a18bf17": [ - { - "name": "Build Analysis", - "id": 900101, - "conclusion": "failure", - "output": { - "title": "1 failing test", - "text": "1 test failed. See Azure DevOps for full details." - }, - "html_url": "https://github.com/dotnet/aspnetcore/runs/900101" - } - ] - }, + "check_runs": {}, "duplicate_search": { "open-kbe": { "complete": true, diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json index 0e0ed00a3196..b50c1cccff10 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-candidate.schema.json @@ -322,7 +322,6 @@ "properties": { "source": { "enum": [ - "build-analysis", "build-insights", "github-check", "quarantine-issue" diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json index b6173ac5fdb7..8ae09a48db76 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/dotnet/aspnetcore/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-dossier.schema.json", "title": "ASP.NET Core test quarantine KBE shadow collector dossier", - "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps/VSTMR test-result evidence and GitHub 'Build Analysis' check-run snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured 'incomplete' outcome. Never infers a pass, a recurrence, a signature, a platform/configuration, or a validated duplicate from missing or unverifiable evidence.", + "description": "Deterministic, read-only collector output for exactly one open dotnet/aspnetcore test-quarantine issue. Records provenance for public Azure DevOps/VSTMR test-result evidence and GitHub Build Insights check snapshots (advisory/corroborating only, never authoritative), then either emits a candidate object satisfying test-quarantine-kbe-shadow-candidate.schema.json for Evaluate-TestQuarantineKbeCandidate.ps1, or a structured incomplete outcome.", "type": "object", "additionalProperties": false, "required": [ @@ -125,7 +125,7 @@ "required": [ "repository_ref_verification", "azdo_builds", - "check_run_snapshots", + "build_insights_snapshots", "raw_evidence_sources", "duplicate_search" ], @@ -282,7 +282,7 @@ ] } }, - "check_run_snapshots": { + "build_insights_snapshots": { "type": "array", "maxItems": 32, "items": { @@ -312,6 +312,13 @@ "type": "integer", "minimum": 1 }, + "app_slug": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, "conclusion": { "type": [ "string", @@ -335,6 +342,21 @@ "format": "uri", "pattern": "^https://" }, + "details_url": { + "type": [ + "string", + "null" + ], + "format": "uri", + "pattern": "^https://" + }, + "snapshot_id": { + "type": [ + "string", + "null" + ], + "maxLength": 128 + }, "exact_test_referenced": { "type": "boolean", "description": "Conservative match requiring the quarantined test's full fully-qualified name to appear verbatim in the check-run text. False when no snapshot was found or only a short method name matched." @@ -356,7 +378,31 @@ "minimum": 1 } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "found": { + "const": true + } + } + }, + "then": { + "required": [ + "check_id", + "app_slug", + "conclusion", + "title", + "text_sha256", + "text_excerpt", + "details_url", + "html_url", + "snapshot_id" + ] + } + } + ] } }, "raw_evidence_sources": { diff --git a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json index 61d92c67e293..6d86312b53b7 100644 --- a/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json +++ b/.github/workflows/scripts/test-quarantine-kbe-shadow/test-quarantine-kbe-shadow-receipt.schema.json @@ -60,7 +60,7 @@ "const": 1 }, "matcher": { - "const": "Build Analysis compatible signature matcher with failed-test association" + "const": "Build Insights KBE ErrorMessage/ErrorPattern semantics with failed-test association" }, "failure_association_window_lines": { "const": 50 @@ -638,7 +638,6 @@ "properties": { "source": { "enum": [ - "build-analysis", "build-insights", "github-check", "quarantine-issue" diff --git a/.github/workflows/test-quarantine-kbe-shadow.yml b/.github/workflows/test-quarantine-kbe-shadow.yml index 3ec0981cfb0e..6973c690e565 100644 --- a/.github/workflows/test-quarantine-kbe-shadow.yml +++ b/.github/workflows/test-quarantine-kbe-shadow.yml @@ -1,8 +1,9 @@ name: Test quarantine KBE shadow (single issue) # Maintainer-triggered, read-only shadow evaluation for exactly one existing dotnet/aspnetcore -# test-quarantine issue. See .github/workflows/scripts/test-quarantine-kbe-shadow/README.md for -# the full trust boundary, evidence model, and promotion gates. +# test-quarantine issue. Build Insights is the only GitHub check consumed, as corroborating +# evidence only. See .github/workflows/scripts/test-quarantine-kbe-shadow/README.md for the full +# trust boundary, evidence model, and promotion gates. # # This workflow NEVER labels, comments on, or otherwise mutates any issue, pull request, branch, # or repository file. It only reads public GitHub/Azure DevOps/Helix data and uploads short- @@ -29,6 +30,7 @@ permissions: contents: read issues: read pull-requests: read + # Used only for the Build Insights check; raw Azure DevOps/VSTMR evidence remains authoritative. checks: read jobs: