CI: accept a missing trust anchor in the Windows runner trust probe

.github/actions/upgrade-ca-certificates/emudhra_intermediates.ps1

@@ -69,3 +69,56 @@ function Test-ExplicitlyDistrusted([System.Security.Cryptography.X509Certificate
         $chain.Dispose()
     }
 }
+
+# Classifies how Windows chain building treats the certificate, using only the
+# local trust state (no revocation, no AIA downloads):
+#   ExplicitDistrust : in LocalMachine\Disallowed or Microsoft's disallowed list
+#   NoTrustAnchor    : the chain no longer ends in a trusted root, and
+#                      UntrustedRoot / PartialChain are the only problems
+#                      reported, e.g. because the root was removed
+#   Trusted          : the chain builds cleanly, so the compromised key is usable
+#   Indeterminate    : the build failed for some other reason; never a pass
+# Verification time is pinned (the instant used by runner_trust_probe.sh) so an
+# expired certificate cannot masquerade as untrusted.
+# A PartialChain only shows that the root is untrusted if the issuers are
+# available to the build, so callers pass the non-root issuers in
+# $ExtraCertificates. Issuers that are roots need not be supplied: a missing
+# root is exactly the condition this function reports as NoTrustAnchor.
+function Get-ChainTrustState(
+    [System.Security.Cryptography.X509Certificates.X509Certificate2]$Cert,
+    [System.Security.Cryptography.X509Certificates.X509Certificate2[]]$ExtraCertificates = @()
+) {
+    $explicitDistrust = [System.Security.Cryptography.X509Certificates.X509ChainStatusFlags]::ExplicitDistrust
+    $noAnchor = [int]([System.Security.Cryptography.X509Certificates.X509ChainStatusFlags]::UntrustedRoot -bor
+        [System.Security.Cryptography.X509Certificates.X509ChainStatusFlags]::PartialChain)
+    $chain = [System.Security.Cryptography.X509Certificates.X509Chain]::new()
+    try {
+        foreach ($extra in $ExtraCertificates) {
+            [void]$chain.ChainPolicy.ExtraStore.Add($extra)
+        }
+        $chain.ChainPolicy.RevocationMode = [System.Security.Cryptography.X509Certificates.X509RevocationMode]::NoCheck
+        $chain.ChainPolicy.VerificationTime = [DateTime]::new(2026, 10, 23, 0, 0, 0, [DateTimeKind]::Utc)
+        # DisableCertificateDownloads does not exist on .NET Framework (Windows PowerShell 5.1).
+        if ($chain.ChainPolicy | Get-Member -Name DisableCertificateDownloads) {
+            $chain.ChainPolicy.DisableCertificateDownloads = $true
+        }
+        $built = $chain.Build($Cert)
+        $statuses = @($chain.ChainStatus) + @($chain.ChainElements | ForEach-Object { $_.ChainElementStatus })
+        $names = @($statuses | ForEach-Object { $_.Status.ToString() } | Where-Object { $_ -ne 'NoError' } | Sort-Object -Unique)
+        if ($statuses | Where-Object { $_.Status.HasFlag($explicitDistrust) }) {
+            $state = 'ExplicitDistrust'
+        } elseif ($built) {
+            $state = 'Trusted'
+        } elseif (
+            ($statuses | Where-Object { ([int]$_.Status -band $noAnchor) -ne 0 }) -and
+            -not ($statuses | Where-Object { ([int]$_.Status -band (-bnot $noAnchor)) -ne 0 })
+        ) {
+            $state = 'NoTrustAnchor'
+        } else {
+            $state = 'Indeterminate'
+        }
+        return [pscustomobject]@{ State = $state; Statuses = ($names -join ',') }
+    } finally {
+        $chain.Dispose()
+    }
+}

.spdx-ignore

@@ -24,3 +24,4 @@ cuda_core/cuda/**/*.pyi
 
 # Third-party eMudhra disclosure data, kept byte-identical to the published files
 ci/tools/runner_trust_probe_chains/**
+ci/tools/runner_trust_probe_issuers/**

ci/tools/runner_trust_probe.ps1

@@ -47,62 +47,107 @@ $inDisallowed = [bool](Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -Erro
     Where-Object { (Get-CertificateSha256 $_) -eq $EmSignG3Hex })
 Record-Check 'emSign ECC Root CA - G3 store state' 'INFO' "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed"
 
-# 2. All compromised intermediates and cross-signs explicitly distrusted (gating)
+# 2. No compromised intermediate or cross-sign is usable as a trust path (gating).
+# Pass = explicitly distrusted, or the chain no longer reaches a trusted root
+# (the hosted images have removed the roots rather than enabling the disallowed CTL).
 $certs = @(Get-EmudhraIntermediates)
+# Supply the non-root issuers so a missing parent is not mistaken for a missing trust
+# anchor: emSign Root TLS CA - G3 comes with the vendored host chains, and
+# emSign Root CS CA - G3 (cross-signed by emSign ECC Root CA - G3) with the issuers.
+$issuerCerts = @()
+foreach ($dirName in 'runner_trust_probe_chains', 'runner_trust_probe_issuers') {
+    $found = @(
+        Get-ChildItem -LiteralPath (Join-Path $PSScriptRoot $dirName) -Filter '*.pem' |
+            ForEach-Object {
+                [regex]::Matches((Get-Content -LiteralPath $_.FullName -Raw), '-----BEGIN CERTIFICATE-----([^-]+)-----END CERTIFICATE-----') |
+                    ForEach-Object {
+                        [System.Security.Cryptography.X509Certificates.X509Certificate2]::new([System.Convert]::FromBase64String(($_.Groups[1].Value -replace '\s', '')))
+                    }
+            }
+    )
+    if ($found.Count -eq 0) {
+        throw "No certificates found in $PSScriptRoot\$dirName."
+    }
+    $issuerCerts += $found
+}
+$usable = @()
+$states = @()
+$statusNames = @()
 try {
-    $notDistrusted = @($certs | Where-Object { -not (Test-ExplicitlyDistrusted $_) } | ForEach-Object { $_.EmudhraLabel })
+    foreach ($c in $certs) {
+        $r = Get-ChainTrustState $c $issuerCerts
+        $states += $r.State
+        $statusNames += $r.Statuses -split ',' | Where-Object { $_ }
+        if ($r.State -notin @('ExplicitDistrust', 'NoTrustAnchor')) {
+            $usable += "$($c.EmudhraLabel) [$($r.State): $($r.Statuses)]"
+        }
+    }
 } finally {
     $certs | ForEach-Object { $_.Dispose() }
+    $issuerCerts | ForEach-Object { $_.Dispose() }
 }
-if ($notDistrusted.Count -eq 0) {
-    Record-Check 'Compromised intermediates distrusted' 'PASS' "All $($certs.Count) explicitly distrusted"
+$counts = ($states | Group-Object | ForEach-Object { "$($_.Name)=$($_.Count)" }) -join ', '
+$flags = ($statusNames | Sort-Object -Unique) -join ','
+if ($usable.Count -eq 0) {
+    Record-Check 'Compromised intermediates distrusted' 'PASS' "All $($certs.Count) rejected ($counts; chain status: $flags)"
 } else {
-    Record-Check 'Compromised intermediates distrusted' 'FAIL' "$($notDistrusted.Count) of $($certs.Count) not explicitly distrusted: $($notDistrusted -join '; ')"
+    Record-Check 'Compromised intermediates distrusted' 'FAIL' "$($usable.Count) of $($certs.Count) usable or indeterminate: $($usable -join '; ')"
     $GatingFailures++
 }
 
-# 3. OS-store client (Invoke-WebRequest) rejects each test endpoint (only a successful connection fails the job).
+# 3. OS-store client (Invoke-WebRequest) rejects each test endpoint (a successful connection, or no verdict after a retry, fails the job).
 # -SkipHttpErrorCheck is PowerShell 7+ only, so HTTP error statuses are caught below instead:
 # any response at all means the TLS handshake (and thus the certificate chain) succeeded.
 $progTmp = $ProgressPreference
 $ProgressPreference = 'SilentlyContinue'
 try {
     foreach ($endpoint in $TestEndpoints) {
         $checkName = "Invoke-WebRequest rejects ``$endpoint``"
-        try {
-            $res = Invoke-WebRequest -Uri $endpoint -UseBasicParsing -TimeoutSec 15
-            Record-Check $checkName 'FAIL' "Connection succeeded with HTTP $($res.StatusCode) (certificate was trusted)"
-            $GatingFailures++
-        } catch {
-            if ($null -ne $_.Exception.Response) {
-                Record-Check $checkName 'FAIL' "Connection succeeded with HTTP $([int]$_.Exception.Response.StatusCode) (certificate was trusted)"
+        # Retry non-certificate/network errors once; if the second attempt also fails,
+        # the probe could not show that the certificate is rejected, so the check fails.
+        for ($attempt = 1; $attempt -le 2; $attempt++) {
+            try {
+                $res = Invoke-WebRequest -Uri $endpoint -UseBasicParsing -TimeoutSec 15
+                Record-Check $checkName 'FAIL' "Connection succeeded with HTTP $($res.StatusCode) (certificate was trusted)"
                 $GatingFailures++
-                continue
-            }
-            $certRejected = $false
-            $hostnameMismatchOnly = $false
-            $ex = $_.Exception
-            while ($null -ne $ex) {
-                if (
-                    ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or
-                    ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure)
-                ) {
-                    $certRejected = $true
+                break
+            } catch {
+                if ($null -ne $_.Exception.Response) {
+                    Record-Check $checkName 'FAIL' "Connection succeeded with HTTP $([int]$_.Exception.Response.StatusCode) (certificate was trusted)"
+                    $GatingFailures++
                     break
                 }
-                if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') {
-                    $hostnameMismatchOnly = $true
+                $certRejected = $false
+                $hostnameMismatchOnly = $false
+                $ex = $_.Exception
+                while ($null -ne $ex) {
+                    if (
+                        ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or
+                        ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure)
+                    ) {
+                        $certRejected = $true
+                        break
+                    }
+                    if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') {
+                        $hostnameMismatchOnly = $true
+                    }
+                    $ex = $ex.InnerException
+                }
+                $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|')
+                if ($certRejected) {
+                    Record-Check $checkName 'PASS' "Rejected with certificate validation error ($cleanMsg)"
+                    break
+                } elseif ($hostnameMismatchOnly) {
+                    Record-Check $checkName 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)"
+                    $GatingFailures++
+                    break
+                } elseif ($attempt -lt 2) {
+                    Write-Output "[INFO] ${checkName}: attempt $attempt failed with a non-certificate or network error ($cleanMsg); retrying"
+                    Start-Sleep -Seconds 5
+                } else {
+                    Record-Check $checkName 'FAIL' "Could not determine whether the certificate is rejected after $attempt attempts: non-certificate or network error ($cleanMsg)"
+                    $GatingFailures++
                 }
-                $ex = $ex.InnerException
-            }
-            $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|')
-            if ($certRejected) {
-                Record-Check $checkName 'PASS' "Rejected with certificate validation error ($cleanMsg)"
-            } elseif ($hostnameMismatchOnly) {
-                Record-Check $checkName 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)"
-                $GatingFailures++
-            } else {
-                Record-Check $checkName 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)"
             }
         }
     }

ci/tools/runner_trust_probe_issuers/README.txt

@@ -0,0 +1,20 @@
+Issuer certificates for the Windows runner trust probe
+======================================================
+
+ci/tools/runner_trust_probe.ps1 passes these certificates to Windows chain
+building, together with the chains in ../runner_trust_probe_chains, so a missing
+parent certificate is not mistaken for a missing trust anchor.
+
+  emsign-root-cs-ca-g3-cross.pem
+      emSign Root CS CA - G3, issued by emSign ECC Root CA - G3 (the
+      cross-certificate of the self-signed emSign Root CS CA - G3). It is the
+      issuer of the two native code-signing intermediates (EM OV CS CA - G3B
+      and G3C), which the test hosts' chains do not cover.
+
+      Source:  https://repository.emsign.com/certs/crosssign/rootcscag3.crt
+               (DER; converted to PEM without other changes)
+      SHA-256: 7137FA71AD46BA686F746D1014AC57E8E6C6E9AF4AA2F73C0C44A5CF27DEBB59
+      Valid:   2024-07-11 to 2039-07-09
+      Checked: same public key as the self-signed emSign Root CS CA - G3, and
+               its key verifies the signatures of both code-signing
+               intermediates in impacted-intermediates.json.

ci/tools/runner_trust_probe_issuers/emsign-root-cs-ca-g3-cross.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDaDCCAu2gAwIBAgIPFMZ1TpPl2UxjjT/JD612MAoGCCqGSM49BAMDMGsxCzAJ
+BgNVBAYTAklOMRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJh
+IFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDExdlbVNpZ24gRUNDIFJvb3Qg
+Q0EgLSBHMzAeFw0yNDA3MTExNjEwMzBaFw0zOTA3MDkxODMwMDBaMFUxCzAJBgNV
+BAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8w
+HQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMHYwEAYHKoZIzj0CAQYFK4EE
+ACIDYgAESl9kM5D0VA/Og76dz5Xzw1hnGQ4fSR9993/qBkkvI48pXQVKOdyHCgAZ
+ThgWejQ8OpFRiTISjpabn4tfRFOYuzhApR/XxGbkg9X+2cEtbyghF9CdoF91W22V
+/R2EVe4xo4IBazCCAWcwHwYDVR0jBBgwFoAUfF0ChBPUzIqbgc4XHC4pHpxIY0Iw
+HQYDVR0OBBYEFGh0A/iFyo1UUFcdFuDI3kUNq80tMBMGA1UdJQQMMAoGCCsGAQUF
+BwMDMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxodHRwOi8vcmVw
+b3NpdG9yeS5lbXNpZ24uY29tMHgGCCsGAQUFBwEBBGwwajAiBggrBgEFBQcwAYYW
+aHR0cDovL29jc3AuZW1zaWduLmNvbTBEBggrBgEFBQcwAoY4aHR0cDovL3JlcG9z
+aXRvcnkuZW1zaWduLmNvbS9jZXJ0cy9lbVNpZ25FQ0NSb290Q0FHMy5jcnQwMwYD
+VR0fBCwwKjAooCagJIYiaHR0cDovL2NybC5lbXNpZ24uY29tP1Jvb3RDQUczLmNy
+bDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATAKBggqhkjOPQQD
+AwNpADBmAjEAuFH6M20dolyKkZ9AD7DsgJyo8MkQTAc05niLQzoyqo60+mYdtseW
+P2tOfXULs3XBAjEA7r4peOh9eWi+vDbjD8kB2lxRGYW2n/I2+KTLKYAOzp/9GBmx
+x0cnzyQU+QX3koAd
+-----END CERTIFICATE-----