From 2177e4b1e832b1df2c12e90ee41b4ada4c8f5e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Sat, 18 Jul 2026 11:46:12 +0200 Subject: [PATCH 1/4] Add GitHub Actions CI pilot (with Codecov) alongside Azure Pipelines Runs in parallel with azure-pipelines.yml so we can compare the two before cutting over. Build once on Windows, then the same 8-leg matrix as Azure: PowerShell 7 on ubuntu/macOS/windows (latest + one previous GA), Windows PowerShell 5.1 on windows (latest + previous). Coverage renders in each run's job summary and a consolidated table, and uploads to Codecov with a flag per leg. GA-only images: floating *-latest plus one pinned previous GA. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 257 +++++++++++++++++++++++++++++++++++++++ codecov.yml | 30 +++++ 2 files changed, 287 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..6e8e9c67b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,257 @@ +# GitHub Actions CI — pilot port of azure-pipelines.yml. +# +# This runs ALONGSIDE Azure Pipelines (we did not remove azure-pipelines.yml), +# so every push/PR triggers both systems and we can compare them before cutting +# over. Mirrors the Azure setup: build once on Windows, publish the built module +# as an artifact, then fan out the test matrix (all 8 legs run in parallel). +# +# Coverage: each leg runs `test.ps1 -CC`, producing a JaCoCo coverage.xml. Azure +# rendered this via PublishCodeCoverageResults@2; here we surface it natively in +# the run's job summaries (per leg, plus a consolidated table in "Done") and keep +# the raw coverage.xml as an artifact. +# +# Image policy: GA only. We float `*-latest` for the newest GA image and pin the +# one previous GA image ("latest + reasonably recent"). No preview images +# (macos-26, windows-2025-vs2026) are used. Note macos-14 is on the deprecation +# clock (brownouts Oct 2026, removal Nov 2 2026) — swap it out before then. +name: CI (GitHub Actions pilot) + +on: + push: + branches: [main, 'rel/*'] + paths-ignore: + - '.devcontainer/**' + - '.vscode/**' + - 'docs/**' + - 'images/**' + - '**/*.md' + pull_request: + branches: [main, 'rel/*', 'dev/*'] + paths-ignore: + - '.devcontainer/**' + - '.vscode/**' + - 'docs/**' + - 'images/**' + - '**/*.md' + workflow_dispatch: + +permissions: + contents: read + +env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: '1' + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_GENERATE_ASPNET_CERTIFICATE: '0' + DOTNET_NOLOGO: '1' + CI: '1' + +# Supersede in-flight runs for the same branch/PR to save minutes. +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build + runs-on: windows-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Setup .NET (global.json) + uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + - name: Build module (inline, locked restore) + shell: pwsh + run: | + "Running .NET SDK v$(dotnet --version)" + ./build.ps1 -LockedRestore -Clean -Inline + + # Publish the built tree so every test leg runs the exact same build. + # Mirrors Azure's `publish: $(Build.SourcesDirectory)` + .artifactignore. + - name: Upload build output + uses: actions/upload-artifact@v4 + with: + name: pester-build + retention-days: 1 + include-hidden-files: true + path: | + bin/ + src/ + tst/ + build.ps1 + test.ps1 + global.json + !src/csharp/**/bin/** + !src/csharp/**/obj/** + !src/csharp/.vs/** + + test: + name: ${{ matrix.name }} + needs: build + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + # PowerShell 7 (psexe: pwsh) — latest + one previous GA per OS. + - { name: 'PS7 - Ubuntu latest', id: ps7-ubuntu-latest, os: ubuntu-latest, psexe: pwsh } + - { name: 'PS7 - Ubuntu 22.04', id: ps7-ubuntu-2204, os: ubuntu-22.04, psexe: pwsh } + - { name: 'PS7 - macOS latest', id: ps7-macos-latest, os: macos-latest, psexe: pwsh } + - { name: 'PS7 - macOS 14', id: ps7-macos-14, os: macos-14, psexe: pwsh } + - { name: 'PS7 - Windows latest', id: ps7-windows-latest, os: windows-latest, psexe: pwsh } + - { name: 'PS7 - Windows 2022', id: ps7-windows-2022, os: windows-2022, psexe: pwsh } + # Windows PowerShell 5.1 (psexe: powershell) — Windows only. + - { name: 'PS5.1 - Windows latest', id: ps51-windows-latest, os: windows-latest, psexe: powershell } + - { name: 'PS5.1 - Windows 2022', id: ps51-windows-2022, os: windows-2022, psexe: powershell } + steps: + # No checkout — we test the published build, exactly like Azure (checkout: none). + - name: Download build output + uses: actions/download-artifact@v4 + with: + name: pester-build + path: . + + # `shell:` cannot take a matrix expression, so run from a fixed pwsh (present + # on every runner) and launch the leg's interpreter: pwsh for PS7, powershell + # (Windows PowerShell 5.1) for the 5.1 legs. exit $LASTEXITCODE so a failure + # in the launched process still fails the job. + - name: Test Pester + shell: pwsh + run: | + & ${{ matrix.psexe }} -NoProfile -Command "& ./test.ps1 -CI -CC -PassThru -NoBuild" + exit $LASTEXITCODE + + # Render this leg's JaCoCo coverage into the job summary (Azure parity). + - name: Coverage summary + if: always() + shell: pwsh + env: + LEG_NAME: ${{ matrix.name }} + run: | + $ErrorActionPreference = 'Continue' + function Append-Summary([string]$text) { + [System.IO.File]::AppendAllText($env:GITHUB_STEP_SUMMARY, $text, (New-Object System.Text.UTF8Encoding($false))) + } + try { + if (-not (Test-Path 'coverage.xml')) { + Append-Summary "## Coverage - $env:LEG_NAME`n`nNo coverage.xml was produced.`n" + return + } + [xml]$xml = Get-Content -LiteralPath 'coverage.xml' + $counters = @($xml.report.counter) # report-level totals (direct children) + $rows = foreach ($type in 'LINE','INSTRUCTION','METHOD','CLASS') { + $c = $counters | Where-Object { $_.type -eq $type } + if ($c) { + $cov = [int]$c.covered; $mis = [int]$c.missed; $tot = $cov + $mis + $pct = if ($tot) { [math]::Round(100.0 * $cov / $tot, 2) } else { 0 } + $pctStr = ([double]$pct).ToString('0.##', [System.Globalization.CultureInfo]::InvariantCulture) + [pscustomobject]@{ Type = $type; Covered = $cov; Missed = $mis; Total = $tot; PctStr = $pctStr } + } + } + $line = $rows | Where-Object { $_.Type -eq 'LINE' } + $sb = [System.Text.StringBuilder]::new() + [void]$sb.AppendLine("## Coverage - $env:LEG_NAME`n") + if ($line) { [void]$sb.AppendLine("**Line coverage: $($line.PctStr)%** ($($line.Covered)/$($line.Total) lines)`n") } + [void]$sb.AppendLine("| Metric | Covered | Missed | Total | % |") + [void]$sb.AppendLine("|---|--:|--:|--:|--:|") + foreach ($r in $rows) { [void]$sb.AppendLine("| $($r.Type) | $($r.Covered) | $($r.Missed) | $($r.Total) | $($r.PctStr)% |") } + Append-Summary ($sb.ToString() + "`n") + if ($line) { Write-Host "Line coverage ($env:LEG_NAME): $($line.PctStr)%" } + } catch { + Append-Summary "## Coverage - $env:LEG_NAME`n`nCoverage summary failed: $($_.Exception.Message)`n" + } + + # Upload this leg's JaCoCo report to Codecov. `flags` keeps a per-leg + # breakdown while Codecov merges all legs into one project report. + # pester/Pester is public, so upload works tokenless (v5 opt-out); a + # CODECOV_TOKEN secret, if set, makes uploads more reliable. + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: ./coverage.xml + disable_search: true + flags: ${{ matrix.id }} + name: ${{ matrix.name }} + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.id }} + retention-days: 7 + if-no-files-found: ignore + path: | + testResults.xml + coverage.xml + + # Single gate after the matrix so branch protection / auto-merge only needs to + # require "Done" instead of listing all 8 legs. Mirrors Azure's Done stage. + # It also renders a consolidated coverage table (one glance at all legs). + done: + name: Done + needs: test + if: always() + runs-on: ubuntu-latest + steps: + - name: Download coverage reports + if: always() + uses: actions/download-artifact@v4 + with: + pattern: results-* + path: coverage-reports + + - name: Consolidated coverage + if: always() + shell: pwsh + run: | + $ErrorActionPreference = 'Continue' + function Append-Summary([string]$text) { + [System.IO.File]::AppendAllText($env:GITHUB_STEP_SUMMARY, $text, (New-Object System.Text.UTF8Encoding($false))) + } + function Get-Pct($counters, [string]$type) { + $c = $counters | Where-Object { $_.type -eq $type } + if (-not $c) { return $null } + $cov = [int]$c.covered; $tot = $cov + [int]$c.missed + $pct = if ($tot) { [math]::Round(100.0 * $cov / $tot, 2) } else { 0 } + ([double]$pct).ToString('0.##', [System.Globalization.CultureInfo]::InvariantCulture) + } + try { + $files = Get-ChildItem -Path 'coverage-reports' -Recurse -Filter 'coverage.xml' -ErrorAction SilentlyContinue + if (-not $files) { Append-Summary "## Coverage by matrix leg`n`nNo coverage reports found.`n"; return } + $sb = [System.Text.StringBuilder]::new() + [void]$sb.AppendLine("## Coverage by matrix leg`n") + [void]$sb.AppendLine("| Leg | Line % | Instruction % |") + [void]$sb.AppendLine("|---|--:|--:|") + foreach ($f in ($files | Sort-Object FullName)) { + $leg = (Split-Path (Split-Path $f.FullName -Parent) -Leaf) -replace '^results-', '' + try { + [xml]$xml = Get-Content -LiteralPath $f.FullName + $counters = @($xml.report.counter) + $lpct = Get-Pct $counters 'LINE' + $ipct = Get-Pct $counters 'INSTRUCTION' + [void]$sb.AppendLine("| $leg | $(if ($null -ne $lpct) { "$lpct%" } else { 'n/a' }) | $(if ($null -ne $ipct) { "$ipct%" } else { 'n/a' }) |") + } catch { + [void]$sb.AppendLine("| $leg | error | error |") + } + } + Append-Summary ($sb.ToString() + "`n") + } catch { + Append-Summary "## Coverage by matrix leg`n`nConsolidated coverage failed: $($_.Exception.Message)`n" + } + + - name: All test legs passed + run: | + echo "Test matrix result: ${{ needs.test.result }}" + [ "${{ needs.test.result }}" = "success" ] || exit 1 + echo "done" diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..9fc23f205 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,30 @@ +# Codecov configuration — added alongside the GitHub Actions CI pilot +# (.github/workflows/ci.yml). Coverage is uploaded per matrix leg using flags. +# +# During the pilot we keep coverage status "informational" so Codecov never +# posts a failing/blocking commit status while we compare against Azure. + +codecov: + # We upload one report per matrix leg (8 legs). Wait for all of them before + # finalizing notifications/PR comment so the merged picture is complete. + notify: + after_n_builds: 8 + +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + +comment: + layout: "header, diff, flags, files" + require_changes: false + +# Per-leg flags (ps7-ubuntu-latest, ps51-windows-2022, ...) uploaded by ci.yml. +# Carry a flag's last-known coverage forward on commits where it didn't upload. +flag_management: + default_rules: + carryforward: true From b9ca9a41283372a57794b966e348f81391c0e40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Sat, 18 Jul 2026 22:07:19 +0200 Subject: [PATCH 2/4] Address review: add test-reporter and Dependabot for actions - Publish NUnit3 test results per matrix leg with dorny/test-reporter, a GitHub-native test report (Azure test-tab parity). test.ps1 gains an env-selectable result format (PESTER_TESTRESULT_FORMAT); Azure keeps the default NUnit 2.5, only the pilot switches to NUnit3. - Add Dependabot for github-actions (weekly, grouped, incl. major) to keep the actions current and ahead of Node.js runtime deprecations. Co-Authored-By: Claude Opus 4.8 --- .github/dependabot.yml | 16 ++++++++++++++++ .github/workflows/ci.yml | 19 +++++++++++++++++++ test.ps1 | 7 +++++++ 3 files changed, 42 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..9f3ed4223 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + # Keep the GitHub Actions used by the workflows current — in particular so we + # stay ahead of Node.js runtime deprecations, which land as major bumps. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + github-actions: + patterns: + - "*" + update-types: + - major + - minor + - patch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e8e9c67b..a5d43c330 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,8 @@ on: permissions: contents: read + actions: read + checks: write # dorny/test-reporter creates a check run with the test report env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: '1' @@ -97,6 +99,10 @@ jobs: needs: build runs-on: ${{ matrix.os }} timeout-minutes: 20 + # NUnit3 so dorny/test-reporter (below) can parse the result file. test.ps1 + # reads this; unset elsewhere (e.g. Azure) keeps the default NUnit 2.5. + env: + PESTER_TESTRESULT_FORMAT: NUnit3 strategy: fail-fast: false matrix: @@ -195,6 +201,19 @@ jobs: testResults.xml coverage.xml + # Render the NUnit3 results as a check run (Azure test-tab parity). + # fail-on-error: false so a parse hiccup — or a fork PR, where the token is + # read-only and can't create checks — doesn't fail the leg. + - name: Publish test report + if: always() + uses: dorny/test-reporter@v3 + with: + name: 'Tests - ${{ matrix.name }}' + path: testResults.xml + reporter: dotnet-nunit + fail-on-error: false + use-actions-summary: 'true' + # Single gate after the matrix so branch protection / auto-merge only needs to # require "Done" instead of listing all 8 legs. Mirrors Azure's Done stage. # It also renders a consolidated coverage table (one glance at all legs). diff --git a/test.ps1 b/test.ps1 index 8939f2e3e..2b0c31f88 100644 --- a/test.ps1 +++ b/test.ps1 @@ -185,6 +185,13 @@ if ($CI) { $configuration.CodeCoverage.Enabled = $false $configuration.TestResult.Enabled = $true + + # Let CI pick the test-result format. Azure Pipelines consumes the default + # NUnit 2.5 schema; the GitHub Actions pilot sets NUnit3 so dorny/test-reporter + # can render it. Unset -> unchanged for every existing caller. + if ($env:PESTER_TESTRESULT_FORMAT) { + $configuration.TestResult.OutputFormat = $env:PESTER_TESTRESULT_FORMAT + } } $r = Invoke-Pester -Configuration $configuration From 247ede194b20057e522c5876b002a88c3f80b9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Sat, 18 Jul 2026 22:26:58 +0200 Subject: [PATCH 3/4] Give test legs a checkout so test-reporter has git context dorny/test-reporter shells out to git and fails with exit 128 when the leg has no repo checkout (the legs are artifact-only). Add a shallow checkout before the artifact download; the artifact overlays it, so the tests still run against the published build. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5d43c330..250879f73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,14 @@ jobs: - { name: 'PS5.1 - Windows latest', id: ps51-windows-latest, os: windows-latest, psexe: powershell } - { name: 'PS5.1 - Windows 2022', id: ps51-windows-2022, os: windows-2022, psexe: powershell } steps: - # No checkout — we test the published build, exactly like Azure (checkout: none). + # Shallow checkout only so dorny/test-reporter (below) has git context — it + # shells out to git and errors without a repo. The artifact overlays this, + # so the tests still run against the published build, like Azure. + - name: Checkout (git context for test-reporter) + uses: actions/checkout@v6 + with: + fetch-depth: 1 + - name: Download build output uses: actions/download-artifact@v4 with: From 4d4136b003f96bf602a9686d231859cb2b1283be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Sun, 19 Jul 2026 10:41:23 +0200 Subject: [PATCH 4/4] Use NUnit3 test results for both Azure and the pilot Azure PublishTestResults@2 supports NUnit3, so test.ps1 can just emit the modern format for both consumers. Drops the PESTER_TESTRESULT_FORMAT env indirection. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 4 ---- test.ps1 | 9 +++------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 250879f73..9d509f1c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,10 +99,6 @@ jobs: needs: build runs-on: ${{ matrix.os }} timeout-minutes: 20 - # NUnit3 so dorny/test-reporter (below) can parse the result file. test.ps1 - # reads this; unset elsewhere (e.g. Azure) keeps the default NUnit 2.5. - env: - PESTER_TESTRESULT_FORMAT: NUnit3 strategy: fail-fast: false matrix: diff --git a/test.ps1 b/test.ps1 index 2b0c31f88..d927c7e8b 100644 --- a/test.ps1 +++ b/test.ps1 @@ -186,12 +186,9 @@ if ($CI) { $configuration.TestResult.Enabled = $true - # Let CI pick the test-result format. Azure Pipelines consumes the default - # NUnit 2.5 schema; the GitHub Actions pilot sets NUnit3 so dorny/test-reporter - # can render it. Unset -> unchanged for every existing caller. - if ($env:PESTER_TESTRESULT_FORMAT) { - $configuration.TestResult.OutputFormat = $env:PESTER_TESTRESULT_FORMAT - } + # Modern NUnit3 schema. Both consumers read it: Azure Pipelines + # (PublishTestResults@2) and the GitHub Actions pilot (dorny/test-reporter). + $configuration.TestResult.OutputFormat = 'NUnit3' } $r = Invoke-Pester -Configuration $configuration