diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e702d9e7b..a12065ea5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,11 +13,18 @@ updates: interval: "weekly" day: "monday" open-pull-requests-limit: 10 - # One grouped PR for all NuGet bumps, to keep review noise down. + # One grouped PR for routine NuGet bumps, to keep review noise down. Majors are + # deliberately NOT in the group (Studio's convention): a breaking change arrives as + # its own PR instead of buried in a routine batch. With central package management + # (Directory.Packages.props) each bump is one line, so grouped PRs can no longer + # ship the #2100 class of multi-project version misalignment. groups: - nuget: + nuget-patch-and-minor: patterns: - "*" + update-types: + - "minor" + - "patch" # GitHub Actions used by the workflows in .github/workflows. - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6dc6cd444..bd8e9924e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,872 +1,876 @@ -name: Build - -on: - push: - branches: [main, dev] - pull_request: - branches: [main, dev] - release: - types: [published] - # Merge-queue runs. Inert until a queue ruleset is enabled on a branch (a repo setting), - # but the required checks must handle the event BEFORE that click, or every queued PR - # stalls on checks that never report. dorny/paths-filter v4.0.1+ resolves merge_group - # diffs from the payload's base_sha/head_sha whenever the base input is empty — exactly - # what the filter steps pass for non-push events — so path classification works - # unchanged in a queue run. - merge_group: - -permissions: - contents: write - id-token: write - actions: read - -# A re-push to a PR cancels that PR's superseded in-flight run, and a push to dev/main -# cancels that BRANCH's superseded in-flight run — newest SHA wins. Finishing a build of -# code that is no longer the head helps nobody, and the shared Windows runner pool is what -# serializes everyone's CI (#1697 sat queued behind two dev builds; on 2026-07-26 a -# ~20-merge train left 13 of the day's 30 dev-push runs finishing SHAs a newer merge had -# already replaced — ~60 reclaimable runner-minutes in one evening). Cancelling a -# superseded PUSH run is safe because a push run produces nothing any other run consumes: -# every upload-artifact step in this workflow is gated to the release event (the SignPath -# signing path) or to failure() (darling-pg diagnostics), nothing in the repo downloads -# cross-run artifacts (no download-artifact, gh run download, or workflow_run consumer -# exists), nightly.yml builds its own tree from its own checkout, and a release compiles -# fresh on the release event. The accepted trade: push builds are diff-scoped, so a -# cancelled run's areas are not re-verified until the next change touches them — the -# nightly and the all-areas dev->main release PR are the backstops. Release and -# merge-queue runs deliberately keep a UNIQUE group per run (run_id) and are NEVER -# cancelled: a release build waits on SignPath's manual approval gate, and a queue -# validation is the last check before its result lands on dev. -concurrency: - group: ${{ github.event_name == 'pull_request' && format('build-pr-{0}', github.event.pull_request.number) || github.event_name == 'push' && format('build-push-{0}', github.ref) || format('build-run-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v7 - - - name: Detect changed paths - id: filter - if: github.event_name != 'release' - uses: dorny/paths-filter@v4 - with: - # On push events, compare against the previous commit on this branch - # (github.event.before). Without this, the action defaults to comparing - # against the default branch on non-default branch pushes, which would - # match every accumulated change and defeat the filter. - base: ${{ github.event_name == 'push' && github.event.before || '' }} - # Emit the matched file list so the fast-path step can NAME what it classified - # as documentation. A fast path that silently under-builds is the failure mode - # worth guarding against, so the reason is always printed, never inferred. - list-files: shell - filters: | - # A change to a root build file (the solution, restore config, or THIS workflow) can - # affect every product, so it forces a full build/test/publish. - root: - - 'PerformanceMonitor.sln' - - 'global.json' - - 'nuget.config' - - 'NuGet.config' - - '.github/workflows/build.yml' - # The shared PerformanceMonitor.* core libraries feed Lite, the Full Dashboard, AND - # Darling (verified via ProjectReference), so a change here fans out to all three. - # NOT the CLI Installer — it references only Installer.Core. - # - # Every area pattern says `dir/**/!(*.md)` — any non-markdown file under the - # area — instead of the old `dir/**` include plus a bare `!**/*.md` exclude. - # That is not style: dorny v4 evaluates each pattern as an INDEPENDENT - # predicate under the default predicate-quantifier 'some' (a filter is true - # when any changed file matches at least one rule), so a bare `!**/*.md` line - # is not a subtraction — it is its own rule meaning "any file that is not - # markdown", which silently made every area filter true for ANY non-markdown - # change anywhere in the repo. Measured proof: a single root .gitignore edit - # built and tested all four products and ran the full Darling PG suite - # (PR #1714, run 30219202642, filter log: "Filter darling = true, Matching - # files: .gitignore"). The extglob keeps the markdown carve-out INSIDE the - # include, where quantifier semantics cannot detach it. - core: - - 'PerformanceMonitor.Alerting/**/!(*.md)' - - 'PerformanceMonitor.Analysis/**/!(*.md)' - - 'PerformanceMonitor.Collectors/**/!(*.md)' - - 'PerformanceMonitor.Common/**/!(*.md)' - - 'PerformanceMonitor.Notifications/**/!(*.md)' - - 'PerformanceMonitor.PlanAnalysis/**/!(*.md)' - - 'PerformanceMonitor.Ui/**/!(*.md)' - # Installer.Core is shared by the CLI Installer AND the Full Dashboard's integrated - # installer — a change rebuilds both, and nothing else. - installer_core: - - 'deprecated/Installer.Core/**/!(*.md)' - dashboard: - - 'deprecated/Dashboard/**/!(*.md)' - - 'deprecated/Dashboard.Tests/**/!(*.md)' - lite: - - 'Lite/**/!(*.md)' - - 'Lite.Tests/**/!(*.md)' - # Same silently-stops-guarding reason as the darling filter's Lite entries below: - # Lite.Tests/ThemeParityLiteDarlingTests.cs READS the Darling viewer's theme - # dictionaries to assert the two apps' shared brush keys still resolve to the same - # colors. A Darling-theme-only edit is exactly the drift that guard exists to catch, - # so it has to reach the suite. - - 'Darling/PerformanceMonitor.Darling.Viewer/Themes/*.xaml' - installer: - - 'deprecated/Installer/**/!(*.md)' - - 'deprecated/Installer.Tests/**/!(*.md)' - - 'install/**/!(*.md)' - - 'upgrades/**/!(*.md)' - darling: - - 'Darling/**/!(*.md)' - # nightly.yml is not a build input, but Darling.Tests PARSES it: the #1888 - # guard reads both workflows' throwaway-cluster settings and compares them - # against the product's worker-sizing formula. Without this, a nightly-only - # edit would change a file the guard asserts on while never running the - # guard — a guard that silently stops guarding, which is the exact failure - # mode the source-parsing tests here exist to prevent. - - '.github/workflows/nightly.yml' - # Same reason, Lite side: the #1949 pin in Darling.Tests asserts every twinned - # query grid carries the SAME column sequence in both front ends, so it reads - # these six Lite files. A Lite-only XAML edit has to reach the suite or the - # parity half of that guard stops guarding. - - 'Lite/Controls/ServerTab.xaml' - - 'Lite/Controls/FinOpsTab.xaml' - - 'Lite/Windows/WaitDrillDownWindow.xaml' - - 'Lite/Windows/ProcedureHistoryWindow.xaml' - - 'Lite/Windows/QueryStatsHistoryWindow.xaml' - - 'Lite/Windows/QueryStoreHistoryWindow.xaml' - # The DOCUMENTATION allowlist: files that cannot affect a build under any - # job in this workflow. Deliberately an allowlist of non-executable content, - # not a "everything that isn't code" subtraction — a new file type defaults - # to being treated as code, which is the safe direction to be wrong in. - # - # NOT here, on purpose: *.sql (the installer and sql-validation compile it), - # *.yml (workflows), *.csproj / *.props / packages.lock.json (build inputs), - # and *.cs regardless of how comment-only the change looks — an XML doc - # comment still recompiles, and the compiler is what proves it still builds. - # - # The docs/ and Screenshots/ entries are extension-explicit rather than bare - # directory globs for the same reason: everything in them today is markdown, - # SVG, or a screenshot image, and a .sql or script dropped into either - # directory tomorrow should default to being code, not inherit a free pass - # from its parent directory. - docs: - - '**/*.md' - - 'LICENSE' - - 'CITATION.cff' - - '.gitignore' - - '.gitattributes' - - 'docs/**/*.{md,svg,png,jpg,jpeg,gif}' - - 'Screenshots/**/*.{md,svg,png,jpg,jpeg,gif}' - # Catch-all COUNTER, not a boolean gate: the classify step below decides - # "documentation-only" by comparing all_count to docs_count — they are equal - # exactly when every changed file sits on the docs allowlist. Stated as a - # count comparison because the previous shape ('**' plus '!' exclusions, - # a code: filter) could never be false under predicate-quantifier 'some' — - # every file matches '**', so the #1712 fast path shipped unable to engage - # (throwaway PR #1714: a .gitignore-only diff still paid setup + restore and, - # via the predicate bug above, a full build). - all: - - '**' - - # Decides the docs fast path ONCE, in one place, and says so out loud. Guards keep - # it off every path where a skipped restore would be a real loss: - # release — the filter step does not even run there, and a release must always - # compile and publish from a cold, fully restored tree. - # push — dev/main pushes are the integration signal for what just merged, so - # they restore unconditionally even for a docs-only commit. Cheap - # insurance: this only forces the restore back on, it does not force - # the per-product build/test steps, which stay path-gated as before. - # merge_group — a queue run is the LAST validation before its result lands on dev, - # so it takes the same always-restore path as a push. - # areas — belt and suspenders: even when the counts say docs-only, any lit - # area filter vetoes the fast path, because an area=true with restore - # skipped would run `dotnet build --no-restore` against nothing. The - # two classifications are built from the same allowlist so they cannot - # disagree today; this guard is for the day someone edits one and not - # the other. - # Everything else (pull_request) is eligible, and engages only when EVERY changed - # file is on the documentation allowlist (all_count == docs_count). - - name: Classify change for the docs fast path - id: fastpath - shell: bash - env: - ALL_COUNT: ${{ steps.filter.outputs.all_count }} - DOCS_COUNT: ${{ steps.filter.outputs.docs_count }} - DOCS_FILES: ${{ steps.filter.outputs.docs_files }} - AREAS: 'root=${{ steps.filter.outputs.root }} core=${{ steps.filter.outputs.core }} installer_core=${{ steps.filter.outputs.installer_core }} dashboard=${{ steps.filter.outputs.dashboard }} lite=${{ steps.filter.outputs.lite }} installer=${{ steps.filter.outputs.installer }} darling=${{ steps.filter.outputs.darling }}' - run: | - set -euo pipefail - - if [ "${{ github.event_name }}" = "release" ]; then - echo "engaged=false" >> "$GITHUB_OUTPUT" - echo "::notice title=Full build::Release event - the docs fast path never applies to a release." - exit 0 - fi - - if [ "${{ github.event_name }}" = "push" ] || [ "${{ github.event_name }}" = "merge_group" ]; then - echo "engaged=false" >> "$GITHUB_OUTPUT" - echo "::notice title=Full build::${{ github.event_name }} on '${{ github.ref_name }}' - integration runs always restore, even for a docs-only change." - exit 0 - fi - - echo "Changed files: ${ALL_COUNT:-0} total, ${DOCS_COUNT:-0} on the documentation allowlist. Areas: ${AREAS}" - - if [ "${ALL_COUNT:-0}" -gt 0 ] && [ "${ALL_COUNT:-0}" -eq "${DOCS_COUNT:-0}" ] && [[ "${AREAS}" != *"=true"* ]]; then - echo "engaged=true" >> "$GITHUB_OUTPUT" - echo "::notice title=DOCS FAST PATH ENGAGED::All ${ALL_COUNT} changed files are on the documentation allowlist, so .NET setup, restore and versioning are skipped. This job still reports its result." - echo "Documentation files classified in this change:" - for f in ${DOCS_FILES}; do echo " - ${f}"; done - else - echo "engaged=false" >> "$GITHUB_OUTPUT" - echo "::notice title=Full build::At least one changed file is off the documentation allowlist (${DOCS_COUNT:-0} of ${ALL_COUNT:-0} classified as documentation)." - fi - - - name: Setup .NET 10.0 - if: steps.fastpath.outputs.engaged != 'true' - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - - name: Restore dependencies - if: steps.fastpath.outputs.engaged != 'true' - run: | - dotnet restore Lite/PerformanceMonitorLite.csproj --locked-mode - dotnet restore Lite.Tests/Lite.Tests.csproj --locked-mode - dotnet restore deprecated/Installer.Tests/Installer.Tests.csproj --locked-mode - dotnet restore deprecated/Dashboard.Tests/Dashboard.Tests.csproj --locked-mode - dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode - dotnet restore Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj --locked-mode - - - name: Build Lite.Tests - if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet build Lite.Tests/Lite.Tests.csproj -c Release --no-restore - - - name: Build Installer.Tests - if: steps.filter.outputs.installer == 'true' || steps.filter.outputs.installer_core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet build deprecated/Installer.Tests/Installer.Tests.csproj -c Release --no-restore - - # The 'dashboard' path filter was defined when the Full Dashboard moved to deprecated/ (#1612) but - # never wired to a step, so its build and tests silently stopped running — which is how a batch of - # compiler warnings and three broken ThemeParityTests accumulated unnoticed (#1643). Deprecated means - # bug-fix-only, not unverified: it still compiles warning-free and its tests still guard cross-app - # parity (the theme palettes it checks are LITE's too). - - name: Build Dashboard.Tests - if: steps.filter.outputs.dashboard == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet build deprecated/Dashboard.Tests/Dashboard.Tests.csproj -c Release --no-restore - - - name: Build Darling - if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: | - dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore - dotnet build Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release --no-restore - - # One step for the whole Lite suite. It was split into fast / analysis-heavy halves when the - # seven analysis classes rebuilt the full DuckDB schema inside every test and their subset - # alone cost ~9 minutes; after the shared class fixtures (#1693, #1698) and batched seeding - # (#1694) that subset runs in ~1 minute, so the split — and the narrower lite_analysis path - # gate that let non-analysis Lite changes skip it — stopped earning its second test-host - # spin-up and its filter-drift risk. - - name: Run Lite tests - if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet test Lite.Tests/Lite.Tests.csproj -c Release --no-build --verbosity normal - - - name: Run Installer tests - if: steps.filter.outputs.installer == 'true' || steps.filter.outputs.installer_core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet test deprecated/Installer.Tests/Installer.Tests.csproj -c Release --no-build --verbosity normal --filter "FullyQualifiedName!~VersionDetectionTests&FullyQualifiedName!~IdempotencyTests&FullyQualifiedName!~AdversarialTests" - - - name: Run Dashboard tests - if: steps.filter.outputs.dashboard == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet test deprecated/Dashboard.Tests/Dashboard.Tests.csproj -c Release --no-build --verbosity normal - - - name: Run Darling tests - if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal - - - name: Get version - if: steps.fastpath.outputs.engaged != 'true' - id: version - shell: pwsh - run: | - $version = ([xml](Get-Content Lite/PerformanceMonitorLite.csproj)).Project.PropertyGroup.Version | Where-Object { $_ } - echo "VERSION=$version" >> $env:GITHUB_OUTPUT - - - name: Publish Lite - if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -o publish/Lite - - - name: Publish Lite (self-contained for Velopack) - if: github.event_name == 'release' - run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -r win-x64 --self-contained -o publish/Lite-velopack - - - name: Publish Darling Service - if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -o publish/DarlingService - - - name: Publish Darling Viewer - if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' - run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -o publish/DarlingViewer - - - name: Publish Darling Viewer (self-contained for Velopack) - if: github.event_name == 'release' - run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -r win-x64 --self-contained -o publish/DarlingViewer-velopack - - # Darling bundles a PostgreSQL 18 + TimescaleDB runtime (pg-runtime.zip) that ships beside - # the service exe; DarlingManagedPostgres extracts it on first run. The fetch script pulls - # ~340MB of pinned EDB/TimescaleDB archives, so this is release-only and cached. The key is - # the fetch script's own content hash (the SHA256 pins live inside it): a re-release with - # unchanged pins restores the assembled zip and skips both the download and the assembly, - # and any pin/version bump edits the script and invalidates the cache automatically. - - name: Cache Darling pg-runtime.zip - if: github.event_name == 'release' - id: cache-pg-runtime - uses: actions/cache@v6 - with: - path: Darling/artifacts/pg-runtime.zip - key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} - - - name: Build Darling pg-runtime.zip - if: github.event_name == 'release' && steps.cache-pg-runtime.outputs.cache-hit != 'true' - shell: pwsh - run: ./Darling/tools/fetch-pg-runtime.ps1 - - - name: Package release artifacts - if: github.event_name == 'release' - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - New-Item -ItemType Directory -Force -Path releases - - # Lite ZIP - portable artifact for advanced/air-gapped users. The README points end - # users at Setup.exe (Velopack); this ZIP is the explicit fallback. - Compress-Archive -Path 'publish/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force - - # upload-artifact is deliberately HELD at v6 (#1653): every signing step below consumes - # `steps.upload-*.outputs.artifact-id`, and v7 changes artifact archiving semantics (the - # `archive` parameter). The signing path only executes on `release: [published]`, so a broken - # bump surfaces at release time — bump only alongside a validated real signing run. - # Dependabot is configured to skip this major (see .github/dependabot.yml). - - name: Upload Lite for signing - if: github.event_name == 'release' - id: upload-lite - uses: actions/upload-artifact@v6 - with: - name: Lite-unsigned - path: publish/Lite/ - - - name: Stage Darling for signing - if: github.event_name == 'release' - shell: pwsh - run: | - $stage = 'publish/Darling-signing' - if (Test-Path $stage) { Remove-Item -Recurse -Force $stage } - New-Item -ItemType Directory -Force -Path "$stage/viewer" | Out-Null - Copy-Item 'publish/DarlingService/*' $stage -Recurse - Copy-Item 'publish/DarlingViewer/*' "$stage/viewer" -Recurse - - - name: Upload Darling for signing - if: github.event_name == 'release' - id: upload-darling - uses: actions/upload-artifact@v6 - with: - name: Darling-unsigned - path: publish/Darling-signing/ - - - name: Sign Lite - if: github.event_name == 'release' - uses: signpath/github-action-submit-signing-request@v2 - with: - api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' - organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' - project-slug: 'PerformanceMonitor' - signing-policy-slug: 'release-signing' - artifact-configuration-slug: 'Lite' - github-artifact-id: '${{ steps.upload-lite.outputs.artifact-id }}' - wait-for-completion: true - output-artifact-directory: 'signed/Lite' - - - name: Sign Darling - if: github.event_name == 'release' - uses: signpath/github-action-submit-signing-request@v2 - with: - api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' - organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' - project-slug: 'PerformanceMonitor' - signing-policy-slug: 'release-signing' - artifact-configuration-slug: 'Darling' - github-artifact-id: '${{ steps.upload-darling.outputs.artifact-id }}' - wait-for-completion: true - output-artifact-directory: 'signed/Darling' - - - name: Replace with signed artifacts - if: github.event_name == 'release' - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - # Re-zip signed files into release archives - Remove-Item "releases/PerformanceMonitorLite-$version.zip" -ErrorAction SilentlyContinue - Compress-Archive -Path 'signed/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force - - - name: Package Darling (signed) - if: github.event_name == 'release' - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - # One product, one zip (mirrors the one-zip-per-product convention above). signed/Darling - # already holds the signed tree in its final layout — the service at the archive root (its - # darling.sample.json alongside), the viewer in a viewer\ subfolder. Drop pg-runtime.zip - # beside the service exe, exactly where DarlingManagedPostgres looks (AppContext.BaseDirectory) - # and extracts it on first run. The EDB PostgreSQL binaries inside pg-runtime.zip are shipped - # as opaque data and were never signed. - Copy-Item 'Darling/artifacts/pg-runtime.zip' 'signed/Darling' - - Remove-Item "releases/PerformanceMonitorDarling-$version.zip" -ErrorAction SilentlyContinue - Compress-Archive -Path 'signed/Darling/*' -DestinationPath "releases/PerformanceMonitorDarling-$version.zip" -Force - - # The Velopack (Setup.exe) path publishes a SEPARATE self-contained build - # (publish/Dashboard-velopack, publish/Lite-velopack -- see the "self-contained - # for Velopack" steps above) that previously went straight into `vpk pack` - # without ever being uploaded to SignPath. Only the framework-dependent trees - # used for the legacy ZIPs were signed, so every Setup.exe shipped unsigned - # since Velopack packaging was introduced. These steps close that gap by - # mirroring the exact upload/sign pattern used for Dashboard/Lite/Installer - # above, and vpk pack below now reads from the signed output. - - name: Upload Lite (Velopack) for signing - if: github.event_name == 'release' - id: upload-lite-velopack - uses: actions/upload-artifact@v6 - with: - name: Lite-Velopack-unsigned - path: publish/Lite-velopack/ - - - name: Upload Darling Viewer (Velopack) for signing - if: github.event_name == 'release' - id: upload-darlingviewer-velopack - uses: actions/upload-artifact@v6 - with: - name: DarlingViewer-Velopack-unsigned - path: publish/DarlingViewer-velopack/ - - - name: Sign Lite (Velopack) - if: github.event_name == 'release' - uses: signpath/github-action-submit-signing-request@v2 - with: - api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' - organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' - project-slug: 'PerformanceMonitor' - signing-policy-slug: 'release-signing' - artifact-configuration-slug: 'Lite' - github-artifact-id: '${{ steps.upload-lite-velopack.outputs.artifact-id }}' - wait-for-completion: true - output-artifact-directory: 'signed/Lite-Velopack' - - # The remote-seat viewer Setup.exe (#1555) is signed with its OWN 'DarlingViewer' artifact - # configuration — the co-located-zip 'Darling' slug signs a service+viewer\ tree layout, which - # does not match this self-contained viewer-at-root publish. Like the 'Darling' slug, the - # 'DarlingViewer' config (which files get signed) lives outside this repo on signpath.io; until - # Erik creates the slug this step fails the release — the standing #1340 SignPath prerequisite. - - name: Sign Darling Viewer (Velopack) - if: github.event_name == 'release' - uses: signpath/github-action-submit-signing-request@v2 - with: - api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' - organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' - project-slug: 'PerformanceMonitor' - signing-policy-slug: 'release-signing' - artifact-configuration-slug: 'DarlingViewer' - github-artifact-id: '${{ steps.upload-darlingviewer-velopack.outputs.artifact-id }}' - wait-for-completion: true - output-artifact-directory: 'signed/DarlingViewer-Velopack' - - - name: Create Velopack releases (Lite + Darling Viewer) - if: github.event_name == 'release' - shell: pwsh - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ steps.version.outputs.VERSION }} - run: | - # Pin vpk to the Velopack library version (keep in sync with the Velopack - # PackageReference in PerformanceMonitorLite.csproj). - dotnet tool install -g vpk --version 1.2.0 - New-Item -ItemType Directory -Force -Path releases/velopack-lite - New-Item -ItemType Directory -Force -Path releases/velopack-darlingviewer - - # Lite: download previous + pack (from the SIGNED velopack output, not the raw publish dir) - vpk download github --repoUrl https://github.com/${{ github.repository }} --channel lite -o releases/velopack-lite --token $env:GH_TOKEN - vpk pack -u PerformanceMonitorLite -v $env:VERSION -p signed/Lite-Velopack -e PerformanceMonitorLite.exe -o releases/velopack-lite --channel lite - - # Darling Viewer remote-seat installer (#1555): download previous + pack (from the SIGNED - # velopack output). Its own 'darlingviewer' channel/delta feed, separate from the co-located - # viewer inside PerformanceMonitorDarling-*.zip (which stays plain-zip only). - vpk download github --repoUrl https://github.com/${{ github.repository }} --channel darlingviewer -o releases/velopack-darlingviewer --token $env:GH_TOKEN - vpk pack -u PerformanceMonitorDarlingViewer -v $env:VERSION -p signed/DarlingViewer-Velopack -e PerformanceMonitor.Darling.Viewer.exe -o releases/velopack-darlingviewer --channel darlingviewer - - - name: Generate checksums - if: github.event_name == 'release' - shell: pwsh - run: | - $checksums = Get-ChildItem releases/*.zip | ForEach-Object { - $hash = (Get-FileHash $_.FullName -Algorithm SHA256).Hash.ToLower() - "$hash $($_.Name)" - } - $checksums | Out-File -FilePath releases/SHA256SUMS.txt -Encoding utf8 - Write-Host "Checksums:" - $checksums | ForEach-Object { Write-Host $_ } - - - name: Upload release assets - if: github.event_name == 'release' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload ${{ github.event.release.tag_name }} releases/*.zip releases/SHA256SUMS.txt --clobber - - - name: Upload Lite Velopack artifacts - if: github.event_name == 'release' - shell: pwsh - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ steps.version.outputs.VERSION }} - run: | - vpk upload github --repoUrl https://github.com/${{ github.repository }} --channel lite -o releases/velopack-lite --releaseName "v$env:VERSION" --tag "v$env:VERSION" --merge --token $env:GH_TOKEN - - - name: Upload Darling Viewer Velopack artifacts - if: github.event_name == 'release' - shell: pwsh - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ steps.version.outputs.VERSION }} - run: | - vpk upload github --repoUrl https://github.com/${{ github.repository }} --channel darlingviewer -o releases/velopack-darlingviewer --releaseName "v$env:VERSION" --tag "v$env:VERSION" --merge --token $env:GH_TOKEN - - # #1587: gated-live Darling coverage BEFORE merge, not only in the nightly. Darling.Tests has - # live-PostgreSQL tests (the *_AgainstDevPostgres classes) gated on DARLING_TEST_PG, which the - # build job above never sets — so those tests only ever ran post-merge in nightly.yml. That is - # exactly how #1586's alter_job(bigint) bug merged AND deployed clean: the test that catches it is - # gated-live, and PR CI did not run it. This job stands up a throwaway PostgreSQL + TimescaleDB - # from the bundled pg-runtime and runs the FULL Darling suite against it — but ONLY when Darling - # code (or this workflow) changed, so a Lite/Dashboard-only PR pays nothing. On a non-Darling - # change every step below no-ops via the path filter and the job still reports SUCCESS, so it can - # be made a required check without blocking unrelated PRs (the same always-runs-reports-a-result - # shape the build job uses for doc-only changes). Mirrors the nightly darling-pg job step-for-step; - # the one live-SQL-Server E2E stays skipped (DARLING_TEST_SQL unset — no SQL Server on the runner). - darling-pg: - name: Darling PostgreSQL tests - runs-on: windows-latest - # Max observed on a warm cache is ~3m40s; a cold pg-runtime cache adds a ~340MB fetch. - # 30 minutes is 3x headroom over the cold path — past that, something is hung (pg_ctl -w - # waiting on a cluster that will never come up), and the default 6h timeout would hold a - # shared-pool Windows runner hostage for the duration. The build job above deliberately - # has NO timeout: on release it waits on SignPath's manual approval gate, which can - # legitimately take hours. - timeout-minutes: 30 - permissions: - contents: read - - steps: - - uses: actions/checkout@v7 - - # Only do the expensive TimescaleDB work when Darling code changed — or when THIS workflow - # changed, so a change to the gate itself is exercised by the gate (this is what makes the PR - # that introduces this job validate itself end-to-end). Doc-only Darling edits don't trigger - # it. Skipped entirely on release: the dev push that produced the release commit already ran - # it, so the filter step doesn't run and every step below no-ops. - - name: Detect changed paths - id: filter - if: github.event_name != 'release' - uses: dorny/paths-filter@v4 - with: - # On push, compare against the previous commit on this branch (mirrors the build job); - # on pull_request, an empty base makes the action diff against the PR base branch. - base: ${{ github.event_name == 'push' && github.event.before || '' }} - # `Darling/**/!(*.md)` instead of a `Darling/**` include plus a `!Darling/**/*.md` - # exclude: dorny v4 treats each pattern as an independent predicate under the - # default quantifier, so the old bare negation was itself a match-all-non-Darling-md - # rule — this job ran the full TimescaleDB suite on every PR, including md-only - # ones (run 30218459544: "Filter darling = true, Matching files: CHANGELOG.md"). - filters: | - darling: - - 'Darling/**/!(*.md)' - - '.github/workflows/build.yml' - # Same reason as the build job's darling filter: the #1888 cluster-sizing - # guard parses nightly.yml, so an edit to it has to reach the suite. - - '.github/workflows/nightly.yml' - - # This job's gate was already correct for documentation — a docs-only change leaves - # 'darling' false and every step below no-ops. What it lacked was SAYING so: a job - # that reports success having quietly run nothing looks identical to one that tested - # everything. Costs one step; buys a log you can point at when asking "did this - # actually get tested?". - - name: Report the Darling PG gate decision - shell: bash - run: | - set -euo pipefail - - if [ "${{ github.event_name }}" = "release" ]; then - echo "::notice title=Darling PG tests skipped::Release event - the dev push that produced this commit already ran them." - elif [ "${{ steps.filter.outputs.darling }}" = "true" ]; then - echo "::notice title=Darling PG tests running::Darling code (or this workflow) changed." - else - echo "::notice title=Darling PG tests skipped::No Darling code changed - documentation-only Darling edits do not trigger the TimescaleDB suite." - fi - - - name: Setup .NET 10.0 - if: steps.filter.outputs.darling == 'true' - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - # Same cache key the release job and nightly.yml use (the fetch script's own content hash), so - # a warm cache from any of the three means no ~340MB EDB/TimescaleDB download here. - - name: Cache Darling pg-runtime.zip - if: steps.filter.outputs.darling == 'true' - id: cache-pg-runtime - uses: actions/cache@v6 - with: - path: Darling/artifacts/pg-runtime.zip - key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} - - - name: Build Darling pg-runtime.zip (cache miss only) - if: steps.filter.outputs.darling == 'true' && steps.cache-pg-runtime.outputs.cache-hit != 'true' - shell: pwsh - run: ./Darling/tools/fetch-pg-runtime.ps1 - - - name: Extract pg-runtime - if: steps.filter.outputs.darling == 'true' - shell: pwsh - run: | - $zip = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime.zip" - $dest = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime" - if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zip, $dest) - if (-not (Test-Path "$dest\pgsql\bin\pg_ctl.exe")) { throw "pg-runtime missing pgsql\bin\pg_ctl.exe" } - - - name: Restore Darling.Tests - if: steps.filter.outputs.darling == 'true' - run: dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode - - - name: Build Darling.Tests - if: steps.filter.outputs.darling == 'true' - run: dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore - - # Throwaway cluster from the bundled runtime. initdb TRUST auth is acceptable ONLY here: an - # ephemeral CI runner, loopback-only, throwaway data (the product default is scram-sha-256 + - # a generated credential). Superuser is "darling" (not "postgres") so CREATE SCHEMA ... - # AUTHORIZATION darling in the V8 schema-split migration resolves. Settings mirror - # DarlingManagedPostgres.BuildConfAppend (timescaledb preload, port 5541, loopback bind) - # and BuildWorkerSizingConfAppend (the two worker settings below). - # - # #1888: the worker settings are NOT optional garnish. PostgreSQL's default - # max_worker_processes = 8 cannot launch TimescaleDB's per-hypertable compression, - # retention and continuous-aggregate policy jobs — the postmaster logs "failed to start a - # background worker" storms and most policy runs never happen. Without them this job tested a - # configuration NO customer runs (the product refuses to: BuildWorkerSizingConfAppend writes - # these on every managed start), and worse, made failures luck-of-the-slot rather than - # reproducible — which is exactly how #1862's compression flake failed twice on CI in two - # unrecognizably different ways and could not be reproduced locally at all. - # - # The values are the product's own derivation from the live hypertable count - # (TimescaleSupport.HypertableCount = the 41-collector catalog + collection_log = 42): - # timescaledb.max_background_workers = HypertableCount + 2 = 44 - # max_worker_processes = 3 + (HypertableCount + 2) + 8 = 55 - # Hard-coded here because a workflow cannot call into the product — so - # CiClusterWorkerSizingTests parses THIS FILE and fails the build if either number stops - # matching the formula as collectors are added, and CiClusterWorkerSizingLiveTests asserts - # the running cluster actually serves them (a conf line that never took effect is - # indistinguishable from one that did, by inspection). - - name: Initialize and start throwaway PostgreSQL - if: steps.filter.outputs.darling == 'true' - shell: pwsh - run: | - $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" - $dataDir = "$env:RUNNER_TEMP\darling-pgdata" - $logFile = "$env:RUNNER_TEMP\darling-pg.log" - & "$bin\initdb.exe" -D $dataDir -U darling -A trust --encoding=UTF8 - if ($LASTEXITCODE -ne 0) { throw "initdb failed ($LASTEXITCODE)" } - Add-Content -Path "$dataDir\postgresql.conf" -Value "shared_preload_libraries = 'timescaledb'" - Add-Content -Path "$dataDir\postgresql.conf" -Value "port = 5541" - Add-Content -Path "$dataDir\postgresql.conf" -Value "listen_addresses = '127.0.0.1'" - Add-Content -Path "$dataDir\postgresql.conf" -Value "timescaledb.max_background_workers = 44" - Add-Content -Path "$dataDir\postgresql.conf" -Value "max_worker_processes = 55" - & "$bin\pg_ctl.exe" -D $dataDir -l $logFile -w start - if ($LASTEXITCODE -ne 0) { if (Test-Path $logFile) { Get-Content $logFile -Tail 50 }; throw "pg_ctl start failed ($LASTEXITCODE)" } - & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling - if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" } - - # DARLING_TEST_PG lights up the [Collection("live-postgres")] classes; DARLING_TEST_PGRUNTIME - # lights up the managed-bootstrap E2E. DARLING_TEST_SQL is intentionally unset — the one - # live-SQL-Server E2E stays skipped (no SQL Server on the runner). Full suite (not a filtered - # subset): the ungated ~3s overlap with the build job is negligible and a filter could hide a - # test the way narrow filters have bitten before. - - name: Run Darling PG tests - if: steps.filter.outputs.darling == 'true' - shell: pwsh - env: - DARLING_TEST_PG: "Host=127.0.0.1;Port=5541;Username=darling;Database=darling" - DARLING_TEST_PGRUNTIME: ${{ github.workspace }}\Darling\artifacts\pg-runtime - run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal --logger "trx;LogFileName=darling-pr.trx" --results-directory TestResults - - - name: Stop PostgreSQL - if: always() && steps.filter.outputs.darling == 'true' - shell: pwsh - run: | - $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" - $dataDir = "$env:RUNNER_TEMP\darling-pgdata" - if (Test-Path "$bin\pg_ctl.exe") { & "$bin\pg_ctl.exe" -D $dataDir -m fast -w stop } - exit 0 - - - name: Upload PG log and test results on failure - if: failure() && steps.filter.outputs.darling == 'true' - uses: actions/upload-artifact@v6 - with: - name: darling-pg-failure - path: | - ${{ runner.temp }}/darling-pg.log - TestResults/ - if-no-files-found: ignore - - # ── Linux service build + container image (#1804) ──────────────────────────────────────────────── - # The Darling service is cross-platform .NET on purpose, but until this job nothing PROVED it on - # every PR — the linux-x64 publish and the container image both built for the first time at release - # time or never. Same path-filter shape as darling-pg above: only runs the expensive work when - # Darling/service code (or this workflow, or the Dockerfile) changed, always reports a result so it - # can be a required check. No tests run here — the test projects are net10.0-windows (they reference - # the WPF apps); the cross-platform behavior they pin is exercised by the Windows jobs, and the - # container smoke lives in the compose quickstart. On PRs/pushes this job answers exactly two - # questions: does the service still publish for linux-x64, and does the image still build. - # - # On the RELEASE event this job is the Linux PUBLISHER: before it, a stable release shipped Windows - # zips and Setup.exes while the linux tar.gz and the ghcr image existed only at nightly quality - # (nightly.yml, tag :nightly) — the compose quickstart pointed released users at a nightly image. - # Now the release uploads PerformanceMonitorDarling-linux-x64-.tar.gz + SHA256SUMS-linux.txt to - # the release and pushes ghcr : and :latest. Linux binaries are NOT SignPath-signed - # (SignPath signs Windows PEs); the checksums file is the integrity story, same as the nightly. - darling-linux: - name: Darling Linux build - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: write - packages: write - # Sigstore keyless signing + GitHub provenance (proven on the nightly first): id-token yields - # the OIDC identity Fulcio certifies against; attestations stores the tarball's provenance. - id-token: write - attestations: write - - steps: - - uses: actions/checkout@v7 - - - name: Detect changed paths - id: filter - if: github.event_name != 'release' - uses: dorny/paths-filter@v4 - with: - base: ${{ github.event_name == 'push' && github.event.before || '' }} - filters: | - darling: - - 'Darling/**/!(*.md)' - - 'PerformanceMonitor.Common/**' - - 'PerformanceMonitor.Collectors/**' - - 'PerformanceMonitor.Analysis/**' - - '.github/workflows/build.yml' - - - name: Report the Linux gate decision - shell: bash - run: | - set -euo pipefail - if [ "${{ github.event_name }}" = "release" ]; then - echo "::notice title=Darling Linux publishing::Release event - packaging the versioned linux tar.gz and pushing the ghcr image." - elif [ "${{ steps.filter.outputs.darling }}" = "true" ]; then - echo "::notice title=Darling Linux build running::Darling/service code (or this workflow) changed." - else - echo "::notice title=Darling Linux build skipped::No Darling/service code changed." - fi - - - name: Setup .NET 10.0 - if: github.event_name == 'release' || steps.filter.outputs.darling == 'true' - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - - name: Publish service (linux-x64) - if: github.event_name == 'release' || steps.filter.outputs.darling == 'true' - run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -r linux-x64 --self-contained false -o publish/DarlingService-linux - - - name: Build container image - if: github.event_name != 'release' && steps.filter.outputs.darling == 'true' - run: docker build -f Darling/Dockerfile -t performancemonitor-darling:pr . - - # ── Release-only publishing (mirrors nightly.yml's linux job, versioned instead of :nightly) ── - - - name: Get version - if: github.event_name == 'release' - id: version - shell: bash - run: | - set -euo pipefail - version="$(grep -oPm1 '(?<=)[^<]+' Lite/PerformanceMonitorLite.csproj)" - echo "VERSION=${version}" >> "$GITHUB_OUTPUT" - - - name: Package linux artifact + checksum - if: github.event_name == 'release' - shell: bash - run: | - set -euo pipefail - version="${{ steps.version.outputs.VERSION }}" - mkdir -p releases - tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . - (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) - - - name: Upload linux artifact to the release - if: github.event_name == 'release' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release upload ${{ github.event.release.tag_name }} releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt --clobber - - - name: Build and push container image (ghcr, versioned + latest) - if: github.event_name == 'release' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - set -euo pipefail - version="${{ steps.version.outputs.VERSION }}" - image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" - echo "$GH_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - docker build -f Darling/Dockerfile -t "${image}:${version}" -t "${image}:latest" . - docker push "${image}:${version}" - docker push "${image}:latest" - - # Keyless Sigstore signature on the released image — the same steps the nightly runs (verified - # end-to-end from a client 2026-08-06: cosign validated the claims, the Rekor log entry, and the - # workflow identity). SignPath's cosign support is edition-gated, and this path needs no keys or - # subscription at all. Verify: - # cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling: \ - # --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' \ - # --certificate-oidc-issuer https://token.actions.githubusercontent.com - - name: Install cosign - if: github.event_name == 'release' - uses: sigstore/cosign-installer@v3 - - - name: Sign container image (keyless, Sigstore) - if: github.event_name == 'release' - shell: bash - run: | - set -euo pipefail - version="${{ steps.version.outputs.VERSION }}" - image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" - digest="$(docker inspect --format='{{index .RepoDigests 0}}' "${image}:${version}")" - cosign sign --yes "${digest}" - - # GitHub-native SLSA provenance for the released tarball: gh attestation verify -R . - - name: Attest the linux tarball (GitHub provenance) - if: github.event_name == 'release' - uses: actions/attest-build-provenance@v3 - with: - subject-path: releases/PerformanceMonitorDarling-linux-x64-*.tar.gz +name: Build + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + release: + types: [published] + # Merge-queue runs. Inert until a queue ruleset is enabled on a branch (a repo setting), + # but the required checks must handle the event BEFORE that click, or every queued PR + # stalls on checks that never report. dorny/paths-filter v4.0.1+ resolves merge_group + # diffs from the payload's base_sha/head_sha whenever the base input is empty — exactly + # what the filter steps pass for non-push events — so path classification works + # unchanged in a queue run. + merge_group: + +permissions: + contents: write + id-token: write + actions: read + +# A re-push to a PR cancels that PR's superseded in-flight run, and a push to dev/main +# cancels that BRANCH's superseded in-flight run — newest SHA wins. Finishing a build of +# code that is no longer the head helps nobody, and the shared Windows runner pool is what +# serializes everyone's CI (#1697 sat queued behind two dev builds; on 2026-07-26 a +# ~20-merge train left 13 of the day's 30 dev-push runs finishing SHAs a newer merge had +# already replaced — ~60 reclaimable runner-minutes in one evening). Cancelling a +# superseded PUSH run is safe because a push run produces nothing any other run consumes: +# every upload-artifact step in this workflow is gated to the release event (the SignPath +# signing path) or to failure() (darling-pg diagnostics), nothing in the repo downloads +# cross-run artifacts (no download-artifact, gh run download, or workflow_run consumer +# exists), nightly.yml builds its own tree from its own checkout, and a release compiles +# fresh on the release event. The accepted trade: push builds are diff-scoped, so a +# cancelled run's areas are not re-verified until the next change touches them — the +# nightly and the all-areas dev->main release PR are the backstops. Release and +# merge-queue runs deliberately keep a UNIQUE group per run (run_id) and are NEVER +# cancelled: a release build waits on SignPath's manual approval gate, and a queue +# validation is the last check before its result lands on dev. +concurrency: + group: ${{ github.event_name == 'pull_request' && format('build-pr-{0}', github.event.pull_request.number) || github.event_name == 'push' && format('build-push-{0}', github.ref) || format('build-run-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v7 + + - name: Detect changed paths + id: filter + if: github.event_name != 'release' + uses: dorny/paths-filter@v4 + with: + # On push events, compare against the previous commit on this branch + # (github.event.before). Without this, the action defaults to comparing + # against the default branch on non-default branch pushes, which would + # match every accumulated change and defeat the filter. + base: ${{ github.event_name == 'push' && github.event.before || '' }} + # Emit the matched file list so the fast-path step can NAME what it classified + # as documentation. A fast path that silently under-builds is the failure mode + # worth guarding against, so the reason is always printed, never inferred. + list-files: shell + filters: | + # A change to a root build file (the solution, restore config, or THIS workflow) can + # affect every product, so it forces a full build/test/publish. + root: + - 'PerformanceMonitor.sln' + - 'global.json' + - 'nuget.config' + - 'NuGet.config' + - '.github/workflows/build.yml' + # The shared PerformanceMonitor.* core libraries feed Lite, the Full Dashboard, AND + # Darling (verified via ProjectReference), so a change here fans out to all three. + # NOT the CLI Installer — it references only Installer.Core. + # + # Every area pattern says `dir/**/!(*.md)` — any non-markdown file under the + # area — instead of the old `dir/**` include plus a bare `!**/*.md` exclude. + # That is not style: dorny v4 evaluates each pattern as an INDEPENDENT + # predicate under the default predicate-quantifier 'some' (a filter is true + # when any changed file matches at least one rule), so a bare `!**/*.md` line + # is not a subtraction — it is its own rule meaning "any file that is not + # markdown", which silently made every area filter true for ANY non-markdown + # change anywhere in the repo. Measured proof: a single root .gitignore edit + # built and tested all four products and ran the full Darling PG suite + # (PR #1714, run 30219202642, filter log: "Filter darling = true, Matching + # files: .gitignore"). The extglob keeps the markdown carve-out INSIDE the + # include, where quantifier semantics cannot detach it. + core: + - 'PerformanceMonitor.Alerting/**/!(*.md)' + - 'PerformanceMonitor.Analysis/**/!(*.md)' + - 'PerformanceMonitor.Collectors/**/!(*.md)' + - 'PerformanceMonitor.Common/**/!(*.md)' + - 'PerformanceMonitor.Notifications/**/!(*.md)' + - 'PerformanceMonitor.PlanAnalysis/**/!(*.md)' + - 'PerformanceMonitor.Ui/**/!(*.md)' + # Installer.Core is shared by the CLI Installer AND the Full Dashboard's integrated + # installer — a change rebuilds both, and nothing else. + installer_core: + - 'deprecated/Installer.Core/**/!(*.md)' + dashboard: + - 'deprecated/Dashboard/**/!(*.md)' + - 'deprecated/Dashboard.Tests/**/!(*.md)' + lite: + - 'Lite/**/!(*.md)' + - 'Lite.Tests/**/!(*.md)' + # Same silently-stops-guarding reason as the darling filter's Lite entries below: + # Lite.Tests/ThemeParityLiteDarlingTests.cs READS the Darling viewer's theme + # dictionaries to assert the two apps' shared brush keys still resolve to the same + # colors. A Darling-theme-only edit is exactly the drift that guard exists to catch, + # so it has to reach the suite. + - 'Darling/PerformanceMonitor.Darling.Viewer/Themes/*.xaml' + installer: + - 'deprecated/Installer/**/!(*.md)' + - 'deprecated/Installer.Tests/**/!(*.md)' + - 'install/**/!(*.md)' + - 'upgrades/**/!(*.md)' + darling: + - 'Darling/**/!(*.md)' + # nightly.yml is not a build input, but Darling.Tests PARSES it: the #1888 + # guard reads both workflows' throwaway-cluster settings and compares them + # against the product's worker-sizing formula. Without this, a nightly-only + # edit would change a file the guard asserts on while never running the + # guard — a guard that silently stops guarding, which is the exact failure + # mode the source-parsing tests here exist to prevent. + - '.github/workflows/nightly.yml' + # Same reason, Lite side: the #1949 pin in Darling.Tests asserts every twinned + # query grid carries the SAME column sequence in both front ends, so it reads + # these six Lite files. A Lite-only XAML edit has to reach the suite or the + # parity half of that guard stops guarding. + - 'Lite/Controls/ServerTab.xaml' + - 'Lite/Controls/FinOpsTab.xaml' + - 'Lite/Windows/WaitDrillDownWindow.xaml' + - 'Lite/Windows/ProcedureHistoryWindow.xaml' + - 'Lite/Windows/QueryStatsHistoryWindow.xaml' + - 'Lite/Windows/QueryStoreHistoryWindow.xaml' + # #2114: XamlStaticResourceHygieneTests scans EVERY Lite XAML file — a StaticResource + # regression in one outside the six named above must still trigger the Darling job + # that runs the guard, or it slips to the nightly. + - 'Lite/**/*.xaml' + # The DOCUMENTATION allowlist: files that cannot affect a build under any + # job in this workflow. Deliberately an allowlist of non-executable content, + # not a "everything that isn't code" subtraction — a new file type defaults + # to being treated as code, which is the safe direction to be wrong in. + # + # NOT here, on purpose: *.sql (the installer and sql-validation compile it), + # *.yml (workflows), *.csproj / *.props / packages.lock.json (build inputs), + # and *.cs regardless of how comment-only the change looks — an XML doc + # comment still recompiles, and the compiler is what proves it still builds. + # + # The docs/ and Screenshots/ entries are extension-explicit rather than bare + # directory globs for the same reason: everything in them today is markdown, + # SVG, or a screenshot image, and a .sql or script dropped into either + # directory tomorrow should default to being code, not inherit a free pass + # from its parent directory. + docs: + - '**/*.md' + - 'LICENSE' + - 'CITATION.cff' + - '.gitignore' + - '.gitattributes' + - 'docs/**/*.{md,svg,png,jpg,jpeg,gif}' + - 'Screenshots/**/*.{md,svg,png,jpg,jpeg,gif}' + # Catch-all COUNTER, not a boolean gate: the classify step below decides + # "documentation-only" by comparing all_count to docs_count — they are equal + # exactly when every changed file sits on the docs allowlist. Stated as a + # count comparison because the previous shape ('**' plus '!' exclusions, + # a code: filter) could never be false under predicate-quantifier 'some' — + # every file matches '**', so the #1712 fast path shipped unable to engage + # (throwaway PR #1714: a .gitignore-only diff still paid setup + restore and, + # via the predicate bug above, a full build). + all: + - '**' + + # Decides the docs fast path ONCE, in one place, and says so out loud. Guards keep + # it off every path where a skipped restore would be a real loss: + # release — the filter step does not even run there, and a release must always + # compile and publish from a cold, fully restored tree. + # push — dev/main pushes are the integration signal for what just merged, so + # they restore unconditionally even for a docs-only commit. Cheap + # insurance: this only forces the restore back on, it does not force + # the per-product build/test steps, which stay path-gated as before. + # merge_group — a queue run is the LAST validation before its result lands on dev, + # so it takes the same always-restore path as a push. + # areas — belt and suspenders: even when the counts say docs-only, any lit + # area filter vetoes the fast path, because an area=true with restore + # skipped would run `dotnet build --no-restore` against nothing. The + # two classifications are built from the same allowlist so they cannot + # disagree today; this guard is for the day someone edits one and not + # the other. + # Everything else (pull_request) is eligible, and engages only when EVERY changed + # file is on the documentation allowlist (all_count == docs_count). + - name: Classify change for the docs fast path + id: fastpath + shell: bash + env: + ALL_COUNT: ${{ steps.filter.outputs.all_count }} + DOCS_COUNT: ${{ steps.filter.outputs.docs_count }} + DOCS_FILES: ${{ steps.filter.outputs.docs_files }} + AREAS: 'root=${{ steps.filter.outputs.root }} core=${{ steps.filter.outputs.core }} installer_core=${{ steps.filter.outputs.installer_core }} dashboard=${{ steps.filter.outputs.dashboard }} lite=${{ steps.filter.outputs.lite }} installer=${{ steps.filter.outputs.installer }} darling=${{ steps.filter.outputs.darling }}' + run: | + set -euo pipefail + + if [ "${{ github.event_name }}" = "release" ]; then + echo "engaged=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Full build::Release event - the docs fast path never applies to a release." + exit 0 + fi + + if [ "${{ github.event_name }}" = "push" ] || [ "${{ github.event_name }}" = "merge_group" ]; then + echo "engaged=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Full build::${{ github.event_name }} on '${{ github.ref_name }}' - integration runs always restore, even for a docs-only change." + exit 0 + fi + + echo "Changed files: ${ALL_COUNT:-0} total, ${DOCS_COUNT:-0} on the documentation allowlist. Areas: ${AREAS}" + + if [ "${ALL_COUNT:-0}" -gt 0 ] && [ "${ALL_COUNT:-0}" -eq "${DOCS_COUNT:-0}" ] && [[ "${AREAS}" != *"=true"* ]]; then + echo "engaged=true" >> "$GITHUB_OUTPUT" + echo "::notice title=DOCS FAST PATH ENGAGED::All ${ALL_COUNT} changed files are on the documentation allowlist, so .NET setup, restore and versioning are skipped. This job still reports its result." + echo "Documentation files classified in this change:" + for f in ${DOCS_FILES}; do echo " - ${f}"; done + else + echo "engaged=false" >> "$GITHUB_OUTPUT" + echo "::notice title=Full build::At least one changed file is off the documentation allowlist (${DOCS_COUNT:-0} of ${ALL_COUNT:-0} classified as documentation)." + fi + + - name: Setup .NET 10.0 + if: steps.fastpath.outputs.engaged != 'true' + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + - name: Restore dependencies + if: steps.fastpath.outputs.engaged != 'true' + run: | + dotnet restore Lite/PerformanceMonitorLite.csproj --locked-mode + dotnet restore Lite.Tests/Lite.Tests.csproj --locked-mode + dotnet restore deprecated/Installer.Tests/Installer.Tests.csproj --locked-mode + dotnet restore deprecated/Dashboard.Tests/Dashboard.Tests.csproj --locked-mode + dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode + dotnet restore Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj --locked-mode + + - name: Build Lite.Tests + if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet build Lite.Tests/Lite.Tests.csproj -c Release --no-restore + + - name: Build Installer.Tests + if: steps.filter.outputs.installer == 'true' || steps.filter.outputs.installer_core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet build deprecated/Installer.Tests/Installer.Tests.csproj -c Release --no-restore + + # The 'dashboard' path filter was defined when the Full Dashboard moved to deprecated/ (#1612) but + # never wired to a step, so its build and tests silently stopped running — which is how a batch of + # compiler warnings and three broken ThemeParityTests accumulated unnoticed (#1643). Deprecated means + # bug-fix-only, not unverified: it still compiles warning-free and its tests still guard cross-app + # parity (the theme palettes it checks are LITE's too). + - name: Build Dashboard.Tests + if: steps.filter.outputs.dashboard == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet build deprecated/Dashboard.Tests/Dashboard.Tests.csproj -c Release --no-restore + + - name: Build Darling + if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: | + dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore + dotnet build Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release --no-restore + + # One step for the whole Lite suite. It was split into fast / analysis-heavy halves when the + # seven analysis classes rebuilt the full DuckDB schema inside every test and their subset + # alone cost ~9 minutes; after the shared class fixtures (#1693, #1698) and batched seeding + # (#1694) that subset runs in ~1 minute, so the split — and the narrower lite_analysis path + # gate that let non-analysis Lite changes skip it — stopped earning its second test-host + # spin-up and its filter-drift risk. + - name: Run Lite tests + if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet test Lite.Tests/Lite.Tests.csproj -c Release --no-build --verbosity normal + + - name: Run Installer tests + if: steps.filter.outputs.installer == 'true' || steps.filter.outputs.installer_core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet test deprecated/Installer.Tests/Installer.Tests.csproj -c Release --no-build --verbosity normal --filter "FullyQualifiedName!~VersionDetectionTests&FullyQualifiedName!~IdempotencyTests&FullyQualifiedName!~AdversarialTests" + + - name: Run Dashboard tests + if: steps.filter.outputs.dashboard == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet test deprecated/Dashboard.Tests/Dashboard.Tests.csproj -c Release --no-build --verbosity normal + + - name: Run Darling tests + if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal + + - name: Get version + if: steps.fastpath.outputs.engaged != 'true' + id: version + shell: pwsh + run: | + $version = ([xml](Get-Content Lite/PerformanceMonitorLite.csproj)).Project.PropertyGroup.Version | Where-Object { $_ } + echo "VERSION=$version" >> $env:GITHUB_OUTPUT + + - name: Publish Lite + if: steps.filter.outputs.lite == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -o publish/Lite + + - name: Publish Lite (self-contained for Velopack) + if: github.event_name == 'release' + run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -r win-x64 --self-contained -o publish/Lite-velopack + + - name: Publish Darling Service + if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -o publish/DarlingService + + - name: Publish Darling Viewer + if: steps.filter.outputs.darling == 'true' || steps.filter.outputs.core == 'true' || steps.filter.outputs.root == 'true' || github.event_name == 'release' + run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -o publish/DarlingViewer + + - name: Publish Darling Viewer (self-contained for Velopack) + if: github.event_name == 'release' + run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -r win-x64 --self-contained -o publish/DarlingViewer-velopack + + # Darling bundles a PostgreSQL 18 + TimescaleDB runtime (pg-runtime.zip) that ships beside + # the service exe; DarlingManagedPostgres extracts it on first run. The fetch script pulls + # ~340MB of pinned EDB/TimescaleDB archives, so this is release-only and cached. The key is + # the fetch script's own content hash (the SHA256 pins live inside it): a re-release with + # unchanged pins restores the assembled zip and skips both the download and the assembly, + # and any pin/version bump edits the script and invalidates the cache automatically. + - name: Cache Darling pg-runtime.zip + if: github.event_name == 'release' + id: cache-pg-runtime + uses: actions/cache@v6 + with: + path: Darling/artifacts/pg-runtime.zip + key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} + + - name: Build Darling pg-runtime.zip + if: github.event_name == 'release' && steps.cache-pg-runtime.outputs.cache-hit != 'true' + shell: pwsh + run: ./Darling/tools/fetch-pg-runtime.ps1 + + - name: Package release artifacts + if: github.event_name == 'release' + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + New-Item -ItemType Directory -Force -Path releases + + # Lite ZIP - portable artifact for advanced/air-gapped users. The README points end + # users at Setup.exe (Velopack); this ZIP is the explicit fallback. + Compress-Archive -Path 'publish/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force + + # upload-artifact is deliberately HELD at v6 (#1653): every signing step below consumes + # `steps.upload-*.outputs.artifact-id`, and v7 changes artifact archiving semantics (the + # `archive` parameter). The signing path only executes on `release: [published]`, so a broken + # bump surfaces at release time — bump only alongside a validated real signing run. + # Dependabot is configured to skip this major (see .github/dependabot.yml). + - name: Upload Lite for signing + if: github.event_name == 'release' + id: upload-lite + uses: actions/upload-artifact@v6 + with: + name: Lite-unsigned + path: publish/Lite/ + + - name: Stage Darling for signing + if: github.event_name == 'release' + shell: pwsh + run: | + $stage = 'publish/Darling-signing' + if (Test-Path $stage) { Remove-Item -Recurse -Force $stage } + New-Item -ItemType Directory -Force -Path "$stage/viewer" | Out-Null + Copy-Item 'publish/DarlingService/*' $stage -Recurse + Copy-Item 'publish/DarlingViewer/*' "$stage/viewer" -Recurse + + - name: Upload Darling for signing + if: github.event_name == 'release' + id: upload-darling + uses: actions/upload-artifact@v6 + with: + name: Darling-unsigned + path: publish/Darling-signing/ + + - name: Sign Lite + if: github.event_name == 'release' + uses: signpath/github-action-submit-signing-request@v2 + with: + api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' + organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' + project-slug: 'PerformanceMonitor' + signing-policy-slug: 'release-signing' + artifact-configuration-slug: 'Lite' + github-artifact-id: '${{ steps.upload-lite.outputs.artifact-id }}' + wait-for-completion: true + output-artifact-directory: 'signed/Lite' + + - name: Sign Darling + if: github.event_name == 'release' + uses: signpath/github-action-submit-signing-request@v2 + with: + api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' + organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' + project-slug: 'PerformanceMonitor' + signing-policy-slug: 'release-signing' + artifact-configuration-slug: 'Darling' + github-artifact-id: '${{ steps.upload-darling.outputs.artifact-id }}' + wait-for-completion: true + output-artifact-directory: 'signed/Darling' + + - name: Replace with signed artifacts + if: github.event_name == 'release' + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + # Re-zip signed files into release archives + Remove-Item "releases/PerformanceMonitorLite-$version.zip" -ErrorAction SilentlyContinue + Compress-Archive -Path 'signed/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force + + - name: Package Darling (signed) + if: github.event_name == 'release' + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + # One product, one zip (mirrors the one-zip-per-product convention above). signed/Darling + # already holds the signed tree in its final layout — the service at the archive root (its + # darling.sample.json alongside), the viewer in a viewer\ subfolder. Drop pg-runtime.zip + # beside the service exe, exactly where DarlingManagedPostgres looks (AppContext.BaseDirectory) + # and extracts it on first run. The EDB PostgreSQL binaries inside pg-runtime.zip are shipped + # as opaque data and were never signed. + Copy-Item 'Darling/artifacts/pg-runtime.zip' 'signed/Darling' + + Remove-Item "releases/PerformanceMonitorDarling-$version.zip" -ErrorAction SilentlyContinue + Compress-Archive -Path 'signed/Darling/*' -DestinationPath "releases/PerformanceMonitorDarling-$version.zip" -Force + + # The Velopack (Setup.exe) path publishes a SEPARATE self-contained build + # (publish/Dashboard-velopack, publish/Lite-velopack -- see the "self-contained + # for Velopack" steps above) that previously went straight into `vpk pack` + # without ever being uploaded to SignPath. Only the framework-dependent trees + # used for the legacy ZIPs were signed, so every Setup.exe shipped unsigned + # since Velopack packaging was introduced. These steps close that gap by + # mirroring the exact upload/sign pattern used for Dashboard/Lite/Installer + # above, and vpk pack below now reads from the signed output. + - name: Upload Lite (Velopack) for signing + if: github.event_name == 'release' + id: upload-lite-velopack + uses: actions/upload-artifact@v6 + with: + name: Lite-Velopack-unsigned + path: publish/Lite-velopack/ + + - name: Upload Darling Viewer (Velopack) for signing + if: github.event_name == 'release' + id: upload-darlingviewer-velopack + uses: actions/upload-artifact@v6 + with: + name: DarlingViewer-Velopack-unsigned + path: publish/DarlingViewer-velopack/ + + - name: Sign Lite (Velopack) + if: github.event_name == 'release' + uses: signpath/github-action-submit-signing-request@v2 + with: + api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' + organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' + project-slug: 'PerformanceMonitor' + signing-policy-slug: 'release-signing' + artifact-configuration-slug: 'Lite' + github-artifact-id: '${{ steps.upload-lite-velopack.outputs.artifact-id }}' + wait-for-completion: true + output-artifact-directory: 'signed/Lite-Velopack' + + # The remote-seat viewer Setup.exe (#1555) is signed with its OWN 'DarlingViewer' artifact + # configuration — the co-located-zip 'Darling' slug signs a service+viewer\ tree layout, which + # does not match this self-contained viewer-at-root publish. Like the 'Darling' slug, the + # 'DarlingViewer' config (which files get signed) lives outside this repo on signpath.io; until + # Erik creates the slug this step fails the release — the standing #1340 SignPath prerequisite. + - name: Sign Darling Viewer (Velopack) + if: github.event_name == 'release' + uses: signpath/github-action-submit-signing-request@v2 + with: + api-token: '${{ secrets.SIGNPATH_API_TOKEN }}' + organization-id: '7969f8b6-d946-4a74-9bac-a55856d8b8e0' + project-slug: 'PerformanceMonitor' + signing-policy-slug: 'release-signing' + artifact-configuration-slug: 'DarlingViewer' + github-artifact-id: '${{ steps.upload-darlingviewer-velopack.outputs.artifact-id }}' + wait-for-completion: true + output-artifact-directory: 'signed/DarlingViewer-Velopack' + + - name: Create Velopack releases (Lite + Darling Viewer) + if: github.event_name == 'release' + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + # Pin vpk to the Velopack library version (keep in sync with the Velopack + # PackageReference in PerformanceMonitorLite.csproj). + dotnet tool install -g vpk --version 1.2.0 + New-Item -ItemType Directory -Force -Path releases/velopack-lite + New-Item -ItemType Directory -Force -Path releases/velopack-darlingviewer + + # Lite: download previous + pack (from the SIGNED velopack output, not the raw publish dir) + vpk download github --repoUrl https://github.com/${{ github.repository }} --channel lite -o releases/velopack-lite --token $env:GH_TOKEN + vpk pack -u PerformanceMonitorLite -v $env:VERSION -p signed/Lite-Velopack -e PerformanceMonitorLite.exe -o releases/velopack-lite --channel lite + + # Darling Viewer remote-seat installer (#1555): download previous + pack (from the SIGNED + # velopack output). Its own 'darlingviewer' channel/delta feed, separate from the co-located + # viewer inside PerformanceMonitorDarling-*.zip (which stays plain-zip only). + vpk download github --repoUrl https://github.com/${{ github.repository }} --channel darlingviewer -o releases/velopack-darlingviewer --token $env:GH_TOKEN + vpk pack -u PerformanceMonitorDarlingViewer -v $env:VERSION -p signed/DarlingViewer-Velopack -e PerformanceMonitor.Darling.Viewer.exe -o releases/velopack-darlingviewer --channel darlingviewer + + - name: Generate checksums + if: github.event_name == 'release' + shell: pwsh + run: | + $checksums = Get-ChildItem releases/*.zip | ForEach-Object { + $hash = (Get-FileHash $_.FullName -Algorithm SHA256).Hash.ToLower() + "$hash $($_.Name)" + } + $checksums | Out-File -FilePath releases/SHA256SUMS.txt -Encoding utf8 + Write-Host "Checksums:" + $checksums | ForEach-Object { Write-Host $_ } + + - name: Upload release assets + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ github.event.release.tag_name }} releases/*.zip releases/SHA256SUMS.txt --clobber + + - name: Upload Lite Velopack artifacts + if: github.event_name == 'release' + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + vpk upload github --repoUrl https://github.com/${{ github.repository }} --channel lite -o releases/velopack-lite --releaseName "v$env:VERSION" --tag "v$env:VERSION" --merge --token $env:GH_TOKEN + + - name: Upload Darling Viewer Velopack artifacts + if: github.event_name == 'release' + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + vpk upload github --repoUrl https://github.com/${{ github.repository }} --channel darlingviewer -o releases/velopack-darlingviewer --releaseName "v$env:VERSION" --tag "v$env:VERSION" --merge --token $env:GH_TOKEN + + # #1587: gated-live Darling coverage BEFORE merge, not only in the nightly. Darling.Tests has + # live-PostgreSQL tests (the *_AgainstDevPostgres classes) gated on DARLING_TEST_PG, which the + # build job above never sets — so those tests only ever ran post-merge in nightly.yml. That is + # exactly how #1586's alter_job(bigint) bug merged AND deployed clean: the test that catches it is + # gated-live, and PR CI did not run it. This job stands up a throwaway PostgreSQL + TimescaleDB + # from the bundled pg-runtime and runs the FULL Darling suite against it — but ONLY when Darling + # code (or this workflow) changed, so a Lite/Dashboard-only PR pays nothing. On a non-Darling + # change every step below no-ops via the path filter and the job still reports SUCCESS, so it can + # be made a required check without blocking unrelated PRs (the same always-runs-reports-a-result + # shape the build job uses for doc-only changes). Mirrors the nightly darling-pg job step-for-step; + # the one live-SQL-Server E2E stays skipped (DARLING_TEST_SQL unset — no SQL Server on the runner). + darling-pg: + name: Darling PostgreSQL tests + runs-on: windows-latest + # Max observed on a warm cache is ~3m40s; a cold pg-runtime cache adds a ~340MB fetch. + # 30 minutes is 3x headroom over the cold path — past that, something is hung (pg_ctl -w + # waiting on a cluster that will never come up), and the default 6h timeout would hold a + # shared-pool Windows runner hostage for the duration. The build job above deliberately + # has NO timeout: on release it waits on SignPath's manual approval gate, which can + # legitimately take hours. + timeout-minutes: 30 + permissions: + contents: read + + steps: + - uses: actions/checkout@v7 + + # Only do the expensive TimescaleDB work when Darling code changed — or when THIS workflow + # changed, so a change to the gate itself is exercised by the gate (this is what makes the PR + # that introduces this job validate itself end-to-end). Doc-only Darling edits don't trigger + # it. Skipped entirely on release: the dev push that produced the release commit already ran + # it, so the filter step doesn't run and every step below no-ops. + - name: Detect changed paths + id: filter + if: github.event_name != 'release' + uses: dorny/paths-filter@v4 + with: + # On push, compare against the previous commit on this branch (mirrors the build job); + # on pull_request, an empty base makes the action diff against the PR base branch. + base: ${{ github.event_name == 'push' && github.event.before || '' }} + # `Darling/**/!(*.md)` instead of a `Darling/**` include plus a `!Darling/**/*.md` + # exclude: dorny v4 treats each pattern as an independent predicate under the + # default quantifier, so the old bare negation was itself a match-all-non-Darling-md + # rule — this job ran the full TimescaleDB suite on every PR, including md-only + # ones (run 30218459544: "Filter darling = true, Matching files: CHANGELOG.md"). + filters: | + darling: + - 'Darling/**/!(*.md)' + - '.github/workflows/build.yml' + # Same reason as the build job's darling filter: the #1888 cluster-sizing + # guard parses nightly.yml, so an edit to it has to reach the suite. + - '.github/workflows/nightly.yml' + + # This job's gate was already correct for documentation — a docs-only change leaves + # 'darling' false and every step below no-ops. What it lacked was SAYING so: a job + # that reports success having quietly run nothing looks identical to one that tested + # everything. Costs one step; buys a log you can point at when asking "did this + # actually get tested?". + - name: Report the Darling PG gate decision + shell: bash + run: | + set -euo pipefail + + if [ "${{ github.event_name }}" = "release" ]; then + echo "::notice title=Darling PG tests skipped::Release event - the dev push that produced this commit already ran them." + elif [ "${{ steps.filter.outputs.darling }}" = "true" ]; then + echo "::notice title=Darling PG tests running::Darling code (or this workflow) changed." + else + echo "::notice title=Darling PG tests skipped::No Darling code changed - documentation-only Darling edits do not trigger the TimescaleDB suite." + fi + + - name: Setup .NET 10.0 + if: steps.filter.outputs.darling == 'true' + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + # Same cache key the release job and nightly.yml use (the fetch script's own content hash), so + # a warm cache from any of the three means no ~340MB EDB/TimescaleDB download here. + - name: Cache Darling pg-runtime.zip + if: steps.filter.outputs.darling == 'true' + id: cache-pg-runtime + uses: actions/cache@v6 + with: + path: Darling/artifacts/pg-runtime.zip + key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} + + - name: Build Darling pg-runtime.zip (cache miss only) + if: steps.filter.outputs.darling == 'true' && steps.cache-pg-runtime.outputs.cache-hit != 'true' + shell: pwsh + run: ./Darling/tools/fetch-pg-runtime.ps1 + + - name: Extract pg-runtime + if: steps.filter.outputs.darling == 'true' + shell: pwsh + run: | + $zip = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime.zip" + $dest = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime" + if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zip, $dest) + if (-not (Test-Path "$dest\pgsql\bin\pg_ctl.exe")) { throw "pg-runtime missing pgsql\bin\pg_ctl.exe" } + + - name: Restore Darling.Tests + if: steps.filter.outputs.darling == 'true' + run: dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode + + - name: Build Darling.Tests + if: steps.filter.outputs.darling == 'true' + run: dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore + + # Throwaway cluster from the bundled runtime. initdb TRUST auth is acceptable ONLY here: an + # ephemeral CI runner, loopback-only, throwaway data (the product default is scram-sha-256 + + # a generated credential). Superuser is "darling" (not "postgres") so CREATE SCHEMA ... + # AUTHORIZATION darling in the V8 schema-split migration resolves. Settings mirror + # DarlingManagedPostgres.BuildConfAppend (timescaledb preload, port 5541, loopback bind) + # and BuildWorkerSizingConfAppend (the two worker settings below). + # + # #1888: the worker settings are NOT optional garnish. PostgreSQL's default + # max_worker_processes = 8 cannot launch TimescaleDB's per-hypertable compression, + # retention and continuous-aggregate policy jobs — the postmaster logs "failed to start a + # background worker" storms and most policy runs never happen. Without them this job tested a + # configuration NO customer runs (the product refuses to: BuildWorkerSizingConfAppend writes + # these on every managed start), and worse, made failures luck-of-the-slot rather than + # reproducible — which is exactly how #1862's compression flake failed twice on CI in two + # unrecognizably different ways and could not be reproduced locally at all. + # + # The values are the product's own derivation from the live hypertable count + # (TimescaleSupport.HypertableCount = the 50-collector catalog + collection_log = 51): + # timescaledb.max_background_workers = HypertableCount + 2 = 53 + # max_worker_processes = 3 + (HypertableCount + 2) + 8 = 64 + # Hard-coded here because a workflow cannot call into the product — so + # CiClusterWorkerSizingTests parses THIS FILE and fails the build if either number stops + # matching the formula as collectors are added, and CiClusterWorkerSizingLiveTests asserts + # the running cluster actually serves them (a conf line that never took effect is + # indistinguishable from one that did, by inspection). + - name: Initialize and start throwaway PostgreSQL + if: steps.filter.outputs.darling == 'true' + shell: pwsh + run: | + $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" + $dataDir = "$env:RUNNER_TEMP\darling-pgdata" + $logFile = "$env:RUNNER_TEMP\darling-pg.log" + & "$bin\initdb.exe" -D $dataDir -U darling -A trust --encoding=UTF8 + if ($LASTEXITCODE -ne 0) { throw "initdb failed ($LASTEXITCODE)" } + Add-Content -Path "$dataDir\postgresql.conf" -Value "shared_preload_libraries = 'timescaledb'" + Add-Content -Path "$dataDir\postgresql.conf" -Value "port = 5541" + Add-Content -Path "$dataDir\postgresql.conf" -Value "listen_addresses = '127.0.0.1'" + Add-Content -Path "$dataDir\postgresql.conf" -Value "timescaledb.max_background_workers = 53" + Add-Content -Path "$dataDir\postgresql.conf" -Value "max_worker_processes = 64" + & "$bin\pg_ctl.exe" -D $dataDir -l $logFile -w start + if ($LASTEXITCODE -ne 0) { if (Test-Path $logFile) { Get-Content $logFile -Tail 50 }; throw "pg_ctl start failed ($LASTEXITCODE)" } + & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling + if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" } + + # DARLING_TEST_PG lights up the [Collection("live-postgres")] classes; DARLING_TEST_PGRUNTIME + # lights up the managed-bootstrap E2E. DARLING_TEST_SQL is intentionally unset — the one + # live-SQL-Server E2E stays skipped (no SQL Server on the runner). Full suite (not a filtered + # subset): the ungated ~3s overlap with the build job is negligible and a filter could hide a + # test the way narrow filters have bitten before. + - name: Run Darling PG tests + if: steps.filter.outputs.darling == 'true' + shell: pwsh + env: + DARLING_TEST_PG: "Host=127.0.0.1;Port=5541;Username=darling;Database=darling" + DARLING_TEST_PGRUNTIME: ${{ github.workspace }}\Darling\artifacts\pg-runtime + run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal --logger "trx;LogFileName=darling-pr.trx" --results-directory TestResults + + - name: Stop PostgreSQL + if: always() && steps.filter.outputs.darling == 'true' + shell: pwsh + run: | + $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" + $dataDir = "$env:RUNNER_TEMP\darling-pgdata" + if (Test-Path "$bin\pg_ctl.exe") { & "$bin\pg_ctl.exe" -D $dataDir -m fast -w stop } + exit 0 + + - name: Upload PG log and test results on failure + if: failure() && steps.filter.outputs.darling == 'true' + uses: actions/upload-artifact@v6 + with: + name: darling-pg-failure + path: | + ${{ runner.temp }}/darling-pg.log + TestResults/ + if-no-files-found: ignore + + # ── Linux service build + container image (#1804) ──────────────────────────────────────────────── + # The Darling service is cross-platform .NET on purpose, but until this job nothing PROVED it on + # every PR — the linux-x64 publish and the container image both built for the first time at release + # time or never. Same path-filter shape as darling-pg above: only runs the expensive work when + # Darling/service code (or this workflow, or the Dockerfile) changed, always reports a result so it + # can be a required check. No tests run here — the test projects are net10.0-windows (they reference + # the WPF apps); the cross-platform behavior they pin is exercised by the Windows jobs, and the + # container smoke lives in the compose quickstart. On PRs/pushes this job answers exactly two + # questions: does the service still publish for linux-x64, and does the image still build. + # + # On the RELEASE event this job is the Linux PUBLISHER: before it, a stable release shipped Windows + # zips and Setup.exes while the linux tar.gz and the ghcr image existed only at nightly quality + # (nightly.yml, tag :nightly) — the compose quickstart pointed released users at a nightly image. + # Now the release uploads PerformanceMonitorDarling-linux-x64-.tar.gz + SHA256SUMS-linux.txt to + # the release and pushes ghcr : and :latest. Linux binaries are NOT SignPath-signed + # (SignPath signs Windows PEs); the checksums file is the integrity story, same as the nightly. + darling-linux: + name: Darling Linux build + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + packages: write + # Sigstore keyless signing + GitHub provenance (proven on the nightly first): id-token yields + # the OIDC identity Fulcio certifies against; attestations stores the tarball's provenance. + id-token: write + attestations: write + + steps: + - uses: actions/checkout@v7 + + - name: Detect changed paths + id: filter + if: github.event_name != 'release' + uses: dorny/paths-filter@v4 + with: + base: ${{ github.event_name == 'push' && github.event.before || '' }} + filters: | + darling: + - 'Darling/**/!(*.md)' + - 'PerformanceMonitor.Common/**' + - 'PerformanceMonitor.Collectors/**' + - 'PerformanceMonitor.Analysis/**' + - '.github/workflows/build.yml' + + - name: Report the Linux gate decision + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "release" ]; then + echo "::notice title=Darling Linux publishing::Release event - packaging the versioned linux tar.gz and pushing the ghcr image." + elif [ "${{ steps.filter.outputs.darling }}" = "true" ]; then + echo "::notice title=Darling Linux build running::Darling/service code (or this workflow) changed." + else + echo "::notice title=Darling Linux build skipped::No Darling/service code changed." + fi + + - name: Setup .NET 10.0 + if: github.event_name == 'release' || steps.filter.outputs.darling == 'true' + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + - name: Publish service (linux-x64) + if: github.event_name == 'release' || steps.filter.outputs.darling == 'true' + run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -r linux-x64 --self-contained false -o publish/DarlingService-linux + + - name: Build container image + if: github.event_name != 'release' && steps.filter.outputs.darling == 'true' + run: docker build -f Darling/Dockerfile -t performancemonitor-darling:pr . + + # ── Release-only publishing (mirrors nightly.yml's linux job, versioned instead of :nightly) ── + + - name: Get version + if: github.event_name == 'release' + id: version + shell: bash + run: | + set -euo pipefail + version="$(grep -oPm1 '(?<=)[^<]+' Lite/PerformanceMonitorLite.csproj)" + echo "VERSION=${version}" >> "$GITHUB_OUTPUT" + + - name: Package linux artifact + checksum + if: github.event_name == 'release' + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + mkdir -p releases + tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . + (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) + + - name: Upload linux artifact to the release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.event.release.tag_name }} releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt --clobber + + - name: Build and push container image (ghcr, versioned + latest) + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" + echo "$GH_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + docker build -f Darling/Dockerfile -t "${image}:${version}" -t "${image}:latest" . + docker push "${image}:${version}" + docker push "${image}:latest" + + # Keyless Sigstore signature on the released image — the same steps the nightly runs (verified + # end-to-end from a client 2026-08-06: cosign validated the claims, the Rekor log entry, and the + # workflow identity). SignPath's cosign support is edition-gated, and this path needs no keys or + # subscription at all. Verify: + # cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling: \ + # --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' \ + # --certificate-oidc-issuer https://token.actions.githubusercontent.com + - name: Install cosign + if: github.event_name == 'release' + uses: sigstore/cosign-installer@v3 + + - name: Sign container image (keyless, Sigstore) + if: github.event_name == 'release' + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" + digest="$(docker inspect --format='{{index .RepoDigests 0}}' "${image}:${version}")" + cosign sign --yes "${digest}" + + # GitHub-native SLSA provenance for the released tarball: gh attestation verify -R . + - name: Attest the linux tarball (GitHub provenance) + if: github.event_name == 'release' + uses: actions/attest-build-provenance@v4 + with: + subject-path: releases/PerformanceMonitorDarling-linux-x64-*.tar.gz diff --git a/.github/workflows/claude-review-guard.yml b/.github/workflows/claude-review-guard.yml new file mode 100644 index 000000000..b8df2007e --- /dev/null +++ b/.github/workflows/claude-review-guard.yml @@ -0,0 +1,285 @@ +name: Claude review guard + +# #2229: the review workflow can finish green and leave nothing an operator can read. Two ways, +# both observed on this repo, both silent: +# +# 1. claude-code-action refuses to run when .github/workflows/claude-review.yml differs from the +# DEFAULT branch -- the check is server-side, during the OIDC token exchange. It exits SUCCESS +# in about four seconds and sets no output a caller can read. +# 2. The review runs for real and posts nothing anyway. PR #2213 did this eleven times; one run +# spent $4.18 over 24 turns with 6 permission denials, reported subtype=success, and left zero +# comments. A 104-file change merged unreviewed and nothing anywhere said so. +# +# What it must NOT do is call a CLEAN review a failure. The prompt tells the reviewer HOW to report +# findings, not to report when it has none, so silence on a one-line change is the correct outcome -- +# and a mark that goes red there is one people learn to ignore, which costs more than it catches. So +# the zero-posted verdict is keyed on whether claude-review.yml actually promises a post. +# +# This guard is a SEPARATE workflow on purpose. Editing claude-review.yml makes this branch's copy +# differ from the default branch, which triggers cause (1) for EVERY pull request in the repo until +# the next release -- so a guard living inside the file it protects would disable review each time +# it was touched. Nothing here invokes claude-code-action, so this file can be edited freely. +# +# It polls rather than using `on: workflow_run` because workflow_run only ever fires from the copy +# of a workflow that is on the default branch: that variant would be dormant, and untestable, until +# a release shipped it. +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [dev, main] + +# Match the review's own cancel-in-progress behavior. A new push cancels the review it is watching, +# so the guard has to go too, or it waits out its deadline on a run that was cancelled by design. +concurrency: + group: claude-review-guard-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + # The review is capped at 30 minutes; leave room to observe one that uses all of it. + timeout-minutes: 40 + # No issues: scope here, despite the tally reading repos/.../issues/$PR/comments for + # top-level output. A pull request IS an issue to that API, and the PullRequests scope + # covers its comments: run 31726598542 was granted exactly Actions/Checks/Contents/ + # Metadata/PullRequests read and counted issue: 1 from that endpoint without a 403. + permissions: + actions: read + checks: read + contents: read + pull-requests: read + steps: + - name: Verify the review posted something + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + R: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + SHA: ${{ github.event.pull_request.head.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + WF: .github/workflows/claude-review.yml + # Coupling to a step NAME in the other workflow. Deliberate: the step's conclusion is the + # only structured way to tell "no token, skipped cleanly" from "ran and said nothing". + REVIEW_STEP: Claude review + # Author of every artifact the review leaves behind (the action reports bot_name). + REVIEW_BOT: claude[bot] + # Whether "posted nothing" is a DEFECT depends on whether the review is obliged to post. + # Today's prompt says "use gh pr comment for top-level feedback" -- an instruction about + # HOW to report findings, not a promise to report when there are none -- so a clean review + # is legitimately silent, and failing on it would make this mark red on every tidy PR. + # Keyed on a sentinel in the prompt rather than on prose so the coupling is exact: add + # MUST_POST_SENTINEL to $WF along with the #2229 allowedTools cure and this guard turns + # strict by itself, with no second edit here to forget. + MUST_POST_SENTINEL: ALWAYS POST + run: | + set -euo pipefail + + # "$*", not "$1": every echo in this step wraps across lines with a trailing backslash, + # which produces SEPARATE ARGUMENTS that echo joins with a space. A helper reading only $1 + # silently truncates at the first wrap -- the summary keeps its opening clause and drops + # the rest, with nothing to show that it did. Identical for the single-argument calls. + summary() { echo "$*" >> "$GITHUB_STEP_SUMMARY"; } + + # #2309: every one-shot catalog lookup below goes through this, because under `set -e` a + # bare $(gh api ...) assignment that fails KILLS the step with one context-free stderr + # line -- which is how a transient 404/503 became a deterministic-looking hard red on + # three PRs in one afternoon, with no annotation naming the call that died. A lookup + # failure is not a review verdict (the same rule the sentinel read at the bottom already + # encodes): name the endpoint, mark the review UNCONFIRMED, exit 0. The helper assigns to + # LOOKUP_OUT instead of being command-substituted because an exit inside $() only leaves + # the subshell -- the one shape that CANNOT work here. + LOOKUP_OUT='' + lookup() { + local what=$1; shift + # stderr goes to its OWN capture, never into LOOKUP_OUT: a SUCCESSFUL gh call that + # prints a warning (deprecation notice, scope hint) must not pollute a value the call + # sites compare ("$step" != "success") or feed to arithmetic -- that would be this + # exact defect class reintroduced through the fix (review catch on #2309's own PR). + local errfile err + # The plumbing itself must not be able to kill the step either (review catch on the + # #2309 fix): a failed mktemp degrades to /dev/null (stderr is then lost but the + # verdict path survives), and the cat/rm are failure-tolerant for the same reason. + errfile=$(mktemp 2>/dev/null || echo /dev/null) + if ! LOOKUP_OUT=$(gh api "$@" 2>"$errfile"); then + err=$(cat "$errfile" 2>/dev/null || true) + [ "$errfile" != /dev/null ] && rm -f "$errfile" || true + echo "::warning title=Guard lookup failed::gh api (${what}) failed: ${err:-$LOOKUP_OUT}."\ + "A lookup failure must not become a review verdict (#2309), so the review is"\ + "UNCONFIRMED -- read the PR's comments to confirm it by eye, or re-run this guard." + summary "- Guard lookup failed at ${what}; review UNCONFIRMED (#2309)." + exit 0 + fi + [ "$errfile" != /dev/null ] && rm -f "$errfile" || true + } + + # Wait for the review run on this exact commit. Nothing appearing inside the grace window + # means review is not operating here at all -- no secret, or a fork PR, which the review + # workflow no-ops by design and which is not this guard's business. + # The poll deliberately does NOT use lookup(): a transient API failure mid-poll should + # consume one attempt and try again, not end the guard -- `|| line=''` keeps `set -e` + # out of it, and an empty line is already the loop's no-run-yet case (#2309). + run_id=''; status=''; conclusion=''; created=''; seen='' + for attempt in $(seq 1 70); do + line=$(gh api \ + "repos/$R/actions/workflows/claude-review.yml/runs?head_sha=$SHA&per_page=1" \ + --jq '.workflow_runs[0] // empty + | "\(.id) \(.status) \(.conclusion // "-") \(.created_at)"' \ + 2>/dev/null || true) + if [ -n "$line" ]; then + read -r run_id status conclusion created <<<"$line" + seen=1 + [ "$status" = "completed" ] && break + # Keyed on never having seen a run, not on this poll being empty: a transient empty + # listing after one was found must not be read as "review is not operating here". + elif [ -z "$seen" ] && [ "$attempt" -ge 6 ]; then + echo "::notice title=No review run::No Claude Auto Review run exists for $SHA, so"\ + "review is not operating on this PR (missing secret, or a fork). Nothing to verify." + exit 0 + fi + sleep 30 + done + + if [ "$status" != "completed" ]; then + echo "::warning title=Review did not finish::Gave up waiting on review run $run_id for"\ + "PR #$PR after 35 minutes. Its own 30-minute timeout will mark it; this PR is NOT"\ + "Claude-reviewed." + summary '### Claude review did not finish inside the guard window' + exit 0 + fi + + if [ "$conclusion" = "cancelled" ] || [ "$conclusion" = "skipped" ]; then + echo "::notice title=Review $conclusion::run $run_id was $conclusion; nothing to verify." + exit 0 + fi + + # The step's own conclusion, which separates a clean no-op from a real run. + lookup "review-run jobs (step conclusion)" "repos/$R/actions/runs/$run_id/jobs" --paginate \ + --jq '[.jobs[].steps[] | select(.name == env.REVIEW_STEP)] | .[0].conclusion // empty' + step=$LOOKUP_OUT + + if [ -z "$step" ]; then + echo "::error title=Guard cannot find the review step::No step named"\ + "'$REVIEW_STEP' in run $run_id. This guard reads that step to verify the review, so"\ + "renaming it in $WF blinds the guard -- update REVIEW_STEP here to match." + summary "### Guard is blind: no step named \`$REVIEW_STEP\` in the review run" + exit 1 + fi + + if [ "$step" = "skipped" ]; then + echo "::notice title=Review skipped cleanly::The '$REVIEW_STEP' step was skipped"\ + "(no CLAUDE_CODE_OAUTH_TOKEN, or a fork PR). Nothing to verify." + exit 0 + fi + + # Did the action invoke Claude at all? Its workflow-validation skip reaches the outside + # world ONLY as a warning annotation: it sets skipped_due_to_workflow_validation_mismatch, + # but that output is not declared in the composite action's outputs block, and it returns + # before execution_file and conclusion are ever set. Actions job ids double as check-run + # ids, which is what makes the annotation readable from a different workflow. + never_ran='' + lookup "review-run jobs (annotation scan)" "repos/$R/actions/runs/$run_id/jobs" --paginate --jq '.jobs[].id' + for job in $LOOKUP_OUT; do + if gh api "repos/$R/check-runs/$job/annotations" --jq '.[].message' 2>/dev/null \ + | grep -qF 'Skipping action due to workflow validation'; then + never_ran=1 + break + fi + done + + if [ -n "$never_ran" ]; then + lookup "PR changed files" "repos/$R/pulls/$PR/files" --paginate --jq '.[].filename' + touched=$(printf '%s' "$LOOKUP_OUT" | { grep -Fx "$WF" || true; }) + summary '### Claude review never ran' + if [ -n "$touched" ]; then + # Expected and unavoidable: the action requires that file to match the default branch + # byte for byte, so a PR editing it cannot be reviewed until it merges. A warning, not + # a failure -- a mark that is always red on workflow PRs is one people learn to ignore. + echo "::warning title=Review skipped - PR edits the workflow::This PR changes $WF,"\ + "which claude-code-action refuses to run until it matches $DEFAULT_BRANCH. This PR"\ + "is NOT Claude-reviewed and cannot be until it merges -- a human must review it." + summary "- Expected: this PR edits \`$WF\`, so a human must review it." + exit 0 + fi + # Nothing in this PR explains the skip, so $WF on this branch has drifted from the + # default branch -- which silently disables review for EVERY PR, not just this one. + echo "::error title=Review disabled repo-wide::Claude never ran and this PR does not"\ + "touch $WF, so that file has drifted from $DEFAULT_BRANCH. Every PR in the repo is"\ + "silently going unreviewed until the two match." + summary "- Unexpected: \`$WF\` has drifted from \`$DEFAULT_BRANCH\`; ALL PRs go unreviewed." + exit 1 + fi + + if [ "$step" != "success" ]; then + echo "::error title=Review step failed::The '$REVIEW_STEP' step reported '$step' in run"\ + "$run_id, so no review happened. Read that run for the cause; do NOT read this PR as"\ + "reviewed." + summary "### Claude review failed (step conclusion: \`$step\`)" + exit 1 + fi + + # Count all three artifact kinds, scoped to the reviewer. Reviews on this repo routinely + # carry an EMPTY body and put every finding in INLINE comments -- #2225 had fifteen + # reviews with body_len=0 and twelve findings inline, so counting review bodies alone + # would call it silent. The AUTHOR filter matters just as much: the review window runs up + # to 30 minutes, so without it any human or unrelated bot commenting inside that window + # satisfies the guard and recreates the false pass this exists to catch. If the app is + # ever renamed this reports a false "posted nothing" -- loud and wrong, which is the safer + # direction for a guard to fail. + mine='select(.user.login == env.REVIEW_BOT)' + lookup "issue-comment tally" "repos/$R/issues/$PR/comments" --paginate \ + --jq "[.[] | $mine | select(.created_at > \"$created\")] | length" + issue=$LOOKUP_OUT + lookup "inline-comment tally" "repos/$R/pulls/$PR/comments" --paginate \ + --jq "[.[] | $mine | select(.created_at > \"$created\")] | length" + inline=$LOOKUP_OUT + lookup "review tally" "repos/$R/pulls/$PR/reviews" --paginate \ + --jq "[.[] | $mine | select(.submitted_at != null + and .submitted_at > \"$created\")] | length" + reviews=$LOOKUP_OUT + total=$(( issue + inline + reviews )) + echo "posted by $REVIEW_BOT since $created -- issue: $issue, inline: $inline,"\ + "reviews: $reviews" + summary '### Claude review output' + summary "- issue comments: $issue" + summary "- inline comments: $inline" + summary "- reviews: $reviews" + + if [ "$total" -ne 0 ]; then + exit 0 + fi + + # Nothing posted. Whether that is a defect or a clean bill of health is NOT decidable from + # the artifact count -- the two are the same observation from out here -- so it is decided + # by the contract in $WF on the DEFAULT branch, which is the copy the action validated and + # ran. A missing file or an unreadable one reads as "no obligation", the same as a prompt + # without the sentinel: a lookup failure must not become a review verdict. + # The trailing `|| true` is load-bearing under `set -euo pipefail`: without it a 404 on + # $WF, or a payload base64 cannot decode, fails the PIPELINE and kills this step -- turning + # a lookup failure into exactly the hard guard failure the comment above forbids. Verified + # against all four inputs: sentinel present, sentinel absent, empty, undecodable. + encoded=$(gh api "repos/$R/contents/$WF?ref=$DEFAULT_BRANCH" --jq '.content' 2>/dev/null \ + || true) + must_post=$(printf '%s' "$encoded" | base64 -d 2>/dev/null \ + | { grep -qF "$MUST_POST_SENTINEL" && echo 1 || true; } || true) + + if [ -z "$must_post" ]; then + # The honest reading of today's prompt. Still surfaced, because "reviewed and clean" and + # "reviewed and swallowed" are the same observation, and a reader deciding whether to + # trust the green check needs to know that which one it is cannot be told from here. + echo "::warning title=Review posted nothing::Review run $run_id completed and left no"\ + "comment, inline comment, or review by $REVIEW_BOT on PR #$PR. $WF does not oblige"\ + "the review to post when it finds nothing, so this is consistent with a clean review"\ + "-- but it is ALSO what the #2229 swallowed-output failure looks like, and the two"\ + "cannot be told apart from here. Treat the review as unconfirmed." + summary '- Nothing posted. Consistent with a clean review, and also indistinguishable'\ + 'from the #2229 failure because the prompt does not require a post. UNCONFIRMED.' + exit 0 + fi + + # The prompt promises a post, so silence is a broken promise and nothing else. + echo "::error title=Review posted nothing::Review run $run_id completed but left no"\ + "comment, inline comment, or review by $REVIEW_BOT on PR #$PR, and $WF requires a post"\ + "even for a clean review. Real money was spent and the output vanished (#2229, and PR"\ + "#2213 which did this eleven times). Do NOT read this PR as reviewed; re-run the review"\ + "workflow." + summary "- Nothing posted, and \`$WF\` requires a post. The output was LOST." + exit 1 diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index a096a7f9b..49ae3da27 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -41,8 +41,9 @@ jobs: REPO: ${{ github.repository }} PR NUMBER: ${{ github.event.pull_request.number }} - Review this pull request. Follow the conventions in CLAUDE.md and the T-SQL style - guide it points to. This repository ships two apps that must stay in parity — Lite + Review this pull request. Follow the conventions in CONTRIBUTING.md — including + its T-SQL style section (AS on table aliases, column_name = expression aliasing, + OPTION(RECOMPILE) on collector queries). This repository ships two apps that must stay in parity — Lite and Darling — so flag any change made to one but not its counterpart. Focus on: - Correctness: bugs, edge cases, null/error handling at system boundaries - Lite/Darling parity drift diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cbdb49c38..dce91ea53 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,487 +1,487 @@ -name: Nightly Build - -on: - schedule: - # 6:00 AM UTC (1:00 AM EST / 2:00 AM EDT) - - cron: '0 6 * * *' - workflow_dispatch: # manual trigger — and the vehicle the scheduled re-dispatch below rides - inputs: - from_schedule: - description: 'Set true by the scheduled re-dispatch so the 24h new-commit check applies. Leave false for manual runs, which always build.' - type: boolean - required: false - default: false - -permissions: - contents: write - # #1804: the linux job pushes the nightly container image to ghcr. - packages: write - # Sigstore keyless signing + GitHub provenance for the linux artifacts: id-token lets the job - # obtain its OIDC identity (Fulcio issues the short-lived signing cert against it), attestations - # lets attest-build-provenance store the tarball's provenance. Neither grants anything else. - id-token: write - attestations: write - -jobs: - # Scheduled workflows always execute the DEFAULT branch's copy of this file, while nightly - # artifacts deliberately build from dev's tree. That skew is how the 2026-07-26 nightly - # failed (run 30194606068): main's stale copy still read Dashboard/Dashboard.csproj, a path - # #1612 moved to deprecated/ on dev, so 'Set nightly version' died on a file missing from - # the tree it had just checked out — and the same trap bit before (#1550/#1551). The cure - # is structural, not another sync: on schedule this workflow does NOTHING but re-dispatch - # itself onto the dev REF, because a workflow_dispatch run executes the dispatched ref's - # copy of this file — dev's, current by definition. Once main carries this shape, its copy - # has exactly one job that must keep working, and that job references no tree paths at - # all; every future change to the real nightly logic lands on dev and takes effect the - # night it merges, no promotion to main needed. GITHUB_TOKEN can create workflow_dispatch - # runs (the Actions recursion guard exempts workflow_dispatch and repository_dispatch), - # and the dispatched run cannot loop back here because it arrives as workflow_dispatch, - # not schedule. Until main is synced once, the scheduled run still executes main's OLD - # copy and keeps failing nightly — the one-time sync is in the PR that introduced this. - redispatch: - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - actions: write - steps: - - name: Re-dispatch this workflow onto the dev ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh workflow run nightly.yml --repo ${{ github.repository }} --ref dev -f from_schedule=true - - # Everything below runs only in a workflow_dispatch run — the scheduled re-dispatch or a - # manual one — which executes from the dispatched ref (dev for the scheduled path). - check: - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - has_changes: ${{ steps.check.outputs.has_changes }} - steps: - - uses: actions/checkout@v7 - with: - ref: dev - fetch-depth: 0 - - - name: Check for new commits in last 24 hours - id: check - run: | - RECENT=$(git log --since="24 hours ago" --oneline | head -1) - if [ -n "$RECENT" ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - echo "New commits found — building nightly" - else - echo "has_changes=false" >> $GITHUB_OUTPUT - echo "No new commits — skipping nightly build" - fi - - build: - needs: check - # Manual dispatches always build (from_schedule defaults false); the scheduled - # re-dispatch sets from_schedule=true and builds only when dev changed in the last - # 24h — the same policy the schedule applied when it ran these jobs directly. - if: needs.check.outputs.has_changes == 'true' || inputs.from_schedule != true - runs-on: windows-latest - # Full pipeline (restore, tests, four publishes, cold pg-runtime fetch, vpk pack, - # release upload) is well under an hour; 90 minutes means hung-not-slow. Nightly ships - # unsigned, so unlike build.yml's release path there is no manual signing gate to wait on. - timeout-minutes: 90 - - steps: - - uses: actions/checkout@v7 - with: - ref: dev - - - name: Setup .NET 10.0 - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - - name: Set nightly version - id: version - shell: pwsh - run: | - $base = ([xml](Get-Content Lite/PerformanceMonitorLite.csproj)).Project.PropertyGroup.Version | Where-Object { $_ } - $date = Get-Date -Format "yyyyMMdd" - $nightly = "$base-nightly.$date" - echo "VERSION=$nightly" >> $env:GITHUB_OUTPUT - echo "Nightly version: $nightly" - - - name: Restore dependencies - run: | - dotnet restore Lite/PerformanceMonitorLite.csproj --locked-mode - dotnet restore Lite.Tests/Lite.Tests.csproj --locked-mode - dotnet restore Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj --locked-mode - - - name: Run tests - run: dotnet test Lite.Tests/Lite.Tests.csproj -c Release --verbosity normal - - - name: Publish Lite - run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -o publish/Lite - - - name: Publish Darling Service - run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -o publish/DarlingService - - - name: Publish Darling Viewer - run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -o publish/DarlingViewer - - # Self-contained viewer publish that feeds the remote-seat Velopack Setup.exe (#1555), the same - # publish shape build.yml uses for the Dashboard/Lite Velopack packs. This is IN ADDITION to the - # framework-dependent "Publish Darling Viewer" above, which still feeds the co-located viewer\ - # folder inside PerformanceMonitorDarling-*.zip — that zip is unchanged. - - name: Publish Darling Viewer (self-contained for Velopack) - run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -r win-x64 --self-contained -o publish/DarlingViewer-velopack - - # Same cache key as build.yml's release path: the fetch script's content hash (the SHA256 - # pins live inside it). The ~340MB EDB/TimescaleDB fetch runs at most once per pin-set per - # branch; nightly and release runs share the assembled zip whenever the cache is visible. - - name: Cache Darling pg-runtime.zip - id: cache-pg-runtime - uses: actions/cache@v6 - with: - path: Darling/artifacts/pg-runtime.zip - key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} - - - name: Build Darling pg-runtime.zip - if: steps.cache-pg-runtime.outputs.cache-hit != 'true' - shell: pwsh - run: ./Darling/tools/fetch-pg-runtime.ps1 - - - name: Package artifacts - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - New-Item -ItemType Directory -Force -Path releases - - Compress-Archive -Path 'publish/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force - - - # Same layout as the release zip (build.yml "Package Darling (signed)"): service at the - # archive root with darling.sample.json alongside, viewer under viewer\, pg-runtime.zip - # beside the service exe where DarlingManagedPostgres extracts it on first run. Nightly - # zips are unsigned across the board, so this stages from publish/ instead of signed/. - $darlingDir = 'publish/Darling' - New-Item -ItemType Directory -Force -Path "$darlingDir/viewer" | Out-Null - Copy-Item 'publish/DarlingService/*' $darlingDir -Recurse - Copy-Item 'publish/DarlingViewer/*' "$darlingDir/viewer" -Recurse - Copy-Item 'Darling/artifacts/pg-runtime.zip' $darlingDir - - Compress-Archive -Path 'publish/Darling/*' -DestinationPath "releases/PerformanceMonitorDarling-$version.zip" -Force - - # Darling viewer remote-seat installer (#1555). Nightly ships it UNSIGNED like every other nightly - # artifact. Mirrors build.yml's release vpk pack (same pack id / exe / channel) but packs from the - # raw self-contained publish (no SignPath), and deliberately does NOT touch the Velopack update - # feed: the nightly GitHub release is deleted + recreated each night, so there is no persistent - # delta chain to `vpk download`/`vpk upload` from — we ship a standalone full Setup.exe as a plain - # release asset. Copied to a deterministic name so the checksum + upload steps below pick it up. - # Purely additive: the co-located viewer inside PerformanceMonitorDarling-*.zip is untouched. - - name: Create Darling Viewer Setup.exe (Velopack, unsigned) - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - dotnet tool install -g vpk --version 1.2.0 - New-Item -ItemType Directory -Force -Path releases/velopack-darlingviewer - vpk pack -u PerformanceMonitorDarlingViewer -v $version -p publish/DarlingViewer-velopack -e PerformanceMonitor.Darling.Viewer.exe -o releases/velopack-darlingviewer --channel darlingviewer - $setup = Get-ChildItem releases/velopack-darlingviewer/*Setup.exe | Select-Object -First 1 - Copy-Item $setup.FullName "releases/PerformanceMonitorDarlingViewer-$version-Setup.exe" - - - name: Generate checksums - shell: pwsh - run: | - $checksums = Get-ChildItem releases/*.zip, releases/*.exe | ForEach-Object { - $hash = (Get-FileHash $_.FullName -Algorithm SHA256).Hash.ToLower() - "$hash $($_.Name)" - } - $checksums | Out-File -FilePath releases/SHA256SUMS.txt -Encoding utf8 - Write-Host "Checksums:" - $checksums | ForEach-Object { Write-Host $_ } - - - name: Delete previous nightly release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release delete nightly --yes --cleanup-tag 2>$null; exit 0 - shell: pwsh - - - name: Create nightly release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: pwsh - run: | - $version = "${{ steps.version.outputs.VERSION }}" - $sha = git rev-parse --short HEAD - $body = @" - Automated nightly build from ``dev`` branch. - - **Version:** ``$version`` - **Commit:** ``$sha`` - **Built:** $(Get-Date -Format "yyyy-MM-dd HH:mm UTC") - - > These builds include the latest changes and may be unstable. - > For production use, download the [latest stable release](https://github.com/erikdarlingdata/PerformanceMonitor/releases/latest). - "@ - - gh release create nightly ` - --target dev ` - --title "Nightly Build ($version)" ` - --notes $body ` - --prerelease ` - releases/*.zip releases/*.exe releases/SHA256SUMS.txt - - # Darling.Tests has live-PostgreSQL tests gated on env vars that the per-PR/push build never - # sets, so regular CI runs only the ungated subset. This job stands up a throwaway PostgreSQL - # from the bundled pg-runtime and runs the full Darling suite against it nightly, so the - # Postgres + managed-bootstrap paths get real coverage. The one live-SQL-Server E2E stays - # skipped (no SQL Server on the runner) — expected. Gated like the build job so it only runs - # when dev actually changed (or on manual dispatch). - darling-pg: - name: Darling PostgreSQL tests - needs: check - # Same gating as the build job: manual dispatches always run, the scheduled - # re-dispatch (from_schedule=true) runs only when dev changed in the last 24h. - if: needs.check.outputs.has_changes == 'true' || inputs.from_schedule != true - runs-on: windows-latest - # Cold pg-runtime fetch + build + the full live-PG suite fits well inside an hour; a - # cluster that never comes up (pg_ctl -w) is the hang this bounds. - timeout-minutes: 60 - - steps: - # Scheduled runs always test dev (schedules execute from the default branch, so ref_name - # would be main). A manual dispatch tests the DISPATCHED ref — the only way to validate a - # branch's gated-pg test changes before merge; the artifact-publishing build job stays - # pinned to dev either way, so a branch dispatch can never ship branch binaries. - - uses: actions/checkout@v7 - with: - ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'dev' }} - - - name: Setup .NET 10.0 - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - # Darling bundles a PostgreSQL 18 + TimescaleDB runtime (pg-runtime.zip). The fetch script - # pulls ~340MB of pinned EDB/TimescaleDB archives, so cache the assembled zip on the SAME - # key build.yml's release job uses (the fetch script's own content hash) — nightly and - # release share one cache entry, so a warm cache means no download here. - - name: Cache Darling pg-runtime.zip - id: cache-pg-runtime - uses: actions/cache@v6 - with: - path: Darling/artifacts/pg-runtime.zip - key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} - - - name: Build Darling pg-runtime.zip (cache miss only) - if: steps.cache-pg-runtime.outputs.cache-hit != 'true' - shell: pwsh - run: ./Darling/tools/fetch-pg-runtime.ps1 - - # One uniform path for hit and miss: we always have the zip (restored or freshly built), - # so always extract it — simpler than branching on the script's -KeepWork assembled tree. - # ExtractToDirectory mirrors the fetch script's own API and reads any zip it writes. - - name: Extract pg-runtime - shell: pwsh - run: | - $zip = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime.zip" - $dest = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime" - if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zip, $dest) - if (-not (Test-Path "$dest\pgsql\bin\pg_ctl.exe")) { throw "pg-runtime missing pgsql\bin\pg_ctl.exe" } - - # The PREVIOUS-major runtime, for the upgraded-in-place fixture (#1706). Without it the gated - # store-upgrade E2E skips, and with it skips the ONLY behavioral coverage of the in-place - # 17-to-18 path: the runtime rescue, the TimescaleDB bridge, the data-directory swap, the - # revert, and the loopback override that stops pg_upgrade dialing ::1 against our IPv4-only - # listen_addresses. String assertions catch that constant being deleted; nothing but this - # catches the override ceasing to TAKE EFFECT, and the failure mode is a fleet-wide hang. - # Only the DOWNLOADS are cached (~340MB of pinned archives): the script re-verifies them by - # SHA256 and re-assembles in seconds, so a warm cache costs no network. - - name: Cache upgrade-fixture downloads (previous-major runtime) - uses: actions/cache@v6 - with: - path: Darling/artifacts/upgrade-fixture/work/downloads - key: upgrade-fixture-${{ runner.os }}-${{ hashFiles('Darling/tools/new-upgraded-store-fixture.ps1') }} - - # -SkipNew: the CURRENT runtime is already built/cached above as pg-runtime.zip, which is what - # DARLING_TEST_PGRUNTIME_NEWZIP points at. This step only needs the old side. - - name: Build previous-major runtime for the upgrade fixture - shell: pwsh - run: ./Darling/tools/new-upgraded-store-fixture.ps1 -SkipNew - - - name: Restore Darling.Tests - run: dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode - - - name: Build Darling.Tests - run: dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore - - # Stand up a throwaway cluster from the bundled runtime. initdb TRUST auth is acceptable - # ONLY here: an ephemeral CI runner, loopback-only, throwaway data. The PRODUCT default is - # the opposite (scram-sha-256 + a generated credential; see DarlingManagedPostgres). The - # appended settings mirror DarlingManagedPostgres.BuildConfAppend (timescaledb preload — - # the Timescale-gated tests detect and use it — plus the port and loopback bind) and - # BuildWorkerSizingConfAppend (the two worker settings). Port 5541 is fixed and distinct; the - # managed-bootstrap E2E starts its OWN postgres on a random free port, so there is no collision. - # - # #1888: the worker settings are load-bearing, not garnish. PostgreSQL's default - # max_worker_processes = 8 cannot launch TimescaleDB's per-hypertable compression, retention - # and continuous-aggregate policy jobs, so without them this job tested a configuration no - # customer runs and made scheduler-racing failures luck-of-the-slot instead of reproducible. - # Values are the product's own derivation from the live hypertable count - # (TimescaleSupport.HypertableCount = the 41-collector catalog + collection_log = 42): - # timescaledb.max_background_workers = HypertableCount + 2 = 44 - # max_worker_processes = 3 + (HypertableCount + 2) + 8 = 55 - # Kept honest by CiClusterWorkerSizingTests (parses this file against the formula) and - # CiClusterWorkerSizingLiveTests (asserts the running cluster serves them). Must configure - # the cluster identically to build.yml's darling-pg job: that guard parses the appended - # settings out of BOTH files and requires the two sets to be equal, so a fix applied to one - # workflow and not the other — the ordinary way these hand-maintained copies drift — fails. - - name: Initialize and start throwaway PostgreSQL - shell: pwsh - run: | - $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" - $dataDir = "$env:RUNNER_TEMP\darling-pgdata" - $logFile = "$env:RUNNER_TEMP\darling-pg.log" - # The bootstrap superuser is named "darling", not "postgres": the V8 schema-split - # migration runs CREATE SCHEMA ... AUTHORIZATION darling (PgSchemaGenerator.OwnerRole), - # so a cluster without that role fails every fresh-store migration. Matching managed - # mode's shape (DarlingManagedPostgres also initdbs its owner as "darling"). - & "$bin\initdb.exe" -D $dataDir -U darling -A trust --encoding=UTF8 - if ($LASTEXITCODE -ne 0) { throw "initdb failed ($LASTEXITCODE)" } - Add-Content -Path "$dataDir\postgresql.conf" -Value "shared_preload_libraries = 'timescaledb'" - Add-Content -Path "$dataDir\postgresql.conf" -Value "port = 5541" - Add-Content -Path "$dataDir\postgresql.conf" -Value "listen_addresses = '127.0.0.1'" - Add-Content -Path "$dataDir\postgresql.conf" -Value "timescaledb.max_background_workers = 44" - Add-Content -Path "$dataDir\postgresql.conf" -Value "max_worker_processes = 55" - & "$bin\pg_ctl.exe" -D $dataDir -l $logFile -w start - if ($LASTEXITCODE -ne 0) { if (Test-Path $logFile) { Get-Content $logFile -Tail 50 }; throw "pg_ctl start failed ($LASTEXITCODE)" } - & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling - if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" } - - # DARLING_TEST_PG lights up the [Collection("live-postgres")] classes; DARLING_TEST_PGRUNTIME - # lights up the managed-bootstrap E2E. DARLING_TEST_SQL is intentionally unset — the one - # live-SQL-Server E2E stays skipped (no SQL Server on the runner). - - name: Run Darling PG tests - shell: pwsh - env: - DARLING_TEST_PG: "Host=127.0.0.1;Port=5541;Username=darling;Database=darling" - DARLING_TEST_PGRUNTIME: ${{ github.workspace }}\Darling\artifacts\pg-runtime - # #1706: the pair that lights up the upgraded-in-place store-upgrade E2E — a real - # previous-major store (hypertable, TOAST-sized plan XML, continuous aggregate, - # compressed chunk) upgraded through the production bootstrap and compared by ordered - # row checksum before and after. This is the [#1705] CI gap: every other job only ever - # sees a FRESH store, so nothing else can catch an upgrade path that breaks. - DARLING_TEST_PGRUNTIME_OLD: ${{ github.workspace }}\Darling\artifacts\upgrade-fixture\old\pg-runtime - DARLING_TEST_PGRUNTIME_NEWZIP: ${{ github.workspace }}\Darling\artifacts\pg-runtime.zip - run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal --logger "trx;LogFileName=darling-nightly.trx" --results-directory TestResults - - - name: Stop PostgreSQL - if: always() - shell: pwsh - run: | - $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" - $dataDir = "$env:RUNNER_TEMP\darling-pgdata" - if (Test-Path "$bin\pg_ctl.exe") { & "$bin\pg_ctl.exe" -D $dataDir -m fast -w stop } - exit 0 - - - name: Upload PG log and test results on failure - if: failure() - uses: actions/upload-artifact@v6 - with: - name: darling-pg-failure - path: | - ${{ runner.temp }}/darling-pg.log - TestResults/ - if-no-files-found: ignore - - # ── Linux artifact + container image (#1804) ───────────────────────────────────────────────────── - # Runs AFTER the windows build job so the nightly release exists to upload into. Publishes the - # linux-x64 service tar.gz with its own checksum file (the windows job owns SHA256SUMS.txt; a - # cross-job rewrite of one file is a race), and pushes the service image to ghcr tagged :nightly. - # The bundled pg-runtime is deliberately absent from the linux artifact — the compose distribution - # pairs the service with the official timescale/timescaledb image, and managed mode stays Windows. - linux: - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v7 - with: - ref: dev - - - name: Setup .NET 10.0 - uses: actions/setup-dotnet@v6 - with: - global-json-file: global.json - cache: true - cache-dependency-path: '**/packages.lock.json' - - - name: Set nightly version - id: version - shell: bash - run: | - set -euo pipefail - base=$(grep -oPm1 '(?<=)[^<]+' Lite/PerformanceMonitorLite.csproj) - date=$(date +%Y%m%d) - echo "VERSION=${base}-nightly.${date}" >> "$GITHUB_OUTPUT" - echo "Nightly version: ${base}-nightly.${date}" - - - name: Publish service (linux-x64) - run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -r linux-x64 --self-contained false -o publish/DarlingService-linux - - - name: Package linux artifact + checksum - shell: bash - run: | - set -euo pipefail - version="${{ steps.version.outputs.VERSION }}" - mkdir -p releases - tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . - (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) - - - name: Upload linux artifact to the nightly release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release upload nightly releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt --clobber - - - name: Build and push container image (ghcr, :nightly) - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - set -euo pipefail - version="${{ steps.version.outputs.VERSION }}" - image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" - echo "$GH_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - docker build -f Darling/Dockerfile -t "${image}:nightly" -t "${image}:${version}" . - docker push "${image}:nightly" - docker push "${image}:${version}" - - # Keyless Sigstore signing: Fulcio issues a short-lived certificate against this job's OIDC - # identity and the signature lands in ghcr next to the image, logged in Rekor. No keys exist - # anywhere to manage or leak — the signature attests "built by this repository's workflow", - # which is the claim a container consumer actually wants verified. (SignPath's cosign support - # is edition-gated; this path has no subscription dependency.) Verify: - # cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling:nightly \ - # --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' \ - # --certificate-oidc-issuer https://token.actions.githubusercontent.com - - name: Install cosign - uses: sigstore/cosign-installer@v3 - - - name: Sign container image (keyless, Sigstore) - shell: bash - run: | - set -euo pipefail - image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" - digest="$(docker inspect --format='{{index .RepoDigests 0}}' "${image}:nightly")" - cosign sign --yes "${digest}" - - # GitHub-native provenance for the tarball (SLSA): verified with - # gh attestation verify PerformanceMonitorDarling-linux-x64-.tar.gz -R erikdarlingdata/PerformanceMonitor - - name: Attest the linux tarball (GitHub provenance) - uses: actions/attest-build-provenance@v3 - with: - subject-path: releases/PerformanceMonitorDarling-linux-x64-*.tar.gz +name: Nightly Build + +on: + schedule: + # 6:00 AM UTC (1:00 AM EST / 2:00 AM EDT) + - cron: '0 6 * * *' + workflow_dispatch: # manual trigger — and the vehicle the scheduled re-dispatch below rides + inputs: + from_schedule: + description: 'Set true by the scheduled re-dispatch so the 24h new-commit check applies. Leave false for manual runs, which always build.' + type: boolean + required: false + default: false + +permissions: + contents: write + # #1804: the linux job pushes the nightly container image to ghcr. + packages: write + # Sigstore keyless signing + GitHub provenance for the linux artifacts: id-token lets the job + # obtain its OIDC identity (Fulcio issues the short-lived signing cert against it), attestations + # lets attest-build-provenance store the tarball's provenance. Neither grants anything else. + id-token: write + attestations: write + +jobs: + # Scheduled workflows always execute the DEFAULT branch's copy of this file, while nightly + # artifacts deliberately build from dev's tree. That skew is how the 2026-07-26 nightly + # failed (run 30194606068): main's stale copy still read Dashboard/Dashboard.csproj, a path + # #1612 moved to deprecated/ on dev, so 'Set nightly version' died on a file missing from + # the tree it had just checked out — and the same trap bit before (#1550/#1551). The cure + # is structural, not another sync: on schedule this workflow does NOTHING but re-dispatch + # itself onto the dev REF, because a workflow_dispatch run executes the dispatched ref's + # copy of this file — dev's, current by definition. Once main carries this shape, its copy + # has exactly one job that must keep working, and that job references no tree paths at + # all; every future change to the real nightly logic lands on dev and takes effect the + # night it merges, no promotion to main needed. GITHUB_TOKEN can create workflow_dispatch + # runs (the Actions recursion guard exempts workflow_dispatch and repository_dispatch), + # and the dispatched run cannot loop back here because it arrives as workflow_dispatch, + # not schedule. Until main is synced once, the scheduled run still executes main's OLD + # copy and keeps failing nightly — the one-time sync is in the PR that introduced this. + redispatch: + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Re-dispatch this workflow onto the dev ref + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run nightly.yml --repo ${{ github.repository }} --ref dev -f from_schedule=true + + # Everything below runs only in a workflow_dispatch run — the scheduled re-dispatch or a + # manual one — which executes from the dispatched ref (dev for the scheduled path). + check: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + has_changes: ${{ steps.check.outputs.has_changes }} + steps: + - uses: actions/checkout@v7 + with: + ref: dev + fetch-depth: 0 + + - name: Check for new commits in last 24 hours + id: check + run: | + RECENT=$(git log --since="24 hours ago" --oneline | head -1) + if [ -n "$RECENT" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + echo "New commits found — building nightly" + else + echo "has_changes=false" >> $GITHUB_OUTPUT + echo "No new commits — skipping nightly build" + fi + + build: + needs: check + # Manual dispatches always build (from_schedule defaults false); the scheduled + # re-dispatch sets from_schedule=true and builds only when dev changed in the last + # 24h — the same policy the schedule applied when it ran these jobs directly. + if: needs.check.outputs.has_changes == 'true' || inputs.from_schedule != true + runs-on: windows-latest + # Full pipeline (restore, tests, four publishes, cold pg-runtime fetch, vpk pack, + # release upload) is well under an hour; 90 minutes means hung-not-slow. Nightly ships + # unsigned, so unlike build.yml's release path there is no manual signing gate to wait on. + timeout-minutes: 90 + + steps: + - uses: actions/checkout@v7 + with: + ref: dev + + - name: Setup .NET 10.0 + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + - name: Set nightly version + id: version + shell: pwsh + run: | + $base = ([xml](Get-Content Lite/PerformanceMonitorLite.csproj)).Project.PropertyGroup.Version | Where-Object { $_ } + $date = Get-Date -Format "yyyyMMdd" + $nightly = "$base-nightly.$date" + echo "VERSION=$nightly" >> $env:GITHUB_OUTPUT + echo "Nightly version: $nightly" + + - name: Restore dependencies + run: | + dotnet restore Lite/PerformanceMonitorLite.csproj --locked-mode + dotnet restore Lite.Tests/Lite.Tests.csproj --locked-mode + dotnet restore Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj --locked-mode + + - name: Run tests + run: dotnet test Lite.Tests/Lite.Tests.csproj -c Release --verbosity normal + + - name: Publish Lite + run: dotnet publish Lite/PerformanceMonitorLite.csproj -c Release -o publish/Lite + + - name: Publish Darling Service + run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -o publish/DarlingService + + - name: Publish Darling Viewer + run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -o publish/DarlingViewer + + # Self-contained viewer publish that feeds the remote-seat Velopack Setup.exe (#1555), the same + # publish shape build.yml uses for the Dashboard/Lite Velopack packs. This is IN ADDITION to the + # framework-dependent "Publish Darling Viewer" above, which still feeds the co-located viewer\ + # folder inside PerformanceMonitorDarling-*.zip — that zip is unchanged. + - name: Publish Darling Viewer (self-contained for Velopack) + run: dotnet publish Darling/PerformanceMonitor.Darling.Viewer/PerformanceMonitor.Darling.Viewer.csproj -c Release -r win-x64 --self-contained -o publish/DarlingViewer-velopack + + # Same cache key as build.yml's release path: the fetch script's content hash (the SHA256 + # pins live inside it). The ~340MB EDB/TimescaleDB fetch runs at most once per pin-set per + # branch; nightly and release runs share the assembled zip whenever the cache is visible. + - name: Cache Darling pg-runtime.zip + id: cache-pg-runtime + uses: actions/cache@v6 + with: + path: Darling/artifacts/pg-runtime.zip + key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} + + - name: Build Darling pg-runtime.zip + if: steps.cache-pg-runtime.outputs.cache-hit != 'true' + shell: pwsh + run: ./Darling/tools/fetch-pg-runtime.ps1 + + - name: Package artifacts + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + New-Item -ItemType Directory -Force -Path releases + + Compress-Archive -Path 'publish/Lite/*' -DestinationPath "releases/PerformanceMonitorLite-$version.zip" -Force + + + # Same layout as the release zip (build.yml "Package Darling (signed)"): service at the + # archive root with darling.sample.json alongside, viewer under viewer\, pg-runtime.zip + # beside the service exe where DarlingManagedPostgres extracts it on first run. Nightly + # zips are unsigned across the board, so this stages from publish/ instead of signed/. + $darlingDir = 'publish/Darling' + New-Item -ItemType Directory -Force -Path "$darlingDir/viewer" | Out-Null + Copy-Item 'publish/DarlingService/*' $darlingDir -Recurse + Copy-Item 'publish/DarlingViewer/*' "$darlingDir/viewer" -Recurse + Copy-Item 'Darling/artifacts/pg-runtime.zip' $darlingDir + + Compress-Archive -Path 'publish/Darling/*' -DestinationPath "releases/PerformanceMonitorDarling-$version.zip" -Force + + # Darling viewer remote-seat installer (#1555). Nightly ships it UNSIGNED like every other nightly + # artifact. Mirrors build.yml's release vpk pack (same pack id / exe / channel) but packs from the + # raw self-contained publish (no SignPath), and deliberately does NOT touch the Velopack update + # feed: the nightly GitHub release is deleted + recreated each night, so there is no persistent + # delta chain to `vpk download`/`vpk upload` from — we ship a standalone full Setup.exe as a plain + # release asset. Copied to a deterministic name so the checksum + upload steps below pick it up. + # Purely additive: the co-located viewer inside PerformanceMonitorDarling-*.zip is untouched. + - name: Create Darling Viewer Setup.exe (Velopack, unsigned) + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + dotnet tool install -g vpk --version 1.2.0 + New-Item -ItemType Directory -Force -Path releases/velopack-darlingviewer + vpk pack -u PerformanceMonitorDarlingViewer -v $version -p publish/DarlingViewer-velopack -e PerformanceMonitor.Darling.Viewer.exe -o releases/velopack-darlingviewer --channel darlingviewer + $setup = Get-ChildItem releases/velopack-darlingviewer/*Setup.exe | Select-Object -First 1 + Copy-Item $setup.FullName "releases/PerformanceMonitorDarlingViewer-$version-Setup.exe" + + - name: Generate checksums + shell: pwsh + run: | + $checksums = Get-ChildItem releases/*.zip, releases/*.exe | ForEach-Object { + $hash = (Get-FileHash $_.FullName -Algorithm SHA256).Hash.ToLower() + "$hash $($_.Name)" + } + $checksums | Out-File -FilePath releases/SHA256SUMS.txt -Encoding utf8 + Write-Host "Checksums:" + $checksums | ForEach-Object { Write-Host $_ } + + - name: Delete previous nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release delete nightly --yes --cleanup-tag 2>$null; exit 0 + shell: pwsh + + - name: Create nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + $sha = git rev-parse --short HEAD + $body = @" + Automated nightly build from ``dev`` branch. + + **Version:** ``$version`` + **Commit:** ``$sha`` + **Built:** $(Get-Date -Format "yyyy-MM-dd HH:mm UTC") + + > These builds include the latest changes and may be unstable. + > For production use, download the [latest stable release](https://github.com/erikdarlingdata/PerformanceMonitor/releases/latest). + "@ + + gh release create nightly ` + --target dev ` + --title "Nightly Build ($version)" ` + --notes $body ` + --prerelease ` + releases/*.zip releases/*.exe releases/SHA256SUMS.txt + + # Darling.Tests has live-PostgreSQL tests gated on env vars that the per-PR/push build never + # sets, so regular CI runs only the ungated subset. This job stands up a throwaway PostgreSQL + # from the bundled pg-runtime and runs the full Darling suite against it nightly, so the + # Postgres + managed-bootstrap paths get real coverage. The one live-SQL-Server E2E stays + # skipped (no SQL Server on the runner) — expected. Gated like the build job so it only runs + # when dev actually changed (or on manual dispatch). + darling-pg: + name: Darling PostgreSQL tests + needs: check + # Same gating as the build job: manual dispatches always run, the scheduled + # re-dispatch (from_schedule=true) runs only when dev changed in the last 24h. + if: needs.check.outputs.has_changes == 'true' || inputs.from_schedule != true + runs-on: windows-latest + # Cold pg-runtime fetch + build + the full live-PG suite fits well inside an hour; a + # cluster that never comes up (pg_ctl -w) is the hang this bounds. + timeout-minutes: 60 + + steps: + # Scheduled runs always test dev (schedules execute from the default branch, so ref_name + # would be main). A manual dispatch tests the DISPATCHED ref — the only way to validate a + # branch's gated-pg test changes before merge; the artifact-publishing build job stays + # pinned to dev either way, so a branch dispatch can never ship branch binaries. + - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.ref_name || 'dev' }} + + - name: Setup .NET 10.0 + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + # Darling bundles a PostgreSQL 18 + TimescaleDB runtime (pg-runtime.zip). The fetch script + # pulls ~340MB of pinned EDB/TimescaleDB archives, so cache the assembled zip on the SAME + # key build.yml's release job uses (the fetch script's own content hash) — nightly and + # release share one cache entry, so a warm cache means no download here. + - name: Cache Darling pg-runtime.zip + id: cache-pg-runtime + uses: actions/cache@v6 + with: + path: Darling/artifacts/pg-runtime.zip + key: pg-runtime-${{ runner.os }}-${{ hashFiles('Darling/tools/fetch-pg-runtime.ps1') }} + + - name: Build Darling pg-runtime.zip (cache miss only) + if: steps.cache-pg-runtime.outputs.cache-hit != 'true' + shell: pwsh + run: ./Darling/tools/fetch-pg-runtime.ps1 + + # One uniform path for hit and miss: we always have the zip (restored or freshly built), + # so always extract it — simpler than branching on the script's -KeepWork assembled tree. + # ExtractToDirectory mirrors the fetch script's own API and reads any zip it writes. + - name: Extract pg-runtime + shell: pwsh + run: | + $zip = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime.zip" + $dest = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime" + if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zip, $dest) + if (-not (Test-Path "$dest\pgsql\bin\pg_ctl.exe")) { throw "pg-runtime missing pgsql\bin\pg_ctl.exe" } + + # The PREVIOUS-major runtime, for the upgraded-in-place fixture (#1706). Without it the gated + # store-upgrade E2E skips, and with it skips the ONLY behavioral coverage of the in-place + # 17-to-18 path: the runtime rescue, the TimescaleDB bridge, the data-directory swap, the + # revert, and the loopback override that stops pg_upgrade dialing ::1 against our IPv4-only + # listen_addresses. String assertions catch that constant being deleted; nothing but this + # catches the override ceasing to TAKE EFFECT, and the failure mode is a fleet-wide hang. + # Only the DOWNLOADS are cached (~340MB of pinned archives): the script re-verifies them by + # SHA256 and re-assembles in seconds, so a warm cache costs no network. + - name: Cache upgrade-fixture downloads (previous-major runtime) + uses: actions/cache@v6 + with: + path: Darling/artifacts/upgrade-fixture/work/downloads + key: upgrade-fixture-${{ runner.os }}-${{ hashFiles('Darling/tools/new-upgraded-store-fixture.ps1') }} + + # -SkipNew: the CURRENT runtime is already built/cached above as pg-runtime.zip, which is what + # DARLING_TEST_PGRUNTIME_NEWZIP points at. This step only needs the old side. + - name: Build previous-major runtime for the upgrade fixture + shell: pwsh + run: ./Darling/tools/new-upgraded-store-fixture.ps1 -SkipNew + + - name: Restore Darling.Tests + run: dotnet restore Darling/Darling.Tests/Darling.Tests.csproj --locked-mode + + - name: Build Darling.Tests + run: dotnet build Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-restore + + # Stand up a throwaway cluster from the bundled runtime. initdb TRUST auth is acceptable + # ONLY here: an ephemeral CI runner, loopback-only, throwaway data. The PRODUCT default is + # the opposite (scram-sha-256 + a generated credential; see DarlingManagedPostgres). The + # appended settings mirror DarlingManagedPostgres.BuildConfAppend (timescaledb preload — + # the Timescale-gated tests detect and use it — plus the port and loopback bind) and + # BuildWorkerSizingConfAppend (the two worker settings). Port 5541 is fixed and distinct; the + # managed-bootstrap E2E starts its OWN postgres on a random free port, so there is no collision. + # + # #1888: the worker settings are load-bearing, not garnish. PostgreSQL's default + # max_worker_processes = 8 cannot launch TimescaleDB's per-hypertable compression, retention + # and continuous-aggregate policy jobs, so without them this job tested a configuration no + # customer runs and made scheduler-racing failures luck-of-the-slot instead of reproducible. + # Values are the product's own derivation from the live hypertable count + # (TimescaleSupport.HypertableCount = the 50-collector catalog + collection_log = 51): + # timescaledb.max_background_workers = HypertableCount + 2 = 53 + # max_worker_processes = 3 + (HypertableCount + 2) + 8 = 64 + # Kept honest by CiClusterWorkerSizingTests (parses this file against the formula) and + # CiClusterWorkerSizingLiveTests (asserts the running cluster serves them). Must configure + # the cluster identically to build.yml's darling-pg job: that guard parses the appended + # settings out of BOTH files and requires the two sets to be equal, so a fix applied to one + # workflow and not the other — the ordinary way these hand-maintained copies drift — fails. + - name: Initialize and start throwaway PostgreSQL + shell: pwsh + run: | + $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" + $dataDir = "$env:RUNNER_TEMP\darling-pgdata" + $logFile = "$env:RUNNER_TEMP\darling-pg.log" + # The bootstrap superuser is named "darling", not "postgres": the V8 schema-split + # migration runs CREATE SCHEMA ... AUTHORIZATION darling (PgSchemaGenerator.OwnerRole), + # so a cluster without that role fails every fresh-store migration. Matching managed + # mode's shape (DarlingManagedPostgres also initdbs its owner as "darling"). + & "$bin\initdb.exe" -D $dataDir -U darling -A trust --encoding=UTF8 + if ($LASTEXITCODE -ne 0) { throw "initdb failed ($LASTEXITCODE)" } + Add-Content -Path "$dataDir\postgresql.conf" -Value "shared_preload_libraries = 'timescaledb'" + Add-Content -Path "$dataDir\postgresql.conf" -Value "port = 5541" + Add-Content -Path "$dataDir\postgresql.conf" -Value "listen_addresses = '127.0.0.1'" + Add-Content -Path "$dataDir\postgresql.conf" -Value "timescaledb.max_background_workers = 53" + Add-Content -Path "$dataDir\postgresql.conf" -Value "max_worker_processes = 64" + & "$bin\pg_ctl.exe" -D $dataDir -l $logFile -w start + if ($LASTEXITCODE -ne 0) { if (Test-Path $logFile) { Get-Content $logFile -Tail 50 }; throw "pg_ctl start failed ($LASTEXITCODE)" } + & "$bin\createdb.exe" -h 127.0.0.1 -p 5541 -U darling darling + if ($LASTEXITCODE -ne 0) { throw "createdb failed ($LASTEXITCODE)" } + + # DARLING_TEST_PG lights up the [Collection("live-postgres")] classes; DARLING_TEST_PGRUNTIME + # lights up the managed-bootstrap E2E. DARLING_TEST_SQL is intentionally unset — the one + # live-SQL-Server E2E stays skipped (no SQL Server on the runner). + - name: Run Darling PG tests + shell: pwsh + env: + DARLING_TEST_PG: "Host=127.0.0.1;Port=5541;Username=darling;Database=darling" + DARLING_TEST_PGRUNTIME: ${{ github.workspace }}\Darling\artifacts\pg-runtime + # #1706: the pair that lights up the upgraded-in-place store-upgrade E2E — a real + # previous-major store (hypertable, TOAST-sized plan XML, continuous aggregate, + # compressed chunk) upgraded through the production bootstrap and compared by ordered + # row checksum before and after. This is the [#1705] CI gap: every other job only ever + # sees a FRESH store, so nothing else can catch an upgrade path that breaks. + DARLING_TEST_PGRUNTIME_OLD: ${{ github.workspace }}\Darling\artifacts\upgrade-fixture\old\pg-runtime + DARLING_TEST_PGRUNTIME_NEWZIP: ${{ github.workspace }}\Darling\artifacts\pg-runtime.zip + run: dotnet test Darling/Darling.Tests/Darling.Tests.csproj -c Release --no-build --verbosity normal --logger "trx;LogFileName=darling-nightly.trx" --results-directory TestResults + + - name: Stop PostgreSQL + if: always() + shell: pwsh + run: | + $bin = "$env:GITHUB_WORKSPACE\Darling\artifacts\pg-runtime\pgsql\bin" + $dataDir = "$env:RUNNER_TEMP\darling-pgdata" + if (Test-Path "$bin\pg_ctl.exe") { & "$bin\pg_ctl.exe" -D $dataDir -m fast -w stop } + exit 0 + + - name: Upload PG log and test results on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: darling-pg-failure + path: | + ${{ runner.temp }}/darling-pg.log + TestResults/ + if-no-files-found: ignore + + # ── Linux artifact + container image (#1804) ───────────────────────────────────────────────────── + # Runs AFTER the windows build job so the nightly release exists to upload into. Publishes the + # linux-x64 service tar.gz with its own checksum file (the windows job owns SHA256SUMS.txt; a + # cross-job rewrite of one file is a race), and pushes the service image to ghcr tagged :nightly. + # The bundled pg-runtime is deliberately absent from the linux artifact — the compose distribution + # pairs the service with the official timescale/timescaledb image, and managed mode stays Windows. + linux: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v7 + with: + ref: dev + + - name: Setup .NET 10.0 + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/packages.lock.json' + + - name: Set nightly version + id: version + shell: bash + run: | + set -euo pipefail + base=$(grep -oPm1 '(?<=)[^<]+' Lite/PerformanceMonitorLite.csproj) + date=$(date +%Y%m%d) + echo "VERSION=${base}-nightly.${date}" >> "$GITHUB_OUTPUT" + echo "Nightly version: ${base}-nightly.${date}" + + - name: Publish service (linux-x64) + run: dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj -c Release -r linux-x64 --self-contained false -o publish/DarlingService-linux + + - name: Package linux artifact + checksum + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + mkdir -p releases + tar -C publish/DarlingService-linux -czf "releases/PerformanceMonitorDarling-linux-x64-${version}.tar.gz" . + (cd releases && sha256sum "PerformanceMonitorDarling-linux-x64-${version}.tar.gz" > SHA256SUMS-linux.txt && cat SHA256SUMS-linux.txt) + + - name: Upload linux artifact to the nightly release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload nightly releases/PerformanceMonitorDarling-linux-x64-*.tar.gz releases/SHA256SUMS-linux.txt --clobber + + - name: Build and push container image (ghcr, :nightly) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + version="${{ steps.version.outputs.VERSION }}" + image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" + echo "$GH_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + docker build -f Darling/Dockerfile -t "${image}:nightly" -t "${image}:${version}" . + docker push "${image}:nightly" + docker push "${image}:${version}" + + # Keyless Sigstore signing: Fulcio issues a short-lived certificate against this job's OIDC + # identity and the signature lands in ghcr next to the image, logged in Rekor. No keys exist + # anywhere to manage or leak — the signature attests "built by this repository's workflow", + # which is the claim a container consumer actually wants verified. (SignPath's cosign support + # is edition-gated; this path has no subscription dependency.) Verify: + # cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling:nightly \ + # --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' \ + # --certificate-oidc-issuer https://token.actions.githubusercontent.com + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Sign container image (keyless, Sigstore) + shell: bash + run: | + set -euo pipefail + image="ghcr.io/${{ github.repository_owner }}/performancemonitor-darling" + digest="$(docker inspect --format='{{index .RepoDigests 0}}' "${image}:nightly")" + cosign sign --yes "${digest}" + + # GitHub-native provenance for the tarball (SLSA): verified with + # gh attestation verify PerformanceMonitorDarling-linux-x64-.tar.gz -R erikdarlingdata/PerformanceMonitor + - name: Attest the linux tarball (GitHub provenance) + uses: actions/attest-build-provenance@v4 + with: + subject-path: releases/PerformanceMonitorDarling-linux-x64-*.tar.gz diff --git a/.gitignore b/.gitignore index 2d4662edc..5bbec5c39 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,9 @@ TestResults/ # holds a LIVE database password in cleartext. viewer-config/ darling.json + +# Local verification harnesses for the PostgreSQL work (Darling/tools/pg-harnesses). Throwaway net10.0 +# console apps that reference the real projects, standing in for test suites that cannot execute on macOS. +# Deliberately untracked for now: whether pincheck in particular belongs in the repo is Erik's call, and it +# should not join a PR by accident. +Darling/tools/pg-harnesses/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f950d1b93..b73c1e085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,143 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.5.0] - 2026-08-19 + +### Added +- **`cpu_attribution` on the top-CPU rankings: what fraction of the box the ranking explains** ([#2320]) - the last unshipped item from #2235's wishlist. `get_top_queries_by_cpu` and `get_top_procedures_by_cpu` (both SKUs) now return the returned rows' summed CPU-seconds, the SQL process's measured CPU-seconds for the same window (avg `cpu_utilization` % x core count x window - both stores already collect every piece), and `attributed_cpu_ratio`. Pre-#2290 the reads explained ~10% of the box and nothing said so - a caller chased the visible tenth assuming it was everything; and the ratio catches impossible claims at a glance (an external comparison died the moment its worker_time sum divided out to 137% of the box's available CPU-seconds - above the process's own measured consumption the note now says to distrust the numbers rather than presenting them). Below half, a note explains where unattributable CPU goes (evictions between snapshots, rows outside the top-N, zero-cost rows, non-query CPU). The degrade rule is explicit: missing CPU series, missing core count, or a series covering under 90% of the window omits the ratio rather than inventing one. One computation in `PerformanceMonitor.Common` (`CpuAttribution`), pinned by the same decision table in both test projects; the denominator read windows on `collection_time` with the same bounds as the rankings, so numerator and denominator share collection gaps. +- **`get_query_store_health`: the MCP read for the new collector, both SKUs** ([#2319]) - the promised follow-up to the `query_store_health` collector: one browsable tool (beside `get_database_scoped_config`, whose latest-snapshot shape it mirrors) returning per-database actual vs desired state with the mismatch pre-folded into `state_matches_desired`, `readonly_reason` both raw and decoded, storage used vs cap with `pct_of_cap`, cleanup mode/thresholds, and the runtime-stats interval length; also exposed as a `/api/read` web endpoint. The `readonly_reason` bit table now lives ONCE in `PerformanceMonitor.Common` (`QueryStoreReadonlyReason`) and both viewers' grids and both MCP servers decode through it - the labels were miswritten from memory once during #2319 review, so a single source is the fix. While counting the tools for the instructions doc, the census sentence turned out to have silently drifted (it said ninety tools while the server exposed one hundred); it is rewritten with accurate digit counts (101 total / 76 shared with Lite / 25 Darling-only) and a new cross-app pin test parses it against the scanned inventory so it can never drift again. +- **Per-database Query Store health: a new `query_store_health` collector, both SKUs, both stores** ([#2319]) - `database_config` knows exactly one bit (`is_query_store_on = true`), which cannot answer the questions an investigation like #2312 needed: is Query Store actually WORKING (the classic silent failure is desired_state READ_WRITE with actual_state READ_ONLY after the storage cap hit - `readonly_reason` says why), how close to the cap is it, and what interval grain is it aggregating at. The new collector reads `sys.database_query_store_options` per database - the same proven enumeration idiom as `database_scoped_config` (list accessible ONLINE primaries, then `[db].sys.sp_executesql` per database), deliberately NOT filtered to QS-on databases: the options view answers one row even when Query Store is off, so OFF is recorded as OFF and an absent row can only mean "not collected". Hourly rather than the config family's on-load cadence, because unlike operator-changed knobs these values change BY THEMSELVES and the cap-hit transition is the whole point of collecting them. Every column exists on 2016+, so there are no version gates. Surfaced as a Query Store sub-tab on the Configuration tab in both apps (V76 store table + `v_query_store_health` passthrough keep the two viewers' SQL byte-identical; Lite's table and archive view generate from the catalog); a `get_query_store_health` MCP read follows separately. The issue asked for the fields on `database_config` itself; they land as a sibling enumerating collector instead because `database_config` is a single `sys.databases` scan and these fields need per-database context - bolting an enumeration onto it would change its execution model and failure isolation, and the codebase already has the per-database config member in `database_scoped_config` to mirror. +- **V75 gives plan CONTENT its own retention horizon, because the fact-coupled one cannot bound a young store** ([#2316]) - the payload dimensions' GC deliberately follows the widest dim-feeding fact retention (90 days) so nothing a live fact references is ever deleted, and that guarantee has a blind spot measured on the dogfood fleet: `query_plan_dim` reached **127 GB - 63% of the store - in its first 22 days**, growing ~6 GB/day of parameter-sniffing recompile churn (344k distinct plan XMLs per day from 5,327 plan SHAPES - 65 variants per shape, the worst single shape producing 57,402 in one day), with the coupled GC unable to delete a single row until the horizon crossed the dimension's birth date - roughly a month AFTER the projected disk-full. Orphan pruning already existed and was healthy; compression was already spent (every row app-gzipped); the inflow is legitimate distinct content by the #1767 design, so the remaining lever is lifetime. The new `config_service.plan_content_retention_days` knob (default 21, clamps [7,365], 0 = disabled = the old behavior byte-for-byte) sets how long a stored plan XML outlives its last sighting: the dimension cutoff becomes the NEWER of the fact-coupled cutoff and `now - (knob + 1)` - the same one-day margin as the measured floor, for the same hourly `last_seen` refresh guard. Facts keep their full retention (metrics, hashes and text stay analyzable); a plan older than the window renders as the missing plan every reader already handles. The horizon governs the PLAN dimension only - query text keeps the fact-coupled cutoff (it is ~40 MB against the plan dim's 127 GB, and shortening it would break "text stays analyzable" for nothing) - and the Query Store plan map's prune learns the knob too, keeping the dimension-outlives-the-map ordering under every knob value so a live map row can never resolve to deleted content. A knob wider than the fact horizon is deliberately a no-op - it must not become a way to keep XML nothing can reference. Deliberately NOT done: shape-keyed latest-wins storage would shrink this 65x but breaks the historical-fact-to-exact-XML contract #1767 preserves on purpose - parameter-variant plans are the product's diagnostic bread and butter. + +- **The generic webhook can now hand automation the alert's structure: `{{context_json}}`, `{{incidents_json}}` and `{{dedup_key}}`** ([#2302]) - everything automation needs already existed structured inside the product, and every channel then flattened a different half: Teams/Slack keep incident structure but bury the scalars in display strings, the generic channel keeps discrete scalars but joins the whole context into one " | " line whose delimiters collide with Victim SQL, and PagerDuty keeps only the first incident's key. The reporting consumer measured the cost precisely: 31 of 49 Logic App actions existed only to undo the flattening, including two silent-failure guesses (deriving the server by splitting the summary on " on ", detecting incident sections by substring). The new tokens are raw JSON VALUES substituted unquoted - they deliberately bypass the per-token JSON escaping, via an explicit raw set that leaves the single-pass MatchEvaluator untouched - and their shape is EXACTLY the AlertContextSerializer projection persisted as alert-history ContextJson, so a consumer parses one shape whether it reads the webhook or the history row (pinned by a round-trip test through the same serializer). `{{dedup_key}}` carries the very key the PagerDuty channel derives - including the stable serverId+metric fallback that existed in code but was never exposed to any consumer, which had forced title-matching heuristics for level/threshold alerts - so tickets correlate across channels. The shipped default template is byte-identical (pinned), unknown tokens stay literal, and a template that quotes a raw token is caught by the existing well-formedness check as a config error. Both SKUs, since the whole channel lives in the shared Notifications project. +- **get_collection_health now carries a sweep_pressure verdict, so half-rate collection stops hiding behind 40 healthy collectors** ([#2296]) - two cross-region servers were collecting at half their configured cadence: their four heaviest collectors averaged ~60.7s of combined execution against a 60s sweep, so the serial collection body could never finish inside its interval, every relaunch was skipped (~50 service-log warnings/hour), and NOTHING else surfaced it - every collector reported HEALTHY, because from each one's own seat nothing was wrong. The tool now rolls the collectors' combined demand (average duration amortized by each collector's own cadence) against the minute the fastest cadence holds and serves busy_ms_per_minute / busy_percent / a verdict (OK, AT_RISK at 75%, SATURATED at 100%) plus the three heaviest contributors - attribution, because "which collectors spend the budget" is the actionable half of the answer. Deliberately built from the collectors' own execution times rather than delivered-gap statistics: at fleet scale the delivered cadence stretches benignly from bounded sweep concurrency (queueing), so gap-based detection would flag every server and drown the two that matter; execution demand is the arithmetic behind the watchdog's own "has not completed after Ns of EXECUTION" line and queueing cannot inflate it. The decision lives in the shared SweepPressureClassifier (PerformanceMonitor.Common) with the same decision table pinned in both suites, and both SKUs' tools serve the identical shape. Root-cause options for the two saturated servers (move them in-region, or lengthen their cadence) stay tracked on the issue - this change makes the condition visible either way. + +- **V74 stores query_store statement text out of the sorted stream, with its own watermarked fetch, prune and Viewer probe** ([#2150]) - the storage half of the seam added earlier. `collect.query_store_text` is keyed `(server_id, database_name, query_id)`, and **`query_id` was chosen precisely because it is already a stored fact column** - so this rung adds a table and alters nothing, readers get the join key for free, and no migration touches `query_store_stats`. Text is stored INLINE rather than as a digest into a content-addressed dimension: `QueryStorePlanMap` earns that machinery because plan XML is enormous and duplicated, whereas Query Store has already de-duplicated text one row per statement per database, so there is nothing to squeeze - and inline removes the dimension GC liveness interlock **whose failure mode is silently missing text**. **The upsert overwrites the text, not just the stamp**, which is load-bearing rather than defensive: `query_id` is unique within a database only until Query Store is RESET, which renumbers from the start, so id 5 afterwards is a different statement than id 5 before - the refresh horizon brings us back and this is where the corrected text lands. Touching only `last_seen` would leave the old statement's text on the new id forever, which reads as a plausible wrong answer rather than as missing data. Pruned on `last_seen` rather than by `drop_chunks` (a keyed store, not a time series), bounded to one chunk-width of the oldest rows per call, with the retention margin **added** to the fact horizon so text outlives the rows referencing it. **Shipped inert and turned ON later in this same release** (see the reader conversion under Changed): the flag stayed false while six reader surfaces still projected `query_text` straight off `query_store_stats`, because the flip and the conversion have to land together or those surfaces silently lose text for new rows - so the storage half got its own reviewable change first, and this rung is what the readers now resolve through. What does ship live is the fetch pass, its per-database watermark under its OWN state owner (the load merges both owners, and writing it under the plan fetch's owner would read back fine and then never be pruned, because the shared prune set pairs `textwm:` with `query_store_text`), and the Viewer's three-place probe edit - a probe column, a reader argument and a map parameter, verified in lockstep at 50/50/50 with contiguous ordinals, because a probe that cannot SEE the newest object maps every fully-migrated store below the required version and the Viewer refuses to open. + +- **The query_store collector can now resolve statement text through a separate watermarked fetch, so the text stops being sorted** ([#2150]) - the payload selects `query_sql_text` (`nvarchar(max)`) inside a `TOP ... WITH TIES ... ORDER BY last_execution_time`, and a Top-N Sort carries every output column through the sort while reading ALL of its input before emitting row one - so choosing the 50,000 rows to ship materialized the text for the entire qualifying set. **Measured with [#2210]'s plan XML already gone and that one column as the only difference**, on a purpose-built Azure SQL DB store: time-to-first-row **4.67s vs 0.45s** at 1,505 rows / 12.8 MB of text and **5.02s vs 0.57s** at 4,037 rows / 34 MB, full drain **8.06s vs 0.50s** and **16.95s vs 1.45s**. So #2210 did not finish this - it removed the larger column (195 KB average plan against 8.5 KB of text on that store) and left the one that still dominates. **Neither knob that looks like it should bound this can**, both measured: `TOP (500)` cost the same as `TOP (50000)` because the sort consumes its input either way, and wall time was flat across a 4 / 8 / 16 / 32 / 64 / 256 MB client budget sweep because the server finishes before the client sees a byte. **Shipped OFF, then enabled for the Darling sweep later in this release** (see the reader conversion under Changed): the flag defaults false, so on the release it landed in the emitted SQL was byte-identical to before - pinned by a test that normalizes the one column out of both forms and requires the remainder to match exactly, which covers "nothing else moved" for every column rather than the handful someone thought to list. **It is a flag rather than a deletion because Lite stores that text inline in DuckDB** and its grid reads it from there, so nulling the column unconditionally would blind Lite; gated, the placeholder keeps the column's ORDINAL, which is load-bearing because the readers index the row by number. **The fetch is watermarked rather than deduped per pass**, which is a decision this collector already made once: [#1556] shipped each plan once per PASS via `ROW_NUMBER` and [#2164] replaced it precisely because that form "ships each plan once per pass but re-ships it every pass forever, and since drain is 94-97% of a pass and is per-row LOB cost, NOT fetching is worth far more than fetching less." `query_id` is an identity, monotonic within a database, so a statement's text is fetched ONCE, ever - and keying on `query_id` rather than `query_text_id` means **no new fact-table column and no migration**, because `query_id` is already a stored payload column and therefore already a join key. **Deliberately simpler than the plan fetch**: no candidate-window estimator, because `SUM(DATALENGTH(query_plan))` forces the server to decompress every plan in the window (`sys.query_store_plan.query_plan` is decompressed BY the view on access) while `query_sql_text` is not, so a flat coarse bound plus the exact running-byte total is enough; and no content hash, because plan XML can be rewritten in place whereas a statement's text is fixed for the life of its id. The bounded refresh horizon is kept for the one hazard that does apply - `query_id` is monotonic in FIRST-SEEN order, not in "we have stored it", so a Query Store reset renumbers from the start and without a horizon would suppress every text forever. + +- **The darling.json reconciliation now tells "never registered" from "deliberately removed"** ([#2258]) - a server named in the file but absent from the registry had two indistinguishable causes: added to the file after the first seed and never registered (the [#2252] field report - the operator expects monitoring and is not getting it), or registered once and then removed via the Viewer, which is a CORRECT state. So the line reported at Information and worded itself for both, which was honest but could neither warn about the first nor stay calm about the second. It is now two lines: a WARNING naming the servers that were never monitored and what to do about it, and an Information line for the ones that were removed on purpose which advises nothing and notes their collected history is kept. **No tombstone table and no migration rung, which is the point.** The issue proposed a `config_removed_servers` table or an `is_removed` flag; the fact both would record already exists - `collect.servers`, the OBSERVED registry, gets a row on every successful connect, the Viewer's Remove deletes only from `config_monitored_servers` (the DESIRED config), and nothing purges the observed registry because it is a registry rather than a time series. So the distinction is answerable today, with no schema change and no second piece of state that could disagree with the first. The `is_removed` flag was worth rejecting explicitly: `is_enabled = FALSE` already means "registered but paused", so a second flag makes `(is_enabled, is_removed)` a four-state space with two meaningless combinations, and every existing reader of that table would have to learn the new flag or silently start including removed servers - the same seam failure [#2280] had to fix in the dedupe gate. **The limits are stated on the method rather than left to be discovered**: a server registered but never successfully connected to reads as never-monitored, which is the right answer to "is this being monitored?" even though it is the wrong answer to "was it ever registered?"; and a rebuilt store has no observed rows, so everything reads as never-monitored until it connects once - degrading to a warning rather than to silence, which is the safe direction for a fresh store where the file genuinely is the intent. + +- **The store's scale test now reports what the compression job DID, not only how long it took** ([#2266] item 1) - that test fails intermittently on diffs that cannot reach it, and the reading that reframed it was `d1=689ms, d10=689ms`: **byte-identical**. Two independent sub-second timings of different workloads do not land on the same millisecond by chance, so the earlier "runner jitter owns the constant factor" reading cannot be right - and that reading is why the assertion was left as bare monotonicity in the first place. The obvious alternative is already ruled out in the helper: `RunJobViaSchedulerAsync` waits for `last_successful_finish` to ADVANCE, so each measurement is of a genuinely new COMPLETED run rather than a stale row. What is left is that both runs did the same amount of work - plausibly close to none, with the duration dominated by fixed per-run overhead - which a duration alone cannot show. The failure message now carries chunk totals, compressed-chunk counts, `total_runs`, `last_run_status` and `last_successful_finish` for BOTH passes, and says outright that equal compressed-chunk counts mean the two runs did the same work and the assertion was never measuring the [#2136] capacity model - a fixture defect rather than a timing-tolerance one. **No threshold was changed**, which is the point: guessing a tolerance is how an intermittent test stops looking broken without becoming correct, and the next recurrence now diagnoses itself instead of costing another re-run. The describe helper is best-effort and cannot throw - an explanation that fails would replace the failure it exists to explain, which is the [#1902] mistake in miniature. + +- **PostgreSQL statement text is stored, so `get_pg_top_queries` returns something readable** ([#2219]) - `pg_statement_stats` identified queries by `queryid` and stored no text, because `aurora_stat_statements`' `showtext` is a real per-collection cost and normalized text is highly repetitive. But `queryid` is **not stable across a major version upgrade**, so afterwards the stored history joined to nothing readable: a list of integers that used to be your slowest queries, and unrecoverable - the live view no longer holds the old ids, and anything else on the instance may have reset `pg_stat_statements` out from under us (pganalyze's collector calls `pg_stat_statements_reset()` on a size budget). V73 adds `collect.pg_statement_text`, one row per `(server_id, queryid)`, refreshed hourly and joined into the read. **Text is INLINE rather than a `query_text_dim` digest, which reverses what V64's comment promised, deliberately.** The dimension route is blocked and expensive to unblock - V38 is GENERATED from `PayloadDimensions.All`, so registering the fact table makes V38 `ALTER` a table it has not created yet on every upgraded store, and it would break V64's own ladder diff - but the deciding reason is that the dimension needs the GC liveness interlock `QueryStorePlanMap` documents at length, whose failure mode is **silently missing text**. Inline cannot dangle. The cost is cross-server dedup: one row per server per queryid rather than one per distinct text, a few hundred MB on the measured 52-server fleet against a store whose Query Store plan XML alone was 43 GB. Paying that to make a silent-loss mode impossible is the trade. **Idempotent by construction**, which is what makes the cadence a free choice: every refresh upserts the same rows, so re-fetching costs one statement and no growth - there is no "which queryids do I have" bookkeeping to get wrong and no watermark to corrupt. Due-ness is asked of the STORE rather than remembered in the service, so a restart cannot re-fetch the fleet and two hosts cannot disagree about when text was last written. `first_seen` is preserved on conflict because it is the one fact that survives a major-version re-key and cannot be reconstructed; `query_text` advances. Hung off the statement-stats collector's success rather than given its own loop - it is meaningless without those rows and must never run against a server whose stats collection is failing - and best-effort, because unreadable text is a degraded read while a failed collection is lost data. The prune margin makes text **outlive** the statistics referencing it, the opposite direction from the plan map's, because the asymmetry is the other way round: text kept past its facts is dead bytes, facts kept past their text is the list-of-integers failure this table exists to fix. Not a hypertable and not a collector table, exactly like V72's plan map - a bespoke upsert path, pruned on `last_seen`. `query_text` is **null**, never an empty string, when nothing has been captured for a queryid yet, so a caller cannot read a blank as the query. + +- **`add_servers` refuses a registration whose connection lands in a database already monitored** ([#2280], the registration-time half of [#2228]) - identity is registration-derived, so N registrations that silently resolve to one database get N identities and N full copies of every collected row: [#2220]'s byte-identical deadlock graphs under six `server_id`s, one incident alerting six times. [#2277] added the connect-time tripwire that REPORTS it; this refuses it where it can be prevented instead of described. The probe already runs in-process at Add, and since [#2277] it returns the database the connection actually reached, so the answer is in hand at exactly the moment the decision has to be made - no comparison of configuration could substitute, because the two colliding registrations genuinely differ, which is why [#2158] and [#2218] could not touch this. Keyed on the FULL identity, which is what keeps two legitimate pairs working: a read-only-intent registration alongside its read-write twin for one database, and a PostgreSQL instance alongside a SQL Server on one host. Silent when the probe reports no database (unknown is not colliding - refusing there would block a registration for a reason nobody could act on) and when the entry reaches the database it names (the ordinary case, already ruled on by the declared-duplicate gate). **It also fixes a regression [#2218] introduced in this same method**: the duplicate gate built its key from host, database and read-only intent only, so once engine and port joined the identity the gate was keying on a NARROWER identity than the store derives - and a PostgreSQL instance on a host that already had a SQL Server registration read as a duplicate and was refused. A valid pair rejected because the gate could not see what distinguished them; it compiled and every existing test passed, which is why it now has an explicit pin. The honest limit, since it bounds what this can promise: existing rows record only the database they DECLARE, so this catches "the new one lands where an existing one lives" and not "both mis-resolve to a database neither names" - [#2277]'s tripwire reports the latter at connect for both. + +- **Add Server warns when a SQL-auth password may not be decryptable by the service** ([#2279], the policy half of [#2255]) - a password is stored as a DPAPI `LocalMachine` blob, decryptable only on the machine that wrote it, and the SERVICE is what has to decrypt it. So a credential saved from a viewer on another PC can never be used and the server fails to connect on every sweep afterwards - the [#2255] report. [#2273] made that failure explain itself; this says so BEFORE the save, in the same place and the same way the dialog already warns about an Azure/Entra mode the service cannot honour, so it lands as the mode is picked rather than after the fact. **Warned, not refused**, and that is the decision rather than a hedge: the signal is whether the viewer's store is reached over LOOPBACK, which is a proxy for "the service runs here" derived from the product's own architecture - the managed deploy builds its store connection on literal `127.0.0.1` (`ViewerSettings` mirroring the service's `DarlingManagedPostgres.BuildConnectionString`) and the service runs where its managed store runs. A non-loopback store does NOT prove the viewer is remote, though: a bring-your-own store on another host with the service local reads identically. That is good enough to decide whether to SAY something and not good enough to decide whether to BLOCK, and inverting it would refuse a legitimate first-run Add on the service host. **Silent for a loopback store**, which is the managed single-box deploy the DPAPI design targets and the overwhelmingly common case - a hint that fires for everyone is a hint nobody reads, so a false positive there would defeat the feature rather than merely annoy. Also silent for an omitted host (Npgsql defaults to localhost, so no `Host` means a local store), for an absent connection string, and for an unparseable one - failing toward silence, since the wrong direction is a warning the operator cannot act on. A host that merely CONTAINS a loopback spelling still warns. The hint names all three ways to produce a usable credential (a viewer on the service's host, `--add-server` there, or an `env:`/`file:` reference, which is not machine-bound at all) and clears itself when the auth mode changes away, the same self-clearing discipline the Azure message uses. + +- **A skipped database-state maintenance cycle now says so, once** ([#2266] follow-on) - `GetDatabaseStateDeviationsAsync` performs its baseline seed, the [#2189] heal, the [#2203] forget and the prune inside a best-effort block: it opens the write connection with a 5-second lock acquisition and, on `TimeoutException`, skips all of it while still running the deviation read. Skipping is the right behaviour - the method's own comment makes the case that it is the only lossless option when archival holds the lock - but it logged NOTHING, so a sustained window of write-lock contention meant baselines quietly stopped being seeded and healed with no evidence anywhere. That matters more than a typical missing log line: [#2189] exists *because* an unhealed baseline inverts the alert permanently, so the failure this maintenance prevents is itself invisible and its absence has to be visible instead. Reported on the TRANSITION - one line when it starts skipping, one when it resumes - rather than per sweep, because a standing contention window would otherwise emit a line every cycle and get filtered, which restores the silence. At **Warn**, not Error: one skipped cycle is the expected benign outcome of colliding with archival and the next sweep re-runs everything, so flagging it as a fault is the fastest way to get the line ignored. The message names the CONSEQUENCE rather than the event ("baselines are not being seeded or healed while this persists"), says the deviations were still read so nobody wonders whether the numbers are stale, and says one occurrence is expected so nobody escalates it. Keyed per server, because the lock is process-wide but the consequence is not. A regression test pins the one property that is a stray `return` away at all times: a skipped maintenance block must still run the deviation read - swallowing it would report every database as recovered and clear the alert memory for all of them. + +- **`server_id` identity now carries engine and port, without re-keying anything that exists** ([#2218]) - the storage name was derived from host, database and read-only intent only, so a SQL Server and a PostgreSQL instance on ONE host collided into a single identity and interleaved their histories, as did two PostgreSQL instances distinguished only by port - both of which [#2213] made first-class configuration. Engine and port are now discriminators. **The interesting constraint is what could NOT change.** Lite derives `server_id` FRESH at runtime from the shared `ServerIdHelper.BuildStorageName`, everywhere, and has no stored-id fallback the way Darling does - so altering what that function returns for an EXISTING server would re-key it in Lite and orphan all of its collected history, silently: the same class of harm as [#2158], arrived at from the other direction. So the new parameters are OPTIONAL and append nothing at their defaults, which keeps Lite's three-argument call byte-identical - verified against a re-statement of the pre-change rule rather than against hand-written strings, so it holds for any input and not just the cases someone thought to list. A SQL Server entry that DOES pass them is unchanged too, which is what lets Darling pass `Engine` and `Port` unconditionally instead of branching: `Engine` folds to no token for SQL Server, and `Port` is a PostgreSQL-only field that stays 0 there because SQL Server carries a non-default port inside the host as `host,1433` and is therefore already discriminated by the host string. Darling's already-registered PostgreSQL targets do not re-key either, for a different reason: their id comes from the store and is only ever derived for an entry with no row yet - which is why [#2158] (identity assigned, never re-derived) was a prerequisite for this rather than a sibling of it. Every spelling of the engine folds to one token (`postgres`, `PostgreSQL`, `pg`, any casing), so a colleague's capitalisation cannot mint a second identity for one instance - a split history nothing downstream could diagnose - and an unrecognised engine appends nothing rather than being interpolated raw, so a typo cannot mint one either. Suffix order is fixed (engine, port, then `:RO`) so two callers supplying the same facts cannot produce two names. + +- **A registration connected to a database it does not name now says so** ([#2228]) - identity is registration-derived and was never checked against the connection, so a registration whose Initial Catalog is absent, misspelled or overridden lands in a different database and every collected row is stored under that registration's identity while describing somewhere else - indefinitely, with nothing anywhere saying so. When a sibling registration names that same database, both collect it and its history exists twice under two identities: [#2220]'s report of byte-identical deadlock graphs under six `server_id`s, one real incident alerting six times. Both engine probes now also return the database the connection ACTUALLY reached - `DB_NAME()` on SQL Server, `current_database()` on PostgreSQL - and the worker compares it against the registration on every connect. **Why this has to happen at connect and not in the registry**: [#2158] established that identity is assigned rather than derived, which fixes the re-key class but cannot touch this one, because the two registrations here genuinely DIFFER in configuration - no amount of care in hashing config can tell that they resolve to one database. Only the server can answer what a connection reached. The message names what is at stake (mis-attributed rows, and duplication when a sibling names the same database) and both places the setting lives, because the log line is the whole diagnosis. Logged at ERROR on the TRANSITION rather than per connect: a mismatch is a standing misconfiguration that persists until someone edits the registration, so repeating it every reconnect would bury the one line that matters, which is how a tripwire gets trained past and stops working; the recovery is logged too, so an operator who fixes it has confirmation rather than silence. **Silent in three cases on purpose**, each a false positive that would have made the feature useless: a registration naming NO database is server-scoped by design and meant to land wherever the login defaults, so it would otherwise fire on every correctly-configured server-scoped registration in the fleet; a null probe answer is unknown rather than mismatched; and comparison is case-insensitive, which is what SQL Server database names are. `DB_NAME()` needs no DMV, so it does not reintroduce the VIEW DATABASE STATE dependency [#1535] removed, and both columns are APPENDED because every read in those probes is positional. The registration-time half of the issue - refusing a new registration whose (host, actual database) pair matches an existing one - is not in this: it needs a connection during Add, which is a different surface. + +- **Editing a server's address keeps its identity, so its collected history stays attached** ([#2158]) - the Add/Edit save re-derived `server_id` from host/database/read-only-intent on EVERY save, so fixing a hostname typo wrote a row under a new id and deleted the old one. That left the registry tidy and every `collect.*` row keyed to the old id orphaned with nothing pointing at it, which is why it went unnoticed: nothing looks broken, the server simply reads as though it had never been monitored. An edit now keeps the row's identity and rewrites the address in place; derivation runs only on Add, where there is no history to lose. **Why assigned rather than derived, argued from consequences rather than preference**: three issues pull on this identity and they pull in opposite directions. This one says a config edit must not change it. [#2228] says two different configs that resolve to the same real database must not be two identities - which no config-derived hash can decide, because the configs genuinely differ. [#2218] says two instances on one host need distinguishing, which wants MORE fields in the derivation and so makes this bug strictly worse. Only one shape satisfies all three: identity is allocated once and never recomputed, the derived address is a lookup key rather than the identity, and what the target actually IS comes from the connection. The groundwork was already in - `StoredServerId` made the store authoritative and `DarlingServerConnector` reads `config.ServerId` rather than re-hashing - so the Viewer's save was the last place identity was recomputed. **Two things the fix had to carry with it.** The collision guard used to look a derived id up in the registry, which only works while every row's id equals the hash of its own address; left alone, an edit could point a second registration at an address another server already monitors and the guard would never fire, because the id it looked up belongs to nobody. It now matches on the ADDRESS columns (`IS NOT DISTINCT FROM` for the nullable database, since `=` never matches NULL and every server-scoped registration would otherwise read as "address free") and compares ids afterwards, which is what still lets a rename or a credential change through. And the darling.json reconcile matched on id alone, so after an edit it would report a server that IS monitored as absent and advise re-adding it - wrong advice, on every start, about the one server the operator had just fixed; it now matches id OR name, either-or rather than name-only so a same-named sibling cannot hide a genuinely unmonitored server. + +- **The database-state heal tests no longer assume a best-effort maintenance cycle always runs** ([#2266] item 2, root-caused from source) - `RebaselinedByHandDuringAnOutage_HealsOnceTheDatabaseRecovers` failed on a PR whose diff could not reach it and passed on a re-run of the same commit, reporting `ExpectedState = SUSPECT` alongside `StateDesc = ONLINE`. That pair is only reachable one way: `GetDatabaseStateDeviationsAsync` performs its seeding, the [#2189] heal, the [#2203] forget and the prune inside a block that opens the write connection with a **5-second** lock acquisition and, on `TimeoutException`, skips the entire maintenance block while still running the deviation read - which is deliberate and documented, because skipping is the only lossless option when archival holds the lock. That write lock is **static, shared by the whole process**, and xunit runs test classes in parallel, so another class can hold it long enough for a cycle to skip its maintenance. The test was therefore asserting that the heal lands in ONE cycle, which the design does not promise; it now sweeps until the expectation settles, bounded, which is the actual contract. It cannot mask a regression: a genuinely broken heal never settles, every cycle runs, and the caller's own assertion fails on the final result with its own message exactly as before. Ruled out first: the fixture mints a unique temp directory per instance so no two classes share a store file, and the server id is used by no other class - the shared resource is the LOCK, not the data. **Not fixed here, and worth knowing**: that `catch (TimeoutException)` logs nothing, so in production a sustained contention window means baselines quietly stop being seeded and healed with no evidence anywhere, which matters because [#2189] exists precisely because an unhealed baseline inverts the alert permanently. Item 1 of that issue (the sub-second scale-test comparison) is deliberately untouched: every candidate fix needs the jitter distribution, and guessing a threshold is how an intermittent test stops looking broken without becoming correct. + +- **`get_top_queries_by_cpu` can rank a procedure's dynamic SQL as ONE statement: `group_by: "host_object"`** ([#2235]) - `query_hash` is a SHAPE hash, so dynamic SQL built with per-value literals fragments one logical statement across as many hashes as there are literal sets. Measured on `prod-pos-use2-apex-01`: **21 hashes** for a single `API.GetInventoryWithLabsV5` `insert #result` statement, whose fragments were **58-65% of the instance's worker_time** in every window sampled - while the hash never entered the 168-hour top 20, and the per-query ranking as a whole accounted for roughly a **tenth** of the box's CPU. A top-N-by-hash list cannot surface that no matter how large N is, and nothing in the output said so. Setting `group_by: "host_object"` collapses every statement of a hosting procedure or function into one row, which is what makes the real consumer rank first; `distinct_query_hashes` reports how many hashes the row rolled up (21, in the reported case) and is the number that explains why the default ranking missed it, with a `rollup_note` saying so in words. `query_hash` and `query_text` in a rolled-up row are one representative fragment, exactly as `query_text` already is when `distinct_texts > 1`. **Ad-hoc statements keep their per-hash grouping in BOTH modes, and that is the load-bearing part**: ad-hoc rows carry `host_object_name = NULL`, so a bare `GROUP BY host_object_name` would pool every unrelated ad-hoc statement in a database into one meaningless row - a worse attribution bug than the one being fixed - and the representative-text lookup would start serving an unrelated statement's text. The grouping key keys those rows on their own `query_hash` instead, identical to the default read. The per-hash grouping ([#2012] stage 2) stays the DEFAULT and is unchanged: two procedures sharing a hash genuinely are different work, which is why that split exists, so this is an additional lens rather than a replacement. Implemented as a sibling SQL const rather than a built clause because Postgres cannot parameterize `GROUP BY` and every read here is a public const so the suite can pin its dialect without a live store; an unrecognised `group_by` is rejected rather than silently falling back, since a caller who asked for a rollup and got a per-hash ranking would read it as "this procedure is not hot", which is the exact wrong conclusion. `group_by` is trailing and optional, so Lite-shaped calls are unaffected, and it is deliberately absent from `get_top_procedures_by_cpu` (already keyed on the object) and `get_query_store_top` (keys on `query_id`, which does not fragment). Pinned by a live-Postgres test that asserts the collapse, the ad-hoc non-pooling, per-row text correctness, and that total CPU and executions are CONSERVED across both groupings - a rollup must redistribute attribution, never invent or lose it. + +- **The Query Store tick and the Query Store backfill no longer run against one server at the same time** ([#2165]) - the per-tick `query_store` collection and the [#2058] first-contact backfill were independent loops with no per-server coordination at all, and both do heavy Query Store text extraction. Dogfood evidence from a 4-core multi-tenant box mid-consolidation: a 64 MB backfill slice for a freshly restored database ran concurrently with the tick's collection of a SIBLING database - a 12:50:58 backfill ship overlapping a 12:51:09 tick completion - so roughly 128 MB of extraction was in flight at once on the box least able to afford it. That overlap is not bad luck: a big catalog arriving is exactly what triggers BOTH the backfill and budget-bound tick passes, so the two loops collide precisely when the server is already drowning. A per-server gate now excludes them, in both apps - Darling's `DarlingWorker` keyed by server id, Lite's `RemoteCollectorService` keyed by server - sharing one `QueryStoreServerGate` primitive beside `AbandonableStep` (that one bounds how long a step may hold a loop; this one bounds what may run beside it). **Nothing ever waits, deliberately.** Both sides try-acquire with a zero timeout and SKIP on failure, because these are shared fleet loops: an in-flight slice runs to a 180-300 second abandonment deadline, so a blocking acquire would let one slow server stall collection for the entire fleet - the [#2148] wedge arriving through a lock instead of a hang. **Skipping is safe for this collector specifically** because its window is a watermark ([#1960]): the next pass resumes from the same boundary, so a skipped pass defers rows rather than dropping them - which is also why the gate must not be reused for a collector whose window is wall-clock derived. The "tick wins, backfill defers" bias is realized by CADENCE rather than preemption (stopping a statement already running on the monitored server would mean killing it): the tick retries on its own ~1-minute interval against the backfill's 5, so it recovers five times faster from a collision, and a slice is byte-budgeted so it is short in the healthy case. The backfill takes the gate OUTSIDE its `AbandonableStep`, so an abandoned-but-still-wedged slice keeps the gate closed - the statement is genuinely still running on the server and the tick must keep yielding to it. Built on an interlocked flag rather than a `SemaphoreSlim`: a gate that never waits needs none of what a semaphore provides, and would otherwise own one undisposed kernel object per monitored server forever. Leases are idempotent on dispose, because a stray second `Dispose()` would otherwise clear a flag the OTHER loop had since taken and let both run at once - the exact condition being prevented, reached from the wrong direction. + +- **A credential the service cannot decrypt now says why, once, instead of `Key not valid for use in specified state` every 60 seconds forever** ([#2255], second fault from the [#2252] report) - that string is `ProtectedData.Unprotect`'s own, surfaced verbatim into the connect-retry warning. It does not say DPAPI, does not name what failed to decrypt, does not mention that a MACHINE boundary is involved, and reads exactly like SQL Server rejecting a login - which is where the operator looked. **What is actually true**, confirmed from source: both the service's `DarlingSecrets` and the Viewer's `ViewerServerSecret` protect with `DataProtectionScope.LocalMachine` and share the entropy string byte-for-byte (pinned by a round-trip test). LocalMachine means ANY user on the writing machine can decrypt and NO other machine ever can - so this is never a service-account permissions problem and never a user boundary, it is purely a machine boundary. Which settles the open question on the issue: a Viewer on a remote PC cannot produce a credential this service can use, by construction, and the Viewer's own source already documents that single-box limitation. The failure now names the credential and the server, states that this is Windows Data Protection on THIS host and that no credential was ever sent to the server, explains the machine binding, names the remote Viewer as the usual cause, and lists the remedies - all of which run on the service host: re-add from a local Viewer, `--add-server` here, `--encrypt-password` here, or an `env:`/`file:` reference, which is the one option that is not machine-bound at all. The original `CryptographicException` is kept as the inner exception, so nothing is lost for a bug report. **And it stops flooding the log**: a DPAPI failure is permanent by construction, so the full explanation logs at ERROR once per distinct cause and repeats as a single terse line while it persists, with the latch cleared on a successful connect so a cause that is fixed and later recurs is not swallowed as a repeat. Still open on the issue and deliberately not done here: the Add Server dialog refusing at WRITE time when it is not running on the service host - a guard needs a reliable "am I on the service host?" test, which a store connection alone does not establish. + +- **The incident readers take an alert's Dedup Key: `get_deadlocks` / `get_deadlock_detail` / `get_blocking` accept `dedup_key`** ([#2159], asked for by @gotqn) - the [#1140] fingerprint already travels end to end and the reporter sets it on their Azure DevOps tickets, but no reader accepted it, so triaging from an alert meant pulling a server+time window and eyeballing rows for the one whose involved objects matched the alert. Slow, and easy to analyze the WRONG deadlock. Paste the key instead and get exactly that incident's graph, queries and stats. The three tools also RETURN a `dedup_key` on every row, so an incident found by browsing correlates back to its alerts and can be handed to another agent as a stable identifier - the [#2138] "agents will read these" direction. **Recomputed on read rather than stored**: the key is a pure function of data the rows already carry, so this covers all retained history immediately, where a new column would have back-filled nothing and left exactly the history an operator triages unsearchable. **It calls the same groupers the alert path calls**, which is the whole design rather than an implementation note - the fingerprint is a SHA-256 over normalized identity members, so any divergence in deriving them yields a different hash and matches NOTHING, and an empty result is indistinguishable from "that incident is outside the window". So the reader builds the same `DeadlockEvent` / `BlockedEvent` the alert builders build and hands them to `DeadlockIncidentGrouper` / `BlockingIncidentGrouper`; pinned by parity tests against the real `AlertContextBuilders` entry points, including the blocking branch that falls back from a contentious object to a normalized query pair. **The trap that made this more than a parameter**: `AlertFingerprint` hashes the SERVER NAME into the key and the alert path passes the DISPLAY name (`Name`, else `Host`), while the MCP resolver returns the STORAGE name (`host[:database][:RO]`) - different strings for any server with a custom display name, and for every registration naming a database or read-only intent. Fingerprinting with the resolved name would have worked on plain hosts and silently returned nothing on exactly the servers most likely to be carefully named, so the readers resolve the display name explicitly. The filter runs over the whole window BEFORE `limit` is applied, because capping first would let `limit` discard the very incident the key names. A no-match answer reports how many rows it examined and names the three causes that look identical from silence - wrong window, wrong server, or a server renamed since the alert fired, which re-keys every fingerprint it has ever produced. `dedup_key` is trailing and optional on all three, so a client written against Lite's parameter contract still calls them correctly; the cross-app pin now asserts Lite's list as a PREFIX, which is the guarantee that was always the point. + +- **An `initdb` loader failure now names WHICH binary could not load** ([#2185]) - [#2186] taught the bootstrap to decode `initdb failed (exit code -1073741515)` into `0xC0000135 STATUS_DLL_NOT_FOUND` and to explain the two causes specific to how this product ships PostgreSQL: the MSVC runtime is bundled beside the binaries, so a missing one means a partial extract rather than an absent prerequisite, and the service runs as a virtual account that a user-profile install tree does not grant. That decode is right and it was not enough. The reporter worked both checks - moved the install to `C:\PerformanceMonitorDarling`, confirmed all three bundled DLLs present - and then ran `initdb --version` by hand, which printed `initdb (PostgreSQL) 18.4`. A binary that dies in the loader cannot print its own version, so that one observation killed both suggested causes, and it existed nowhere but the operator's shell: four exchanges in, the message the product emitted was still describing causes the operator had already eliminated. So the product now makes that observation itself. On a **loader** status only, the bootstrap probes `initdb.exe --version` and `postgres.exe --version` - the one invocation that loads a binary and its whole dependency chain while touching neither the data directory, the port, nor the cluster - and appends which of them could not load. Four verdicts, because each implies a different fix: only `postgres.exe` dead is the reported shape and the interesting one, since a real `initdb` run spawns `postgres.exe` in bootstrap mode to build the template database and passes its status back, so a dependency missing for **`postgres.exe` alone** fails the bootstrap while leaving `initdb --version` working - and that verdict says outright that the bundle is the suspect, not the install location or the service account, rather than letting the generic paragraph above it stand after the probe has disproved it. Both dead is the shared runtime, not one binary. Only `initdb.exe` dead is a single damaged file, re-extract. Both **alive** is a finding rather than a shrug: it rules out a permanently missing dependency and sends the operator to the one source that names the offending module, Event Viewer > Windows Logs > Application. Gated on a loader status because ungated it would launch two processes on every ordinary `initdb` failure - non-empty data directory, bad locale, permissions - and append a paragraph about DLL loading to an error that has nothing to do with loading, which is worse than silence for pointing somewhere wrong. The probe cannot make things worse: it runs on a path that has already failed fatally, and every fault mode of its own - missing file, cancellation, its five-second timeout - resolves to the empty string, composing to exactly the message shipped before it existed. **This does not diagnose the open report**; it means the next occurrence arrives already carrying the fact that took this one four exchanges to surface. + +- **A headless host can register a monitored server: `--add-server`** ([#2256], from the report in [#2252]) - the store is authoritative after the first seed, so `darling.json` edits are ignored ([#2254]); the web dashboard deliberately keeps the registry-writing tools off its read-only surface; and there was no CLI verb. A service on Windows Server 2012 - which cannot run the Viewer at all - therefore had **no supported way to add a server**, leaving a GUI on another machine or standing up an MCP client as the only routes. `--add-server` (also `--add-servers`) reads a JSON array of servers from **stdin** in the same shape the `add_servers` MCP tool takes, and goes through that same code path: validation, dedupe, the in-process connection probe, password encryption and the `server_id` identity computation are shared rather than reimplemented - and the identity in particular is a hash of the storage name, which is exactly the part an operator cannot safely produce by hand (get it wrong and the collectors write history under one identity while the registry points at another, the split-history class in [#2158]). Stdin rather than an argument because a password in argv is visible in the process list and in shell history, the same reason `--encrypt-password` reads it that way; empty stdin prints the JSON shape and two copy-paste pipelines to STDOUT rather than hanging, per the [#2097] lesson that STDERR is invisible in the ISE and some integrated terminals. It reports one line per server (`ADDED` / `SKIP` / `FAIL` / `INVALID`) with the probe detail, and says that no restart is needed - the registry write bumps the `config_version` beacon the worker polls every sweep. Exit 0 requires that something landed and nothing failed, so it is usable as a deployment gate: a re-run of the same file is idempotent and exits 0 on pure duplicates, while a batch where nothing landed exits 1 rather than reporting success for having changed nothing. Windows is required only for a MANAGED store credential (DPAPI), which the verb checks itself - a Linux host with bring-your-own Postgres can use it. + +- **Darling captures PostgreSQL blocking chains** ([#2213] follow-on) - an eighth PostgreSQL collector, `pg_blocking`, storing who is blocked by whom as an **edge list** (one row per blocked/blocking pair) rather than a rendered tree, so root blocker, chain depth and fan-out are ordinary SQL over the stored rows instead of string parsing. Both sides of every edge carry their own state, because the remedy depends on it: an `idle in transaction` root is an application defect while an `active` root is a query-tuning problem, and the pid alone does not distinguish them - capturing only pids is the most common gap in homegrown PostgreSQL blocking monitoring, since you get a number, go looking, and by then it is gone. `get_pg_blocking` assembles the chains, names the root, reports how many separate captures that same backend has been the root of (keyed on a synthetic `backend_start`+pid identity, because a pid is reused and a 30-day history keyed on it silently merges two different backends), and returns a specific remedy per root state. Runs on any PostgreSQL target **including standbys**, where recovery conflicts are blocking that happens nowhere else. **It is a sample, not an event log, and the tool says so in every response**: SQL Server's blocked-process report is written by the engine when blocking crosses a threshold, whereas PostgreSQL records nothing unless something asks - so blocking shorter than the one-minute interval is never seen, and the response carries its own capture counts so an empty answer cannot be misread as an all-clear. `pg_blocking_pids()` takes ShareLock on the lock manager partitions per call, so it is evaluated only for backends already waiting on a lock; on a 5,000-connection instance the ungated form is the monitoring query that becomes the incident. Lock **cycles** are reported separately and had to be: the chain read finds a root by absence, and in a cycle every participant is blocked, so a sampled deadlock would otherwise have been stored and then silently dropped by the read. Store rung V71, `collect.pg_blocking_edges`, 1 min / 30 d. +- **Every PostgreSQL migration rung is now pinned identical to the generated schema** - `EveryPostgresRung_IsIdenticalToTheGeneratedSchema` diffs all eight rungs against `PgSchemaGenerator`. A fresh store's tables come from V1's generated schema and an existing store's come from the rungs, and nothing forced the two texts to agree: the suite checked that the generator emits every table and that each rung is well-formed, but never that they said the same thing. One drifted column type would have been a permanent, invisible divergence between the two populations of store. +- **The viewer's store-schema probe is pinned to its reader's arity** - `StoreSchemaProbe_ColumnCount_MatchesTheMapArity`. The probe SQL's columns are read positionally, so adding a sentinel without a parameter made a fully-migrated store report one rung short, and adding a parameter without a column threw `IndexOutOfRange` at connect time. Both were runtime-only against a live store, and the existing arity test builds its arguments from the signature so it agreed with itself either way. +- **Darling monitors PostgreSQL and Amazon Aurora PostgreSQL** ([#2213]) - a monitored server can now declare `"engine": "postgres"` and is collected by seven PostgreSQL collectors instead of the T-SQL ones, into the same store, on the same naive-UTC contract and the same `server_id` identity. A mixed fleet is one store, one viewer, one MCP endpoint; nothing is partitioned by engine. Every collector definition declares the engine it targets and is never dispatched against the other one, so a PostgreSQL target is never sent T-SQL and a SQL Server target never sees `pg_stat_statements`. What gets collected: cumulative wait events and per-query-shape execution statistics (both from Aurora's own functions, because core PostgreSQL has no cumulative wait counters in any version), `pg_stat_io` attributed to a (backend type, object, context) triple rather than to a file, per-table autovacuum state stored beside each table's OWN computed trigger threshold, and three outage predictors that have no SQL Server counterpart at all - transaction-ID and MultiXact freeze headroom, what is holding the vacuum horizon back attributed to the specific holder, and replication-slot retained WAL with whether it is still growing. Those three ALERT, graded against the target's own settings rather than a constant, because each names a condition that stops the server outright and each is silent until it is nearly too late. Permissions are one `GRANT pg_monitor` and nothing is created on the monitored instance - no Extended Events sessions to provision, no server setting to bootstrap. `--test-connection` reports what a PostgreSQL target actually is (version, writer or reader, Aurora or not) and how many of the seven collectors will really run against it, which is the difference between "this is configured" and "this will collect": an Aurora writer clears all seven, a reader clears six, a self-hosted PostgreSQL 15 reader clears three. There is a step-by-step runbook with a proof point at every step in `docs/postgres-first-target-runbook.md`. + +- **The store can keep plan XML readable over plain SQL: `plan_xml_compression` ([#2171], asked for by @argpna)** - V54 moved plan XML into gzip bytes (`query_plan_dim.query_plan_gz`), which every app surface and MCP tool decompresses client-side - and which nothing reading the store DIRECTLY over SQL can decompress at all, because PostgreSQL exposes no inflate: the field workaround was `plpython3u` plus a hand-rolled gunzip UDF, an untrusted-language extension in a monitoring store just to read your own data. The store now carries a service setting, `plan_xml_compression`, default `gzip` (today's behavior, unchanged): set it to `none` and the dimension writer stores plans as PLAIN TEXT in `query_plan_xml` instead - lz4 TOAST does the compressing (~8.9x measured, vs gzip's 14.0x), and Grafana-class consumers read the column bare, no extension, no UDF, no one-shot container. Flipping it affects NEW rows only, in either direction: the readers' text-first-else-gz resolution already covers every mix of eras and modes, the dimension stays content-addressed so dedup is codec-independent, and nothing rewrites existing rows. The setting rides `config_service` (V62) like its sibling knobs, hot-reloads within one poll, and anything that is not exactly `none` reads as `gzip` - a hand-edited row fails toward the shipped default, and a CHECK constraint enforces the same set DB-side. `--recompress-plan-dim` now refuses to run against a store set to `none`, naming why: it would convert exactly the rows the live writer keeps producing as text, the two fighting forever - set `gzip` back first if that is what you want. The default stays `gzip` because the cost is real at fleet scale (the measured 52-server plan dimension would grow roughly 100 GB to 160 GB); a store whose plans are read by people rather than only by the apps is exactly who should flip it. + +- **Alerts carry a monotonic occurrence total per incident, not just a rolling-window count** ([#2216], reported by @gotqn) - the `Occurrences` fact on a grouped incident counts events inside the collectors' one-hour read window, so it rises as events arrive and falls as they age out. A consumer that only sees throttled deliveries - one per #1154 per-fingerprint cooldown - cannot recover the true total from a series of those readings, because a 3 followed by a 3 is indistinguishable from nothing-happened and three-happened-while-three-aged-out. Blocking and deadlock incidents now also carry **Total Occurrences**, accumulated per dedup fingerprint over the life of the incident, plus **Incident Since** so a total that restarts can be told from one that continues. Both ride every surface that iterates the alert details (Teams facts, Slack fields, email, the in-app dialog) and persist in the history row's context JSON. The existing `Occurrences` fact is deliberately unchanged: downstream automation keys on fact NAMES, so redefining that one from a gauge to a total would have broken every current consumer silently - the two answer different questions and both are emitted. Keyed per FINGERPRINT rather than per (server, metric), because a deadlock over one object set and a deadlock over another are separate incidents whose totals must not pool. Persisted in both stores (Darling V61 `config.incident_occurrences`, Lite v54 `config_incident_occurrences`) rather than as columns on the existing watermark row, for two independent reasons: the key is wrong, and Lite writes that row with `INSERT OR REPLACE` over a partial column list, which would reset a counter living there to zero on every fired alert - precisely when it is read. **The honest bound**: this is a lower bound on occurrences, exact whenever the read window outlives the gap between OBSERVATIONS - the loss is one occurrence per event that ages out of the window between two of them, because a retirement and an arrival cancel in the gauge. The accumulation therefore runs on every SWEEP, not on every delivery: with a one-hour window and a sweep measured in seconds nothing can arrive and age out inside one interval, so it is exact in practice, whereas observing only at delivery time undercounts a long incident by roughly the number of events the window retired while a cooldown was suppressing delivery. What no window gauge can recover is an occurrence that both arrives AND ages out between two observations - that needs an event-identity watermark at the collector, which is a different feature. A row left behind by a host that died mid-incident is ignored once it is older than the read window, so a stranded counter can never undercount the NEXT incident on the same fingerprint under a stale start time. + +- **A failing forced plan now alerts** ([#2157]) - when Query Store cannot reproduce a forced plan, the query silently runs on whatever the optimizer picks instead, and nothing in the product witnessed it: the only trace was a counter climbing inside Query Store. The new **Forced Plan Failing** alert fires per plan when `force_failure_count` RISES between collections, carrying the database, query and plan ids, whether the force was MANUAL or automatic plan correction, the engine's own failure reason, and how many failures are new. It is a rise and never a level, because the counter is cumulative AND travels with a restored database - alerting on the level would page forever about failures that happened on hardware you may no longer own. A counter that drops (an unforce/re-force cycle) is silence, and a plan seen for the first time waits one cycle, since 'new' is not knowable from a single observation. Severity is Warning for every rise: a failing force is not an outage, it is somebody's mitigation quietly not working. + +- **Every previously-hardcoded alert threshold is now a real setting** ([#2107], the split-out from gotqn's #2101 - "it was fine to hardcode these for development but any serious monitoring allows configuring of alert thresholds") - six new knobs ride the store control plane (V55), the Viewer's Settings window, and `get_alert_settings`/`update_alert_settings`, clamped on read like their siblings: the monitor store volume's self-alert warning percent (was 10), the Collection Stopped staleness window (was 30 minutes) and consecutive-failure fast path (was 10), the low-disk CRITICAL severity tier's percent and GB floors (were 3% / 2 GB - these grade the target-volume alert in BOTH apps, and Lite reads its pair from `settings.json` as `alert_disk_critical_free_percent` / `alert_disk_critical_free_gb`), and the analysis notification cooldown (was a hardcoded 360 in Darling while Lite always honored a configured value - the parity gap closed). MCP shape: `low_disk.critical_free_percent` / `low_disk.critical_free_gb`, a new `self_alerts` group, and `analysis.notify_cooldown_minutes`. +- **Per-database collection timing now separates server think-time from row streaming** ([#2164]) - the per-database Query Store line reports `sql:Xms = wm:Wms + open:Yms + drain:Zms`, where wm is the watermark refresh (a monitor-store round trip the timer already started before), open is everything before the first row arrives, and drain is streaming those rows to the collector. This exists because of a measurement that overturned an assumption: cutting the text budget from 64 MB to 12 MB on a production server moved 5x fewer bytes and roughly 7x fewer rows, and the batch clock did not move at all. That says the cost lives upstream of shipping - in Query Store's own aggregation before the first row - which no client-side budget or payload trimming can shorten. The blended number could not show that, so the split is now visible: a pass that is nearly all `open` needs the server-side query narrowed, and a pass that is mostly `drain` is the one a smaller budget or a shorter network path helps. +- **The two collector memory bounds are now operator knobs** ([#2164], [#2170]) - the per-database Query Store text budget (was a hardcoded 64 MB) and the fleet sweep width (was a hardcoded 4 servers) ride the store control plane (V59), the Viewer's Settings window, and the service's live reload, clamped [4,256] MB and [1,16] on read. Defaults reproduce the old constants exactly, so an upgrade changes nothing until a dial moves. Why both at once: peak transient memory is roughly the two multiplied, so an operator moving one needs the other in front of them. Lower the budget when the monitored fleet is a network hop away - the budget bounds memory, but it also sets how long one collector query holds the monitored server open draining to the client, which over a cross-region link is the tenant-visible cost (a smaller budget trades catch-up latency for shorter statements, never data, because every cut is resumable). Raise the sweep width when a large fleet queues behind 4-wide collection on a host with headroom, which is what makes the Fleet Health screen report staleness while every collector reports healthy. Narrowing the width never interrupts a running collection - the retiring permits are absorbed as bodies finish. +- **The Query Store backfill has an off switch** ([#2167]) - the #2058 backfill previously ran unconditionally, and during a fleet consolidation a freshly restored database's imported catalog put it into sustained byte-budget drains against a cross-region production primary with no way to stop it short of disabling plan capture everywhere. `config_service.query_store_backfill_enabled` (V58, default on) is read live by the service's backfill loop - flip it in the Viewer's Settings window (new checkbox beside plan capture) and the loop idles from its next cycle, no restart; re-enabling resumes exactly where the watermarks left off. Live collection is never affected. Lite gets the same control as a Settings checkbox ("Fill Query Store history gaps in the background"), read live so it takes effect without restarting the app - the store column exists on the Darling side because a headless service has no window to click. +- **The store measures its own background jobs** ([#2136], the visibility half) - the hourly #2068 self-metrics sweep now writes one row per TimescaleDB background job (object_kind `background_job`, V56 columns): last run duration, schedule interval, total runs, total failures. Why: the store's heaviest recurring work is its own job machinery - on the production 52-server store the four most expensive jobs are all the query_store_stats family (compression 157s, interval_hourly refresh 96s) - their runtimes scale SERIALLY with raw volume (the finalize hash-aggregate runs in one process), and a job that outgrows its own cadence compounds refresh lag silently. With the interval stored beside the duration, 'how close is each job to its ceiling' is one division over a 400-day series instead of archaeology - the number an onboarding wave moves first. A threshold alert on the series is the issue's next half. +- **The store alerts when its own background jobs outgrow their schedule** ([#2136], the alert half) - a new fleet-level self-alert, Store Job Over Cadence: WARNING when a job's last successful run reaches a store-backed percent of its own schedule interval (V57 knob `store_job_cadence_warn_percent`, default 25, clamped 5-100, on the Settings window and `get_alert_settings`/`update_alert_settings` under `self_alerts`), CRITICAL fixed at 100 - past that the job is still running when its next run is due, so runs back up behind each other and everything it maintains (CAGG freshness, compression, retention) falls further behind every cycle. Judged hourly from timescaledb_information.job_stats on the same connection as the compression-health check, successful runs only (a failed run's duration is not a cadence signal); standing condition with cooldown re-fires and a Store Job Cadence Recovered resolution row; the alert text points at the V56 duration series in collect.store_metrics for the trend. Calibration: the production 52-server store's worst job runs at ~7% of cadence, so the default warns at 3.5x today's ceiling but far ahead of real compounding. +- **The #2136 capacity model is now proven by a synthetic scale test, not asserted from one observation** - a live end-to-end drives a throwaway hypertable's compression job at 1x and then 4x row volume (parked policy, deterministic run_job - the #1888 discipline, so the scheduler can never race the measurement) and pins the whole loop: job runtime GROWS with volume (monotonicity, not a ratio - runner jitter owns the constant factor, the direction is the claim), the V56 telemetry series records both readings in order, and the Store Job Over Cadence alert fires its Critical tier from REAL store readings once the schedule interval is shrunk under the measured duration. + +### Changed +- **The query_store per-database log split now names the plan-XML and text fetches** ([#2312] investigation) - the per-item sql: stopwatch wraps the whole read, which since the separate fetches landed includes two more queries against the Query Store catalogs after the payload drain - so on a closed-only cycle that shipped ZERO rows, a 298-second bill (measured, ayr-01) had nowhere visible to live: drain silently absorbed it. Cycles that ran a separate fetch now log `... + plan_fetch:Nms + text_fetch:Nms`, the fetch phases come out of drain in the one shipped DrainMsFrom arithmetic (pinned like the #2164 split it extends), and every collector line without a separate fetch is byte-identical to before. Darling-only by construction - Lite runs no separate fetches and its zeros mean exactly that. +- **Query Store statement text is now resolved from `collect.query_store_text`, and the separate fetch is ON** ([#2150]) - the flip of `FetchQueryTextSeparately` and the conversion of every reader that projects `query_text`, in one change, because they cannot land apart: flipping the flag nulls the payload's inline `query_sql_text`, so any reader still reading that column would show BLANK text for newly collected rows while looking perfectly healthy - no error, no empty result, just a grid with the statement missing. Six blocks across five files now resolve the side table first and fall back to the inline column: the stored-plan resolver behind Get Actual Plan, the MCP `get_query_store_top` read, the Viewer's Query Store grid, its current-vs-baseline comparison, its regressions grid, and the PLAN_REGRESSION drill-down. **The fallback is permanent, not a migration step**: rows collected before this carry their text inline and nothing backfills them, so removing the fallback later would blank all existing history - the two arms are the two populations, not an old way and a new way. **Where the resolution goes was chosen to keep the filters honest.** Three of these queries also FILTER on the text (the [#1565] `WAITFOR` self-exclusion, and the resolver's `IS NOT NULL`), and testing the raw column there would have excluded every post-cutover row - the whole set the change exists to serve - so each one resolves the text ONCE, inside the existing lateral or a derived table, and the filter tests the resolved value under its original name. That also keeps the diffs to one block each rather than a projection edit plus a filter edit plus a repeated `COALESCE` in the `WHERE`. The comparison read needed `query_id` projected through its dedup CTEs (it groups by `query_hash`, but text is stored per `query_id`) and both its arms converted, since the final projection coalesces current over baseline and converting one arm would leave a GONE row with nothing to fall back to. **Verified against the live 52-server store rather than by reading the SQL**, which is the only instrument that can see any of this - there is no CI job that executes these Postgres strings. All six shipped query bodies were extracted from source, planned with `EXPLAIN (GENERIC_PLAN)` (six plans, every one containing `query_store_text` nodes), then run against real data twice: with the side table EMPTY, each converted query returned a byte-identical md5 to its pre-change form over 330 rows (1 / 50 / 50 / 179 / 50 / 1), proving the fallback arm and proving no keyed join fans out; then with the post-cutover condition INDUCED by shadowing the table with distinctive rows, all 331 rows resolved from the side table at identical counts, proving the arm that will serve every row once this ships. `FetchRowsAsync` deliberately stays OFF - it hands rows straight to its caller and writes nothing, so nulling the inline column there would lose the text outright instead of relocating it - and Lite is untouched, its DuckDB store having no side table to resolve from. + +- **Lite's Query Store prune names the state keys it retired, like Darling's** ([#2205]) - Darling's prune uses `DELETE ... RETURNING` and logs which keys went; Lite's DuckDB twin summed the affected-row counts into a number. Correctness was already identical - same anti-join, same freshness guard, pinned by the [#2195] tests - so this is purely forensic, and it matters on this path specifically because the symptom of a *wrong* delete here is a silent refetch, which leaves nothing else behind to diagnose it with. A count of three cannot be told apart from a mistaken prune of the same size. + +- **Query Store plan XML is stored once per plan instead of re-shipped every pass, and moves into the shared plan dimension** ([#2164] / [#2210]) - plan XML lived INLINE on `query_store_stats`, re-sent on every collection pass forever: 871,196 XML-carrying rows against 175,328 distinct (database, plan) pairs in a day on a 52-server fleet, or 5.0x the same content, on a table that had grown to 33 GB. Collection now fetches plans in `plan_id` order under a per-database byte budget and lands each one ONCE, keyed through a new map table into the same gzip-compressed dimension the other two plan collectors already share - so a Query Store plan byte-identical to one collected via `query_stats` costs nothing extra fleet-wide. Measured on production catalogs before shipping rather than reasoned about: converting inside the candidate window rather than joining back for the text halved the fetch (133ms against 274ms, identical 114 rows and 1.7 MB out of both, with a plan-id-only floor of 114ms), and the byte-budget predicate admits a plan on the total BEFORE it so a single plan larger than the whole budget ships alone and advances rather than wedging that database forever. **This is a cutover, not a rewrite**: existing inline plan XML is NOT migrated, readers resolve map-to-dimension first and fall back to the legacy column, and the old rows age out within their existing retention - so the transition window is one full `query_store` retention period (30 days by default) during which both shapes are readable and no operator action is required. `query_plan_text` is deliberately retained for that window; dropping it is a separate later migration. + +- **Query Store collection stops re-shipping plan XML the store already holds** ([#2164]) - 97% of the execution-plan XML shipped in a three-hour fleet window (197,113 of 202,790 rows) was for plans the store had already held for over an hour. The #1556 dedupe lands each plan once per PASS, but it re-lands it on every pass forever, and since streaming rows is 94-97% of a pass and costs per-row LOB bytes, NOT fetching a plan is worth far more than fetching less of one - cutting the text budget from 64 MB to 12 MB moved 5.3x less text and left the clock unchanged, which is what pointed here. Collection now carries a per-database watermark on `plan_id` (monotonic within one database's Query Store) and asks only for plans above the highest one whose XML actually landed. An absent, malformed or expired watermark renders no predicate at all, so the conservative path is byte-identical to the old query, and the watermark never applies to the history backfill - backfill reads intervals OLDER than anything collected, whose plans are numbered below the watermark, so honoring it there would suppress the very plans that pass exists to fetch. Measured against four production Query Store catalogs before shipping: identical row counts, 28-51% less elapsed time, and plan XML rows down 42-45% with the watermark set at only the MEDIAN plan_id. One full refetch per database per DAY still happens by design, which is what bounds the three things a permanent watermark would get wrong: a plan whose XML is rewritten in place without a new id, recovery after somebody clears Query Store (plan_id restarts at 1, so every new plan would sort below a stale watermark), and a plan compiled before monitoring began that stays dormant through every collected window and so is never seen above the mark. + +- **A deliberately offline database alerts once instead of every cooldown** ([#2166], reported by @gotqn) - the Database State alert re-fired for as long as a database stayed deviated, so parking one OFFLINE for a month produced hundreds of identical alerts for a single intended action. OFFLINE, RESTORING, RECOVERING and STANDBY are now edge-triggered: they alert on the transition and stay quiet until the state changes. The integrity states (SUSPECT, RECOVERY_PENDING, EMERGENCY) are unchanged and keep re-firing on the cooldown, because nobody parks a database in SUSPECT and there the repetition IS the signal. The memory is persisted per database (V60), not held in RAM, so a service restart cannot re-announce every parked database - and it composes with the per-database expected-state override exactly as reported: a database you park stays silent while still alerting the moment it turns SUSPECT or comes back ONLINE. **This changes default behavior**: if you were relying on the repeat-nag to track deliberate offlines, it now goes quiet after the first alert. Lite gets the same behavior in this release via [#2203]. +- **Force-plan findings carry a machine-first verdict for MCP consumers** ([#2138]; Erik: agents will read these more than people) - analyze_server and get_analysis_findings (both apps) now emit structured_remediation alongside the copy-paste command: per target, eligible plus NAMED blockers (parameter_sensitivity_cofired, secondary_replica_evidence), the evidence numbers, join keys, and split force_sql / unforce_sql / verify_sql artifacts (verify asks both post-force questions: did the force stick, and the per-interval cost since). The blocker list comes from FactRemediation.ForcePlanBlockers - the ONE policy gate a future auto-force feature would consult, so what agents inspect today is what any later automation enforces, as a testable data contract. Computed at read time from the persisted targets (never persisted itself - one source of truth, no DTO mirror to forget). +- **Plan-regression detection scores CPU as the primary signal and gates on absolute spend** ([#2138] Phase 0, the advise-only foundation for the auto-force-plan bot) - the old score was GREATEST(cpu ratio, duration ratio), so a plan whose CPU never moved could fire on duration alone - but duration is confounded by blocking, IO waits and machine contention that no plan choice caused, exactly the false positive a plan-forcing bot must never act on. Now a CPU regression scores at its own ratio; a duration-dominant one fires only when EXTREME (>= 4x) AND corroborated by at least mild CPU worsening (>= 1.25x), scored at HALF the duration ratio so it competes honestly with CPU-detected rows. A new noise floor drops offenders whose latest plan burned under 10 CPU-seconds across the whole 14-day comparison window - a 12x ratio on a query costing 12ms per run is sampling jitter, not a finding. The regressed-queries drill-down previously kept its OWN copy of the old score and could surface rows the fact never counted; it now runs the same scoring, in both SKUs, pinned by split-signal tests that move CPU and duration independently. +- **The force-plan recommendation now warns when the regressed query is parameter-sensitive** ([#2138] gap 3) - each regressed_queries row carries a `parameter_sensitivity_cofired` flag, computed inside the drill-down with the PARAMETER_SENSITIVITY detector's own thresholds (one cached plan whose per-execution cost varies >= 10x across parameter values, same floors, same window) joined by query hash - so the flag can never claim evidence the detector would not report. A flagged target's force-plan preview gains a caution block naming the risk (forcing pins ONE shape for every parameter value; the population that preferred the other plan inherits the wrong one permanently, quietly, because a forced plan no longer recompiles away) and the gentler first levers (statistics updates; PSP optimization / Query Store hints on 2022+), and the copy-paste surface gets a compact two-line version of the same warning. Unflagged targets render byte-identically to before. This flag is also the standing gate for the future auto-force bot: a flagged target is never auto-forced. Both SKUs, pinned by live tests in both stores. + +### Fixed +- **Darling Viewer crash on Queries -> Query Store by Duration** ([#2181], [#2331]) - the same uncatchable crash class as Lite's #2114, on the OTHER SKU: the grid's inline View Plan button referenced `DarkButton`, a key that IS defined in the Viewer - in `MainWindow.xaml`'s window resources, a scope a UserControl's templates cannot see, because StaticResource resolves lexically at load rather than through the runtime tree. The miss inside a cell template stack-overflows the process the moment the grid renders a row, which is also why it survived dogfooding: an EMPTY Query Store grid never applies its cell template. #2181 reported this against the Darling Viewer and was closed as a duplicate of the Lite fix on a wrong premise; #2331 re-proved it on 3.4.0. The button uses default chrome now (Lite's exact fix), and the XAML hygiene test's model is widened from per-app to per-FILE resolution (own keys + merged dictionaries + App.xaml scope - WPF's actual lookup), which flags exactly this class and produced zero false positives across both apps. +- **The store self-metrics sweep's ~5-a-day "Exception while reading from stream" ERRORs were command timeouts in a network-fault costume** ([#2317]) - the sweep's sizing queries (`hypertable_detailed_size` across every hypertable - its inner `hypertable_local_size` is the frame the server log names - and `pg_database_size` over the whole store) ran on Npgsql's default 30-second timeout, which they outgrew under load on a 141-object store with a 100+ GB plan dimension. Npgsql enforces its deadline by cancelling the statement (the store side logs `canceling statement due to user request` - confirmed at the exact failure timestamps in the managed server's own log) and the client is left holding a torn stream, so the ERROR read as "the network broke" - the same misdirection #2294 named on the baseline path, one layer over. Every sweep statement now carries a five-minute timeout, and the worker caps the WHOLE sweep at the same five minutes through a linked cancellation (the sweep is awaited on the main loop, so five sequential statement timeouts must not stack into a 25-minute stall of per-server dispatch), the statement count is pinned to the timeout count so a sixth statement cannot ride the default back in, and the worker's catch names a timeout as a timeout - a sweep that still cannot finish skips the tick and the series gains a self-healing one-hour gap, deliberately NOT retrying into the same load. +- **Gapped charts no longer bury their neighbours under opaque black fill** ([#2324]) - the #1944 gap markers (a NaN Y injected mid-gap so lines break across an outage) shipped first in 3.4.0, and they collide with the gradient area fill: reproduced headlessly against ScottPlot 5.1.59, one NaN in a FillY + ColorPositions series renders the ribbon as opaque black polygons with straight chord edges crossing the gap - the fill path closes its contours through the break, and its fill paint under ColorPositions is hardcoded black with the gradient shader expected to paint over it, which a NaN-bearing series defeats. On the reporter's dark theme that black buried every other series on every tab whose data had a collection gap; the one healthy tab was the one with gapless data. A gap-marked series now renders line-only - the break stays visible, nothing is buried - and continuous series keep the full gradient ribbon, pinned in both directions so the fix cannot quietly repeal the fill feature. +- **The plan fetch's adaptive candidate sizing is finally wired** ([#2312] Finding 1) - QueryStorePlanXmlState.CandidatePlanCount was designed to size each pass's decompression window from the database's OWN observed average plan size (the doc argues an 11x fleet spread makes any constant wrong somewhere), and the single call site passed null - so every pass on every database sized its window from the 160KB first-contact seed, K ~ 116, always. The runner now carries a per-database estimate in memory (like the adaptive-shrink counters: a restart costs one seed-sized pass) folded by a pure, pinned Learn: an empty pass proves the walk caught up and clears the catch-up flag while keeping the average; a pass cut by either bound (window consumed or byte budget reached) proves a backlog and sets it; an ordinary pass learns its average. Downstream effect on the measured fleet shapes: a genuine 15KB-average database re-sizes from 116 to ~1,259 candidates per pass and walks its backlog an order of magnitude faster, while the catch-up floor still pins the window at seed size during exactly the biased-sample window the overload's doc describes. +- **max_cpu_ms can no longer masquerade as a this-window number** ([#2235]) - the min/max CPU and elapsed columns on get_top_queries_by_cpu and get_top_procedures_by_cpu are lifetime extremes for the plan's time in cache (sys.dm_exec_query_stats never lowers its high-water marks, and a max cannot be delta'd), while totals and avgs are windowed deltas - so a lifetime max could EXCEED the window's whole total, inviting a reader to quote an extreme from an arbitrary earlier period as if it happened this week (8 of 20 rows on one production box). Both tools' descriptions now state the lifetime semantics the way max_dop's always has, and rows where an extreme provably predates the window (max greater than the windowed total) carry an extremes_note naming the offending column - equality deliberately stays silent, since a single-execution window has max equal to total by construction. Same shape in both apps via the shared QueryStatExtremes helper. +- **The query_store collector stops re-aggregating the open interval every cycle** ([#2312]) - on a large multi-tenant primary the collector cost 40-110 s per run around the clock (554 s worst case, measured on the prod fleet the day the primaries came under monitoring), and the mechanism was not a missing closed-interval watermark: the standing HAVING already excludes any interval fully collected. The bill is the OPEN interval - Query Store rows are cumulative per-interval snapshots, the collector re-fetches the open interval every cycle by design so the read side can collapse to the latest snapshot, and at the 5-minute cadence that re-aggregates the entire current hour across every tenant database ~12 times per interval, each pass pricier as the interval fills, every snapshot but the last discarded by the read side's rn = 1 (and every redundant re-read a full duplicate row-set appended to the store). Most cycles now ship CLOSED intervals only (`i.end_time <= SYSUTCDATETIME()` - server-evaluated, so the single-parameter sp_executesql contract is untouched), which are immutable and therefore final on first collection; the open interval is refreshed on a per-database 15-minute stamp (QueryStoreOpenIntervalState, the fourth query_store state family: own owner, qsowm: prefix, registered in the shared prune set). Correctness leans on the cumulative-snapshot contract twice: a newly closed interval whose content moved past our last snapshot must carry executions newer than the watermark, so the standing time filter readmits it, and one whose content did not change IS our last snapshot. Include-open stays the conservative default - a first run, a restarted host, a broken store and a clock-skewed stamp all behave exactly like the old collector - and the backfill path ignores the flag entirely. Both hosts wire the per-database decision at their watermark seams (enumerated and Azure arms); readers change nothing. The current hour's view lags real time by at most the refresh horizon; closed history loses nothing. #2312's yardstick decides the rest: multi-53 at ~50 s/run before. +- **FinOps column filters no longer follow you to the next server** ([#2306]) - a column filter set while viewing server A survived switching the FinOps tab to server B, where it could match nothing: the grid rendered zero rows while every count indicator (computed from the unfiltered list) stayed full, and Refresh could not clear it because UpdateData deliberately re-applies active filters - correct for same-server refresh, which is untouched. The only tell was the gold funnel icon on one column header. A server switch now clears every FinOps grid's filters (sort preserved, funnel icons dimmed) through the same manager map every FinOps grid already registers into, so a future grid inherits the clear without a second edit; DataGridFilterManager gains the ClearFilters seam on the shared interface. Found while running down #2300, where it was ruled out as that reporter's mechanism but produces the identical symptom. +- **A clean service stop no longer reads as seven faults** ([#2299]) - every operator `Stop-Service` logged a burst of ERRORs after "collection loop stopped": the analysis pass was started per sweep but neither awaited nor cancellable - the entire Analysis project carried no CancellationToken - so shutdown disposed the loop's data source underneath the still-running pass and then stopped the managed postmaster, and the abandoned pass's next store reads logged five baseline failures, an anomaly-detection failure and a mute-filter failure. Seven of that day's nine ERROR lines were this burst; the two real errors were the needles. The pass now observes the host's stopping token (carried on `AnalysisContext`, so Lite and every token-less caller behave byte-identically), a stopping sweep holds the pass open for a bounded 5s grace so it unwinds BEFORE the data source goes away, and residue from a race it still loses - `ObjectDisposedException`, SQLSTATE 57P01/02/03, `OperationCanceledException`, with the token SIGNALLED - collapses to one Information line stating the loss honestly (this pass's findings are gone; the next pass recomputes them). The same exceptions with the token NOT signalled keep their ERROR, because a data source disposed while the service is meant to be running is a real bug whose only evidence is exactly this text; and a command timeout is never relabelled shutdown, so the [#2294] growth signal survives the coincidence. A category pin counts every ERROR-logging catch on the pass and goes red if a new one arrives unclassified. +- **Index Analysis could show 2,000+ recommendations and an empty Recommendations grid** ([#2300]) - not a data bug: the analyzer emits one Note banner per stated limitation (six on a real server), and the banner strip sat in an uncapped Auto row. Banners plus the rollup grid consumed the whole tab, the Recommendations grid's star row collapsed to its header and horizontal scrollbar, and the rows it was actually holding had zero viewport - the count indicator said "2078 recommendation(s)" while the grid under it read as empty, which is exactly how it was reported. The banner strip now lives in a scroll viewer capped at 150px (caveats must never evict the content they qualify), and the grid carries a MinHeight backstop so it can always prove it has data. +- **The MCP host can read its server registry again - by not reading it** ([#2298], the second half of [#2293]) - #2293 skipped the notification row and the failure moved to the next denied column: `ReadMonitoredServersAsync` selects `encrypted_password`, which the section-6 secret ACL deliberately SELECT-carves from the `mcp` role, so the whole config view read still failed with 42501 and live plan fetch still fell back to darling.json - on a seeded box, exactly the set of servers the file does not know about. Skipping columns one 42501 at a time was chasing the carve; the durable agreement with the boundary is that the MCP host performs NO config read of its own. The worker already loads the same rows over its privileged connection (it must, or it could not collect), and now publishes the effective server set through a `MonitoredServerRegistryState` seam - the same publish/observe pattern as the #1560 control-plane knobs - which the plan-fetch resolver reads PER FETCH. That per-fetch read also fixes a quieter defect: the old map was built once at host start, so a server added later through `add_servers` or the Viewer never reached the resolver until an MCP restart; now it arrives on the worker's next reload. The security property is preserved, not weakened: the `mcp` database role's grants are untouched, no MCP tool exposes the state, and a token-holder still cannot obtain a stored credential - the carve was never about keeping credentials out of this process (the worker holds them), it is about keeping them off the MCP wire, which they remain. +- **The FinOps provisioning verdict said UNDER_PROVISIONED for every server alive** ([#2246], from the field report on [#2150]) - the rule tested `total_server_memory_mb / target_server_memory_mb > 0.95`. Those are the perfmon **Total** and **Target** Server Memory counters - Target is what SQL Server wants, Total is what it holds - and they converge at 1.0 the moment an instance is warmed. Measured across 42 production servers: median **1.0000**, min 0.9997, max 1.0002, so **42 of 42** tripped it, and `OVER_PROVISIONED`, whose arm needs the same ratio below **0.5**, was unreachable at any workload. The report was a user seeing every server flagged in Darling while the same fleet read `RIGHT_SIZED` in the older dashboards, which had never carried this rule. The predicate is now one shared `ProvisioningVerdict` in `PerformanceMonitor.Common` rather than six near-identical copies (Darling's point-in-time, trend and inventory reads plus Lite's three), and it reads signals that mean something: workspace-memory **grant waiters**, **grant timeouts** and **forced grants** for memory pressure; sustained CPU at p95 > 85%; and **worker-thread saturation** above 0.8 of the ceiling - a term the Full Dashboard's view has always had and which both app copies had dropped, so a worker-starved server was invisible to both. Idleness now requires quiet CPU *and* grant utilization below 50%, so a server working its semaphore hard is not recommended for downsizing. Every threshold has the fleet distribution behind it, recorded on the type: highest p95 51.0 against a limit of 85, highest worker ratio 0.635 against 0.8, peak grant utilization 18.8% against 50% - reachable rather than academic. The verdict is also no longer decided in SQL anywhere, and the UI stopped inventing its own explanation: it used to render "p95 > 85 ? CPU : blame the memory ratio", so once grant pressure and worker saturation became causes, every one of them would have been explained by a threshold the code no longer checks. Because the fleet has **no** memory pressure at all - 0 nonzero of 2,938,711 grant rows, 0 of 1,000,560 `Memory Grants Pending` samples - the positive control that the alarm still fires is built in tests, one input at a time, rather than borrowed from production. + +- **Half of every delta collector's output was a zero that never happened** ([#2233], [#2234]) - `CollectorDeltaCalculator` discarded a delta whenever the gap between samples exceeded 300 seconds, and reported it as a zero. Measured against the live fleet store before touching the threshold - 99,717 consecutive perfmon gaps across 52 servers over 7 days - the fleet's **median** gap is **299 s**: p90 580, p99 830, p99.9 1,190, max 2,514. The guard was sitting exactly on the median, so it rejected **50.0%** of ordinary sweeps; at the new 3,600 s it rejects **0%** while still catching a genuine collection outage. The number is now a published `DefaultMaxGapSeconds` constant with that distribution recorded beside it instead of forty-one copies of a bare `300`. Two related defects went with it: a counter **reset** (current < previous) zeroed the delta but left the interval populated, so a consumer could divide by a window in which nothing was actually known - the invariant is now `interval == 0` exactly when no delta is knowable; and `perfmon_stats` wrote a hard-coded 60-second `sample_interval_seconds` on every row regardless of the real gap, so any rate computed from the stored data was wrong by whatever the sweep actually took. It now records the measured interval, and the perfmon trend read aggregates it with `MAX` rather than `SUM` - the rows for one sample are 12 to 17 instances of the same interval, and summing them made every derived rate 12 to 17 times too low. + +- **`tempdb_stats` failed forever on Azure SQL Database, and could never have succeeded** ([#2150], field report) - an elastic-pool database logged `tempdb_stats: 11x consecutive - SQL Error #262: VIEW DATABASE PERFORMANCE STATE permission denied in database 'tempdb'` and would have kept doing so indefinitely. The collector's first result set reads `tempdb.sys.dm_db_file_space_usage`, a **three-part reference out of the connected database**, which on Azure SQL DB requires `VIEW DATABASE PERFORMANCE STATE` **in tempdb** - and a non-administrative login cannot hold it there, because tempdb permissions are not persistable on Azure SQL DB and in an elastic pool the databases do not share one. So no grant, no role and no configuration change could have fixed it; the only honest answer is not to run it. The collector is now gated off Azure SQL Database entirely, which means **no rows and no failures** rather than a permanent red entry in collection health - the same posture the other inapplicable collectors already take, where a skipped collector leaves no trace instead of a fake success. + +- **The installer's mapped-drive refusal failed open when WMI was unavailable** ([#2201]) - `Get-NetworkPathKind` catches UNC paths lexically but identifies a mapped **drive letter** by asking WMI for the drive type, and if that call failed for any reason the catch returned "local" - so an install onto a mapped drive proceeded silently, producing exactly the unreadable-install-tree service the [#2187] refusal exists to prevent. It now falls back to `Get-PSDrive`'s `DisplayRoot`, which identifies a mapped letter without WMI at all, and no longer treats an inconclusive answer as proof of a local disk: `Win32_LogicalDisk.DriveType` **0 means unknown**, and unknown is falsy, so the previous check let it short-circuit to local. + +- **The service now names the `darling.json` servers it is not monitoring** ([#2254], from the report in [#2252]) - `darling.json` seeds `config_monitored_servers` only while that table is empty, so a server added to the file after the first successful start is a permanent no-op, and restarting the service cannot change that. Nothing said so - and `--test-connection` reads the FILE, so it validated the newly added server as PASS while the service never collected it: two outputs each correct about a different question, with no way to see the disagreement. The report cost a config edit, a service restart and a support round trip before the seed behavior surfaced. Startup now reconciles the two and names the servers present in the file whose identity the store does not hold, stating both things the operator could not know - that the store is authoritative after the first seed so a restart will not help, and that `--test-connection` will keep reporting them as PASS regardless. The comparison is on `server_id`, not name, because that is the identity the collectors and the registry key on: a file entry whose host or read-only intent differs from the stored row is genuinely unmonitored under the identity the file describes, and a name match would call it present - the same identity confusion behind [#2158]. It reports at INFORMATION and its wording covers both causes, because it cannot tell them apart: the Viewer's Remove action hard-deletes the store row and deliberately never edits `darling.json`, so an operator who removed a server on purpose is in a correct state and a warning telling them to re-add it would be wrong advice repeated on every start. Adding a server to a running install remains the Viewer's Add Server dialog or the MCP `add_servers` tool; a headless host has no verb for it yet ([#2256]). + +- **Microsoft Entra MFA can actually connect - Lite** ([#2184], reported by @joshdbe) - current Microsoft.Data.SqlClient routes interactive Entra auth through the Windows WAM broker, and WAM requires the application to hand it the window that will own its account picker. Lite set `ActiveDirectoryInteractive` on the connection string but never registered an auth provider, so every Entra MFA connection died with `0xwindow_handle_required` instead of prompting - on every machine, not some particular tenant, and old builds only worked because they predate WAM being SqlClient's default. Lite now registers a process-wide provider at startup that resolves the owning window per prompt (the Add/Edit Server dialog in front, not the main window behind it), marshaled to the UI thread because MSAL asks from whatever thread the connection open happens to run on - collector loops included. One behavior worth knowing on sight: when the Windows session already satisfies MFA, the broker succeeds silently and no picker appears at all - that is WAM working, not the prompt failing. Same seam as Performance Studio's fix (PerformanceStudio#426), which the reporter verified against a real Entra-MFA tenant; the silent-SSO behavior above is exactly what his verification observed. + +- **Dropped databases no longer leave Query Store collection state behind forever - both apps** ([#2188]) - Query Store collection writes one `collector_state` row per database and never deleted any of them for a database that was dropped or renamed: the #2164 plan-XML watermark (`planwm:`, Darling only) and the #2022/#2058 backfill worker's tail and gap markers (`done:` / `hole:`, BOTH apps - the worker deletes a hole when it services or expires it, but a dropped database can never do either). On a server with churny database lifecycles - dev/test, or multi-tenant provisioning - those accumulated with no pruning path, since `collector_state` is a keyed registry rather than a hypertable and has no retention policy to catch them. Both apps now retire them on the query_store cycle, from one shared list of which keys are database-keyed, so a future key cannot end up pruned on one app and orphaning on the other. The delete is keyed on `database_states` - the unfiltered `sys.databases` snapshot - and deliberately NOT on query_store's own enumeration, which screens out offline databases, AG secondaries, excluded databases and anything that failed its probe: pruning on absence from that list would delete LIVE state on exactly the servers that keep databases parked or excluded, and the only symptom would be a silent re-fetch. It also requires the snapshot to be NEWER than the row it judges, so a server whose database_states collection has stopped cannot have every database created since then pruned on repeat. A server with no snapshot at all prunes nothing rather than everything, which is what keeps an Azure SQL Database server (where `database_states` is not collected, [#2191]) and a server whose snapshots have aged out from being mass deletes. Cosmetic either way - an orphaned row is simply never read again - and a same-name recreate is bounded by the watermark's own one-day refresh horizon, not by this. +- **A managed-Postgres bootstrap failure now says what went wrong instead of printing a Win32 number** ([#2186], from @jovon44's report in #2185) - a store that could not start reported `initdb failed (exit code -1073741515) for ... Output:` and nothing else. `-1073741515` is `0xC0000135`, `STATUS_DLL_NOT_FOUND`: Windows killed the process in the loader, before a line of its own code ran - which is also why `Output:` was blank and always would be, so the one field an operator reads was guaranteed empty exactly when the failure was a load failure. Their attention then went to the follow-on missing-credential message and `darling.json`, neither of which was the fault. Every bundled-binary failure (initdb, pg_ctl start/status/stop/reload, pg_upgrade and its `--check`, the post-upgrade analyze) now decodes a Windows status into its name, names the two causes that account for nearly all of them - the bundled MSVC runtime absent from `pg-runtime\pgsql\bin`, which means a partial or damaged extract rather than a missing prerequisite, or a service account that cannot read an install tree sitting under a user profile - and gives the two checks that tell them apart, including the caveat that these tools re-execute themselves under a restricted token that drops Administrators, so running the binary by hand succeeding does not clear the permissions theory. An empty `Output:` now says it is empty **because** the process was killed before it could write, rather than looking like data that failed to arrive. Two messages also stopped blaming the wrong thing: `pg_ctl status` no longer calls the data directory unusable when the verdict came from Windows rather than from pg_ctl (that phrasing pointed at deleting a healthy store to fix a missing DLL), and `pg_upgrade --check` no longer reports clusters as incompatible when pg_upgrade never ran to form an opinion. The related [#1738] refusal now quotes and decodes the probe exit code it always had, instead of asserting that the binaries did not run without saying how it knew. +- **A missing store credential no longer reads as a first run after a bootstrap has already failed** ([#2197], the other half of @jovon44's #2185) - when a managed bootstrap dies, the operator's last message is rarely the bootstrap error; it is whatever CLI verb they run next, and every one of those said `Start the PerformanceMonitor Darling service once so its first run initializes the store`. That is correct advice for a genuine first run and a dead end for the case that actually produces it in the field - the service HAS been started, its bootstrap failed, and starting it again fails the same way. In #2185 that is the message the reporter led with, and it is what sent them to `darling.json`, which was never the fault. The six copies of that sentence (five CLI verbs, plus the Viewer's managed-mode parse whose text the main window shows) are now one shared message that decides between the two from the STORE'S OWN FILES: an initialized cluster, a `pg.log`, or a credential file beside the data directory - in particular the store's own credential, which the service writes immediately before it runs initdb and which therefore survives the exact failure [#2186] decodes. With evidence it says this is not a first run, quotes what it found so the verdict is checkable, and points at `%ProgramData%\PerformanceMonitorDarling\logs\darling-service_yyyyMMdd.log` - where, since [#2186], a bundled Postgres tool that Windows killed explains itself in words rather than as a bare exit code. Without evidence the first-run advice is unchanged, and gains the one sentence it was missing for the operator who has already started it. What it deliberately never infers from is a machine-global signal such as the service log directory existing: that would tell somebody standing up a SECOND store on a working box that their bootstrap had failed, which is this same defect pointed somewhere new, and an empty data directory an operator pre-created is not evidence either. The four store-credential verbs also name the credential file's path now, which none of them did. +- **A database observed mid-restore no longer learns RESTORING as its expected state and then alerts forever for being healthy** ([#2189]) - found by dogfooding: the production 52-server box fired 636 Database State alerts in 24 hours from 5 databases, every one of them reading `Expected: RESTORING, Current: ONLINE`. Each had been swept into monitoring during a consolidation while a restore was running, had RESTORING written as its accepted baseline, and had been "deviating" ever since by being healthy - permanently, since the only escape was an operator noticing and re-baselining by hand. Two halves. The seed now refuses to learn the TRANSIENT operational states (RESTORING, RECOVERING) the same way it already refused the integrity ones: a database in the middle of an operation is not a steady state anybody would choose as expected, so it stays pending and silent until it settles into one, and when the restore completes ONLINE is what gets learned. That governs rows that do not exist yet and cannot touch the ones already written, so the second half applies the same rule after the fact - an AUTO-seeded baseline recording a state the seed would refuse to learn is not a baseline anyone chose, and once that database's effective state reaches ONLINE the steady state is learned instead. The already-poisoned rows therefore repair themselves on the next sweep, with no migration and no manual step, and the route that stays open forever is covered too: pressing "reset to current" while a restore is running (or during an outage) records whatever it sees with no state filter at all, and that now un-writes itself. What the repair deliberately does NOT touch is as load-bearing as what it does, because every case here is one where being wrong means silence. It never rewrites a baseline an OPERATOR declared, so #2166's composition contract holds exactly as specified: a database you park at expected OFFLINE stays quiet while parked and still alerts the moment it comes back ONLINE. It never touches an OFFLINE or STANDBY baseline even though the machine inferred those, because both are steady states and leaving one is real news - a STANDBY secondary that turns up truly ONLINE has stopped being a secondary, which means somebody recovered it and log shipping is broken, and an auto-baselined OFFLINE database brought up for an hour of maintenance and re-parked would otherwise come back deviating forever against a baseline it never had. And it matches the EFFECTIVE state rather than `state_desc`, because a standby secondary reports `state_desc = ONLINE` with `is_in_standby` set - matching the raw column would have re-baselined every such secondary and then alerted it forever for being STANDBY, this same bug recreated for the one database family the alert works hardest to keep quiet. A log-shipping secondary restored WITH NORECOVERY, which sits in RESTORING permanently, stays silent forever as before; it now gets there by never being baselined rather than by baselining RESTORING, and an operator who wants deviation coverage on one sets its expected state explicitly. Both SKUs. (This is the seed and baseline logic, not #2166's alerted-state memory, whose Lite half lands separately in [#2203].) +- **Lite's database-state alert now goes quiet after announcing a chosen state, like Darling's** ([#2203], the Lite half of [#2166]) - #2182 made OFFLINE/RESTORING/RECOVERING/STANDBY edge-triggered, but only in Darling: the memory that makes an edge trigger work is persisted per database, Lite's table had no such columns, and its state-store methods were documented no-ops. So `alreadyAnnounced` was always false in Lite and a database parked OFFLINE for a month still produced an alert every cooldown, forever - which is the complaint #2166 was filed about, still live in the SKU most likely to hit it. Lite gains the same two nullable columns (schema v53, NULL meaning never announced, which is what a first observation, a fresh store and a recovered database all look like), the two store methods actually write, and the deviation read returns the value so the shared engine can use it. The clear is **store-derived** rather than taken only from the engine's in-memory set, matching the fix #2182 needed for the same reason: that set empties on restart, so a restart landing between an alert and the recovery would leave the memory stuck and silently swallow the next parking of that database. Worth stating what this is NOT: it does not touch the integrity states (SUSPECT, RECOVERY_PENDING, EMERGENCY), which keep repeating on the cooldown because nobody parks a database in SUSPECT and continued nagging there is the signal rather than the noise. + +- **The doc-comment hygiene pin now catches a stacked summary whose first block is never closed** ([#2190]) - `NoMemberCarriesTwoStackedSummaryBlocks` keyed off a `` immediately followed by a reopening, so it only ever saw a stacked pair when the FIRST block was closed, and two instances were sitting on dev unseen: a duplicated opening tag on `ApplyProcessEnvironment` in the Darling service, and a doc block that #1912's restructure split in `QueryStoreSliceRepairService`, stranding its unclosed head on top of `PromoteRewrittenFileAsync`. The rule now counts `` OPENINGS inside each contiguous run of `///` lines - a run documents exactly one member, so two openings means two summaries whether or not either is closed, and whether they are written single-line or spread over many. That mixed form is the one a closing-tag matcher cannot see at all, and it is what defeated the first attempt at this fix. Both live instances are repaired: the duplicate tag is deleted, and the stranded head is deleted rather than moved back, because the same restructure had already re-documented `FlushExternalFileCacheAsync` in place with a superset of that one sentence, so moving it would have recreated the exact duplicate this rule exists to forbid. The detector now carries its own self-test as well, pinning the five stacked shapes it must catch and the four legitimate ones it must leave alone, since this was a blind spot in the DETECTOR rather than in anyone's reading of the tree. +- **The installer refuses an install directory the service could never read** ([#2187], split out of #2185 reported by @jovon44) - a zip extracted to `C:\Users\\Desktop\PerformanceMonitorDarling-3.2.0\` - a completely reasonable thing to do with a download - installed without a complaint and produced a service that could not work, and nothing told the operator why. The service deliberately runs as the unprivileged virtual account `NT SERVICE\PerformanceMonitor Darling` and never as LocalSystem, because the bundled PostgreSQL refuses to run with administrative privileges; that account is not the installing user, not SYSTEM, and not Administrators, which is approximately everyone a user profile grants anything to. Measured on Windows 11, a directory created under a profile inherits exactly SYSTEM / Administrators / the profile owner - no `BUILTIN\Users`, no Authenticated Users, no CREATOR OWNER - so the service cannot read its own program files, and cannot read back even what it writes there itself; the bundled PostgreSQL's `initdb.exe` died at exit code -1073741515 (`0xC0000135`, STATUS_DLL_NOT_FOUND) before writing a word of output. `install-darling.ps1` now refuses a fresh install anywhere under the machine's profile root (read from `ProfileList\ProfilesDirectory` rather than assumed to be `C:\Users`, since it is relocatable) or on a UNC / mapped-drive path - where a virtual account reaches the network as the computer account rather than as the operator, and cannot see a drive letter belonging to someone else's logon session at all. The refusal names the account, the mechanism, and `C:\PerformanceMonitorDarling` as the fix, and it runs before the pre-flight, the Event Log source, and `sc create`, so a doomed location costs nothing and leaves nothing behind. ACLing the install tree to make a profile path work instead was considered and rejected: it would mean the product silently changing permissions inside somebody's profile. An **upgrade** of a service that already lives there asks rather than refuses, on the same reasoning the script already applies to a re-homed logon account - that location may have been made to work by hand, and refusing outright would strand a deployment that runs today. +- **The store's compressed plan format is now documented** ([#2171], reported by @argpna) - execution-plan XML has been gzip-compressed in `query_plan_dim.query_plan_gz` since 3.4.0, with `query_plan_xml` nullable, and the release notes never said so. A consumer reading the store directly over SQL therefore got nothing back for anything collected by a current build, with no clue why. The store section of the Darling README now states the format (gzip, magic `1f 8b`), that `query_plan_xml IS NULL` means read the compressed column rather than "no plan", and the three practical ways to get XML back - ask the product via `get_plan_xml`, decompress client-side, or ship a UDF into your own store if your tooling is SQL-only. In-product paths were never affected; this was a contract change for direct SQL consumers that shipped silently. +- **tempdb no longer reports more than 100% used in FinOps Database Sizes** ([#2169], reported by @CatastropheOps) - the used percentage divided in-database usage by the size recorded in `sys.master_files`, which is the size set at configuration time and does not track autogrowth for tempdb. A tempdb that had grown was therefore measured against its startup size and rendered above 100%. The per-database probe now captures the file's current size in the same round trip it already makes for space-used, and the payload prefers it, so both halves of the ratio come from one snapshot; a database whose probe fails still falls back to the old source rather than vanishing from the grid. Affects the on-prem, RDS, and Managed Instance path - the Azure SQL Database path already read both numbers in-database. +- **Backing out of Custom Range no longer strands an open calendar - both apps** ([#2154], reported in #2153) - a DatePicker's calendar dropdown is a popup living outside the visual tree's visibility, so collapsing the pickers when the user switched back to a preset range left an already-open calendar floating on screen; the dropdowns now close explicitly alongside the collapse, in Lite's ServerTab and the Darling Viewer's twin alike. +- **One wedged background task can no longer stop collection - in either app** ([#2148], reported on an Azure elastic pool minutes after upgrading) - Lite's collection ladder runs its steps sequentially, and while every step's exceptions were contained, nothing bounded a HANG: one stuck task (the new Query Store backfill was the prime suspect on the reporter's timeline) silently froze every collector, and the CPU chart going blank was just where it got noticed. Every backfill slice now runs under a PER-SERVER abandonment deadline with an in-flight guard (the scheduled-analysis idiom, extracted as a reusable primitive) in BOTH apps - Lite's tick and Darling's fleet loop share the exact shape: a wedged slice is abandoned so everything else continues, quarantined to ITS server only (never relaunched on top of itself, never blocking a neighbor), and self-restoring when the stuck task actually ends; Lite's fleet-wide connection check gets the same treatment. Abandonment logs at ERROR naming the issue, and an abandoned task's LATE exception - the one that explains the wedge - is surfaced instead of discarded. The deadlines are generous multiples of a single healthy slice, so those lines are always defect signals worth reporting. +- **The retention purge retries once on a deadlock instead of wasting the cycle** ([#2143], caught live by the nightly's purge e2e) - drop_chunks can lose a deadlock to a TimescaleDB background job whose chunk locks clear within milliseconds of the abort; previously that one transient loss pushed the table to the row-by-row DELETE fallback (which can deadlock against the same partner) and burned the whole cycle. Now 40P01 earns exactly ONE immediate retry - a second deadlock in a row is standing contention, where the fallback-plus-next-cycle posture is right and a retry loop camped on a lock queue is not. Non-deadlock failures keep the original single-shot behavior. +- **Remote viewers can finally use the exact connection string `--print-viewer-connection` prints** ([#2117], diagnosed to the trust-chain layer by jonchapman-usrc) - the store's TLS certificate was a single self-signed end-entity cert with critical `CA=false` Basic Constraints - a shape the reporter's Windows environment refused as its own trust anchor under the custom-root trust Npgsql applies to `Root Certificate=...` (stock Windows CI tolerates it, per the new end-to-end pins, so the refusal is environmental - strict chain-policy hardening is the likely class - but a shape that only works on unhardened machines is not a shape to ship), with the real error swallowed behind the generic "is the Darling service running?" message (the reporter burned hours eliminating everything else; importing the same cert into the OS trust store - their workaround - keeps working). Three changes: the service now generates a REAL two-cert chain (a throwaway local CA signs the server leaf and its private key is discarded on the spot, so the distributable `root.crt` still pins exactly one server identity), the print/export verbs emit that root, and the viewer's store-unreachable message now carries the underlying error text so a chain rejection, a wrong password, a pg_hba refusal, and a dead host stop reading identically. **Existing stores are deliberately NOT auto-rotated** - operators who imported the old cert keep a working setup, and the service logs the rotation recipe (stop, delete server.crt + server.key, start, redistribute) instead. Chain validity is pinned end-to-end - a real NpgsqlConnection at VerifyFull against an in-test TLS listener - on every CI platform, and the legacy shape's per-platform behavior is recorded in every run's log so a platform change shows up in CI diffs. The reporter's third finding (relative `Root Certificate` resolving against the process working directory) was already fixed on dev by #1970. +- **`--collapse-legacy-slices` no longer dies at TimescaleDB's decompression rail on compressed chunks** ([#2105] round two, ghauan again - the previous fix got the repair four minutes in, far enough to hit `53400: tuple decompression limit exceeded`) - a store old enough to need this repair has had its compression policy running the whole time, so the repair's DELETE necessarily touches compressed chunks, and TimescaleDB caps decompression at 100k tuples per DML transaction by default. Each slice transaction now lifts the cap for itself (`SET LOCAL`, dying with the transaction) - the same rail-lift the retention purge's fallback DELETE has always done, because deliberate bulk decompression is exactly this verb's job. +- **A Query Store member whose catch-up window can't fit the command timeout now shrinks it until one does** ([#2111] promoted from reserve on box evidence - one database sat 3+ hours stuck through quiet overnight hours, its 1h window intermittently exceeding the 60s timeout every cycle) - after N consecutive live failures a database's catch-up window halves per failure toward a 15-minute floor, and the range the tighter window skips rides the SAME hole records the clamp already writes: deferred to the backfill trickle, never dropped. Success resets to full width. The backfill worker gets the mirror treatment - a server whose hour-wide slices keep dying digs in progressively narrower chunks until one fits. One shared pure policy (`AdaptiveSpan`, pinned) drives both paths in both SKUs and both engine arms, so nothing can drift on how fast it backs off. +- **`--collapse-legacy-slices` no longer dies with "Exception while reading from stream" on a store fresh off a large catch-up** ([#2105] follow-up, ghauan again) - the repair's staging aggregation ran on Npgsql's default 30-second command timeout, which a heavy day-slice blows through (the verb runs beside the live service by necessity - stopping a managed store's service stops Postgres - so collector writes and compression jobs contend for the same chunks), and an Npgsql timeout surfaces as a bare stream exception that says nothing about time. Every repair statement now runs on a 15-minute per-statement timeout: generous because the slice is doing real work, bounded because the slice transaction holds chunk locks the compression policy also wants. The dry-run survey gets the same treatment, and the repair remains idempotent and resumable exactly as the failure message promises. +- **Query Store catch-up can no longer spiral a big database into permanent timeout** ([#2102], found dogfooding the use1 migration) - the live path's incremental window was `(watermark, now)` with only a 24h clamp, and the watermark only advances when a cycle SUCCEEDS. The per-database query aggregates, window-functions, and sorts its WHOLE window before `TOP` or the client byte budget can bound anything - a row cap is not a cost cap - so one missed 60s cycle (a flush burst, a CPU blip, a migration cutover) widened the next window, which cost more and timed out again, growing without bound below a clamp that sat far above the tipping point. On the field evidence the big tenants wedged at 0.5-6.5h stale and re-ran the same doomed query every cycle forever while small databases on the same servers stayed current - and the backfill worker's slices had the same latent flaw one layer down, querying a hole's full remaining range in one statement. Catch-up now trickles the way it was always meant to: the clamp is one hour (the envelope the fleet proves every day under Query Store's 900s flush cadence - and it slides forward with now, so recovery is immediate no matter how stale the watermark got), the skipped range is recorded as a hole exactly as before, and every backfill slice windows at most one hour at a time, with an empty chunk SHRINKING the persisted ceiling past the quiet hour instead of wrongly declaring the range complete (a quiet chunk on a derived-boundary tail converts the remainder to a hole record, because MIN over stored rows cannot walk through quiet space). Both SKUs, both the SQL Server and Azure SQL DB arms. +- **Multi-incident alerts render as labeled, self-contained units instead of one flat fact list** ([#2108], reported with the diagnosis by gotqn) - when one alert covered several deadlock fingerprints, the Teams card serialized every victim's fields first and every fingerprint's fields after, in one undifferentiated facts[] block - there was no way to tell which victim belonged to which fingerprint, and downstream automation could not split the card into per-incident work items. Two changes, one per layer where the association was lost: each fingerprinted deadlock is now ONE self-contained item (its Database, Victim SQL, Processes, Dedup Key, Involved Objects, and occurrence count together, headed "Deadlock N of M"), with the standalone victim items kept only for deadlocks the fingerprint cannot see (no parseable lock objects - they would otherwise vanish); and the Teams payload gives every fields-carrying item its OWN sections[] entry, titled by the item's heading and carrying only that item's facts, on every alert type - advice prose and remediation-T-SQL hints stay folded into the lead section, because they are commentary on the whole alert. One compatibility note for payload consumers: automation keyed specifically on sections[0].facts[] containing every fact should iterate sections[] instead. +- **Every database-scoped alert exposes a discrete Database fact** ([#2109], also gotqn) - only Blocking Detected and Long-Running Query carried { "name": "Database" } in their webhook facts; Deadlocks named databases only inside the involved-object strings, Version Store (PVS) only in the item heading, and Database State and the two AG database alerts (Suspended / Sync Fell Behind) only in prose - so routing or tagging by database meant parsing display text. Now: deadlock items and incidents carry the distinct databases parsed from the graph's own process list (comma-separated when a deadlock spans databases), PVS items lead with the database, Database State fires with a structured context (Database / Current State / Expected State), and the AG database alerts carry Database / Availability Group / Replica (plus Suspend Reason where it applies) through one shared builder in both SKUs, so the fact names cannot drift. All additive - no existing fact changes. +- **Query Store backfill yields to the live path on contended replicas** ([#2111], found validating #2102 on the prod monitor fleet) - the hour-chunked catch-up recovered most wedged databases immediately, but servers where the backfill worker ran a slice every tick alongside the live sweep stayed in a failure churn: both paths scan the same QS internal tables, the replicas are often MAXDOP-1, and the live query that normally finishes in seconds died at the command timeout behind the slice's scan - recovery-phase contention, self-inflicted, and on some servers the slices themselves timed out so their holes never drained. The backfill worker now skips a server's slice whenever that server's live query_store collection failed within the last 10 minutes (two poll cycles - "failing NOW"), in both SKUs, judged by one shared policy so the workers cannot drift. This is the class doc's own contract - backfill can be slow forever without delaying collection - enforced at the moment it matters: the hole waits, live recovers, backfill resumes. The signal is server-grain on purpose (any database's live failure vouches for the whole replica being contended) and in-memory on purpose (a restart forgetting it costs one slice racing one cycle, once). +- **Version stamps are single-sourced from ``** ([#2113], reported by SalmanRajwani) - the 3.4.0 release bumped `` in each app project but left the hand-pinned `AssemblyVersion` / `FileVersion` / `InformationalVersion` at 3.3.0, so the 3.4.0 packages installed binaries whose FILE metadata reports 3.3.0.0. The code was genuinely 3.4.0, but the lie was not cosmetic: the in-app update check compares the entry assembly's version (the stale pin) against the latest release tag, so a user already ON 3.4.0 would be told an update is available forever. Four hand-maintained copies of one fact is a release-day trap. The three derived properties are now deleted and derive from `` at build time (with the CI source-revision suffix suppressed so InformationalVersion stays a clean semver); a release bump is now ONE line per project. +- **Lite no longer crashes with an uncatchable stack overflow on Queries > Query Store by Duration** ([#2114], diagnosed nearly end-to-end by SalmanRajwani - WER excerpt, module analysis, and the exact XAML candidate) - #1980 ported the Query Store grid's inline View Plan button into Lite still carrying the Darling Viewer's `DarkButton` style key, which Lite never defines. A missing StaticResource inside a DataGrid cell template is not a cosmetic miss: realizing the template throws `XamlParseException` during measure, WPF re-attempts realization on every layout pass, and the recursion kills the process with `0xc00000fd` - uncatchable, unloggable, the moment the grid renders. The button now uses Lite's default chrome, and a new hygiene test scans both apps' XAML trees so a StaticResource key referenced in one app but defined only in the other can never ship again. +- **Upgrading a 3.3.0-era Darling store to 3.4.0 no longer fails the migration ladder** ([#2119], field report by ghauan on #2105) - migration rung 51 is assembled at runtime from the live view generator, which since #2069 emits the `query_plan_gz` column that rung 54 adds - so a store at or below V50 replayed rung 51 referencing a column three rungs before it existed, the ladder halted with `42703`, and the service stopped. Nothing was harmed (each rung runs in its own transaction) but the upgrade could not complete. Rung 51 now pre-adds the column with V54's own idempotent ALTERs (rung 54's copy no-ops after it), the retry just works on an already-failed store, and new ladder pins make the generated-rung replay hazard - a later column teaching a generator new SQL that an earlier rung re-emits - fail in CI instead of on an operator's store. +- **The upgrade path itself is now release-gated** ([#2119] follow-up) - the 3.4.0 ladder failure escaped because every pre-release check ran either a fresh store (full generator, no ladder) or the dogfood box (which walks each rung in the era it ships); no check pointed new binaries at a store a RELEASED build had made, the one path that replays old rungs and the one path users take. Two gates now: a committed fixture of the previous release's ladder exactly as that release resolved it (generators frozen at the tag), which `MigrationUpgradeLadderLiveTests` builds on scratch Postgres and climbs with the current ladder on every CI run - verified two-sided at birth (fails with the exact field 42703 on the pre-fix build, climbs V39->V54 clean on the fixed one, including from a mid-failure retry) - and a release-cut step that boots the previous release's container image and then the candidate on the SAME volume (docs/releasing.md). +- **Store Disk Pressure no longer re-notifies every cooldown while free space sits at an unchanged level** ([#2101], reported with the diagnosis by gotqn) - the self-alert re-fired on the plain cooldown, so a store volume parked at 7.3% free produced an identical CRITICAL notification every ~15 minutes for as long as the condition stood - one static condition, dozens of notifications. It now runs behind the same worsening gate the target-server volume alert has had since the #754 follow-up: fire once on entry, re-fire only when free% drops at least a full point below the last-alerted level (still cooldown-limited), one resolution row on recovery - which also clears the watermark, so a volume that oscillates around the threshold cannot go permanently silent. Deliberately NOT applied to the state-only self-alerts (Collection Stopped, Agent Not Running, Capture Down): those have no level to worsen, and their per-cooldown "still broken" reminder is wanted. The reporter's second ask - exposing the hardcoded self-alert thresholds as store-backed settings - is real but is its own design decision (control-plane knobs, Viewer rows, MCP settings groups) and stays tracked on the issue. +- **Compression Job Stuck no longer false-alarms on a job it caught mid-run** - measured live on TimescaleDB 2.x: from the moment the scheduler picks up a due job until its run completes, `job_stats.next_start` reads `-infinity` with `job_status = 'Running'`, and the real next start is only computed at completion. The detector's first arm treated `-infinity` unconditionally as "the scheduler will never run it again", so any healthy compression run the self-alert check happened to sample got flagged as stuck, alerted, and "self-healed" with a pointless re-arm - the transient stuck-then-self-healed alert pairs the field has been shrugging off were this false positive, and the stuck-detector live test's CI flake was the same race (it re-arms with `next_start => now()` and then read a single snapshot while the run it had just triggered was still executing). A RUNNING job's `-infinity` now defers to the elapsed-bound arm, which is what actually distinguishes a hung run from a healthy one - a genuinely dead job (`-infinity`, not running) still alerts exactly as before, and a hung run still trips the bound. +- **The Job History tab speaks display names** ([#2126], asked by ghauan) - both the Server filter dropdown and the Server column showed the raw collected server name while every other tab shows the operator's alias, so a fleet navigated by aliases turned into a memory quiz on exactly the tab an operator visits during an incident. Both readers (job history and the Agent status header) now resolve through the servers registry - the alias when one exists, the raw name otherwise - so the filter, the column, the per-column filter popup, and the CSV export all speak the same names as the rest of the viewer, and the Agent roll-up sorts by them. Lite's Job History tab had the same gap through a different mechanism (review catch): Lite's display-name concept lives at the CONFIG layer, not in DuckDB (the stored servers.display_name column is unpopulated by design), so the shell now passes a server_id-to-alias snapshot into the tab Overview-style and rows swap in the alias on every refresh - a server no longer in config keeps its raw collected name, the durable-record case. +- **The long-query completion XE session actually gets created now** ([#2129], from ghauan's field report on #2061 - they enabled the collector on two servers and the Long Queries tab stayed empty forever) - the session DDL SET a customizable attribute `collect_object_name` on `sqlserver.rpc_completed`, and no such attribute exists on that event on ANY version (it belongs to `sp_statement_completed`) - `object_name` is one of rpc_completed's DEFAULT data fields, collected with no SET at all. So the CREATE failed on every server, the session never existed, and the reconcile's follow-up START surfaced as the confusing second error ('Cannot alter the event session... does not exist'). Never caught in dogfood because the collector ships OFF by design, and the DDL test pin asserted the wrong claim, so CI enforced the bug. The SET is gone (the reader already shreds the default field generically - no reader or table change), and the pin now asserts the attribute is ABSENT, with the story attached. Anyone who flipped the collector on before this fix: it starts working on the next reconcile tick after upgrading, no re-toggle needed. +- **`--collapse-legacy-slices` narrows its slice instead of dying when a day does not fit the statement timeout** ([#2105] round three, ghauan once more - with the decompression rail lifted, the run made it ~15 minutes in and died at the NEW wall: a day-wide stage aggregation on a store carrying 60k split intervals blows through the 15-minute per-statement timeout, and the operator got the same bare stream exception) - the verb's fixed day-per-slice loop now runs the same adaptive schedule the Query Store backfill worker shipped this week (`AdaptiveSpan`, 24h base): a failed slice halves the window and retries the SAME start (announced with a [RETRY] line naming the error, so narrowing reads as progress rather than a hang), a completed slice resets to full width, and only a slice that fails at the ~22-minute floor gives up to the existing idempotent re-run message. Healthy stores still repair in a handful of day-wide slices - the narrowing costs nothing until a slice actually fails. +- **Query Store collection no longer has a fixed cost that big catalogs cannot pay** ([#2133], the actual root cause under the whole catch-up saga - #2102's death spiral, #2111's yield, and #2125's adaptive shrink were all mitigating it) - the collector joined its slice aggregate straight into the query_store_plan/query/text catalog TVFs, handing the optimizer nothing but fixed-guess cardinalities, and the plan it picked re-materialized a TVF per probe: on an 82k-plan catalog that was a fixed 30-second-plus cost that NO catch-up window width could reduce - which is exactly why the fleet's big databases (echo, oak, Surge, spruce, insa...) pinned at the 15-minute shrink floor and never converged while their smaller neighbors on the same servers stayed current. Bisected live: the aggregate alone ran in 81 ms and each TVF scanned bare in ~300 ms, yet aggregate-JOIN-plan could not finish in 30 s, hinted or not. The payload now STAGES the aggregate in a temp table and joins FROM it - real row counts instead of guesses, each TVF scanned exactly once, sp_QuickieStore's architecture for the same reason - and the old LOOP JOIN hint is gone for good (looping from the temp into the TVFs is the same per-probe re-materialization by another name). The interval pre-filter also resolves ids from the tiny interval catalog now instead of scanning runtime_stats itself (20 ms vs 426 ms, same id set). Measured end to end on the wedged field store: the full 55-column batch with plan capture completed a one-hour backlog in 21.3 s where the old shape never finished inside 60; the staged core is 524 ms. Same batch = one result set, TOP WITH TIES / derived-watermark / byte-budget semantics unchanged, both SKUs, both engine arms, live and backfill. +- **`--collapse-legacy-slices` runs ~40% fewer window scans and narrates its progress** (#2105 operator feedback - ghauan's successful run 'took some time', all of it against a silent console) - each slice bracketed its work with two window-wide COUNT(*) scans purely to compute the removed-row figure, and on the catch-up-bloated stores this verb exists for, one such scan measures ~12 seconds (hash-aggregate spill to temp disk plus a backward index scan over the uncompressed hot chunk) - paid twice more per slice for bookkeeping. The removed count now derives from the DELETE and INSERT statements' own affected-row counts (deleted minus reinserted IS the net removal, same number by construction), and every completed slice prints an [OK] line with its window, removal count, and percent-of-span - so a long backlog reads as visible progress instead of a hang. + ## [3.4.0] - 2026-08-06 ### Important @@ -204,6 +341,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **MCP no longer loses the monitored-server registry over an SMTP password it never reads** - caught on the dogfood box: startup logged `42501: permission denied for table config_notification` followed by "MCP could not read the monitored-server registry - live plan fetch will use darling.json". Two defects met. The least-privilege carve is correct and unchanged: `DarlingManagedRoles` deliberately REVOKEs table-wide SELECT on `config_notification` from BOTH `viewer` and `mcp` and re-grants only the non-secret columns, so the SMTP password and username and the Teams/Slack/generic/PagerDuty bearer URLs stay unreadable. The MCP host was simply asking for the whole row - and **a column-level denial answers for the TABLE**. **The second defect is why one password cost the registry**: every section of `LoadViewAsync` shares ONE try/catch, so the failed notification read discarded the four reads that had already SUCCEEDED, and MCP fell back to `darling.json` for live plan fetches - a silent capability loss whose logged cause named a table MCP does not use. Fixed by making the reader agree with the boundary rather than by widening the grant: a caller that does not DELIVER alerts skips the notification row, and the MCP surface references neither `Smtp` nor `Webhooks` anywhere. Pinned three ways so the two cannot drift apart again - the host passes `includeNotification: false`, the MCP surface is asserted to use neither type, and the notification SELECT is asserted to still name carved secret columns, so narrowing that SELECT fails the guard and tells whoever did it that the skip became unnecessary. + +- **A baseline query that runs out of time now SAYS so instead of reading as a broken connection** - found on the dogfood box. The service logged `Failed to compute baselines for io_latency: Exception while reading from stream`, which points an investigation at the network; the store's own log, in a different file, showed `ERROR: canceling statement due to user request` **267 ms earlier**. Npgsql enforces its command timeout by CANCELLING the statement, so the server reports the cancellation and the client is left holding a torn stream - the real cause was a query outgrowing its deadline on a store grown to 184 GB, and establishing that took correlating two logs by timestamp. Now classified **structurally** (`57014` query_canceled, or a `TimeoutException` anywhere in the chain) rather than by message text - which is the very thing that was ambiguous - and the timeout arm names the consequence the old line left implicit: the metric has NO baseline that pass, so its anomaly detection is silent while the collected data looks perfectly healthy. Both failure paths now report elapsed seconds, so "it nearly made it" and "it never had a chance" are distinguishable. **The other direction is pinned too**: a genuine connection fault must keep saying so, because labelling one a timeout is the identical defect aimed the other way and would send the next investigation at the query instead of the network. + +- **A recompiled plan's CPU is no longer discarded, so per-query attribution stops under-reporting a plan-churning instance** ([#2235]) - `query_stats` keys its deltas on the full `dm_exec_query_stats` row identity, which includes `plan_handle`, and `plan_handle` changes on every recompile. So a statement whose dynamic SQL mints a new plan constantly presents a **new key on nearly every sighting**, and the first sighting of a key reports 0. Field measurement on a plan-churning readable secondary: a query Datadog measured at **43.1% / 43.9% / 42.5%** of an 8-vCPU box across three windows read through these collectors as **18 executions and 2,824 ms over 168 hours**, and the top-25 procedures accounted for ~49M ms against roughly **498M core-ms** available - about a tenth of the instance. **The worse half is that it was invisible.** The calculator already had an honest path for cache churn: the counter-reset branch reports `interval = 0` precisely so a reader can tell a fabricated zero from an idle one, the invariant [#2234] rests on. But that branch needs the SAME key to reappear with a lower value, and a recompile never does - it arrives under a new key and takes the baseline path, so the loss was indistinguishable from a query that simply had not run. Same class of harm as the 300-second gap policy [#2233] replaced: it did not merely lose data, it invented quiet. **The fix uses a discriminator that was already being collected** - `creation_time` has been in this collector's SELECT all along - so there is no new collection and no schema change: the row now also carries how long ago its plan was compiled, and when a series demonstrably began since the previous pass its whole counter accrued inside that window, making the delta the full value rather than 0. Sent as an AGE rather than `creation_time` itself because a DMV `creation_time` is in the monitored server's local time while collection times are UTC - comparing them client-side is a timezone bug on every server that is not UTC, so `DATEDIFF` is evaluated where both clocks are the same one. **The ceiling is stated rather than left to be discovered**: a plan compiled AND evicted between two passes never appears in the DMV at all, so no keying scheme can recover it. **Why `plan_handle` was not simply dropped from the key**, since that is the obvious first move and it is wrong: parameter-sensitive variants of one statement coexist in cache, so a statement-only key would collide across several live rows in a single pass and each delta would be computed against whichever row happened to be processed last - the multi-statement cross-contamination [#2012] fixed, reintroduced from the other direction. The new entry point is default-implemented on the interface so the other forty-odd delta call sites and every existing implementer are byte-identical, and all eight of the row's counters take the rule together because crediting only some would make one row's metrics disagree about how much work it did. + - **`--encrypt-password` and `--configure-network` explain themselves on a non-interactive console instead of silently doing nothing** ([#2097], reported with the diagnosis by gotqn) - in the PowerShell ISE, remote/PSRemoting sessions, and some integrated terminals, stdin is not an interactive console: `ReadLine()` returns null immediately, and the prompts and errors these verbs wrote to STDERR are not surfaced at all in those hosts - so the very first setup step (encrypting the SQL password) read as a hung or broken tool. Both verbs now write an actionable explanation to STDOUT (the one stream every host shows) naming the cause and the ways forward - run from a real console, or pipe the value: `Read-Host -Prompt 'password' | PerformanceMonitor.Darling.Service.exe --encrypt-password` - and the wizard now tells EOF apart from an explicit quit (guidance + exit 1 vs the quiet \"No changes made.\" + exit 0). - **Linux: `add_servers` accepts `env:`/`file:` secret references, unblocking onboarding on compose deployments** ([#2087], found during the 3.4.0 release smoke) - the MCP/web onboarding path refused every SQL-auth password off-Windows ("Storing a SQL-auth password requires Windows (DPAPI)"), which dead-ended the DESIGNED way to add servers to a running Linux deployment: the control plane is store-authoritative after first seed, so darling.json edits do not add servers. A reference is a pointer, not a secret - the secret stays in the mounted file or environment variable, the #1804 contract - so references are now stored verbatim in the encrypted-password slot and resolved at connect time (a reference can never be confused with a DPAPI blob). Literal passwords still require Windows, and the refusal now says exactly what to pass instead. @@ -2598,3 +2741,83 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2090]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2090 [#2093]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2093 [#2097]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2097 +[#2101]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2101 +[#2105]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2105 +[#2107]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2107 +[#2102]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2102 +[#2108]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2108 +[#2109]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2109 +[#2111]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2111 +[#2113]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2113 +[#2114]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2114 +[#2117]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2117 +[#2119]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2119 +[#2126]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2126 +[#2129]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2129 +[#2133]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2133 +[#2136]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2136 +[#2143]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2143 +[#2148]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2148 +[#2154]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2154 +[#2157]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2157 +[#2164]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2164 +[#2210]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2210 +[#2188]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2188 +[#2191]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2191 +[#2171]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2171 +[#2170]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2170 +[#2169]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2169 +[#2166]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2166 +[#2167]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2167 +[#2213]: https://github.com/erikdarlingdata/PerformanceMonitor/pull/2213 +[#2216]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2216 +[#2138]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2138 +[#2190]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2190 +[#2186]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2186 +[#2185]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2185 +[#2258]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2258 +[#2219]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2219 +[#2280]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2280 +[#2277]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2277 +[#2279]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2279 +[#2273]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2273 +[#2220]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2220 +[#2228]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2228 +[#2218]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2218 +[#2266]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2266 +[#2235]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2235 +[#2165]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2165 +[#2255]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2255 +[#2159]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2159 +[#2197]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2197 +[#2203]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2203 +[#2187]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2187 +[#2189]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2189 +[#2184]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2184 +[#2254]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2254 +[#2252]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2252 +[#2256]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2256 +[#2158]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2158 +[#2246]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2246 +[#2319]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2319 +[#2331]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2331 +[#2181]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2181 +[#2317]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2317 +[#2320]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2320 +[#2316]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2316 +[#2324]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2324 +[#2300]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2300 +[#2312]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2312 +[#2306]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2306 +[#2302]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2302 +[#2296]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2296 +[#2299]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2299 +[#2294]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2294 +[#2298]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2298 +[#2293]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2293 +[#2233]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2233 +[#2234]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2234 +[#2150]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2150 +[#2201]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2201 +[#2205]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2205 +[#2195]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/2195 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c20d71679..189ede1c7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -230,7 +230,35 @@ ORDER BY OPTION(RECOMPILE); ``` -The full T-SQL style guide is in [CLAUDE.md](CLAUDE.md). +A few more rules that come up in review, and the reasoning behind the ones that are +not obvious: + +- **Unicode literals**: prefix with `N` (`N'ONLINE'`, not `'ONLINE'`). +- **`ON` continues its `JOIN` at two spaces**, so the join graph reads down the left edge. +- **`AND` / `OR` align their predicates** (`AND d.state_desc = N'ONLINE'`), so a `WHERE` + clause reads as a list rather than as prose. +- **`GROUP BY` / `ORDER BY` put each term on its own indented line**, so adding one is a + one-line diff. +- **Never suggest missing-index DMV recommendations.** `sys.dm_db_missing_index_*` output + is not used in this project and changes proposing it will not be accepted. +- **No full-text search.** + +Collector queries specifically: + +- **`OPTION(RECOMPILE)` on collector queries.** These run with parameters whose selectivity + varies enormously between a first-run catch-up window and a steady-state minute, and a plan + cached from one is wrong for the other. A statement added to an existing batch needs its own + hint — one on a neighbouring statement does not cover it. +- **Comments explain WHY, at length.** This codebase's comments carry measurements, issue + numbers, and the failure the line prevents. A comment restating the code is noise; one + recording "this threshold was 300s and the fleet's median gap is 299s, so it discarded half + of every sweep" is what stops the next person undoing it. + +Darling's PostgreSQL store (not T-SQL — the Darling service stores to PostgreSQL/TimescaleDB): + +- **Schema-qualify every object in a migration** (`collect.*`, `config.*`). The migrate session's + `search_path` resolves bare names to a different schema, so an unqualified `CREATE` or `ALTER` + can land an object in the wrong one silently. ### C# Style diff --git a/Darling/Darling.Tests/AddServerVerbTests.cs b/Darling/Darling.Tests/AddServerVerbTests.cs new file mode 100644 index 000000000..bb8406325 --- /dev/null +++ b/Darling/Darling.Tests/AddServerVerbTests.cs @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2256: the --add-server verb, which exists because a headless host had no supported way to register a +/// monitored server at all. +/// +/// Why it was impossible. darling.json seeds the registry only while it is empty, so file +/// edits after the first start are ignored (#2254); the web surface keeps add_servers off +/// /api/read/* deliberately because it writes; and there was no CLI verb. The field report ran the +/// service on Windows Server 2012, which cannot run the Viewer, so the only remaining routes were a GUI on +/// another machine or standing up an MCP client. +/// +/// What is pinned here is the part that has no store in it: verb recognition, the stdin contract, +/// and the result formatting plus its exit-code policy. The registration itself is +/// DarlingMcpServerAdminTools.AddServers, already covered by its own tests — the verb deliberately adds +/// no second implementation of validation, dedupe, probing, encryption or identity computation. +/// +public sealed class AddServerVerbTests +{ + [Theory] + [InlineData("--add-server")] + [InlineData("--add-servers")] + [InlineData("--ADD-SERVER")] + public void BothSpellingsAreRecognized_CaseInsensitively(string arg) + { + Assert.True(DarlingCliCommands.IsAddServerVerb(arg)); + + /* And the classifier must dispatch it rather than fall through to starting the host — the #1581 + incident was a verb that reached a real startup and spawned a second instance. */ + Assert.Equal(StartupAction.RunKnownVerb, DarlingCliCommands.ClassifyStartupArgs(new[] { arg })); + } + + [Fact] + public void TheVerbIsDiscoverable_FromHelp() + { + Assert.Contains("--add-server", DarlingCliCommands.UsageText(), StringComparison.Ordinal); + } + + /// + /// Empty stdin must EXPLAIN itself on stdout and change nothing. + /// + /// Stdout rather than stderr is the [#2097] lesson: in the PowerShell ISE, remoting sessions and some + /// integrated terminals stderr is not surfaced, so a verb that writes only there reads as hung — which is + /// exactly how the first setup step was reported as broken. + /// + [Fact] + public async Task EmptyStdin_ExplainsItselfOnStdout_AndChangesNothing() + { + var output = new StringWriter(); + var error = new StringWriter(); + + /* No config path and no store: it must return before touching either, which is itself the assertion — + a store connection here would throw rather than return 1. */ + var exit = await DarlingCliCommands.AddServerAsync( + configPath: null, input: new StringReader(string.Empty), output: output, error: error, + cancellationToken: CancellationToken.None); + + Assert.Equal(1, exit); + Assert.Equal(string.Empty, error.ToString()); + + var text = output.ToString(); + Assert.Contains("stdin", text, StringComparison.OrdinalIgnoreCase); + Assert.Contains("--add-server", text, StringComparison.Ordinal); + /* The reason the password is not an argument, stated where the operator will look for it. */ + Assert.Contains("process list", text, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task WhitespaceOnlyStdin_IsTreatedAsEmpty() + { + var output = new StringWriter(); + var exit = await DarlingCliCommands.AddServerAsync( + null, new StringReader(" \r\n "), output, new StringWriter(), CancellationToken.None); + + Assert.Equal(1, exit); + Assert.Contains("stdin", output.ToString(), StringComparison.OrdinalIgnoreCase); + } + + /// A server that landed: the ADDED line, and the sentence that saves the operator a restart they do + /// not need — the registry write bumps config_version, which the worker polls every sweep. + [Fact] + public void AnAddedServer_ReportsItAndSaysNoRestartIsNeeded() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome( + """{"added":1,"skipped":0,"failed":0,"results":[{"server":"sql01","status":"added","detail":"SQL major version 16"}]}"""); + + Assert.Equal(0, exit); + Assert.Contains(lines, l => l.Contains("[ADDED] sql01", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("SQL major version 16", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("1 added, 0 already registered, 0 failed.", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("no restart is needed", StringComparison.OrdinalIgnoreCase)); + } + + /// Re-running the same file is idempotent, not a failure — so a batch of pure duplicates exits 0 + /// and does NOT claim a restart is pending, because nothing changed. + [Fact] + public void PureDuplicates_ExitZero_AndPromiseNoReload() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome( + """{"added":0,"skipped":2,"failed":0,"results":[{"server":"a","status":"duplicate","detail":"already registered"},{"server":"b","status":"duplicate","detail":"already registered"}]}"""); + + Assert.Equal(0, exit); + Assert.Equal(2, lines.Count(l => l.Contains("[SKIP]", StringComparison.Ordinal))); + Assert.DoesNotContain(lines, l => l.Contains("no restart", StringComparison.OrdinalIgnoreCase)); + } + + /// Anything that failed is a non-zero exit, even alongside a success — the verb is usable as a + /// deployment gate, and a partial batch must not read as clean. + [Fact] + public void AnyFailure_ExitsNonZero_EvenBesideASuccess() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome( + """{"added":1,"skipped":0,"failed":1,"results":[{"server":"good","status":"added","detail":"ok"},{"server":"bad","status":"connection_failed","detail":"login failed"}]}"""); + + Assert.Equal(1, exit); + Assert.Contains(lines, l => l.Contains("[FAIL] bad", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("login failed", StringComparison.Ordinal)); + } + + /// Nothing landed at all — an empty array, or every entry rejected — must not report success to a + /// script. A verb that changed nothing and exits 0 is the failure mode this policy exists for. + [Theory] + [InlineData("""{"added":0,"skipped":0,"failed":0,"results":[]}""")] + [InlineData("""{"added":0,"skipped":0,"failed":1,"results":[{"server":"x","status":"invalid","detail":"unsupported auth"}]}""")] + public void NothingLanded_ExitsNonZero(string json) + { + var (_, exit) = DarlingCliCommands.FormatAddServerOutcome(json); + + Assert.Equal(1, exit); + } + + /// The whole-payload rejection shape carries no results array (the tool returns it without + /// opening the store when the JSON is not an array at all), so the formatter must render its message rather + /// than throw on the missing property. + [Fact] + public void AWholePayloadRejection_IsRenderedNotThrown() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome( + """{"status":"invalid","message":"servers_json must be a JSON array"}"""); + + Assert.Equal(1, exit); + Assert.Contains(lines, l => l.Contains("must be a JSON array", StringComparison.Ordinal)); + } + + /// + /// A store failure AFTER the request parsed does not arrive as JSON at all: AddServersAsync's + /// catch-all returns McpHelpers.FormatError, which is plain text. That text IS the message the + /// operator needs, so it must be surfaced verbatim rather than buried under a "could not parse" wrapper — + /// which is what happened before, precisely when the verb is being used as a deployment gate. + /// + [Fact] + public void APlainTextStoreError_IsSurfacedVerbatim_NotWrapped() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome( + "Error during add_servers: 57P01: terminating connection due to administrator command"); + + Assert.Equal(1, exit); + Assert.Contains(lines, l => l.Contains("57P01", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains("Could not parse", StringComparison.Ordinal)); + } + + /// Something that LOOKED like JSON and was not still says so, and still shows the payload — the + /// two cases are told apart by shape so neither hides the other. + [Fact] + public void MalformedJson_SaysSo_AndShowsThePayload() + { + var (lines, exit) = DarlingCliCommands.FormatAddServerOutcome("{\"added\":1, oops"); + + Assert.Equal(1, exit); + Assert.Contains(lines, l => l.Contains("Could not parse", StringComparison.Ordinal)); + Assert.Contains(lines, l => l.Contains("oops", StringComparison.Ordinal)); + } +} diff --git a/Darling/Darling.Tests/AlertEngineTests.cs b/Darling/Darling.Tests/AlertEngineTests.cs index 84138166b..c8c12353a 100644 --- a/Darling/Darling.Tests/AlertEngineTests.cs +++ b/Darling/Darling.Tests/AlertEngineTests.cs @@ -8,7 +8,9 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using PerformanceMonitor.Alerting; @@ -46,6 +48,7 @@ test switches on exactly the check it pins (a disabled check must not even fetch public bool FailedJobEnabled { get; set; } public bool PvsEnabled { get; set; } public bool DatabaseStateEnabled { get; set; } + public bool ForcePlanFailureEnabled { get; set; } = true; public int CpuThresholdPercent { get; set; } = 80; public int BlockingCountThreshold { get; set; } = 1; /* #1839: 0 = off, the shipped default — a test must opt in for the wait gate to run at all. */ @@ -62,6 +65,12 @@ test switches on exactly the check it pins (a disabled check must not even fetch public int TempDbSpaceThresholdPercent { get; set; } = 80; public int LowDiskThresholdPercent { get; set; } = 10; public int LowDiskThresholdGb { get; set; } = 5; + /* #2107: the previously-hardcoded knobs, at their shipped defaults. */ + public int DiskCriticalFreePercent { get; set; } = 3; + public int DiskCriticalFreeGb { get; set; } = 2; + public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int CollectionStaleMinutes { get; set; } = 30; + public int CollectionFailureThreshold { get; set; } = 10; /* #1984: DarlingConfig defaults (40% / 1 GB); enable stays the class's opt-in OFF. */ public int PvsThresholdPercent { get; set; } = 40; public int PvsFloorGb { get; set; } = 1; @@ -152,6 +161,18 @@ public Task> GetDatabaseStatesAsync(string serverKey, Ca DatabaseStateFetches++; return Task.FromResult(new List(DatabaseStates)); } + + /* #2157: plantable rows + a fetch counter, mirroring the database-state seam above so the + forced-plan alert's tests can assert both what fired and that the read happened. */ + public List ForcePlanFailures { get; } = new(); + + public int ForcePlanFetches { get; private set; } + + public Task> GetForcePlanFailuresAsync(string serverKey, CancellationToken cancellationToken = default) + { + ForcePlanFetches++; + return Task.FromResult(new List(ForcePlanFailures)); + } } private sealed class FakeStateStore : IAlertStateStore @@ -180,6 +201,57 @@ public Task SaveFailedJobWatermarkAsync(string serverKey, DateTime watermark) SavedFailedJob.Add((serverKey, watermark)); return Task.CompletedTask; } + + /* #2216: real per-fingerprint occurrence state, so the engine tests can assert what the accumulator + wrote AND seed a prior incident to accumulate against. */ + public Dictionary<(string Key, string Metric), Dictionary> Occurrences { get; } = new(); + public List<(string Key, string Metric, int Count)> SavedOccurrences { get; } = new(); + + public Task> LoadIncidentOccurrencesAsync(string serverKey, string metricName) => + Task.FromResult>( + Occurrences.TryGetValue((serverKey, metricName), out var states) + ? states + : new Dictionary(StringComparer.Ordinal)); + + public Task SaveIncidentOccurrencesAsync(string serverKey, string metricName, IReadOnlyDictionary states) + { + /* Replace-the-set, exactly like both real stores: whatever arrives IS the metric's state, so an + empty map clears it. A fake that merged instead would hide the falling-edge bug class. */ + Occurrences[(serverKey, metricName)] = new Dictionary(StringComparer.Ordinal); + foreach (var entry in states) + { + Occurrences[(serverKey, metricName)][entry.Key] = entry.Value; + } + SavedOccurrences.Add((serverKey, metricName, states.Count)); + return Task.CompletedTask; + } + + /* #2166 */ + public List<(string Server, string Db, string State)> DatabaseStateAlerted { get; } = new(); + + public Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState) + { + DatabaseStateAlerted.Add((serverKey, databaseName, effectiveState)); + Memory[databaseName] = effectiveState; + return Task.CompletedTask; + } + + public List<(string Server, string Db)> DatabaseStateCleared { get; } = new(); + + /// + /// What the store would HOLD, not merely which calls arrived. The engine's edge trigger is a + /// round trip — write on fire, read back through the adapter next cycle — and a stub that only + /// counts calls cannot fail when one direction of that trip is missing. A test can feed this + /// back in as LastAlertedState to exercise the real composition. + /// + public Dictionary Memory { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName) + { + DatabaseStateCleared.Add((serverKey, databaseName)); + Memory.Remove(databaseName); + return Task.CompletedTask; + } } private sealed class RecordingDeliverer : IAlertDeliverer @@ -734,6 +806,151 @@ public async Task Deadlock_WatermarkSeededFromStore_PreventsThePostRestartRefire Assert.Empty(h.Deliverer.Outcomes); } + [Fact] + public async Task Deadlock_TotalOccurrences_AccumulateAcrossThrottledDeliveries() + { + /* #2216 end to end. Delivery one carries one deadlock; two more happen before the next eligible + delivery. The window gauge reads 3 and the monotonic total reads 3 — the number a consumer that + missed the middle of the incident needs, and the number the gauge alone cannot give it (a reading + of 3 could equally mean "three new" or "one new, two aged out"). */ + var h = new Harness(); + h.Settings.DeadlockEnabled = true; + var engine = h.Build(); + + h.Adapter.Deadlocks.Add(DeadlockRow()); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var first = Assert.Single(h.Deliverer.Outcomes); + var firstIncident = Assert.Single(first.Context!.Incidents!); + Assert.Equal(1, firstIncident.OccurrenceCount); + Assert.Equal(1L, firstIncident.TotalOccurrences); + Assert.Equal(h.Now, firstIncident.IncidentStartedUtc); + + /* Same fingerprint (identical graphs), so this is the same incident continuing. */ + h.Adapter.Deadlocks.Add(DeadlockRow()); + h.Adapter.Deadlocks.Add(DeadlockRow()); + var openedAt = h.Now; + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(2, h.Deliverer.Outcomes.Count); + var second = h.Deliverer.Outcomes[1]; + var secondIncident = Assert.Single(second.Context!.Incidents!); + Assert.Equal(3, secondIncident.OccurrenceCount); + Assert.Equal(3L, secondIncident.TotalOccurrences); + + /* The start time did NOT move — that is how the consumer tells a continuation from a new incident + that happens to read 3. */ + Assert.Equal(openedAt, secondIncident.IncidentStartedUtc); + + var persisted = h.StateStore.Occurrences[(Key, AlertEngine.DeadlockWatermarkMetric)]; + Assert.Equal(3L, persisted[secondIncident.DedupKey].TotalOccurrences); + } + + [Fact] + public async Task Deadlock_OccurrencesAreObservedOnSweepsThatDeliverNothing() + { + /* PR #2221's review: with the accumulation inside the Fire branch, no sweep between two deliveries + observed anything, so an event the window retired during the cooldown cancelled an arrival and the + arrival was never counted. The observation now runs on every sweep that fetched rows. */ + var h = new Harness(); + h.Settings.DeadlockEnabled = true; + var engine = h.Build(); + + h.Adapter.Deadlocks.Add(DeadlockRow()); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* Two more deadlocks INSIDE the cooldown — no delivery, but the count must still be observed. */ + h.Adapter.Deadlocks.Add(DeadlockRow()); + h.Adapter.Deadlocks.Add(DeadlockRow()); + h.Now = h.Now.AddMinutes(1); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); /* the cooldown suppressed the delivery */ + + var persisted = h.StateStore.Occurrences[(Key, AlertEngine.DeadlockWatermarkMetric)]; + Assert.Equal(3L, Assert.Single(persisted).Value.TotalOccurrences); + } + + [Fact] + public async Task Deadlock_OccurrenceStateSeededFromStore_ContinuesTheIncidentAcrossARestart() + { + /* The reason the counter is persisted at all: a total that reset on every service restart would be a + second gauge wearing a total's name. A fresh engine (new in-memory state, as after a restart) + seeded from the store must keep counting the incident it finds there. */ + var discovery = new Harness(); + discovery.Settings.DeadlockEnabled = true; + discovery.Adapter.Deadlocks.Add(DeadlockRow()); + await discovery.Build().EvaluateServerAsync(Harness.Snapshot()); + var dedupKey = Assert.Single(Assert.Single(discovery.Deliverer.Outcomes).Context!.Incidents!).DedupKey; + + var restarted = new Harness(); + restarted.Settings.DeadlockEnabled = true; + var openedAt = restarted.Now.AddMinutes(-20); + restarted.StateStore.Occurrences[(Key, AlertEngine.DeadlockWatermarkMetric)] = + new Dictionary(StringComparer.Ordinal) + { + [dedupKey] = new( + TotalOccurrences: 9, + ObservedWindowCount: 2, + IncidentStartedUtc: openedAt, + LastObservedUtc: restarted.Now.AddMinutes(-5)), + }; + + restarted.Adapter.Deadlocks.Add(DeadlockRow()); + restarted.Adapter.Deadlocks.Add(DeadlockRow()); + restarted.Adapter.Deadlocks.Add(DeadlockRow()); + await restarted.Build().EvaluateServerAsync(Harness.Snapshot()); + + var incident = Assert.Single(Assert.Single(restarted.Deliverer.Outcomes).Context!.Incidents!); + + /* 9 already counted, the window rose from 2 to 3, so one new occurrence: 10. */ + Assert.Equal(10L, incident.TotalOccurrences); + Assert.Equal(openedAt, incident.IncidentStartedUtc); + } + + [Fact] + public async Task Deadlock_FallingEdge_ClearsTheOccurrenceState() + { + /* When the condition clears, the incident is over and its counters go with it — otherwise the next + incident on the same fingerprint reads as a continuation of this one, reporting an undercount + under a start time that points at an incident the user already saw resolve. */ + var h = new Harness(); + h.Settings.DeadlockEnabled = true; + var engine = h.Build(); + + h.Adapter.Deadlocks.Add(DeadlockRow()); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.NotEmpty(h.StateStore.Occurrences[(Key, AlertEngine.DeadlockWatermarkMetric)]); + + h.Adapter.Deadlocks.Clear(); + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Contains(h.Resolutions, r => r.MetricName == "Deadlocks Detected"); + Assert.Empty(h.StateStore.Occurrences[(Key, AlertEngine.DeadlockWatermarkMetric)]); + } + + [Fact] + public async Task Blocking_TotalOccurrences_RideOnTheBlockingIncidentsToo() + { + /* Both count gates go through the same accumulator — the blocking half is not an afterthought, it is + the other half of the reported feature. */ + var h = new Harness(); + h.Settings.BlockingEnabled = true; + var engine = h.Build(); + + h.Adapter.Blocking.Add(BlockingRow(55)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Blocking Detected"); + var incident = Assert.Single(fired.Context!.Incidents!); + Assert.Equal(1L, incident.TotalOccurrences); + Assert.Equal(h.Now, incident.IncidentStartedUtc); + Assert.Contains((Key, AlertEngine.BlockingWatermarkMetric, 1), h.StateStore.SavedOccurrences); + } + [Fact] public async Task Deadlock_WhollyExcludedDatabaseGraphs_DontCount() { @@ -1156,6 +1373,9 @@ public Task GetAnomalousJobsAsync(string serverKey, int mul throw new InvalidOperationException("store down"); public Task> GetDatabaseStatesAsync(string serverKey, CancellationToken cancellationToken = default) => throw new InvalidOperationException("store down"); + + public Task> GetForcePlanFailuresAsync(string serverKey, CancellationToken cancellationToken = default) => + throw new InvalidOperationException("store down"); } [Fact] @@ -1176,6 +1396,405 @@ public async Task StatePerServer_IsIndependent() /* ---------------- database state (baseline deviation) ---------------- */ + [Fact] + public async Task ForcePlanFailure_Disabled_DoesNotFetch() + { + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = false; + h.Adapter.ForcePlanFailures.Add(new ForcePlanFailureInfo { DatabaseName = "Sales", QueryId = 11, PlanId = 22, ForcingType = "MANUAL", FailureReason = "NO_INDEX", FailureDelta = 3, TotalFailures = 3 }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + /* The gate must skip the READ, not just the fire — a disabled alert should cost nothing. */ + Assert.Equal(0, h.Adapter.ForcePlanFetches); + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task ForcePlanFailure_FiresPerPlan_CarryingReasonForcingTypeAndDelta() + { + /* Two plans in the SAME database are two independent conditions — if the alert keyed per server or + per database, the second would be swallowed by the first's cooldown and an operator would never + learn about it. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Adapter.ForcePlanFailures.Add(new ForcePlanFailureInfo { DatabaseName = "Sales", QueryId = 11, PlanId = 22, ForcingType = "MANUAL", FailureReason = "NO_INDEX", FailureDelta = 4, TotalFailures = 9 }); + h.Adapter.ForcePlanFailures.Add(new ForcePlanFailureInfo { DatabaseName = "Sales", QueryId = 33, PlanId = 44, ForcingType = "AUTO", FailureReason = "NO_PLAN", FailureDelta = 1, TotalFailures = 1 }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.All(h.Deliverer.Outcomes, o => Assert.Equal("Forced Plan Failing", o.MetricName)); + /* Warning for every rise — no Critical tier exists yet, on purpose (ForcePlanTokens). */ + Assert.All(h.Deliverer.Outcomes, o => Assert.Equal(PerformanceMonitor.Notifications.AlertSeverityLevel.Warning, o.Severity)); + + var manual = h.Deliverer.Outcomes.Single(o => o.CurrentValue.Contains("plan 22")); + Assert.Contains("NO INDEX", manual.DetailText, StringComparison.Ordinal); + Assert.Contains("MANUAL", manual.DetailText, StringComparison.Ordinal); + /* The delta, not the total, is what says 'happening now'. */ + Assert.Contains("4", manual.DetailText, StringComparison.Ordinal); + + var auto = h.Deliverer.Outcomes.Single(o => o.CurrentValue.Contains("plan 44")); + Assert.Contains("AUTO", auto.DetailText, StringComparison.Ordinal); + } + + [Fact] + public async Task ForcePlanFailure_CooldownSuppressesSecondFire_ThenResolvesWhenTheCounterStops() + { + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Adapter.ForcePlanFailures.Add(new ForcePlanFailureInfo { DatabaseName = "Sales", QueryId = 11, PlanId = 22, ForcingType = "MANUAL", FailureReason = "NO_INDEX", FailureDelta = 2, TotalFailures = 2 }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* Still failing next sweep, inside the cooldown — one alert, not two. */ + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* The adapter stops returning it: the counter stopped rising. That covers unforced, reproducible + again, and query-no-longer-running alike — hence 'no longer failing' rather than 'fixed'. */ + h.Adapter.ForcePlanFailures.Clear(); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + var resolution = Assert.Single(h.Resolutions, r => r.MetricName == "Forced Plan Failing"); + /* The recovery text is read by a human in a toast, an email and a history row, so it must name + the plan the way the firing message did — NOT the internal key. The first version of this + test only asserted the message contained "22", which the leaked key 'forceplan:Sales:11:22' + satisfied, so it passed while operators would have seen gibberish (review catch). */ + Assert.DoesNotContain(ForcePlanTokens.KeyPrefix, resolution.Message, StringComparison.Ordinal); + Assert.Contains("Sales", resolution.Message, StringComparison.Ordinal); + Assert.Contains("query 11", resolution.Message, StringComparison.Ordinal); + Assert.Contains("plan 22", resolution.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ForcePlanFailure_ExcludedDatabase_IsNeverAlerted() + { + /* Parity with every other database-scoped family: the shared exclusion list wins, case-insensitively. + A monitored-but-excluded database must not produce alerts an operator cannot mute per-database. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.ExcludedDatabasesList.Add("sAlEs"); + h.Adapter.ForcePlanFailures.Add(new ForcePlanFailureInfo { DatabaseName = "Sales", QueryId = 11, PlanId = 22, ForcingType = "MANUAL", FailureReason = "NO_INDEX", FailureDelta = 5, TotalFailures = 5 }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task DatabaseState_ChosenState_AlreadyAnnounced_StaysQuiet() + { + /* #2166: the reporter's case — a database parked OFFLINE for a month generated hundreds of + identical alerts. With the state already recorded as announced, a fresh evaluation must be + SILENT even though the deviation is still present and no cooldown is in play. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "OFFLINE" }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task DatabaseState_ChosenState_FirstObservation_FiresAndRecordsIt() + { + /* The transition still alerts — edge-triggered, not silenced — and the state is recorded so the + NEXT evaluation is the quiet one. Recording is what makes the silence survive a restart. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "" }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + Assert.Contains(h.StateStore.DatabaseStateAlerted, r => r.Db == "Archive" && r.State == "OFFLINE"); + } + + [Fact] + public async Task DatabaseState_IntegrityState_StillRepeats_EvenWhenAlreadyAnnounced() + { + /* Nobody parks a database in SUSPECT, so continued repetition IS the signal there. An already- + announced integrity state must keep firing on the cooldown — if this ever goes quiet, a real + corruption stops nagging, which is the failure mode worth protecting against. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Payments", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "SUSPECT" }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(PerformanceMonitor.Notifications.AlertSeverityLevel.Critical, fired.Severity); + } + + [Fact] + public async Task DatabaseState_ChosenState_ChangingToADifferentState_FiresAgain() + { + /* The composition property the reporter identified: going quiet for a parked state must NOT mean + going blind. A database announced as OFFLINE that turns SUSPECT is a different state, so it + alerts — and at Critical, not inheriting the quiet treatment of the state it left. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "OFFLINE" }); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(PerformanceMonitor.Notifications.AlertSeverityLevel.Critical, fired.Severity); + } + + [Fact] + public async Task DatabaseState_TransitionToADifferentState_IsNotSuppressedByThePriorStatesCooldown() + { + /* The safety property, tested where it actually breaks. Both evaluations happen inside one cooldown + window (they run back to back, so no wall-clock time passes), which is exactly the case the old + per-database cooldown key swallowed: OFFLINE fires and stamps the database's only clock, then the + flip to SUSPECT finds that clock still running and goes silent — permanently, now that a chosen + state no longer re-fires every cooldown. SUSPECT is the state this alert must never lose. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + var engine = h.Build(); + + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* Same database, still deviating, but a DIFFERENT state — and the memory now says OFFLINE, which is + what makes alreadyAnnounced false while the OFFLINE cooldown is still warm. */ + h.Deliverer.Outcomes.Clear(); + h.Adapter.DatabaseStates.Clear(); + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "OFFLINE" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(PerformanceMonitor.Notifications.AlertSeverityLevel.Critical, fired.Severity); + } + + [Fact] + public async Task DatabaseState_MutedFire_DoesNotRecordItAsAnnounced_SoUnmutingStillNotifies() + { + /* A mute must be reversible. The four edge-triggered states gate ALL future firing on the announced + memory, so stamping it under a mute made the mute permanent: mute a parked database, remove the + mute, and the alert never came back for as long as the state held. Muting suppresses delivery, not + the engine's honesty about whether anyone was actually told. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + h.Muted = true; + var engine = h.Build(); + + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var muted = Assert.Single(h.Deliverer.Outcomes); + Assert.True(muted.Muted, "the fire itself must still be marked muted"); + Assert.DoesNotContain(h.StateStore.DatabaseStateAlerted, r => r.Db == "Archive"); + Assert.False(h.StateStore.Memory.ContainsKey("Archive"), + "a muted fire must not record the state as announced — nobody was told"); + + /* Mute removed, cooldown elapsed, same state still deviating. The adapter reports what the store + holds, which is still nothing — so this must notify for real. */ + h.Muted = false; + h.Now = h.Now.AddDays(1); + h.Deliverer.Outcomes.Clear(); + h.Adapter.DatabaseStates.Clear(); + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo + { + DatabaseName = "Archive", + StateDesc = "OFFLINE", + ExpectedState = "ONLINE", + LastAlertedState = h.StateStore.Memory.TryGetValue("Archive", out var remembered) ? remembered : "", + }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var announced = Assert.Single(h.Deliverer.Outcomes); + Assert.False(announced.Muted, "unmuting must produce a real, deliverable alert"); + Assert.Contains(h.StateStore.DatabaseStateAlerted, r => r.Db == "Archive" && r.State == "OFFLINE"); + } + + [Fact] + public async Task DatabaseState_RecoveryDoesNotClearACooldown_ForADatabaseWhoseNameContainsTheOldDelimiter() + { + /* SQL Server permits '|' in a database name, so while the cooldown key was a delimited STRING, + recovering "Foo" prefix-matched and wiped "Foo|Bar"'s clock as well. Keying by tuple removes the + bug class rather than documenting it. + + Observable via an integrity state: SUSPECT is not edge-suppressed (RepeatsAreNoise is false), so + its cooldown is the ONLY thing keeping it quiet on the second evaluation. If the recovery sweep + wrongly cleared it, "Foo|Bar" fires again here. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + var engine = h.Build(); + + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Foo", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "" }); + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Foo|Bar", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + + /* Foo returns to expected and drops out; Foo|Bar is untouched and still SUSPECT. */ + h.Deliverer.Outcomes.Clear(); + h.Adapter.DatabaseStates.Clear(); + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Foo|Bar", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "SUSPECT" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + Assert.DoesNotContain(h.StateStore.DatabaseStateCleared, r => r.Db == "Foo|Bar"); + } + + [Fact] + public async Task DatabaseState_SameState_StillRateLimitsItself_WithinOneCooldown() + { + /* The other side of keying by state: it must not have turned the cooldown off. An integrity state + repeats deliberately (RepeatsAreNoise is false for SUSPECT), so the only thing standing between it + and an alert per evaluation is its own cooldown — which must still hold inside one window. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + var engine = h.Build(); + + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Payments", StateDesc = "SUSPECT", ExpectedState = "ONLINE", LastAlertedState = "SUSPECT" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Deliverer.Outcomes.Clear(); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task DatabaseState_RepeatEpisode_OfTheSameState_FiresAgainAfterRecovery() + { + /* The falling-edge property, driven as a full round trip through the store's MEMORY rather than + through call counting — the decoupling that let the first cut of #2166 ship with a permanent + memory. Park, recover, park again in the SAME state: the repeat soft-delete workflow this alert + exists for. If recovery does not clear what firing recorded, evaluation 3 reads OFFLINE == + OFFLINE, judges itself already-announced, and the second parking is swallowed for good. */ + var h = new Harness(); + h.Settings.DatabaseStateEnabled = true; + var engine = h.Build(); + + /* Episode 1: parked. No memory yet, so it announces and records. */ + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo { DatabaseName = "Archive", StateDesc = "OFFLINE", ExpectedState = "ONLINE", LastAlertedState = "" }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("OFFLINE", h.StateStore.Memory["Archive"]); + + /* Recovery: back to expected, so it stops deviating and drops out of the adapter's results. */ + h.Adapter.DatabaseStates.Clear(); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Contains(h.StateStore.DatabaseStateCleared, r => r.Db == "Archive"); + Assert.False(h.StateStore.Memory.ContainsKey("Archive"), + "recovery must forget the announced state, or the edge can never trigger a second time"); + + /* Episode 2: parked again, same state. The adapter reports whatever the store now holds — which is + the whole point — so this fires only if the clear above actually happened. */ + h.Deliverer.Outcomes.Clear(); + h.Adapter.DatabaseStates.Add(new DatabaseStateInfo + { + DatabaseName = "Archive", + StateDesc = "OFFLINE", + ExpectedState = "ONLINE", + LastAlertedState = h.StateStore.Memory.TryGetValue("Archive", out var remembered) ? remembered : "", + }); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + } + + [Fact] + public void DatabaseState_AlertedStamp_IsAnUpdate_NeverAnInsert() + { + /* A row is only absent when the database was first observed in an integrity state, which the seed + logic deliberately refuses to baseline. An INSERT here must supply expected_state (NOT NULL) and + the only value on hand is the state being alerted ON — so inserting would baseline a SUSPECT + database as "expected SUSPECT", stop it deviating, report it RECOVERED while still corrupt, and + silence it permanently. Strictly worse than the repetition being fixed, so it is pinned. */ + var source = ReadStateStoreSource(); + var method = source[source.IndexOf("public async Task SaveDatabaseStateAlertedAsync", StringComparison.Ordinal)..]; + var body = method[..method.IndexOf("public async Task ClearDatabaseStateAlertedAsync", StringComparison.Ordinal)]; + + Assert.Contains("UPDATE config.database_state_expected", body, StringComparison.Ordinal); + Assert.DoesNotContain("INSERT INTO config.database_state_expected", body, StringComparison.Ordinal); + Assert.DoesNotContain("ON CONFLICT", body, StringComparison.Ordinal); + } + + [Fact] + public void DatabaseState_TheNeverBaselinedList_IsWiderThanTheCriticalOne() + { + /* #2189. Two lists that look interchangeable and are not, which is exactly why they are worth + pinning: one answers "bad enough to page about with no baseline to compare against", the other + "must never be LEARNED as this database's normal". A transient state belongs only in the second. + + Collapsing them either way is a shipped bug. Widen the critical list and every restore in progress + pages. Narrow the never-baselined list and a database observed mid-restore learns RESTORING as + expected, then alerts forever for being ONLINE — the reported bug, 636 fires in 24 hours. */ + Assert.Contains(DatabaseStateTokens.Suspect, DatabaseStateTokens.CriticalSqlList, StringComparison.Ordinal); + Assert.Contains(DatabaseStateTokens.RecoveryPending, DatabaseStateTokens.CriticalSqlList, StringComparison.Ordinal); + Assert.Contains(DatabaseStateTokens.Emergency, DatabaseStateTokens.CriticalSqlList, StringComparison.Ordinal); + + /* A pending database in a transient state must stay SILENT, so these must not reach the critical arm. */ + Assert.DoesNotContain(DatabaseStateTokens.Restoring, DatabaseStateTokens.CriticalSqlList, StringComparison.Ordinal); + Assert.DoesNotContain(DatabaseStateTokens.Recovering, DatabaseStateTokens.CriticalSqlList, StringComparison.Ordinal); + + Assert.StartsWith(DatabaseStateTokens.CriticalSqlList, DatabaseStateTokens.NeverBaselinedSqlList, StringComparison.Ordinal); + Assert.Contains($"'{DatabaseStateTokens.Restoring}'", DatabaseStateTokens.NeverBaselinedSqlList, StringComparison.Ordinal); + Assert.Contains($"'{DatabaseStateTokens.Recovering}'", DatabaseStateTokens.NeverBaselinedSqlList, StringComparison.Ordinal); + + /* STANDBY is synthetic and stable by construction — the whole reason it exists is to give a + log-shipping secondary one steady token instead of the RESTORING flicker underneath it. Refusing to + learn it would leave every standby secondary permanently pending for no benefit. */ + Assert.DoesNotContain(DatabaseStateTokens.Standby, DatabaseStateTokens.NeverBaselinedSqlList, StringComparison.Ordinal); + } + + [Fact] + public void DatabaseState_BothDarlingSeedSites_ShareTheOneStateList() + { + /* The viewer's editor seeds and heals baselines with its own copy of this SQL because that project + cannot reference the service's. Two hand-kept copies of "what must never be learned" is the drift + that lets the editor write a baseline the alert refuses to — silently, and only for operators who + happen to open the editor mid-restore. Both sites interpolate the shared constant instead, and + BOTH statements use it: the seed to refuse those states, the heal to un-write them (#2189). A copy + that widened only one of the two would be the subtler half of the same bug. */ + var viewer = ReadRepoFile(Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.DatabaseStates.cs")); + var service = ReadRepoFile(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingAlertReadAdapter.cs")); + + foreach (var source in new[] { viewer, service }) + { + Assert.Contains("NOT IN ({DatabaseStateTokens.NeverBaselinedSqlList})", source, StringComparison.Ordinal); + Assert.Contains("expected_state IN ({DatabaseStateTokens.NeverBaselinedSqlList})", source, StringComparison.Ordinal); + Assert.DoesNotContain($"NOT IN ('{DatabaseStateTokens.Suspect}'", source, StringComparison.Ordinal); + + /* The heal must never be reachable for a state somebody DECLARED, in either copy. */ + Assert.Contains("is_user_override = false", source, StringComparison.Ordinal); + } + } + + private static string ReadStateStoreSource() => + ReadRepoFile(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "PgAlertStateStore.cs")); + + /// Reads a repo-relative source file, walking up from this test file to find the repo root. + private static string ReadRepoFile(string relative, [CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } + [Fact] public async Task DatabaseState_Disabled_DoesNotFetch() { diff --git a/Darling/Darling.Tests/AlertStoredValueTests.cs b/Darling/Darling.Tests/AlertStoredValueTests.cs index d1137e5fd..fc876c649 100644 --- a/Darling/Darling.Tests/AlertStoredValueTests.cs +++ b/Darling/Darling.Tests/AlertStoredValueTests.cs @@ -71,6 +71,7 @@ private sealed class Settings : IAlertEngineSettings public bool FailedJobEnabled { get; set; } public bool PvsEnabled { get; set; } public bool DatabaseStateEnabled { get; set; } + public bool ForcePlanFailureEnabled { get; set; } = true; public int CpuThresholdPercent { get; set; } = 80; public int BlockingCountThreshold { get; set; } = 1; public int BlockingWaitSecondsThreshold { get; set; } @@ -86,6 +87,12 @@ private sealed class Settings : IAlertEngineSettings public int TempDbSpaceThresholdPercent { get; set; } = 80; public int LowDiskThresholdPercent { get; set; } = 10; public int LowDiskThresholdGb { get; set; } = 5; + /* #2107: the previously-hardcoded knobs, at their shipped defaults. */ + public int DiskCriticalFreePercent { get; set; } = 3; + public int DiskCriticalFreeGb { get; set; } = 2; + public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int CollectionStaleMinutes { get; set; } = 30; + public int CollectionFailureThreshold { get; set; } = 10; public int PvsThresholdPercent { get; set; } = 40; public int PvsFloorGb { get; set; } = 1; public int LongRunningJobMultiplier { get; set; } = 3; diff --git a/Darling/Darling.Tests/AnalysisShutdownResidueTests.cs b/Darling/Darling.Tests/AnalysisShutdownResidueTests.cs new file mode 100644 index 000000000..5e5d11932 --- /dev/null +++ b/Darling/Darling.Tests/AnalysisShutdownResidueTests.cs @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; +using Npgsql; +using PerformanceMonitor.Darling.Analysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// A clean stop must not read as seven faults (#2299). +/// +/// Observed on the dogfood box, 2026-08-16, on two separate stops. The analysis pass is +/// started per sweep but was neither awaited nor cancellable, so Stop-Service disposed the loop's +/// data source underneath it and then pg_ctl stop -m fast-ed the managed postmaster. The abandoned +/// pass's next store reads logged five Failed to compute baselines, one anomaly-detection failure +/// and one FilterMutedFindingsAsync failed — all ERROR, all after "collection loop stopped", and +/// 7 of that day's 9 ERROR lines. The two genuine errors were the needles. +/// +/// The repair has two halves and BOTH are pinned here: shutdown residue with the stopping token +/// signalled collapses to one Information line, and the SAME exceptions with the token NOT signalled stay +/// ERRORs — because a data source disposed while the service is meant to be running is a real bug whose +/// only evidence is exactly this text. +/// +public sealed class AnalysisShutdownResidueTests +{ + private static readonly CancellationToken s_fired = new(canceled: true); + + private static PostgresException SqlState(string state) => new( + messageText: "terminating connection due to administrator command", + severity: "FATAL", + invariantSeverity: "FATAL", + sqlState: state); + + /// + /// The shapes a stop actually produces: the token observed properly, the disposed data source + /// (bare and Npgsql-wrapped), and the postmaster going away server-side (the 57P0x trio + /// PostgresTargetProvider already classifies as connection-fatal). + /// + [Fact] + public void EveryShutdownShapeIsAbandonedOnceTheTokenFires() + { + Assert.True(AnalysisShutdown.IsShutdownAbandon(new OperationCanceledException(), s_fired)); + Assert.True(AnalysisShutdown.IsShutdownAbandon(new ObjectDisposedException("NpgsqlDataSource"), s_fired)); + Assert.True(AnalysisShutdown.IsShutdownAbandon( + new NpgsqlException("wrapper", new ObjectDisposedException("NpgsqlDataSource")), s_fired)); + Assert.True(AnalysisShutdown.IsShutdownAbandon(SqlState("57P01"), s_fired)); + Assert.True(AnalysisShutdown.IsShutdownAbandon(SqlState("57P02"), s_fired)); + Assert.True(AnalysisShutdown.IsShutdownAbandon(SqlState("57P03"), s_fired)); + } + + /// + /// The other half of the agreement: with the token NOT signalled, the identical exceptions mean a + /// data source was disposed (or a connection administratively killed) mid-run — a real bug — and + /// must keep their ERROR. Quieting them unconditionally would erase that bug's only evidence. + /// + [Fact] + public void TheSameShapesStayErrorsWhileTheServiceIsRunning() + { + Assert.False(AnalysisShutdown.IsShutdownAbandon(new OperationCanceledException(), CancellationToken.None)); + Assert.False(AnalysisShutdown.IsShutdownAbandon(new ObjectDisposedException("NpgsqlDataSource"), CancellationToken.None)); + Assert.False(AnalysisShutdown.IsShutdownAbandon(SqlState("57P01"), CancellationToken.None)); + } + + /// + /// A command timeout coinciding with a stop is still a query that outgrew its deadline — the growth + /// signal #2294 made visible must survive the coincidence, so a timeout is never relabelled shutdown. + /// + [Fact] + public void ATimeoutIsNeverRelabelledAsShutdown() + { + Assert.False(AnalysisShutdown.IsShutdownAbandon(new TimeoutException("deadline"), s_fired)); + Assert.False(AnalysisShutdown.IsShutdownAbandon( + new NpgsqlException("Exception while reading from stream", new TimeoutException()), s_fired)); + Assert.False(AnalysisShutdown.IsShutdownAbandon(SqlState("57014"), s_fired)); + } + + /// Ordinary faults during a stop are still faults — structural shapes only, never "anything goes". + [Fact] + public void OrdinaryFaultsAreNeverShutdownResidueEvenMidStop() + { + Assert.False(AnalysisShutdown.IsShutdownAbandon(SqlState("42703"), s_fired)); + Assert.False(AnalysisShutdown.IsShutdownAbandon( + new NpgsqlException("Exception while reading from stream", new IOException("reset")), s_fired)); + Assert.False(AnalysisShutdown.IsShutdownAbandon(new InvalidOperationException("something else"), s_fired)); + } + + /// + /// The CATEGORY pin, learned from finding the same defect shape nine times in one file: every + /// ERROR-logging catch on the analysis pass must classify shutdown, or the next detector quietly + /// reintroduces the noise. Counted from the shipped source so a new bare catch (Exception ex) + /// in the detector goes red here with instructions, not silently at the next dogfood stop. + /// + [Fact] + public void EveryErrorLoggingCatchOnTheAnalysisPassClassifiesShutdown() + { + const string contextFilter = "when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken))"; + + /* The detector: NO bare catch is permitted at all — its nine identical per-detector catches were + the bulk of the burst, and a tenth detector must arrive classified. Every `catch (Exception ex)` + must therefore BE one of the two filtered forms (the detectors carry the context token; the + baseline-data gate carries its own parameter), so the counts are equal by construction. */ + var detector = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Analysis", "PgAnomalyDetector.cs")); + Assert.Equal( + Count(detector, "catch (Exception ex)"), + Count(detector, "catch (Exception ex) " + contextFilter) + + Count(detector, "catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken))")); + Assert.True(Count(detector, contextFilter) >= 9, "a detector catch lost its shutdown classification"); + + /* The baseline provider: its single catch produced five of the seven lines. */ + var baseline = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Analysis", "PgBaselineProvider.cs")); + Assert.Equal(1, Count(baseline, "when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken))")); + + /* The finding store: only its two PASS methods run under the worker's token; its read-back + surfaces serve other lifetimes and are deliberately untouched. */ + var findingStore = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Analysis", "PgFindingStore.cs")); + Assert.Equal(2, Count(findingStore, contextFilter)); + + /* The drill-down: one per-finding catch, plus the between-findings abandon point. */ + var drillDown = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Analysis", "PgDrillDownCollector.cs")); + Assert.Equal(1, Count(drillDown, contextFilter)); + Assert.Contains("context.CancellationToken.ThrowIfCancellationRequested();", drillDown, StringComparison.Ordinal); + + /* The service: the ONE Information line a stop is allowed to cost, and the data-span probe must + not convert shutdown residue into a bogus "0 hours of history" skip. */ + var service = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Analysis", "DarlingAnalysisService.cs")); + Assert.Equal(1, Count(service, "when (AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken))")); + Assert.Equal(1, Count(service, "when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken))")); + Assert.Contains("Analysis abandoned at shutdown", service, StringComparison.Ordinal); + + /* The worker: the pass must RECEIVE the stopping token (an uncancellable pass makes every filter + above unreachable), and the stop path must hold the sweep open for the unwind grace with the + already-fired token deliberately not forwarded. */ + var worker = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs")); + Assert.Contains("AnalyzeAsync(serverId, storageName, hoursBack: 4, stoppingToken)", worker, StringComparison.Ordinal); + Assert.Contains("WaitAsync(s_analysisShutdownGrace, CancellationToken.None)", worker, StringComparison.Ordinal); + } + + private static int Count(string source, string needle) + { + var count = 0; + var at = 0; + while ((at = source.IndexOf(needle, at, StringComparison.Ordinal)) >= 0) + { + count++; + at += needle.Length; + } + + return count; + } + + private static string RepoRoot([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + while (dir is not null && !File.Exists(Path.Combine(dir, "PerformanceMonitor.sln")) && !Directory.Exists(Path.Combine(dir, ".git"))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return dir!; + } + + private static string ReadSource(string relative) => File.ReadAllText(Path.Combine(RepoRoot(), relative)); +} diff --git a/Darling/Darling.Tests/AzureForeignStatePruneTests.cs b/Darling/Darling.Tests/AzureForeignStatePruneTests.cs new file mode 100644 index 000000000..345a3b439 --- /dev/null +++ b/Darling/Darling.Tests/AzureForeignStatePruneTests.cs @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2191: the Azure SQL DB arm of the #2188 per-database state prune. +/// +/// Why it was unreachable, and why it is not now. The on-prem prune anti-joins +/// database_states, which is an unfiltered sys.databases read — the only list that answers "does +/// this name still exist" without confusing a dropped database for an offline, excluded or unprobeable one. +/// DatabaseStateCollector.AppliesTo is !IsAzureSqlDb, so an Azure server has no such snapshot and +/// the prune correctly no-opped there, leaving planwm: / done: / hole: rows to accumulate. +/// #2191 asked for "an authoritative unfiltered sys.databases read from master, used only on the success +/// path", and rejected the per-cycle enumeration because it filters ONLINE, applies the excluded-database +/// list, and falls back to a single database on a master-access error. +/// +/// #2220 removed the need for any of that. A registration that names a database now sweeps only +/// that database, so its one legitimate key is derivable from the connection string's own catalog — current by +/// construction, nothing filtered, nothing that can go stale. The question changed from "which databases still +/// exist on the instance" to "which database does this registration own", and the second needs no snapshot. +/// +/// What it deletes in the field. Mostly #2220's residue: before that fix every Azure registration +/// swept each sibling database and wrote a watermark for it under its own server_id. +/// collector_state carries no retention, so unlike the collected rows those orphans would persist +/// indefinitely instead of ageing out — which is why this matters even though the contamination has stopped. +/// +public sealed class AzureForeignStatePruneTests +{ + /// + /// The statement keeps the registration's OWN key and nothing else — asserted on the SQL because the + /// delete is the whole behaviour and it has no in-process seam. + /// + [Fact] + public void ThePruneKeepsExactlyTheRegistrationsOwnKey() + { + var sql = DarlingCollectorRunner.PruneForeignDatabaseStateKeysSql; + + /* Scoped to one server, one state owner, and one key prefix, so it cannot reach another collector's + state or another server's. */ + Assert.Contains("s.server_id = $1", sql, StringComparison.Ordinal); + Assert.Contains("s.collector_name = $2", sql, StringComparison.Ordinal); + Assert.Contains("starts_with(s.state_key, $3)", sql, StringComparison.Ordinal); + + /* The rule itself: everything under the prefix EXCEPT this registration's own database. */ + Assert.Contains("s.state_key <> $3 || $4", sql, StringComparison.Ordinal); + + /* Names what it deleted. The only symptom of a wrong delete here is a silent plan-XML refetch, so a + rows-affected count would leave nothing to diagnose with. */ + Assert.Contains("RETURNING s.state_key", sql, StringComparison.Ordinal); + } + + /// + /// It must NOT carry the on-prem statement's snapshot machinery. Pinned as an absence because copying + /// those guards over would be the natural-looking mistake, and they cannot work here: there is no + /// snapshot to be empty or stale, so an anti-join against database_states — which is empty on + /// every Azure server by design — would delete every row instead of none. + /// + [Fact] + public void ThePruneDoesNotReachForASnapshotThatCannotExistOnAzure() + { + var sql = DarlingCollectorRunner.PruneForeignDatabaseStateKeysSql; + + Assert.DoesNotContain("database_states", sql, StringComparison.Ordinal); + Assert.DoesNotContain("updated_at", sql, StringComparison.Ordinal); + Assert.DoesNotContain("NOT EXISTS", sql, StringComparison.Ordinal); + } + + /// + /// The on-prem statement keeps ITS guards. Same file, opposite requirement — pinned together so a future + /// edit cannot "simplify" the two into one shape, which would either delete every on-prem row when a + /// snapshot is missing or leave Azure orphaning again. + /// + [Fact] + public void TheOnPremPruneStillGuardsOnTheSnapshotAndItsFreshness() + { + var sql = DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql; + + Assert.Contains("database_states", sql, StringComparison.Ordinal); + Assert.Contains("snapshot.newest IS NOT NULL", sql, StringComparison.Ordinal); + Assert.Contains("s.updated_at < snapshot.newest", sql, StringComparison.Ordinal); + } + + /// + /// THE SAFETY PROPERTY. A registration naming no database — or naming master, where a + /// catalog-less Azure connection lands — is a registration of the logical SERVER, whose legitimate + /// database set is everything on it. Pruning against a single name there would delete every live + /// watermark it has, so the caller's gate is and this + /// pins that the gate closes for exactly those cases. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("master")] + [InlineData("MASTER")] + public void ALogicalServerRegistrationIsNeverPrunedAgainstASingleName(string? catalog) + { + Assert.Empty(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// And it opens for a registration that does name one, which is the case #2191 is about. + [Theory] + [InlineData("db1")] + [InlineData("Payments")] + public void ADatabaseNamedRegistrationIsPruned(string catalog) + { + Assert.Single(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// Both arms iterate the SHARED prefix set, so a prefix cannot be pruned on one target type and left + /// orphaning on the other — the drift this list exists to prevent. + /// + [Fact] + public void BothArmsPruneEveryPerDatabasePrefix() + { + Assert.Equal(5, QueryStorePerDatabaseState.PrunableKeys.Count); + Assert.Contains(QueryStorePerDatabaseState.PrunableKeys, + k => k.Prefix == QueryStorePlanXmlState.WatermarkKeyPrefix); + Assert.Contains(QueryStorePerDatabaseState.PrunableKeys, + k => k.Prefix == QueryStoreBackfillState.DoneKeyPrefix); + Assert.Contains(QueryStorePerDatabaseState.PrunableKeys, + k => k.Prefix == QueryStoreBackfillState.HoleKeyPrefix); + /* #2150: the text watermark, keyed prefix + databaseName exactly like the plan watermark. */ + Assert.Contains(QueryStorePerDatabaseState.PrunableKeys, + k => k.Prefix == QueryStoreTextState.WatermarkKeyPrefix); + /* #2312: the open-interval refresh stamp, the fifth per-database prefix. */ + Assert.Contains(QueryStorePerDatabaseState.PrunableKeys, + k => k.Prefix == QueryStoreOpenIntervalState.WatermarkKeyPrefix); + + /* Nothing may sit outside both lists — a server-scoped key added to PrunableKeys by reflex would be + deleted every cycle. */ + Assert.Empty(QueryStorePerDatabaseState.NotKeyedByDatabase); + } +} diff --git a/Darling/Darling.Tests/AzureSweepScopeTests.cs b/Darling/Darling.Tests/AzureSweepScopeTests.cs new file mode 100644 index 000000000..a0473e8a0 --- /dev/null +++ b/Darling/Darling.Tests/AzureSweepScopeTests.cs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2220: which databases one Azure SQL DB registration's per-database sweep covers. +/// +/// The field report. A single real deadlock in one Azure SQL Database produced near-identical +/// "Deadlocks Detected" alerts on every OTHER monitored database sharing the same logical server — and the +/// stored data matched: byte-identical deadlock graphs and the same top query, with counters within ~1%, +/// under six unrelated server_ids. Azure SQL DB engines are isolated per database, so one database's +/// sessions cannot block another's; the rows were not cross-talk, they were the same rows collected six +/// times. +/// +/// The cause, and why it is not a typo. The enumeration read master unconditionally and +/// swept every online database on the logical server, storing all of it under whichever registration ran the +/// sweep. Two parts of the product hold incompatible ideas of what a registration IS, both deliberate: the +/// enumeration assumes one registration = one LOGICAL SERVER (#857's shape), while identity assumes one +/// registration = one DATABASE (server_id hashes host[:database][:RO], and the Azure +/// query_store path needs a per-database connection anyway, #1836). The second shape silently behaved like +/// the first, N times over — N registrations of N databases is N² collection. +/// +/// Pinned in BOTH suites against the shared implementation. A scoping rule that disagrees between Lite +/// and Darling is the same class of defect as the one being fixed, and both runners previously carried their +/// own private copy of it. +/// +public sealed class AzureSweepScopeTests +{ + /// + /// THE FIX. A registration naming a database sweeps exactly that database — the reported case, where + /// fifteen registrations on one logical server each swept all fifteen. + /// + [Theory] + [InlineData("db1")] + [InlineData("AdventureWorks")] + [InlineData("Sibling-A")] + public void ARegistrationThatNamesADatabase_SweepsOnlyThatDatabase(string catalog) + { + Assert.Equal(new[] { catalog }, AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// A registration naming NO database is a registration of the logical server, so it must enumerate — + /// signalled by the empty list rather than by a separate flag, because the caller's next step is a list + /// either way. This is the behaviour #857 was written for and it is deliberately unchanged. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + public void ARegistrationThatNamesNoDatabase_StillEnumeratesTheServer(string? catalog) + { + Assert.Empty(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// master counts as naming none, in any casing. A connection string with no + /// Initial Catalog lands in master on Azure SQL DB, so treating it as a named database + /// would scope such a registration to the one database holding none of the user's data — collecting + /// nothing and looking healthy while doing it. + /// + [Theory] + [InlineData("master")] + [InlineData("MASTER")] + [InlineData("Master")] + public void MasterIsNotADatabaseAnyoneRegisteredFor(string catalog) + { + Assert.Empty(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// A database that merely CONTAINS "master" is a real database and is swept. Guards the obvious + /// over-match, which would silently stop collecting from it. + /// + [Theory] + [InlineData("mastermind")] + [InlineData("paymaster")] + [InlineData("master_archive")] + public void ADatabaseNamedLikeMasterIsStillItsOwnDatabase(string catalog) + { + Assert.Equal(new[] { catalog }, AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// The returned list is the caller's to keep: both runners hand it straight to their per-database loop, + /// and a shared or cached instance would let one sweep's mutation reach another's. + /// + [Fact] + public void EachCallReturnsItsOwnList() + { + var first = AzureSweepScope.OwnDatabaseOrEmpty("db1"); + var second = AzureSweepScope.OwnDatabaseOrEmpty("db1"); + + Assert.NotSame(first, second); + first.Add("mutated"); + Assert.Single(second); + } +} diff --git a/Darling/Darling.Tests/BackfillSwitchTests.cs b/Darling/Darling.Tests/BackfillSwitchTests.cs new file mode 100644 index 000000000..bf6dd2046 --- /dev/null +++ b/Darling/Darling.Tests/BackfillSwitchTests.cs @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the V58 Query Store backfill off switch (#2167): the migration's identity, the probe/gate rung, +/// and the config plumbing from store row to the worker's live-read seam. The switch exists because the +/// #2058 backfill previously ran unconditionally — a freshly restored catalog on a cross-region server +/// had no stop short of gutting plan capture fleet-wide. +/// +public sealed class BackfillSwitchTests +{ + [Fact] + public void V58_MigrationIdentity_AndColumnDefaultOn() + { + var v58 = PgMigrations.Scripts.Single(m => m.Version == 58); + + Assert.Equal("qs-backfill-switch", v58.Name); + /* Idempotent, on config_service, and DEFAULT TRUE — an upgraded store keeps backfilling until an + operator turns it off; the switch must never flip as a side effect of the upgrade itself. */ + Assert.Contains( + "ALTER TABLE config.config_service\r\n ADD COLUMN IF NOT EXISTS query_store_backfill_enabled boolean NOT NULL DEFAULT TRUE;" + .Replace("\r\n", "\n", StringComparison.Ordinal), + v58.Sql.Replace("\r\n", "\n", StringComparison.Ordinal), + StringComparison.Ordinal); + } + + [Fact] + public void ProbeAndGate_KnowTheV58Rung() + { + /* The viewer NAMES the column in ServiceConfigSelectSql/UpdateFlagsSql, so a V57 store must be + refused (42703 otherwise) and a fully-migrated V58 store must map to exactly the required + version — the connect-time-gate trap the V53/V56/V57 pins guard. */ + Assert.Contains("column_name = 'query_store_backfill_enabled'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + Assert.Contains("query_store_backfill_enabled", ViewerDataService.ServiceConfigSelectSql, StringComparison.Ordinal); + Assert.Contains("query_store_backfill_enabled = $6", ViewerDataService.ServiceConfigUpdateFlagsSql, StringComparison.Ordinal); + + /* A store carrying the switch but NOT the later V59 knobs is exactly V58 — stating the newer flag + false is what makes this rung's pin survive the next migration instead of silently becoming a + test of the newest rung. */ + Assert.Equal(58, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true, hasJobCadenceKnob: true, + hasBackfillSwitch: true, hasCollectorMemoryKnobs: false)); + Assert.Equal(57, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true, hasJobCadenceKnob: true, + hasBackfillSwitch: false, hasCollectorMemoryKnobs: false)); + } + + [Fact] + public void ApplyToConfig_CarriesTheSwitch_AndDefaultsStayOn() + { + /* Defaults: a fresh config and an unread view both leave backfill ON (the SKU default). */ + var config = new DarlingConfig(); + Assert.True(config.QueryStoreBackfillEnabled); + Assert.True(new StoreConfigView().QueryStoreBackfillEnabled); + + /* The store flip reaches the held config by reference — the worker's live Func seam reads + this field, so this IS the path a Settings-window toggle takes to the running loop. */ + StoreConfigProvider.ApplyToConfig(config, new StoreConfigView { QueryStoreBackfillEnabled = false }); + Assert.False(config.QueryStoreBackfillEnabled); + + StoreConfigProvider.ApplyToConfig(config, new StoreConfigView { QueryStoreBackfillEnabled = true }); + Assert.True(config.QueryStoreBackfillEnabled); + } +} diff --git a/Darling/Darling.Tests/BaselineTimeoutIsNamedTests.cs b/Darling/Darling.Tests/BaselineTimeoutIsNamedTests.cs new file mode 100644 index 000000000..8ebc85df3 --- /dev/null +++ b/Darling/Darling.Tests/BaselineTimeoutIsNamedTests.cs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Npgsql; +using PerformanceMonitor.Darling.Analysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// A baseline query that ran out of time must SAY so, instead of reading as a broken connection. +/// +/// Observed on the dogfood box, 2026-08-16. The service logged +/// Failed to compute baselines for io_latency: Exception while reading from stream — which reads as +/// a network fault. The store's own log, in a different file, showed +/// ERROR: canceling statement due to user request 267 ms earlier. Npgsql enforces its command +/// timeout by CANCELLING the statement, so the server reports the cancellation and the client is left +/// holding a torn stream. The real cause was a query outgrowing its deadline on a store that had grown to +/// 184 GB, and finding that out took correlating two logs by timestamp. +/// +/// The consequence of mislabelling it is not cosmetic: "connection broke" invites someone to look at +/// the network, while "did not finish within its command timeout" points at the query and the window it +/// scans. The metric also silently loses its baseline for that pass, so anomaly detection for it goes quiet +/// while the collected data looks perfectly healthy. +/// +public sealed class BaselineTimeoutIsNamedTests +{ + /// + /// 57014 is query_canceled — the server telling us it cancelled the statement, which for this + /// caller only ever happens because Npgsql's own deadline asked it to. + /// + [Fact] + public void APostgresCancellationCountsAsATimeout() + => Assert.True(PgBaselineProvider.IsCommandTimeout(new PostgresException( + messageText: "canceling statement due to user request", + severity: "ERROR", + invariantSeverity: "ERROR", + sqlState: "57014"))); + + /// Npgsql's own client-side deadline, direct and wrapped. + [Fact] + public void ATimeoutExceptionCountsEitherDirectlyOrWrapped() + { + Assert.True(PgBaselineProvider.IsCommandTimeout(new TimeoutException("timed out"))); + Assert.True(PgBaselineProvider.IsCommandTimeout( + new NpgsqlException("Exception while reading from stream", new TimeoutException()))); + } + + /// + /// The other direction matters just as much: a GENUINE connection fault must keep saying so. Labelling + /// it a timeout would be the identical defect aimed the other way, and it would send the next + /// investigation at the query instead of the network. + /// + [Fact] + public void ARealConnectionFaultIsNotCalledATimeout() + { + /* The bare message with no timeout in the chain — a reset socket, not a deadline. */ + Assert.False(PgBaselineProvider.IsCommandTimeout( + new NpgsqlException("Exception while reading from stream", new System.IO.IOException("reset")))); + Assert.False(PgBaselineProvider.IsCommandTimeout(new NpgsqlException("Exception while reading from stream"))); + + /* And an ordinary SQL error is neither. */ + Assert.False(PgBaselineProvider.IsCommandTimeout(new PostgresException( + messageText: "column does not exist", + severity: "ERROR", + invariantSeverity: "ERROR", + sqlState: "42703"))); + + Assert.False(PgBaselineProvider.IsCommandTimeout(new InvalidOperationException("something else"))); + } + + /// + /// Classified STRUCTURALLY, not by message text. The message is the very thing that was ambiguous, so a + /// guard that matched on it would be pinning the symptom that caused the confusion. + /// + [Fact] + public void TheMessageTextAloneDecidesNothing() + { + const string ambiguous = "Exception while reading from stream"; + + Assert.True(PgBaselineProvider.IsCommandTimeout(new NpgsqlException(ambiguous, new TimeoutException()))); + Assert.False(PgBaselineProvider.IsCommandTimeout(new NpgsqlException(ambiguous))); + } +} diff --git a/Darling/Darling.Tests/CollectorMemoryKnobTests.cs b/Darling/Darling.Tests/CollectorMemoryKnobTests.cs new file mode 100644 index 000000000..394442389 --- /dev/null +++ b/Darling/Darling.Tests/CollectorMemoryKnobTests.cs @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the V59 collector memory knobs (#2164 query_store text budget + #2170 fleet sweep width): the +/// migration's identity and behavior-preserving defaults, the read clamps, the probe/gate rung, the +/// context override the shared read loop honors, and the invariant that binds the two — peak transient +/// memory is their product, which is why they ship on one rung. +/// +public sealed class CollectorMemoryKnobTests +{ + [Fact] + public void V59_MigrationIdentity_AndDefaultsReproduceTheOldConstants() + { + var v59 = PgMigrations.Scripts.Single(m => m.Version == 59); + Assert.Equal("collector-memory-knobs", v59.Name); + + var sql = v59.Sql.Replace("\r\n", "\n", StringComparison.Ordinal); + /* Idempotent adds on config_service, and the defaults are the constants they replace — an upgraded + store must behave identically until an operator turns a dial. */ + Assert.Contains("ADD COLUMN IF NOT EXISTS query_store_text_budget_mb integer NOT NULL DEFAULT 64", sql, StringComparison.Ordinal); + Assert.Contains("ADD COLUMN IF NOT EXISTS max_concurrent_sweeps integer NOT NULL DEFAULT 4", sql, StringComparison.Ordinal); + + /* The defaults must equal what the code did before the knobs existed, or an upgrade silently + re-tunes every existing deployment. */ + Assert.Equal(64 * 1024 * 1024, QueryStoreCollector.MaxTextBytesPerDatabase); + Assert.Equal(4, DarlingWorker.MaxConcurrentServerSweeps); + } + + [Theory] + [InlineData(0, 4)] // corrupt/unset floors to the minimum, never to "ship nothing" + [InlineData(-16, 4)] + [InlineData(4, 4)] // inclusive bounds + [InlineData(8, 8)] + [InlineData(64, 64)] + [InlineData(256, 256)] + [InlineData(4096, 256)] // an over-generous value caps instead of reintroducing the #1556 balloon + public void TextBudgetClamp_KeepsTheKnobInsideTheMemoryBound(int stored, int expected) => + Assert.Equal(expected, StoreConfigProvider.ClampTextBudgetMb(stored)); + + /// #2171: the codec knob fails toward the shipped default - anything that is not + /// exactly 'none' (any casing, padded) is 'gzip', so a hand-edited row cannot put the writer + /// in an undefined mode. Mirrors the V62 CHECK constraint; both fail the same direction. + [Theory] + [InlineData(null, "gzip")] + [InlineData("", "gzip")] + [InlineData("gzip", "gzip")] + [InlineData(" GZIP ", "gzip")] + [InlineData("none", "none")] + [InlineData(" NONE ", "none")] + [InlineData("zstd", "gzip")] + public void PlanXmlCompression_NormalizesToGzipOrNone_FailingTowardGzip(string? stored, string expected) => + Assert.Equal(expected, StoreConfigProvider.NormalizePlanXmlCompression(stored)); + + [Theory] + [InlineData(0, 1)] // never zero — that would stop collection entirely + [InlineData(1, 1)] + [InlineData(4, 4)] + [InlineData(16, 16)] + [InlineData(64, 16)] // capped at the gate ceiling the semaphore is built with + public void SweepClamp_StaysWithinTheGateCeiling(int stored, int expected) + { + Assert.Equal(expected, StoreConfigProvider.ClampConcurrentSweeps(stored)); + /* The clamp ceiling and the semaphore's construction ceiling are the SAME number by contract: + the gate cannot grow past the permits it was built with, so a clamp above the ceiling would + silently cap and the knob would lie about its effective value. */ + Assert.Equal(DarlingWorker.SweepGateCeiling, StoreConfigProvider.MaxConcurrentSweepsLimit); + } + + [Fact] + public void ApplyToConfig_CarriesBothKnobs_AndDefaultsMatchTheConstants() + { + var config = new DarlingConfig(); + Assert.Equal(64, config.QueryStoreTextBudgetMb); + Assert.Equal(4, config.MaxConcurrentSweeps); + + StoreConfigProvider.ApplyToConfig(config, new StoreConfigView { QueryStoreTextBudgetMb = 8, MaxConcurrentSweeps = 12 }); + Assert.Equal(8, config.QueryStoreTextBudgetMb); + Assert.Equal(12, config.MaxConcurrentSweeps); + } + + [Fact] + public void ContextOverride_WinsOverTheCollectorConstant_AndZeroMeansNoOverride() + { + /* The override is what the shared read loop consults (QueryStoreCollector's budget line), so this + pins the precedence Lite depends on: Lite passes no override and must keep the constant. */ + var noOverride = new CollectorContext + { + ServerId = 1, ServerName = "s", CollectionTime = new DateTime(2026, 8, 10, 0, 0, 0, DateTimeKind.Utc), + Deltas = new CollectorDeltaCalculator(), + }; + Assert.Null(noOverride.TextByteBudgetOverride); + + var overridden = new CollectorContext + { + ServerId = 1, ServerName = "s", CollectionTime = new DateTime(2026, 8, 10, 0, 0, 0, DateTimeKind.Utc), + Deltas = new CollectorDeltaCalculator(), + TextByteBudgetOverride = 8 * 1024 * 1024, + }; + Assert.Equal(8 * 1024 * 1024, overridden.TextByteBudgetOverride); + Assert.True(overridden.TextByteBudgetOverride < QueryStoreCollector.MaxTextBytesPerDatabase, + "the override must be able to LOWER the budget — that is the entire point of the knob"); + } + + [Fact] + public async Task SweepGate_NarrowThenWiden_LandsAtTheConfiguredWidth_NoPermitStealing() + { + /* The review catch: the first cut fired a per-call absorb loop, so a still-running NARROWING task + would immediately re-take the permit a later WIDENING had just released — pinning the gate below + the configured width forever. This drives the real reconcile through that exact sequence with all + permits held (nothing to absorb yet), then widens, and asserts the gate actually reaches the new + width. Reflection because the gate plumbing is private worker state, and pinning behavior beats + making it public just to observe it. */ + var worker = (DarlingWorker)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(DarlingWorker)); + var lockField = typeof(DarlingWorker).GetField("_gateLock", BindingFlags.NonPublic | BindingFlags.Instance)!; + lockField.SetValue(worker, new object()); + var loggerField = typeof(DarlingWorker).GetField("_logger", BindingFlags.NonPublic | BindingFlags.Instance)!; + loggerField.SetValue(worker, NullLogger.Instance); + + var reconcile = typeof(DarlingWorker).GetMethod("ReconcileSweepGate", BindingFlags.NonPublic | BindingFlags.Instance)!; + var absorbed = typeof(DarlingWorker).GetField("_gateAbsorbed", BindingFlags.NonPublic | BindingFlags.Instance)!; + + using var gate = new SemaphoreSlim(DarlingWorker.SweepGateCeiling, DarlingWorker.SweepGateCeiling); + /* Every permit checked out, exactly like a fully busy fleet — a narrowing absorber must park. */ + for (var i = 0; i < DarlingWorker.SweepGateCeiling; i++) + { + await gate.WaitAsync(TestContext.Current.CancellationToken); + } + + reconcile.Invoke(worker, new object[] { gate, 2, TestContext.Current.CancellationToken }); // narrow, absorber parks + reconcile.Invoke(worker, new object[] { gate, 12, TestContext.Current.CancellationToken }); // widen while it is parked + + /* Give every permit back: a healthy gate now offers 12, and the parked absorber must NOT keep any. */ + gate.Release(DarlingWorker.SweepGateCeiling); + var deadline = DateTime.UtcNow.AddSeconds(5); + while (DateTime.UtcNow < deadline && (int)absorbed.GetValue(worker)! != DarlingWorker.SweepGateCeiling - 12) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + + Assert.Equal(DarlingWorker.SweepGateCeiling - 12, (int)absorbed.GetValue(worker)!); + Assert.Equal(12, gate.CurrentCount); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + [InlineData(16)] + public void SweepGate_IsBornAtTheConfiguredWidth_NotTheCeiling(int configured) + { + /* Review catch on the startup path: reconciling DOWN only starts the absorber, so a gate born at the + ceiling would offer ceiling-many permits until it retired them — and a restart with many servers + simultaneously due is exactly when that window gets spent. The worker constructs the gate with the + configured width as its INITIAL count and the ceiling as its MAX, so the window cannot exist. This + pins that SemaphoreSlim actually supports that shape and that the arithmetic seeding the absorbed + count is self-consistent. */ + using var gate = new SemaphoreSlim(configured, DarlingWorker.SweepGateCeiling); + Assert.Equal(configured, gate.CurrentCount); + + var absorbedAtBirth = DarlingWorker.SweepGateCeiling - configured; + Assert.Equal(DarlingWorker.SweepGateCeiling, gate.CurrentCount + absorbedAtBirth); + + /* And the gate can still be widened all the way back to the ceiling later — Release past MAX would + throw, so this is what makes "born narrow, widen later" safe rather than a one-way door. Guarded + because Release(0) also throws: at the ceiling there is nothing absorbed to give back, which is + exactly why ReconcileSweepGate's release is behind `toRelease > 0`. */ + if (absorbedAtBirth > 0) + { + gate.Release(absorbedAtBirth); + } + + Assert.Equal(DarlingWorker.SweepGateCeiling, gate.CurrentCount); + } + + [Fact] + public void ProbeAndGate_KnowTheV59Rung() + { + Assert.Contains("column_name = 'query_store_text_budget_mb'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + Assert.Contains("query_store_text_budget_mb", ViewerDataService.ServiceConfigSelectSql, StringComparison.Ordinal); + Assert.Contains("max_concurrent_sweeps", ViewerDataService.ServiceConfigSelectSql, StringComparison.Ordinal); + Assert.Contains("query_store_text_budget_mb = $7", ViewerDataService.ServiceConfigUpdateFlagsSql, StringComparison.Ordinal); + Assert.Contains("max_concurrent_sweeps = $8", ViewerDataService.ServiceConfigUpdateFlagsSql, StringComparison.Ordinal); + + Assert.Equal(59, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true, hasJobCadenceKnob: true, + hasBackfillSwitch: true, hasCollectorMemoryKnobs: true, hasDatabaseStateEdgeMemory: false)); + /* Invariant form, no literal to go stale (the fourth such pin found in two days of version + bumps): the gate always requires exactly the build's schema version. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + } +} diff --git a/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs b/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs new file mode 100644 index 000000000..dcbf4edcb --- /dev/null +++ b/Darling/Darling.Tests/CollectorRunnerConnectionEngineTests.cs @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Microsoft.Data.SqlClient; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// THE MISSING ASSERTION. The collector runner must get its connection from the target's own provider, for +/// every collector, not only the ones that fan out per database. +/// +/// What this exists to catch. The runner had two paths: the per-database branch resolved +/// TargetProviders.For(target) correctly, and the branch serving everything else did +/// new SqlConnection(server.ConnectionString) literally. Six of the seven PostgreSQL collectors take +/// the second path — only pg_autovacuum_stats fans out — so they were handed a SQL Server connection. +/// SqlClient rejects Npgsql's keywords while PARSING the connection string, before any query runs +/// ("Keyword not supported: 'host'"), and the resulting ArgumentException is neither +/// SqlException nor PostgresException, so it missed both fault-classification arms and recorded +/// a raw ERROR every sweep, forever, including for all three Tier 0 outage predictors. +/// +/// Both providers were correct and both were individually tested. Every test on both sides passed. +/// Nothing asserted that the RUNNER asked the provider — the seam itself was untested, and a passing suite +/// plus green CI reported a feature that could not collect a single row from a PostgreSQL target. It took +/// pointing the service at a live target to find it. +/// +public sealed class CollectorRunnerConnectionEngineTests +{ + private static ServerRuntime Runtime(CollectorTargetEngine engine, string connectionString) => new() + { + Config = new MonitoredServer { Name = "t", Host = "h" }, + ConnectionString = connectionString, + Target = new CollectorTargetInfo { Engine = engine }, + StorageName = "h", + ServerId = 1, + }; + + /// + /// The pin. Nothing is opened — the TYPE is the whole assertion, and it is enough: a connection of the + /// wrong type cannot even parse the other engine's connection string. + /// + [Fact] + public void PostgresTarget_GetsAnNpgsqlConnection() + { + using var connection = DarlingCollectorRunner.CreateTargetConnection( + Runtime(CollectorTargetEngine.PostgreSql, "Host=pg1;Database=postgres;Username=monitor")); + + Assert.IsType(connection); + } + + [Fact] + public void SqlServerTarget_StillGetsASqlConnection() + { + using var connection = DarlingCollectorRunner.CreateTargetConnection( + Runtime(CollectorTargetEngine.SqlServer, "Server=sql1;Integrated Security=true")); + + Assert.IsType(connection); + } + + /// + /// The failure mode itself, pinned: handing a PostgreSQL connection string to SqlClient throws while + /// PARSING it. Stated as a test so the reason the type matters is not just prose — and so nobody + /// "simplifies" the provider indirection away believing the driver would cope. + /// + [Fact] + public void ASqlConnectionCannotEvenParseAPostgresConnectionString() + { + var ex = Assert.ThrowsAny( + () => new SqlConnection("Host=pg1;Database=postgres;Username=monitor")); + + Assert.Contains("Keyword not supported", ex.Message, System.StringComparison.Ordinal); + + /* And this is why it evaded classification: not a SqlException, so DarlingWorker's SqlException arm + never saw it, and not a PostgresException either. */ + Assert.IsNotType(ex); + Assert.IsNotType(ex); + } + + /// + /// Belt and braces over the helper: the runner must not construct an engine-specific connection directly + /// anywhere in its collector paths. A future edit that bypasses + /// would pass the type tests above while + /// reintroducing the bug, so the source is scanned too — the same idiom the viewer-coverage and + /// alert-wiring pins already use in this suite. + /// + [Fact] + public void NoServiceFileConstructsAnEngineSpecificConnectionDirectly() + { + /* The precise hazard is not "constructs a connection" — it is "constructs a connection from + server.ConnectionString", the engine-AMBIGUOUS value, which is exactly what the bug did. A + connection built from a string an explicitly SQL-Server-only plan produced is fine and must stay + allowed: the Azure SQL master hop calls SqlServerTargetProvider.Instance.BuildDatabaseListPlan and + then opens its own SqlConnection, because per-database enumeration on Azure SQL DB is a SQL Server + feature by definition. A blunter "no constructions at all" rule flags that and teaches the next + person to suppress the test rather than read it. + + DIRECTORY-scoped, not runner-scoped, because the single-file form missed the round-2 live catch: + the identical construction sat in DarlingXeSessions.cs, out of scan reach, and failed once a + minute on every PostgreSQL target. Files listed below are ALLOWED to carry the construction + because every path into them is engine-gated, and each entry names the gate that makes it true — + an allowlisted file whose gate is later removed is a live bug this test can no longer see, so the + entry must name something a reviewer can check. */ + var allowed = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + /* BOTH public entry points self-gate on Engine == SqlServer at their first line + (EnsureAllAsync and ReconcileLongQueryCompletionsAsync) — XE is SQL Server only, and + every construction site in the file sits behind one of those two gates. */ + ["DarlingXeSessions.cs"] = "self-gates at EnsureAllAsync and ReconcileLongQueryCompletionsAsync entries", + + /* Backfill dispatch runs behind CollectorCatalog.AppliesTo (the composed engine gate) at + QueryStoreBackfill's work-selection, so a PostgreSQL target never reaches the SQL branch. */ + ["QueryStoreBackfill.cs"] = "composed AppliesTo gate at backfill dispatch", + }; + + var offenders = new List(); + /* Recursive: Mcp/, Targets/ and friends are exactly one directory down, and "the single-file + scan is how this one hid" applies verbatim to a single-directory scan. */ + foreach (var path in Directory.EnumerateFiles(ServiceSourceDirectory(), "*.cs", SearchOption.AllDirectories)) + { + var name = Path.GetFileName(path); + if (allowed.ContainsKey(name)) + { + continue; + } + + foreach (Match match in Regex.Matches( + File.ReadAllText(path), + @"new\s+(?:SqlConnection|NpgsqlConnection)\s*\(\s*server\.ConnectionString\s*\)")) + { + offenders.Add(name + ": " + match.Value); + } + } + + Assert.True( + offenders.Count == 0, + "Service code constructs an engine-specific connection from server.ConnectionString: " + + string.Join(", ", offenders) + + ". That value's engine is whatever the target is — route it through " + + "CreateTargetConnection/TargetProviders.For(server.Target), or gate every path into the file " + + "on engine and add an allowlist entry NAMING the gate. A hardcoded SqlConnection here is what " + + "made six of seven PostgreSQL collectors fail in the connection-string parser every sweep — " + + "and the seventh occurrence hid in a file the old single-file scan never read."); + } + + private static string ServiceSourceDirectory([CallerFilePath] string thisFile = "") + { + var testsDir = Path.GetDirectoryName(thisFile)!; + return Path.GetFullPath(Path.Combine(testsDir, "..", "PerformanceMonitor.Darling.Service")); + } + + /// + /// The scheduled ANALYSIS pass is gated by engine too, and for a reason worth stating: the pass cannot + /// gate itself. RunAnalysisPassAsync takes a serverId and a storage name, not the target, so the + /// decision has to be made at the call site. + /// Ungated, a PostgreSQL target got a full pass — a fresh analysis service and up to 120 seconds — + /// reading SQL Server tables that will never have rows for its server_id. It would hit the 24-hour + /// data-span gate and persist insufficient_data = true forever, so the Recommendations tab would + /// read "still collecting" for the life of the deployment: exactly the state analysis_state exists + /// to tell apart from a genuine all-clear. + /// + [Fact] + public void TheScheduledAnalysisPassIsGatedByEngine() + { + var source = File.ReadAllText(WorkerSourcePath()); + + var gateAt = source.IndexOf( + "server.Runtime?.Target.Engine == CollectorTargetEngine.PostgreSql", StringComparison.Ordinal); + var callAt = source.IndexOf("await RunScheduledAnalysisAsync(", StringComparison.Ordinal); + + Assert.True(gateAt > 0, "the analysis call site must test the target engine"); + Assert.True( + gateAt < callAt, + "the engine test must come BEFORE the analysis call, or the pass runs and then discovers it " + + "should not have"); + + /* And the PostgreSQL arm must say why rather than leaving the tab blank. */ + Assert.Contains("does not apply to a PostgreSQL target", source, StringComparison.Ordinal); + + /* ONE message, shared by the scheduled pass and the manual "Generate now" path. There were two + hand-maintained copies and they had already drifted — adding get_pg_blocking to the scheduled one + left the manual one listing seven tools, so the same product gave different guidance depending on + which door the operator came through. The list grows with every PostgreSQL read, so the drift + recurs by construction unless there is only one copy. */ + Assert.Equal( + 2, + Regex.Matches(source, @"message: PostgresAnalysisNotApplicable,").Count); + Assert.DoesNotContain( + "Scheduled analysis does not apply to a PostgreSQL target: its findings are \"", + source, + StringComparison.Ordinal); + } + + private static string WorkerSourcePath([CallerFilePath] string thisFile = "") + { + var testsDir = Path.GetDirectoryName(thisFile)!; + return Path.Combine(testsDir, "..", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs"); + } +} diff --git a/Darling/Darling.Tests/CollectorTableCatalogShadowTests.cs b/Darling/Darling.Tests/CollectorTableCatalogShadowTests.cs new file mode 100644 index 000000000..80c1a1df6 --- /dev/null +++ b/Darling/Darling.Tests/CollectorTableCatalogShadowTests.cs @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// No collector table may share a name with an object in pg_catalog or information_schema. +/// +/// Why this test exists. A collector table was named pg_replication_slots, which is also +/// the name of a system view. pg_catalog is searched IMPLICITLY AND FIRST — ahead of every entry in +/// search_path, and there is no way to demote it short of naming schemas explicitly everywhere — so an +/// unqualified reference to that name resolves to the system view no matter what the store holds. That +/// produced two failures with very different characters: +/// +/// LOUD: the generated fresh-store schema emits an unqualified CREATE INDEX ... ON pg_replication_slots, +/// which is 42809 "cannot create index on relation" against a view. The migration aborted, so the store never +/// came up and 531 tests failed behind the fixture. +/// QUIET, and worse: a reader's unqualified FROM pg_replication_slots against the STORE returns +/// the monitoring store's own slot list — normally empty. The tool would have reported "no replication slots" +/// forever and the retention alert would never have fired. A silently muted outage predictor is worse than +/// no predictor at all. +/// +/// +/// Schema-qualifying every reference would fix the loud half and leave the quiet half one forgotten +/// qualifier away, which is why the rule is about the NAME. Asserted against a real catalog rather than a +/// hardcoded list of reserved names: the catalog is authoritative, differs between majors, and grows. +/// +[Collection("live-postgres")] +public sealed class CollectorTableCatalogShadowTests +{ + private readonly LivePostgresStoreFixture _fixture; + + public CollectorTableCatalogShadowTests(LivePostgresStoreFixture fixture) => _fixture = fixture; + + /// + /// Every distinct collector table name, checked against the live store's own catalog. Also covers + /// functions, since a set-returning function of the same name would shadow it for a FROM too. + /// + [Fact] + public async Task NoCollectorTableShadowsASystemCatalogObject() + { + Assert.SkipWhen(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DARLING_TEST_PG")), + "Set DARLING_TEST_PG to a Postgres connection string to run the catalog-shadow check."); + Assert.True(_fixture.Established, "The live-postgres fixture did not establish the store."); + + var tables = CollectorCatalog.All.Select(d => d.TargetTable).Distinct().ToArray(); + + await using var connection = new NpgsqlConnection(_fixture.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + + var shadowed = new List(); + await using (var command = new NpgsqlCommand(@" +SELECT c.relname, n.nspname, c.relkind::text +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname IN ('pg_catalog', 'information_schema') +AND c.relname = ANY($1) +UNION ALL +SELECT p.proname, n.nspname, 'function' +FROM pg_catalog.pg_proc p +JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname IN ('pg_catalog', 'information_schema') +AND p.proname = ANY($1) +ORDER BY 1", connection)) + { + command.Parameters.AddWithValue(tables); + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + while (await reader.ReadAsync(TestContext.Current.CancellationToken)) + { + shadowed.Add($"{reader.GetString(0)} shadows {reader.GetString(1)}.{reader.GetString(0)} [{reader.GetString(2)}]"); + } + } + + Assert.True( + shadowed.Count == 0, + "Collector table name(s) collide with a system catalog object. pg_catalog is searched before " + + "search_path, so an unqualified reference resolves to the SYSTEM object — CREATE INDEX fails " + + "42809 and, far worse, a reader silently returns the system view's contents instead of collected " + + "history. RENAME the table (a `_stats` suffix is the house pattern: query_store -> " + + "query_store_stats, pg_replication_slots -> pg_replication_slot_stats); do not try to fix it by " + + "schema-qualifying every reference.\n" + string.Join("\n", shadowed)); + } + + /// + /// The check has teeth: a name that IS a catalog object must be detected. Without this, the test above + /// would pass just as happily against a query that silently matched nothing. + /// + [Fact] + public async Task TheShadowCheckDetectsAKnownCatalogName() + { + Assert.SkipWhen(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DARLING_TEST_PG")), + "Set DARLING_TEST_PG to a Postgres connection string to run the catalog-shadow check."); + Assert.True(_fixture.Established, "The live-postgres fixture did not establish the store."); + + await using var connection = new NpgsqlConnection(_fixture.ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + + await using var command = new NpgsqlCommand(@" +SELECT count(*) +FROM pg_catalog.pg_class c +JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = 'pg_catalog' +AND c.relname = ANY($1)", connection); + command.Parameters.AddWithValue(new[] { "pg_replication_slots", "pg_class" }); + + var found = (long)(await command.ExecuteScalarAsync(TestContext.Current.CancellationToken))!; + Assert.Equal(2, found); + } +} diff --git a/Darling/Darling.Tests/CpuAttributionTests.cs b/Darling/Darling.Tests/CpuAttributionTests.cs new file mode 100644 index 000000000..baa732b0c --- /dev/null +++ b/Darling/Darling.Tests/CpuAttributionTests.cs @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// Decision-table pins for the shared (#2320) — the attributed-CPU +/// disclosure both SKUs' get_top_queries_by_cpu / get_top_procedures_by_cpu serve. The contract under +/// pin: the ratio is measured-or-omitted (never invented — missing samples, missing core count, or +/// thin coverage all degrade to null + a reason), the low note fires under half, and above the +/// process's own measured CPU the note calls the number impossible rather than presenting it — +/// the 137%-of-the-box claim is the whole reason the marker exists. This SAME table is pinned +/// identically in Lite.Tests so the two SKUs cannot drift. +/// +public sealed class CpuAttributionTests +{ + private static readonly DateTime Start = new(2026, 8, 18, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime End = Start.AddHours(1); + + /// Full coverage, healthy ratio: 25% of 8 cores over an hour = 7,200 CPU-seconds; + /// 5,000 ranked seconds is 0.694 — present, rounded to 3, no note. + [Fact] + public void HealthyRatio_NoNote() + { + var result = CpuAttribution.Compute( + rankedCpuSeconds: 5000, Start, End, + sampleCount: 60, firstSampleUtc: Start, lastSampleUtc: End, avgSqlCpuPercent: 25, cpuCount: 8); + + Assert.Equal(5000, result.RankedCpuSeconds); + Assert.Equal(7200, result.SqlCpuSecondsInWindow); + Assert.Equal(0.694, result.AttributedCpuRatio); + Assert.Null(result.Note); + } + + /// The pre-#2290 shape this feature exists for: the ranking explains ~10% of the box, + /// and now something says so instead of letting the caller chase the visible tenth. + [Fact] + public void LowRatio_SaysNotTheWholeStory() + { + var result = CpuAttribution.Compute(720, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(0.1, result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("10%", result.Note, StringComparison.Ordinal); + Assert.Contains("not the whole story", result.Note, StringComparison.Ordinal); + } + + /// The 137% case — worker_time summing to more CPU than the process consumed is an + /// impossible claim, and the note must say to distrust the numbers, not decorate them. + [Fact] + public void OverAttribution_IsFlaggedImpossible() + { + var result = CpuAttribution.Compute(9864, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1.37, result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("137%", result.Note, StringComparison.Ordinal); + Assert.Contains("impossible-claim", result.Note, StringComparison.Ordinal); + } + + /// Just above 1.0 is sampling noise between two independent series, not a lie — + /// the impossible flag waits for the slack threshold. + [Fact] + public void SlightlyOverOne_CarriesNoNote() + { + var result = CpuAttribution.Compute(7500, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1.042, result.AttributedCpuRatio); + Assert.Null(result.Note); + } + + [Fact] + public void NoSamples_OmitsRatio_AndSaysWhy() + { + var result = CpuAttribution.Compute(5000, Start, End, 0, null, null, null, 8); + + Assert.Equal(5000, result.RankedCpuSeconds); + Assert.Null(result.SqlCpuSecondsInWindow); + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("no cpu_utilization samples", result.Note, StringComparison.Ordinal); + } + + [Fact] + public void NoCoreCount_OmitsRatio_AndSaysWhy() + { + var result = CpuAttribution.Compute(5000, Start, End, 60, Start, End, 25, cpuCount: 0); + + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("core count unavailable", result.Note, StringComparison.Ordinal); + } + + /// #2320's explicit degrade rule: a server whose CPU series starts mid-window (added, + /// or monitoring resumed) would deflate the denominator and inflate the ratio — omit instead. + [Fact] + public void PartialCoverage_OmitsRatio_WithThePercentage() + { + var result = CpuAttribution.Compute(5000, Start, End, 30, Start.AddMinutes(30), End, 25, 8); + + Assert.Null(result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("50%", result.Note, StringComparison.Ordinal); + Assert.Contains("partial denominator", result.Note, StringComparison.Ordinal); + } + + /// Samples straddling the window edges clamp to full coverage — a series wider than the + /// window is the NORMAL case (the store holds more history than any one read). + [Fact] + public void SamplesBeyondTheWindow_ClampToFullCoverage() + { + var result = CpuAttribution.Compute( + 5000, Start, End, 120, Start.AddHours(-1), End.AddHours(1), 25, 8); + + Assert.Equal(0.694, result.AttributedCpuRatio); + } + + /// An idle box measures zero CPU-seconds; a ratio against zero is undefined, and the + /// measured zero is still reported so the caller sees WHY. + [Fact] + public void ZeroMeasuredCpu_OmitsRatio_ReportsTheZero() + { + var result = CpuAttribution.Compute(5000, Start, End, 60, Start, End, avgSqlCpuPercent: 0, cpuCount: 8); + + Assert.Equal(0, result.SqlCpuSecondsInWindow); + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("zero", result.Note, StringComparison.Ordinal); + } + + [Fact] + public void EmptyWindow_OmitsRatio() + { + var result = CpuAttribution.Compute(5000, Start, Start, 60, Start, End, 25, 8); + + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("window is empty", result.Note, StringComparison.Ordinal); + } + + /// The numerator is rounded for emission but the ratio divides the RAW value — rounding + /// before dividing would move the third decimal on big windows. + [Fact] + public void RankedSecondsRoundToOneDecimal_RatioToThree() + { + var result = CpuAttribution.Compute(1234.5678, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1234.6, result.RankedCpuSeconds); + Assert.Equal(0.171, result.AttributedCpuRatio); + } +} diff --git a/Darling/Darling.Tests/Darling.Tests.csproj b/Darling/Darling.Tests/Darling.Tests.csproj index f6f3993c8..4395a579d 100644 --- a/Darling/Darling.Tests/Darling.Tests.csproj +++ b/Darling/Darling.Tests/Darling.Tests.csproj @@ -13,12 +13,12 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs b/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs index 2491427a8..c6a275606 100644 --- a/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs +++ b/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs @@ -30,6 +30,41 @@ namespace Darling.Tests; out; this comment is here so the next sweep does not "fix" it. */ public sealed class DarlingAlertTuningKnobsTests { + /* ---------------- #2107: the previously-hardcoded thresholds through the settings seam ---------------- */ + + [Fact] + public void SelfAlertKnobs_DefaultsAreTheConstantsTheyReplaced_AndReadsClampLikeSiblings() + { + var config = new DarlingConfig(); + var settings = new DarlingAlertSettings(config); + + /* Defaults mirror the V55 DDL — the compile-time constants these knobs replaced. */ + Assert.Equal(10, settings.SelfDiskFreeWarnPercent); + Assert.Equal(30, settings.CollectionStaleMinutes); + Assert.Equal(10, settings.CollectionFailureThreshold); + Assert.Equal(3, settings.DiskCriticalFreePercent); + Assert.Equal(2, settings.DiskCriticalFreeGb); + Assert.Equal(360, settings.AnalysisNotifyCooldownMinutes); + + /* Live reload through the by-reference seam, clamped on read — a hand-edited store value + can't drive a nonsense threshold: a 0-minute staleness window would fire every sweep, a + 0 failure threshold on the fast path would fire on any single failure, and the analysis + cooldown keeps the shared engine's documented [30, 10080]. */ + config.Alerts.SelfDiskFreeWarnPercent = 150; + config.Alerts.CollectionStaleMinutes = 0; + config.Alerts.CollectionFailureThreshold = 0; + config.Alerts.DiskCriticalFreePercent = -5; + config.Alerts.DiskCriticalFreeGb = -1; + config.Alerts.AnalysisNotifyCooldownMinutes = 99999; + + Assert.Equal(100, settings.SelfDiskFreeWarnPercent); + Assert.Equal(5, settings.CollectionStaleMinutes); + Assert.Equal(1, settings.CollectionFailureThreshold); + Assert.Equal(0, settings.DiskCriticalFreePercent); + Assert.Equal(0, settings.DiskCriticalFreeGb); + Assert.Equal(10080, settings.AnalysisNotifyCooldownMinutes); + } + /* ---------------- pure: the long-running-query read shape through the settings seam ---------------- */ [Fact] diff --git a/Darling/Darling.Tests/DarlingAnalysisPipelineTests.cs b/Darling/Darling.Tests/DarlingAnalysisPipelineTests.cs index 75cbe3f26..b6403e479 100644 --- a/Darling/Darling.Tests/DarlingAnalysisPipelineTests.cs +++ b/Darling/Darling.Tests/DarlingAnalysisPipelineTests.cs @@ -231,6 +231,17 @@ public void DrillDown_AllSql_EveryFromJoinTarget_ResolvesToAV4ViewOrACollectorTa var tables = CollectorCatalog.All.Select(s => s.TargetTable).ToHashSet(StringComparer.Ordinal); + /* Keyed SIDE tables are a third legal category, and deliberately not in either set above: they + are not collector tables (nothing collects INTO them per sweep; a bespoke upsert path writes + them, and they are pruned on last_seen rather than by drop_chunks) and they are not V4 + passthrough views. #2150's query_store_text is the first one a drill-down reads, because + statement text moved out of the fact row and has to be resolved back. Sourced from the store + class's own TableName rather than spelled here, so a rename cannot leave this guard asserting + against a table that no longer exists. */ + var sideTables = new[] { QueryStoreTextStore.TableName } + .Select(t => t.Contains('.', StringComparison.Ordinal) ? t.Split('.')[^1] : t) + .ToHashSet(StringComparer.Ordinal); + foreach (var sql in PgDrillDownCollector.AllSql) { /* Scan the SQL the SERVER sees, not the comments explaining it. Two things in the raw text @@ -257,8 +268,9 @@ so its right-hand operand parses as a relation name. { var target = m.Groups[1].Value; Assert.True( - views.Contains(target) || tables.Contains(target) || ctes.Contains(target), - $"FROM/JOIN target '{target}' resolves to no V4 view, collector table, or CTE in:\n{sql}"); + views.Contains(target) || tables.Contains(target) || ctes.Contains(target) + || sideTables.Contains(target), + $"FROM/JOIN target '{target}' resolves to no V4 view, collector table, keyed side table, or CTE in:\n{sql}"); } } } @@ -297,6 +309,11 @@ public void Worker_AnalysisCadence_MirrorsLitesAppDefaults() Assert.True(analysis.NotificationsEnabled); Assert.Equal(1.5, analysis.NotifySeverity); Assert.Equal(TimeSpan.FromSeconds(120), DarlingWorker.AnalysisTimeout); + /* #2299: the shutdown grace a stopping sweep grants its in-flight analysis pass. Must stay WELL + inside the worker's 15s shutdown drain budget and the host's 30s ShutdownTimeout — the await + runs inside a drained sweep body, so a grace near either ceiling would turn a clean stop into + a force-kill. */ + Assert.Equal(TimeSpan.FromSeconds(5), DarlingWorker.AnalysisShutdownGrace); } [Fact] diff --git a/Darling/Darling.Tests/DarlingCliCommandsTests.cs b/Darling/Darling.Tests/DarlingCliCommandsTests.cs index f3d9913ec..94f767381 100644 --- a/Darling/Darling.Tests/DarlingCliCommandsTests.cs +++ b/Darling/Darling.Tests/DarlingCliCommandsTests.cs @@ -9,9 +9,11 @@ using System; using System.Linq; using System.IO; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using PerformanceMonitor.Collectors; using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Service.Hosting; using Xunit; @@ -96,6 +98,96 @@ public void FormatProbeLine_Failure_ShowsError() Assert.Contains("Login failed", line, StringComparison.Ordinal); } + /// + /// A PostgreSQL target has no SQL major version, no engine edition and no msdb, so the line must not + /// claim any of them. Before the engine branch existed this printed "SQL major version 0, + /// Unknown (0), msdb access: yes" for a perfectly healthy Aurora cluster — a PASS that reads like a + /// misconfiguration, on the one verb whose whole job is to be trusted as a deployment gate. + /// + [Fact] + public void FormatProbeLine_PostgresTarget_ReportsPostgresFactsAndNoSqlServerOnes() + { + var line = DarlingCliCommands.FormatProbeLine("aurora-writer", PostgresProbe()); + + Assert.Contains("[PASS]", line, StringComparison.Ordinal); + Assert.Contains("PostgreSQL 17", line, StringComparison.Ordinal); + Assert.Contains("170007", line, StringComparison.Ordinal); + Assert.Contains("writer", line, StringComparison.Ordinal); + Assert.Contains("Aurora", line, StringComparison.Ordinal); + + Assert.DoesNotContain("SQL major version", line, StringComparison.Ordinal); + Assert.DoesNotContain("msdb", line, StringComparison.Ordinal); + Assert.DoesNotContain("Unknown (0)", line, StringComparison.Ordinal); + } + + /// An Aurora writer clears every gate, so the count says so rather than listing nothing. + [Fact] + public void FormatProbeLine_AuroraWriter_ReportsEveryPostgresCollectorApplies() + { + var expected = CollectorCatalog.All.Count(d => d.TargetEngine == CollectorTargetEngine.PostgreSql); + + var line = DarlingCliCommands.FormatProbeLine("aurora-writer", PostgresProbe()); + + Assert.Contains($"all {expected} PostgreSQL collectors apply", line, StringComparison.Ordinal); + Assert.DoesNotContain("skipped", line, StringComparison.Ordinal); + } + + /// + /// The case the count exists for. A stock-PostgreSQL 15 reader is the worst realistic target: no + /// Aurora functions, no pg_stat_io, and autovacuum stats that read as all zeros on a standby. Finding + /// that out at pre-flight is the difference between "this is configured" and "this will collect". + /// + [Fact] + public void FormatProbeLine_StockPostgresReader_NamesTheCollectorsThatWillNotRun() + { + var probe = PostgresProbe() with + { + PostgresMajorVersion = 15, + PostgresVersionNum = 150012, + IsAurora = false, + IsInRecovery = true, + }; + + var line = DarlingCliCommands.FormatProbeLine("selfhosted-replica", probe); + + Assert.Contains("reader (in recovery)", line, StringComparison.Ordinal); + Assert.Contains("not Aurora", line, StringComparison.Ordinal); + + /* The Aurora-only pair, the writer-only one and the 16+ one — each named, so nobody has to + reverse-engineer an empty table later. */ + Assert.Contains("skipped:", line, StringComparison.Ordinal); + Assert.Contains("pg_wait_stats", line, StringComparison.Ordinal); + Assert.Contains("pg_statement_stats", line, StringComparison.Ordinal); + Assert.Contains("pg_autovacuum_stats", line, StringComparison.Ordinal); + Assert.Contains("pg_io_stats", line, StringComparison.Ordinal); + } + + /// + /// The count is derived from the real gate, not a parallel list that can rot. Asking the catalog the + /// same question the runner asks must give the same answer. + /// + [Fact] + public void ToTargetInfo_RoundTripsTheFactsTheGateReads() + { + var target = PostgresProbe().ToTargetInfo(); + + Assert.Equal(CollectorTargetEngine.PostgreSql, target.Engine); + Assert.Equal(17, target.PostgresMajorVersion); + Assert.Equal(170007, target.PostgresVersionNum); + Assert.True(target.IsAurora); + Assert.False(target.IsInRecovery); + + Assert.All( + CollectorCatalog.All.Where(d => d.TargetEngine == CollectorTargetEngine.SqlServer), + d => Assert.False(CollectorCatalog.AppliesTo(d, target))); + } + + private static ConnectionProbeResult PostgresProbe() => new( + Success: true, MajorVersion: 0, EngineEdition: 0, EngineEditionDescription: null, + IsAzureSqlDb: false, IsAzureManagedInstance: false, IsAwsRds: false, HasMsdbAccess: true, Error: null, + Engine: CollectorTargetEngine.PostgreSql, PostgresMajorVersion: 17, PostgresVersionNum: 170007, + IsAurora: true, IsInRecovery: false); + [Fact] public void DescribeEngineEdition_MapsKnownEditions() { @@ -104,6 +196,40 @@ public void DescribeEngineEdition_MapsKnownEditions() Assert.Equal("Azure SQL Managed Instance", DarlingServerConnector.DescribeEngineEdition(8)); Assert.Contains("Unknown", DarlingServerConnector.DescribeEngineEdition(999), StringComparison.Ordinal); } + + /* ---- the collapse verb's adaptive narrowing decision (#2105 round three) — pure pins ---- */ + + private static readonly TimeSpan Day = TimeSpan.FromDays(1); + + [Fact] + public void NextNarrowingFailureCount_FullWidthSlice_TakesTheFirstHalvingStep() + { + /* A failed 24h slice narrows to 12h — one more failure than before. */ + Assert.Equal(1, DarlingCliCommands.NextNarrowingFailureCount(Day, 0, Day)); + /* And a 12h slice that fails again narrows to 6h. */ + Assert.Equal(2, DarlingCliCommands.NextNarrowingFailureCount(Day, 1, TimeSpan.FromHours(12))); + } + + [Fact] + public void NextNarrowingFailureCount_ClampedTail_SkipsStepsThatWouldRerunTheSameWindow() + { + /* The review catch: a clamped 30-minute final slice is already narrower than the 12h/6h/3h/1.5h/45m + nominal steps — re-running any of them is the identical window. The first step that actually + narrows 30m is the 22.5m floor (failure count 6). */ + Assert.Equal(6, DarlingCliCommands.NextNarrowingFailureCount(Day, 0, TimeSpan.FromMinutes(30))); + } + + [Fact] + public void NextNarrowingFailureCount_AtOrBelowTheFloor_ReturnsNull_TheSameWidthRetryTakesOver() + { + /* The 24h schedule floors at 22.5m (6 halvings). A slice at or under that width cannot be + narrowed — the caller's one fresh-connection same-width retry is the only move left, and it + must NOT be skipped just because narrowing is impossible (the run's usual last slice is a + partial-day clamp of arbitrary width). */ + Assert.Null(DarlingCliCommands.NextNarrowingFailureCount(Day, 0, TimeSpan.FromMinutes(22.5))); + Assert.Null(DarlingCliCommands.NextNarrowingFailureCount(Day, 0, TimeSpan.FromMinutes(5))); + Assert.Null(DarlingCliCommands.NextNarrowingFailureCount(Day, 6, TimeSpan.FromMinutes(22.5))); + } } /// @@ -437,6 +563,69 @@ public async Task PrintViewerConnectionAsync_ManagedViewer_PrintsConnection_Cert root.Delete(recursive: true); } } + + /// + /// #2117's print-verb half, pinned on the CHAIN-shaped store the sibling test cannot see (it lays down + /// only the legacy server.crt): when root.crt exists beside server.crt, the verb must emit the ROOT — + /// that is what verify-full's Root Certificate anchors on against a chain-serving store — and the + /// header must name the file whose content is actually below (the review-caught label lie: it said + /// server.crt over root.crt's bytes). + /// + [Fact] + public async Task PrintViewerConnectionAsync_ChainShapedStore_EmitsTheRoot_AndLabelsItHonestly() + { + Assert.SkipUnless(OperatingSystem.IsWindows(), "DPAPI requires Windows."); + + var root = Directory.CreateTempSubdirectory("darling-printconn-chain-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "pg"); + var viewerCredential = PerformanceMonitor.Darling.Service.DarlingManagedPostgres.ViewerCredentialPathFor(dataDirectory); + File.WriteAllText(viewerCredential, PerformanceMonitor.Darling.Service.DarlingSecrets.Protect("viewer-secret-pw")); + + var certPath = Path.Combine( + Path.GetDirectoryName(viewerCredential)!, + PerformanceMonitor.Darling.Service.DarlingManagedPostgres.ServerCertFileName); + const string leafPem = "-----BEGIN CERTIFICATE-----\nMIIBLEAFCHAINPEM\n-----END CERTIFICATE-----"; + const string rootPem = "-----BEGIN CERTIFICATE-----\nMIIBROOTCAPEM\n-----END CERTIFICATE-----"; + File.WriteAllText(certPath, leafPem); + File.WriteAllText( + PerformanceMonitor.Darling.Service.DarlingManagedPostgres.RootCertificatePathFor(certPath), rootPem); + + var configPath = Path.Combine(root.FullName, "darling.json"); + var json = $$""" + { + "postgres": { + "managed": true, + "port": 5641, + "dataDirectory": {{JsonSerializer.Serialize(dataDirectory)}}, + "network": { "listen": "192.168.1.205", "allowFrom": "192.168.1.0/24", "role": "viewer" } + }, + "servers": [ { "name": "SQL2022", "host": "SQL2022" } ] + } + """; + await File.WriteAllTextAsync(configPath, json); + + var output = new StringWriter(); + var exit = await DarlingCliCommands.PrintViewerConnectionAsync(configPath, output, new StringWriter(), CancellationToken.None); + var stdout = output.ToString(); + + Assert.Equal(0, exit); + + /* The ROOT's content, labeled as root.crt — never the leaf chain the server serves. */ + Assert.Contains(rootPem, stdout, StringComparison.Ordinal); + Assert.DoesNotContain(leafPem, stdout, StringComparison.Ordinal); + Assert.Contains("(root.crt)", stdout, StringComparison.Ordinal); + + /* The client-side FILE name stays server.crt (ViewerClientCertificateFileName) on purpose — + the save-as path in the connection string does not change with the store's shape. */ + Assert.Contains("Root Certificate=server.crt", stdout, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } } /// @@ -900,3 +1089,437 @@ private static int CountOccurrences(string haystack, string needle) return count; } } + +/// +/// #2197 — the missing-credential refusals every managed-store verb shares. Absence of a credential has two +/// causes that want OPPOSITE advice (a genuine first run, and a bootstrap that has already failed), and +/// before this every verb gave the first-run advice to both. The pure tests pin the evidence probe (including +/// the two ways it must NOT fire, since a wrong "your bootstrap failed" is the same defect pointed somewhere +/// new) and the two message voices; the end-to-end tests drive both branches through two real verbs, because +/// a correct builder nothing calls is exactly what the sibling #1738 defect already was. +/// +public sealed class DarlingMissingCredentialMessageTests +{ + /* ---------------- pure: what counts as evidence, and what must not ---------------- */ + + [Fact] + public void FindBootstrapEvidence_NothingOnDisk_FindsNone() + { + var root = Directory.CreateTempSubdirectory("darling-evidence-none-"); + try + { + /* The store folder itself was never created — a genuine first run. */ + Assert.Null(DarlingStoreBootstrapEvidence.FindBootstrapEvidence( + Path.Combine(root.FullName, "store", "pg"))); + } + finally + { + root.Delete(recursive: true); + } + } + + /// + /// The field case (#2185): initdb died in the Windows loader, and the service writes the store's own + /// credential IMMEDIATELY BEFORE running initdb — so that one file survives the exact failure that + /// produces the role-credential refusal, and is what makes the sharper branch reachable at all. + /// + [Fact] + public void FindBootstrapEvidence_StoreCredentialWrittenBeforeInitdb_IsTheFieldCase() + { + var root = Directory.CreateTempSubdirectory("darling-evidence-cred-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(Path.Combine(root.FullName, "store")); + var storeCredential = DarlingManagedPostgres.CredentialPathFor(dataDirectory); + File.WriteAllText(storeCredential, "not-a-real-credential"); + + var evidence = DarlingStoreBootstrapEvidence.FindBootstrapEvidence(dataDirectory); + + Assert.NotNull(evidence); + Assert.Contains(storeCredential, evidence, StringComparison.Ordinal); + Assert.Contains("immediately before it runs initdb", evidence, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void FindBootstrapEvidence_InitializedCluster_NamesTheCluster() + { + var root = Directory.CreateTempSubdirectory("darling-evidence-pgver-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(dataDirectory); + File.WriteAllText(Path.Combine(dataDirectory, "PG_VERSION"), "18\n"); + + var evidence = DarlingStoreBootstrapEvidence.FindBootstrapEvidence(dataDirectory); + + Assert.NotNull(evidence); + Assert.Contains(dataDirectory, evidence, StringComparison.Ordinal); + Assert.Contains("already initialized", evidence, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void FindBootstrapEvidence_ServerLog_NamesIt() + { + var root = Directory.CreateTempSubdirectory("darling-evidence-pglog-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(Path.Combine(root.FullName, "store")); + var serverLog = Path.Combine(root.FullName, "store", DarlingManagedPostgres.ServerLogFileName); + File.WriteAllText(serverLog, "FATAL: something\n"); + + var evidence = DarlingStoreBootstrapEvidence.FindBootstrapEvidence(dataDirectory); + + Assert.NotNull(evidence); + Assert.Contains(serverLog, evidence, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /// + /// The false-positive guard. An operator who pre-creates the data directory before the first run is + /// still ON their first run, and telling them to go read a service log that does not exist would be the + /// same misdirection this issue is about, merely pointed somewhere new. + /// + [Fact] + public void FindBootstrapEvidence_EmptyDataDirectory_IsNotEvidence() + { + var root = Directory.CreateTempSubdirectory("darling-evidence-empty-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(dataDirectory); + + Assert.Null(DarlingStoreBootstrapEvidence.FindBootstrapEvidence(dataDirectory)); + } + finally + { + root.Delete(recursive: true); + } + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void FindBootstrapEvidence_EmptyPath_FindsNone_WithoutProbingTheWorkingDirectory(string dataDirectory) + { + /* An empty path must never become a RELATIVE one, which would answer about whatever directory the + operator happened to run the verb from. */ + Assert.Null(DarlingStoreBootstrapEvidence.FindBootstrapEvidence(dataDirectory)); + } + + /* ---------------- pure: the two voices ---------------- */ + + [Fact] + public void MissingCredentialMessage_NoEvidence_KeepsTheFirstRunAdvice_AndHedgesForAnAlreadyStartedService() + { + var root = Directory.CreateTempSubdirectory("darling-msg-firstrun-"); + try + { + var message = DarlingStoreBootstrapEvidence.MissingCredentialMessage( + @"The 'viewer' role credential (C:\store\pg-viewer-credential.dpapi)", + "provisions the least-privilege roles and their credentials", + Path.Combine(root.FullName, "store", "pg")); + + /* The advice that is CORRECT for a genuine first run is unchanged — and still searchable. */ + Assert.Contains("does not exist yet", message, StringComparison.Ordinal); + Assert.Contains("Start the PerformanceMonitor Darling service once", message, StringComparison.Ordinal); + Assert.Contains("provisions the least-privilege roles and their credentials", message, StringComparison.Ordinal); + + /* Plus the sentence the old message was missing entirely: the operator who has ALREADY started + it is told where the reason is, and told it is not in darling.json. */ + Assert.Contains("ALREADY started it", message, StringComparison.Ordinal); + Assert.Contains(DarlingStoreBootstrapEvidence.ServiceLogPath, message, StringComparison.Ordinal); + Assert.Contains("darling.json", message, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void MissingCredentialMessage_BootstrapAlreadyAttempted_PointsAtTheLog_AndNeverAtStartingTheServiceAgain() + { + var root = Directory.CreateTempSubdirectory("darling-msg-failed-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(Path.Combine(root.FullName, "store")); + File.WriteAllText(DarlingManagedPostgres.CredentialPathFor(dataDirectory), "not-a-real-credential"); + + var message = DarlingStoreBootstrapEvidence.MissingCredentialMessage( + @"The 'viewer' role credential (C:\store\pg-viewer-credential.dpapi)", + "provisions the least-privilege roles and their credentials", + dataDirectory); + + /* The whole point: this operator must NOT be sent to start the service again, and must not be + sent to darling.json either. */ + Assert.DoesNotContain("Start the PerformanceMonitor Darling service once", message, StringComparison.Ordinal); + Assert.DoesNotContain("does not exist yet", message, StringComparison.Ordinal); + Assert.Contains("NOT a first run", message, StringComparison.Ordinal); + Assert.Contains("starting it again is not the fix", message, StringComparison.Ordinal); + + /* Where to look, named — and why the log is worth reading now (#2194 decodes a bundled tool + that Windows killed instead of printing a bare number). */ + Assert.Contains(DarlingStoreBootstrapEvidence.ServiceLogPath, message, StringComparison.Ordinal); + Assert.Contains("FIRST error", message, StringComparison.Ordinal); + Assert.Contains("bare exit code", message, StringComparison.Ordinal); + Assert.Contains("Nothing in darling.json produces this", message, StringComparison.Ordinal); + + /* The evidence is QUOTED rather than asserted, so the verdict is checkable by the operator. */ + Assert.Contains(DarlingManagedPostgres.CredentialPathFor(dataDirectory), message, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /// The lead clause is the same in both voices: field reports and the issue tracker are + /// searchable by it, so the branch changes what FOLLOWS it, never what an operator pastes into a + /// search box. + [Fact] + public void MissingCredentialMessage_BothVoices_KeepTheSameSearchableLead() + { + var root = Directory.CreateTempSubdirectory("darling-msg-lead-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + const string subject = "The managed store credential (C:\\store\\pg-credential.dpapi)"; + + var firstRun = DarlingStoreBootstrapEvidence.MissingCredentialMessage( + subject, "initializes the store", dataDirectory); + + Directory.CreateDirectory(dataDirectory); + File.WriteAllText(Path.Combine(dataDirectory, "PG_VERSION"), "18\n"); + var attempted = DarlingStoreBootstrapEvidence.MissingCredentialMessage( + subject, "initializes the store", dataDirectory); + + Assert.StartsWith(subject + " does not exist", firstRun, StringComparison.Ordinal); + Assert.StartsWith(subject + " does not exist", attempted, StringComparison.Ordinal); + Assert.NotEqual(firstRun, attempted); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void MissingStoreCredentialMessage_NamesTheCredentialPath_WhichTheOldMessageNeverDid() + { + var root = Directory.CreateTempSubdirectory("darling-msg-storecred-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + var postgres = DarlingConfig.Parse($$""" + { + "postgres": { + "managed": true, + "dataDirectory": {{JsonSerializer.Serialize(dataDirectory)}} + } + } + """).Postgres; + + var message = DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres); + + Assert.Contains("The managed store credential", message, StringComparison.Ordinal); + Assert.Contains(DarlingManagedPostgres.CredentialPathFor(dataDirectory), message, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /* ---------------- end-to-end: both branches, through two real verbs ---------------- */ + + [Fact] + public async Task PrintViewerConnection_TrueFirstRun_StillTellsThemToStartTheService() + { + var root = Directory.CreateTempSubdirectory("darling-e2e-firstrun-"); + try + { + /* The store folder does not exist at all — nothing has ever run against it. */ + var configPath = WriteManagedConfig(root.FullName, Path.Combine(root.FullName, "store", "pg")); + + var error = new StringWriter(); + var exit = await DarlingCliCommands.PrintViewerConnectionAsync( + configPath, new StringWriter(), error, CancellationToken.None); + var stderr = error.ToString(); + + Assert.Equal(1, exit); + Assert.Contains("pg-viewer-credential.dpapi", stderr, StringComparison.Ordinal); + Assert.Contains("Start the PerformanceMonitor Darling service once", stderr, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public async Task PrintViewerConnection_AfterAFailedBootstrap_NamesTheLogInsteadOfTheService() + { + var root = Directory.CreateTempSubdirectory("darling-e2e-failed-"); + try + { + /* The #2185 shape, on disk: the service ran, wrote the store credential, and its initdb died — + so the role credentials were never provisioned. */ + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(Path.Combine(root.FullName, "store")); + File.WriteAllText(DarlingManagedPostgres.CredentialPathFor(dataDirectory), "not-a-real-credential"); + var configPath = WriteManagedConfig(root.FullName, dataDirectory); + + var error = new StringWriter(); + var exit = await DarlingCliCommands.PrintViewerConnectionAsync( + configPath, new StringWriter(), error, CancellationToken.None); + var stderr = error.ToString(); + + Assert.Equal(1, exit); + Assert.Contains("pg-viewer-credential.dpapi", stderr, StringComparison.Ordinal); + Assert.DoesNotContain("Start the PerformanceMonitor Darling service once", stderr, StringComparison.Ordinal); + Assert.Contains(DarlingStoreBootstrapEvidence.ServiceLogPath, stderr, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public async Task EnableMcp_TrueFirstRun_StillTellsThemToStartTheService() + { + var root = Directory.CreateTempSubdirectory("darling-e2e-mcp-firstrun-"); + try + { + var configPath = WriteManagedConfig(root.FullName, Path.Combine(root.FullName, "store", "pg")); + + var error = new StringWriter(); + var exit = await DarlingCliCommands.EnableMcpAsync( + configPath, new StringWriter(), error, CancellationToken.None); + var stderr = error.ToString(); + + Assert.Equal(1, exit); + Assert.Contains("The managed store credential", stderr, StringComparison.Ordinal); + Assert.Contains("Start the PerformanceMonitor Darling service once", stderr, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /// + /// The store-credential verbs' own nasty state: a cluster EXISTS but its credential does not, so initdb + /// will never run again (it only runs on an empty data directory) and no number of restarts produces the + /// file. "Start the service once so its first run initializes the store" was a closed loop there. + /// + [Fact] + public async Task EnableMcp_ClusterExistsButCredentialDoesNot_NamesTheLogInsteadOfTheService() + { + var root = Directory.CreateTempSubdirectory("darling-e2e-mcp-failed-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(dataDirectory); + File.WriteAllText(Path.Combine(dataDirectory, "PG_VERSION"), "18\n"); + var configPath = WriteManagedConfig(root.FullName, dataDirectory); + + var error = new StringWriter(); + var exit = await DarlingCliCommands.EnableMcpAsync( + configPath, new StringWriter(), error, CancellationToken.None); + var stderr = error.ToString(); + + Assert.Equal(1, exit); + Assert.DoesNotContain("Start the PerformanceMonitor Darling service once", stderr, StringComparison.Ordinal); + Assert.Contains("NOT a first run", stderr, StringComparison.Ordinal); + Assert.Contains(DarlingStoreBootstrapEvidence.ServiceLogPath, stderr, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /* ---------------- wiring: no verb keeps a private copy of the old advice ---------------- */ + + /// + /// The defect was five INDEPENDENT copies of one sentence, so the fix is only real if none of them + /// survives. Parsed at the source because four of the five sit behind a store that a test cannot stand + /// up, and a sixth copy added later would reintroduce the bug silently. + /// + [Fact] + public void NoVerbStillCarriesItsOwnFirstRunAdvice() + { + var source = ReadRepoFile(Path.Combine( + "Darling", "PerformanceMonitor.Darling.Service", "DarlingCliCommands.cs")); + + Assert.DoesNotContain("so its first run initializes the store", source, StringComparison.Ordinal); + Assert.DoesNotContain("first run provisions the least-privilege roles", source, StringComparison.Ordinal); + + /* Every managed missing-credential refusal goes through the shared builder instead — one for the + role credentials, FIVE for the store's own (--add-server became the fifth in #2256; the count grows + with each new store verb, and growing it is the point — a verb that grew its OWN copy of the advice + instead would fail the two DoesNotContain assertions above). */ + Assert.Equal(1, CountOccurrences(source, "DarlingStoreBootstrapEvidence.MissingCredentialMessage(")); + Assert.Equal(5, CountOccurrences(source, "DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(")); + } + + private static string WriteManagedConfig(string directory, string dataDirectory) + { + var configPath = Path.Combine(directory, "darling.json"); + File.WriteAllText(configPath, $$""" + { + "postgres": { + "managed": true, + "port": 5641, + "dataDirectory": {{JsonSerializer.Serialize(dataDirectory)}} + }, + "servers": [] + } + """); + return configPath; + } + + /* Locate the repo from this file — the DarlingEnumerationProbeFailureTests idiom; no build-output copying. */ + private static string ReadRepoFile(string relative, [CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } + + private static int CountOccurrences(string haystack, string needle) + { + var count = 0; + var index = 0; + while ((index = haystack.IndexOf(needle, index, StringComparison.Ordinal)) >= 0) + { + count++; + index += needle.Length; + } + + return count; + } +} diff --git a/Darling/Darling.Tests/DarlingDeltaSeederTests.cs b/Darling/Darling.Tests/DarlingDeltaSeederTests.cs index 8cc23cf0b..d17c0eade 100644 --- a/Darling/Darling.Tests/DarlingDeltaSeederTests.cs +++ b/Darling/Darling.Tests/DarlingDeltaSeederTests.cs @@ -173,7 +173,7 @@ public async Task EndToEnd_SeedFromStore_LatestRowBecomesBaseline_AgainstDevPost await DeleteTestRowsAsync(connection, TestContext.Current.CancellationToken); /* Two rows for the same wait type: an older baseline and the latest one, both recent - enough to stay inside the wait-stats 300-second gap policy. */ + enough to stay inside the 300 s gap this call passes explicitly. */ var olderTime = DateTime.SpecifyKind(DateTime.UtcNow.AddMinutes(-2), DateTimeKind.Unspecified); var latestTime = DateTime.SpecifyKind(DateTime.UtcNow.AddMinutes(-1), DateTimeKind.Unspecified); await InsertWaitStatsRowAsync(connection, olderTime, waitingTasks: 10, waitTimeMs: 2000, signalWaitTimeMs: 500); diff --git a/Darling/Darling.Tests/DarlingDimensionGcBoundTests.cs b/Darling/Darling.Tests/DarlingDimensionGcBoundTests.cs index f4005b409..c4c9b19da 100644 --- a/Darling/Darling.Tests/DarlingDimensionGcBoundTests.cs +++ b/Darling/Darling.Tests/DarlingDimensionGcBoundTests.cs @@ -84,4 +84,145 @@ public void V39Indexes_UseExactlyTheProbePredicates() Assert.Contains($"WHERE {predicate}", v39, StringComparison.Ordinal); } } + + /// + /// #2210 / #1969: timestamps bound to the map's naive ::timestamp parameters must carry + /// , or Npgsql infers timestamptz from the Kind and Postgres + /// converts into the session zone on the way in — landing last_seen at the wrong hour silently. For a + /// liveness column that ages a row out ahead of the facts referencing it, which is the silent-missing-plans + /// outcome by way of a timezone. + /// + /// Pins that the helper relabels without SHIFTING: same ticks, Kind cleared. A version that converted + /// would be worse than the bug, since it would look correct in isolation. + /// + [Fact] + public void MapTimestamps_AreRelabelledNaive_WithoutShiftingTheInstant() + { + var utc = new DateTime(2026, 8, 12, 10, 52, 43, DateTimeKind.Utc); + + var naive = QueryStorePlanMap.Naive(utc); + + Assert.Equal(DateTimeKind.Unspecified, naive.Kind); + Assert.Equal(utc.Ticks, naive.Ticks); + + /* Idempotent, and a Local input is relabelled rather than converted — the caller's contract is that it + passes UTC, and this must not quietly "fix" a value it was handed. */ + Assert.Equal(naive, QueryStorePlanMap.Naive(naive)); + Assert.Equal( + utc.Ticks, + QueryStorePlanMap.Naive(DateTime.SpecifyKind(utc, DateTimeKind.Local)).Ticks); + } + + /// + /// #2210, the both-orders race: whichever order the two prunes run in, there must be no reachable state + /// where a surviving map row resolves to an absent digest. + /// + /// Driven off the real cutoffs rather than a narrative. Plant a (map row, dim row) pair sharing one + /// stale last_seen, then ask both cutoffs about it. Because the map's cutoff is strictly LATER, the + /// only orderings available are "map goes, dim stays" (a plan renders as not-collected, self-correcting) or + /// "both go" — never "dim goes, map stays", which is the reader-resolves-to-nothing case. Order of execution + /// cannot produce the bad state because the eligibility windows themselves are nested. + /// + [Theory] + [InlineData(30)] + [InlineData(7)] + public void NeitherPruneOrder_CanLeaveAMapRowResolvingToAnAbsentDigest(int factRetentionDays) + { + var dimCutoff = DarlingRetention.ComputeDimensionCutoff(Now, factRetentionDays, oldestSurvivingDigestFact: null); + var mapCutoff = Now.AddDays(-(factRetentionDays + QueryStorePlanMap.PruneMarginDays)); + + /* Every last_seen from well inside retention to well past both horizons. */ + for (var age = 0; age <= factRetentionDays + TimescaleSupport.ChunkIntervalDays + 4; age++) + { + var lastSeen = Now.AddDays(-age); + var mapEligible = lastSeen < mapCutoff; + var dimEligible = lastSeen < dimCutoff; + + /* The forbidden combination: the dim row is takeable while the map row that points at it is not. */ + Assert.False(dimEligible && !mapEligible, + $"at {age}d the dim row is prunable while its map row survives — a live fact would resolve to absent content"); + } + } + + /// + /// #2210: the re-verify cursor paces itself off RefreshAfter and NEVER touches the watermark. The + /// slice is a row count over an id range, which is the whole point — the old expiry walked BYTES and could + /// not finish inside a day on the catalogs that mattered (15.9 to 107.5 hours measured), so those restarted + /// forever. Redstone's 77k ids at a 5-minute cadence over a 1-day sweep is ~267 ids per pass. + /// + [Fact] + public void CursorSlice_PacesASweepWithinTheRefreshPeriod_AndNeverReturnsZeroForALiveCatalog() + { + var day = TimeSpan.FromDays(1); + var cadence = TimeSpan.FromMinutes(5); + + var redstone = QueryStorePlanMap.CursorSliceWidth(77_176, day, cadence); + Assert.InRange(redstone, 200, 350); + + /* A sweep must actually cover the range within the period: slice * passes >= watermark. */ + var passes = day.Ticks / cadence.Ticks; + Assert.True(redstone * passes >= 77_176, "the sweep must cover the id range inside one refresh period"); + + /* Never zero for a live catalog, and never wider than the range itself. */ + Assert.True(QueryStorePlanMap.CursorSliceWidth(10, day, cadence) > 0); + Assert.Equal(10, QueryStorePlanMap.CursorSliceWidth(10, day, cadence)); + + /* A fresh database has no watermark to re-verify, so there is nothing to slice. */ + Assert.Equal(0, QueryStorePlanMap.CursorSliceWidth(0, day, cadence)); + } + + /// + /// #2210: the DIMENSION must outlive the MAP, expressed the way it actually matters — as cutoff DATES from + /// the two real code paths, not as the margin constants they happen to be derived from. An earlier cutoff + /// deletes fewer rows, so the dim's cutoff has to be strictly earlier than the map's. + /// + /// The asymmetry is the reason this is pinned. A pruned map row whose dim row survives renders a plan + /// as "not collected" and leaves some bytes unreclaimed until the dim's own horizon passes — visible and + /// self-correcting. A pruned DIM row whose map row survives is a reader resolving a live fact to absent + /// content, silently, weeks after the cause. Only one of those is recoverable, and the margin ordering is + /// what makes it the only reachable one. + /// + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(30)] + [InlineData(90)] + public void DimensionOutlivesTheMap_AtEveryFactRetention(int factRetentionDays) + { + var dimCutoff = DarlingRetention.ComputeDimensionCutoff(Now, factRetentionDays, oldestSurvivingDigestFact: null); + var mapCutoff = Now.AddDays(-(factRetentionDays + QueryStorePlanMap.PruneMarginDays)); + + Assert.True(dimCutoff < mapCutoff, + $"the dim GC would take content the map still points at: dim cutoff {dimCutoff:o} is not earlier " + + $"than map cutoff {mapCutoff:o} at {factRetentionDays}d retention"); + } + + /// + /// The invariant's own guard, and the direction it fails in. + /// is where the dim's margin comes from, so shrinking it is the realistic way somebody inverts this without + /// touching either margin deliberately — at 0 the two margins meet and the ordering is gone. + /// + [Fact] + public void MarginOrdering_HoldsAtTheLiveChunkInterval_AndFailsWhenTheMarginsMeet() + { + Assert.True(QueryStorePlanMap.MarginOrderingHolds(TimescaleSupport.ChunkIntervalDays)); + Assert.False(QueryStorePlanMap.MarginOrderingHolds(0)); + } + + /// + /// The measured clamp cannot protect Query Store digests, which is why the batch-touch refresh is the whole + /// protection (#2210). The clamp reads the oldest surviving DIGEST-CARRYING fact, and the digest-carrying + /// fact tables are exactly the two in — Query Store is deliberately not + /// among them, because its facts resolve through the map instead of carrying a digest column. + /// + /// Pinned so that adding a query_store entry to All — the tempting way to "fix" the blindness — + /// fails here and sends the reader to the comment explaining that the entry would describe a column that + /// does not exist. + /// + [Fact] + public void TheMeasuredClamp_IsBlindToQueryStoreFacts_ByConstruction() + { + Assert.DoesNotContain("query_store_stats", PayloadDimensions.All.Select(d => d.TargetTable)); + Assert.DoesNotContain("query_store_stats", PayloadDimensions.DigestPredicateByTable.Keys); + } } diff --git a/Darling/Darling.Tests/DarlingEmptyEnumerationInventoryTests.cs b/Darling/Darling.Tests/DarlingEmptyEnumerationInventoryTests.cs index 9fcff4c1c..3313f4957 100644 --- a/Darling/Darling.Tests/DarlingEmptyEnumerationInventoryTests.cs +++ b/Darling/Darling.Tests/DarlingEmptyEnumerationInventoryTests.cs @@ -107,7 +107,7 @@ quietly turn this pin into a comparison of two empty sets. */ /* Named outright as well as compared, so the failure message names the drift rather than a set. */ Assert.Equal( - new HashSet(StringComparer.OrdinalIgnoreCase) { "query_store", "database_scoped_config", "index_object_stats", "plan_correction" }, + new HashSet(StringComparer.OrdinalIgnoreCase) { "query_store", "database_scoped_config", "index_object_stats", "plan_correction", "query_store_health" }, enumerators); foreach (var name in CollectorCatalog.All.Select(c => c.Name)) diff --git a/Darling/Darling.Tests/DarlingIncidentFingerprintTests.cs b/Darling/Darling.Tests/DarlingIncidentFingerprintTests.cs new file mode 100644 index 000000000..cb729df21 --- /dev/null +++ b/Darling/Darling.Tests/DarlingIncidentFingerprintTests.cs @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2159: recomputing the #1140 dedup fingerprint for STORED incident rows, so an operator can paste an alert's +/// Dedup Key into get_deadlocks / get_deadlock_detail / get_blocking and land on that one +/// incident instead of scanning a server+time window and eyeballing which deadlock matched. +/// +/// The only failure mode worth testing hard is silent disagreement. The key is a SHA-256 over +/// normalized identity members, so a reader that derives those members even slightly differently from the alert +/// path produces a different hash and matches NOTHING — and an empty result is indistinguishable from "that +/// incident is outside the window". So the centrepiece here is parity against the REAL alert entry points +/// ( / ), +/// not against a hand-computed hash: a hand-computed expectation would pin my reimplementation rather than the +/// product's agreement with itself. +/// +public sealed class DarlingIncidentFingerprintTests +{ + private const string ServerName = "SQLPROD01"; + private static readonly IReadOnlyList NoExclusions = Array.Empty(); + + /// A deadlock graph with two lock resources, the shape the object extractor reads. + private static string Graph(string firstObject, string secondObject) => $""" + + + + UPDATE x + UPDATE y + + + + + + + + + + + + + """; + + private static DeadlockAlertRow DeadlockRow(string xml) => + new() { VictimProcessId = "process1", VictimSqlText = "UPDATE x", DeadlockGraphXml = xml }; + + /// + /// THE TEST THIS FEATURE RESTS ON. For the same stored graph, the reader's recomputed key must equal the key + /// the alert path emits — compared against itself, which + /// is what actually feeds the Dedup Key fact an operator pastes in. + /// + [Fact] + public void DeadlockKey_MatchesTheAlertPathsKeyForTheSameGraph() + { + var xml = Graph("SalesDB.dbo.Orders.PK_Orders", "SalesDB.dbo.LineItems.PK_LineItems"); + + var fromAlert = AlertContextBuilders.DeadlockIncidents(ServerName, new[] { DeadlockRow(xml) }, NoExclusions); + var fromReader = DarlingIncidentFingerprint.DeadlockKeys(ServerName, new[] { xml }); + + Assert.Single(fromAlert); + Assert.Single(fromReader); + Assert.Equal(fromAlert[0].DedupKey, fromReader[0]); + /* Guard the vacuous pass: two nulls would satisfy Equal and prove nothing. */ + Assert.False(string.IsNullOrEmpty(fromReader[0])); + } + + /// + /// The blocking twin. Kept separate because the two paths differ in a way that matters: a blocking key comes + /// from the identity bucket's REPRESENTATIVE and falls back from the contentious object to a query-pair key, + /// so it is the one with real room to disagree. + /// + [Fact] + public void BlockingKey_MatchesTheAlertPathsKeyForTheSameRow() + { + var row = new BlockedProcessAlertRow + { + DatabaseName = "SalesDB", + ContentiousObject = "SalesDB.dbo.Orders", + BlockedSqlText = "SELECT * FROM dbo.Orders WHERE id = 1", + BlockingSqlText = "UPDATE dbo.Orders SET total = 5 WHERE id = 1", + WaitTimeMs = 41_000, + LockMode = "X", + }; + + var fromAlert = AlertContextBuilders.BlockingIncidents(ServerName, new[] { row }, NoExclusions); + var fromReader = DarlingIncidentFingerprint.BlockingKeys( + ServerName, + new[] + { + new BlockingIncidentGrouper.BlockedEvent( + row.DatabaseName, row.ContentiousObject, row.BlockedSqlText, row.BlockingSqlText, + row.WaitTimeMs, row.LockMode), + }); + + Assert.Single(fromAlert); + Assert.Single(fromReader); + Assert.Equal(fromAlert[0].DedupKey, fromReader[0]); + Assert.False(string.IsNullOrEmpty(fromReader[0])); + } + + /// + /// The blocking fallback path: no contentious object, so the key comes from the normalized query pair rather + /// than an object set. Exercised separately because it is a different branch of the grouper and the branch a + /// reimplementation would most likely get wrong. + /// + [Fact] + public void BlockingKey_MatchesTheAlertPath_WhenNoContentiousObjectResolved() + { + var row = new BlockedProcessAlertRow + { + DatabaseName = "SalesDB", + ContentiousObject = "", + BlockedSqlText = "SELECT * FROM dbo.Orders WHERE id = 99", + BlockingSqlText = "UPDATE dbo.Orders SET total = 7 WHERE id = 99", + WaitTimeMs = 12_000, + LockMode = "X", + }; + + var fromAlert = AlertContextBuilders.BlockingIncidents(ServerName, new[] { row }, NoExclusions); + var fromReader = DarlingIncidentFingerprint.BlockingKeys( + ServerName, + new[] + { + new BlockingIncidentGrouper.BlockedEvent( + row.DatabaseName, row.ContentiousObject, row.BlockedSqlText, row.BlockingSqlText, + row.WaitTimeMs, row.LockMode), + }); + + Assert.Single(fromAlert); + Assert.Equal(fromAlert[0].DedupKey, fromReader[0]); + Assert.False(string.IsNullOrEmpty(fromReader[0])); + } + + /// + /// Several rows, several incidents: every row's key must equal the alert path's key for its OWN objects, and + /// the two recurrences over the same object set must collapse to one key. Positional correctness is the + /// contract the callers rely on — they filter their own row list by index. + /// + [Fact] + public void DeadlockKeys_ArePositionalAndRecurrencesShareAKey() + { + var ordersGraph = Graph("SalesDB.dbo.Orders.PK_Orders", "SalesDB.dbo.LineItems.PK_LineItems"); + var swapped = Graph("SalesDB.dbo.LineItems.PK_LineItems", "SalesDB.dbo.Orders.PK_Orders"); + var otherGraph = Graph("SalesDB.dbo.Customers.PK_Customers", "SalesDB.dbo.Addresses.PK_Addresses"); + + var keys = DarlingIncidentFingerprint.DeadlockKeys(ServerName, new[] { ordersGraph, otherGraph, swapped }); + + Assert.Equal(3, keys.Count); + /* Same objects in the other order is the SAME incident — the fingerprint sorts its members. */ + Assert.Equal(keys[0], keys[2]); + Assert.NotEqual(keys[0], keys[1]); + + var fromAlert = AlertContextBuilders.DeadlockIncidents( + ServerName, new[] { DeadlockRow(otherGraph) }, NoExclusions); + Assert.Equal(fromAlert[0].DedupKey, keys[1]); + } + + /// + /// THE SCOPE TRAP, pinned. The fingerprint hashes the server name, and the alert path passes the DISPLAY + /// name while the MCP resolver returns the STORAGE name. If those two strings ever fed the same hash the + /// filter would appear to work in testing and return nothing in the field for every renamed server — so + /// assert they genuinely differ, which is what makes FingerprintNameOf load-bearing rather than + /// decorative. + /// + [Fact] + public void TheKeyIsScopedToTheServerName_SoDisplayAndStorageNamesDisagree() + { + var xml = Graph("SalesDB.dbo.Orders.PK_Orders", "SalesDB.dbo.LineItems.PK_LineItems"); + + var underDisplayName = DarlingIncidentFingerprint.DeadlockKeys("Sales Primary", new[] { xml })[0]; + var underStorageName = DarlingIncidentFingerprint.DeadlockKeys("sqlprod01.contoso.com:SalesDB", new[] { xml })[0]; + + Assert.NotEqual(underDisplayName, underStorageName); + } + + /// + /// A graph the extractor finds no objects in yields NO key, so it can never match a filter. That is correct + /// rather than a gap: the alerting layer emits no incident for it either, so there is no key it could be + /// searched by. Must not throw, and must keep its position. + /// + [Theory] + [InlineData("")] + [InlineData(null)] + [InlineData("")] + [InlineData("not xml at all")] + public void ARowWithNoExtractableObjects_HasNoKeyAndDoesNotThrow(string? xml) + { + var keys = DarlingIncidentFingerprint.DeadlockKeys(ServerName, new[] { xml }); + + Assert.Single(keys); + Assert.Null(keys[0]); + } + + /// + /// Keys arrive by copy-and-paste out of a ticket or an alert card, so surrounding whitespace and upper-casing + /// must not defeat the filter. The stored form is lowercase hex. + /// + [Theory] + [InlineData(" ABCDEF0123 ", "abcdef0123")] + [InlineData("ABCDEF0123", "abcdef0123")] + [InlineData("abcdef0123", "abcdef0123")] + [InlineData("\tabcdef0123\n", "abcdef0123")] + public void APastedKeyIsNormalizedBeforeComparison(string pasted, string expected) + { + Assert.Equal(expected, DarlingIncidentFingerprint.NormalizeKey(pasted)); + } + + /// + /// Absent, blank and whitespace-only all mean "no filter requested" — the tools must not treat a blank + /// string as a key that matches nothing, which would turn an omitted argument into an empty result. + /// + [Theory] + [InlineData(null, true)] + [InlineData("", true)] + [InlineData(" ", true)] + [InlineData("abcdef", false)] + public void BlankMeansNoFilterRatherThanAKeyThatMatchesNothing(string? dedupKey, bool expected) + { + Assert.Equal(expected, DarlingIncidentFingerprint.NoFilter(dedupKey)); + } + + /// + /// The no-match message has to be diagnostic, because the three causes are indistinguishable from silence: + /// wrong window, wrong server, or a server renamed since the alert fired. It must name the rename — that one + /// is invisible and permanent — and report how many rows were actually examined, which separates "nothing to + /// match against" from "plenty, none matching". + /// + [Fact] + public void TheNoMatchMessageExplainsWhyRatherThanJustSayingEmpty() + { + var message = DarlingIncidentFingerprint.NoMatchMessage("deadlocks", " ABCDEF ", "Sales Primary", 17); + + Assert.Contains("abcdef", message, StringComparison.Ordinal); + Assert.Contains("Examined 17 deadlocks", message, StringComparison.Ordinal); + Assert.Contains("Sales Primary", message, StringComparison.Ordinal); + Assert.Contains("renamed", message, StringComparison.Ordinal); + Assert.Contains("hours_back", message, StringComparison.Ordinal); + } + + /// + /// FingerprintNameOf reproduces the alert path's choice: the display name when there is one, the + /// storage name only as the fallback for a registry row written without one. + /// + [Theory] + [InlineData("host1:SalesDB", "Sales Primary", "Sales Primary")] + [InlineData("host1:SalesDB", null, "host1:SalesDB")] + [InlineData("host1:SalesDB", "", "host1:SalesDB")] + [InlineData("host1:SalesDB", " ", "host1:SalesDB")] + public void FingerprintNamePrefersTheDisplayNameTheAlertPathHashes( + string storageName, string? displayName, string expected) + { + var server = new DarlingServerResolver.RegisteredServer(42, storageName, displayName); + + Assert.Equal(expected, DarlingServerResolver.FingerprintNameOf(server)); + } +} diff --git a/Darling/Darling.Tests/DarlingInstallLocationTests.cs b/Darling/Darling.Tests/DarlingInstallLocationTests.cs new file mode 100644 index 000000000..7a988735f --- /dev/null +++ b/Darling/Darling.Tests/DarlingInstallLocationTests.cs @@ -0,0 +1,447 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using Xunit; + +namespace Darling.Tests; + +/// +/// install-darling.ps1's install-location guard (#2187). +/// +/// The defect. #2185 extracted the zip to +/// C:\Users\username\Desktop\PerformanceMonitorDarling-3.2.0\ — a completely reasonable thing to do +/// with a download — and the installer created a service that could never work. The service runs as the +/// virtual account NT SERVICE\PerformanceMonitor Darling and never as LocalSystem, because the +/// bundled PostgreSQL refuses to run with administrative privileges; that account is not the installing +/// user, not SYSTEM, and not Administrators, and a profile directory grants access to approximately those +/// three and nobody else. Measured on Windows 11, a directory created under a profile inherits exactly +/// SYSTEM / Administrators / the profile owner, with no BUILTIN\Users, no Authenticated Users and no +/// CREATOR OWNER. So the install succeeded and the bundled PostgreSQL's initdb.exe died at 0xC0000135 +/// (STATUS_DLL_NOT_FOUND) before writing a word of output. +/// +/// Why these tests are split the way they are. The ORDERING and the VERDICT are pinned +/// structurally, the same way this project pins every other PowerShell invariant it cannot compile. The +/// path BOUNDARY is not: it is a decision computed from a string, where the failure that matters most — +/// refusing C:\UsersData because it starts with the same characters as C:\Users — is +/// invisible to any source-parsing assertion and strands an install that would have worked. So that one is +/// executed, against the function as it ships, under Windows PowerShell 5.1: the host an operator on +/// Windows Server 2019 (the reporter's OS) actually gets. +/// +public class DarlingInstallLocationTests +{ + private static string InstallScript => ReadRepoFile(Path.Combine("Darling", "tools", "install-darling.ps1")); + + /// + /// The guard must run before anything that changes machine state. A check that fires after the Event + /// Log source is registered, or after sc create, leaves exactly the debris #2187 exists to + /// prevent: a service on the box that can never start. + /// + [Fact] + public void LocationGuard_RunsBeforeAnythingIsInstalled() + { + var script = InstallScript; + + var guard = script.IndexOf("if ($underProfile -or $networkKind) {", StringComparison.Ordinal); + Assert.True(guard >= 0, "install-darling.ps1 no longer guards the install location (#2187)"); + + /* Every state-changing step in the script, in the order it runs. The pre-flight is included on + purpose: probing every configured SQL Server before saying "this folder cannot work" spends the + operator's time on a question that is already answered. */ + foreach (var (marker, what) in new[] + { + ("& $serviceExe --test-connection", "the --test-connection pre-flight"), + ("Copy-Item $samplePath $configPath", "copying darling.sample.json to darling.json"), + ("New-EventLog -LogName Application", "registering the Event Log source"), + ("& sc.exe create $serviceName", "creating the service"), + ("Set-Acl -Path $secretFile", "hardening the config's ACL"), + ("Start-Service -Name $serviceName", "starting the service"), + }) + { + var at = script.IndexOf(marker, StringComparison.Ordinal); + Assert.True(at >= 0, $"install-darling.ps1 no longer contains {what} ('{marker}')"); + Assert.True(guard < at, $"the install-location guard must run BEFORE {what}, or a doomed location still leaves debris behind (#2187)"); + } + } + + /// + /// A FRESH install in an unreadable location is refused outright, not warned about. #2187 chose this + /// deliberately over warning: the install cannot work, so proceeding is never right — and the reporter's + /// experience is precisely that of an install nothing stopped. + /// + /// An UPGRADE is the one case that asks instead, and that asymmetry is the same one the script + /// already applies to the service's logon account: the upgrade path exists to preserve installs + /// operators have customized (a re-homed domain account or gMSA, #1802/#1823), and granting the service + /// account read on the tree by hand — #2187's rejected option 2 — is exactly the customization that can + /// make this location work. Refusing there would strand a deployment that runs today. + /// + [Fact] + public void LocationGuard_RefusesAFreshInstall_AndOnlyAsksOnAnUpgrade() + { + var script = InstallScript; + var guard = ExtractBracedBlock(script, "if ($underProfile -or $networkKind) {"); + + var refusal = guard.IndexOf("if (-not $existing) {", StringComparison.Ordinal); + Assert.True(refusal >= 0, "the guard no longer distinguishes a fresh install from an upgrade (#2187)"); + + var fresh = ExtractBracedBlock(guard, "if (-not $existing) {"); + Assert.Contains("Fail", fresh, StringComparison.Ordinal); + Assert.DoesNotContain("Read-Host", fresh, StringComparison.Ordinal); + + /* The upgrade branch asks — and defaults to NO, so an operator who hits Enter through an unattended + run does not silently re-point a service at a folder it cannot read. */ + Assert.Contains("Read-Host 'Point the service at this folder anyway? [y/N]'", guard, StringComparison.Ordinal); + Assert.Contains("if ($answer -notmatch '^[Yy]') { exit 4 }", guard, StringComparison.Ordinal); + + /* $existing has to be resolved before the guard consults it, or the fresh/upgrade split silently + collapses to "always fresh" — which would refuse every upgrade of an install already living + there, the one outcome that breaks a working deployment. */ + var resolved = script.IndexOf("$existing = Get-Service -Name $serviceName -ErrorAction SilentlyContinue", StringComparison.Ordinal); + Assert.True(resolved >= 0 && resolved < script.IndexOf("if ($underProfile -or $networkKind) {", StringComparison.Ordinal), + "$existing must be resolved BEFORE the location guard, which branches on it"); + } + + /// + /// The profile root is read from ProfileList\ProfilesDirectory rather than hardcoded to + /// C:\Users. It is relocatable, and a literal would stop matching on exactly the box that moved + /// it — the box where a missed check costs the most. + /// + [Fact] + public void LocationGuard_ReadsTheProfileRootFromWindows_RatherThanAssumingCUsers() + { + var script = InstallScript; + + Assert.Contains(@"HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList", script, StringComparison.Ordinal); + Assert.Contains("ProfilesDirectory", script, StringComparison.Ordinal); + + /* And the current user's own profile is checked as well: a profile redirected outside + ProfilesDirectory is still a profile, and it is the one whose owner is most likely running this. */ + Assert.Contains("Test-PathIsAtOrUnder $root $env:USERPROFILE", script, StringComparison.Ordinal); + } + + /// + /// #2187 weighed fixing the tree's ACLs against refusing, and rejected the fix: it would mean the + /// product starts silently ACLing directories inside somebody's profile. This pins that decision where + /// it can actually be broken — the installer's only Set-Acl targets are the credential files it + /// has always hardened, never the install tree. + /// + [Fact] + public void TheInstaller_NeverAclsTheInstallTree_OnlyTheCredentialFiles() + { + var script = InstallScript; + + var applications = 0; + for (var at = script.IndexOf("Set-Acl -Path ", StringComparison.Ordinal); at >= 0; at = script.IndexOf("Set-Acl -Path ", at + 1, StringComparison.Ordinal)) + { + applications++; + Assert.Contains("Set-Acl -Path $secretFile", script.Substring(at, Math.Min(30, script.Length - at)), StringComparison.Ordinal); + } + + Assert.True(applications == 2, $"expected exactly 2 Set-Acl calls (the hardened DACL and the owner), found {applications}"); + + /* And the install root is never granted anything, by either route. The icacls lines the script does + contain are remediation instructions PRINTED for the operator, which is a different thing from the + product reaching into a profile and changing permissions itself. */ + Assert.DoesNotContain("Set-Acl -Path $root", script, StringComparison.Ordinal); + Assert.DoesNotContain("icacls $root", script, StringComparison.Ordinal); + } + + /// + /// The boundary, executed rather than read. C:\UsersData is not under C:\Users, and a + /// prefix test that says it is would refuse an install that works — a worse failure than the one the + /// guard exists to catch, and one no structural pin can see. + /// + /// Run under powershell.exe (Windows PowerShell 5.1) deliberately: it is what the + /// reporter's Windows Server 2019 gives an operator by default, so this pins 5.1 compatibility of the + /// guard's syntax at the same time. + /// + [Fact] + public void TestPathIsAtOrUnder_DecidesTheBoundary_AsShipped() + { + var cases = new (string Candidate, string Parent, bool Expected)[] + { + /* The reported shape, and the profile root itself — an install root AT the root is as + unreadable as one below it. */ + (@"C:\Users\username\Desktop\PerformanceMonitorDarling-3.2.0", @"C:\Users", true), + (@"C:\Users\username\Desktop\PerformanceMonitorDarling-3.2.0\", @"C:\Users", true), + (@"C:\Users", @"C:\Users", true), + (@"C:\Users\", @"C:\Users", true), + /* Windows paths are case-insensitive, and neither a relative segment nor a forward slash is a + way out of the profile. */ + (@"c:\users\bob\x", @"C:\Users", true), + (@"C:\Users\bob\..\bob\x", @"C:\Users", true), + ("C:/Users/bob/x", @"C:\Users", true), + /* The false-refusal cases. A bare StartsWith fails every one of these. */ + (@"C:\UsersData\Darling", @"C:\Users", false), + (@"C:\UsersData", @"C:\Users", false), + (@"C:\Users2\Darling", @"C:\Users", false), + /* The documented location, and the machine-scoped data root #2187 asked about explicitly. */ + (@"C:\PerformanceMonitorDarling", @"C:\Users", false), + (@"C:\ProgramData\PerformanceMonitorDarling", @"C:\Users", false), + (@"D:\PerformanceMonitorDarling", @"C:\Users", false), + /* Nothing is not somewhere. */ + (@"C:\Users\bob", "", false), + ("", @"C:\Users", false), + }; + + var probe = new StringBuilder(); + probe.AppendLine(ExtractFunction(InstallScript, "Test-PathIsAtOrUnder")); + foreach (var (candidate, parent, _) in cases) + { + probe.AppendLine($"if (Test-PathIsAtOrUnder '{candidate}' '{parent}') {{ 'True' }} else {{ 'False' }}"); + } + + var answers = RunWindowsPowerShell(probe.ToString()); + Assert.Equal(cases.Length, answers.Count); + + var wrong = new List(); + for (var i = 0; i < cases.Length; i++) + { + var (candidate, parent, expected) = cases[i]; + if (!string.Equals(answers[i], expected.ToString(), StringComparison.Ordinal)) + { + wrong.Add($"under('{candidate}', '{parent}') returned {answers[i]}, expected {expected}"); + } + } + + Assert.True(wrong.Count == 0, "install-darling.ps1's path-containment boundary is wrong:\n " + string.Join("\n ", wrong)); + } + + /// + /// The network half, also executed. \\?\C:\... is the long-path prefix on a LOCAL path, not a + /// server name, and treating it as a share would refuse a perfectly ordinary install root. + /// + [Fact] + public void GetNetworkPathKind_SeparatesAShareFromAnExtendedLengthLocalPath_AsShipped() + { + var cases = new (string Path, string Expected)[] + { + (@"\\fileserver\share\PerformanceMonitorDarling", "UNC"), + (@"\\?\C:\PerformanceMonitorDarling", ""), + (@"C:\PerformanceMonitorDarling", ""), + (@"C:\Users\bob\Desktop\PerformanceMonitorDarling", ""), + ("", ""), + }; + + var probe = new StringBuilder(); + probe.AppendLine(ExtractFunction(InstallScript, "Get-NetworkPathKind")); + foreach (var (path, _) in cases) + { + probe.AppendLine($"$k = Get-NetworkPathKind '{path}'; if ($null -eq $k) {{ '' }} else {{ $k }}"); + } + + var answers = RunWindowsPowerShell(probe.ToString()); + Assert.Equal(cases.Length, answers.Count); + + for (var i = 0; i < cases.Length; i++) + { + Assert.Equal(cases[i].Expected, answers[i]); + } + } + + /// + /// #2201: the mapped-drive probe must not fail OPEN when WMI is unavailable. + /// + /// The defect. The probe asks Get-CimInstance Win32_LogicalDisk for the drive type. + /// On a WMI-restricted image (locked-down or Server Core) that call throws, or answers with nothing at + /// all, and the guard then returns "not network" — for exactly the drive-letter-mapped share it exists + /// to refuse. UNC paths are caught lexically before this point, so the exposure is only mapped LETTERS + /// on boxes where WMI is restricted. + /// + /// Why executed rather than structural. The fix is a fallback ORDER — try WMI, and only on + /// its silence consult Get-PSDrive, whose DisplayRoot names the share a letter maps to + /// without touching WMI. A source scan can see that both cmdlets appear; it cannot see which answer + /// wins, nor that the fallback is skipped when WMI already answered. Both cmdlets are shadowed by + /// functions here, which PowerShell resolves ahead of the real ones, so the WMI-restricted box is + /// simulated rather than described. + /// + /// Case 4 is the one that keeps the fix honest: when WMI answers "local", Get-PSDrive is rigged + /// to THROW. A terminating error would surface as stderr and fail this test, so the case passing proves + /// the fallback was never consulted — the guard still trusts a definite WMI answer, and does not invent + /// a refusal from a drive that merely has a DisplayRoot. + /// + /// Case 5 draws the line around what "definite" means. DriveType 0 is WMI's unknown, + /// not local, and a partial row is likeliest on precisely the restricted images this fallback exists + /// for — so it must fall through rather than short-circuit. It works because 0 is falsy in PowerShell, + /// which is worth pinning rather than trusting to stay true. + /// + [Fact] + public void NetworkPathKind_WhenWmiIsUnavailable_FallsBackToPSDriveDisplayRoot() + { + /* Shadows need [CmdletBinding()] so the shipped call sites can pass -ErrorAction, which is a common + parameter and not one a plain function accepts. */ + const string Shadows = @" +function Get-CimInstance { + [CmdletBinding()] + param([Parameter(ValueFromRemainingArguments = $true)] $Rest) + if ($env:PM_WMI -eq ""throw"") { throw ""WMI is not available on this image"" } + if ($env:PM_WMI -eq ""silent"") { return $null } + return [pscustomobject]@{ DriveType = [int]$env:PM_WMI } +} +function Get-PSDrive { + [CmdletBinding()] + param([Parameter(ValueFromRemainingArguments = $true)] $Rest) + if ($env:PM_PSDRIVE -eq ""throw"") { throw ""Get-PSDrive must not be consulted when WMI answered"" } + return [pscustomobject]@{ DisplayRoot = $env:PM_PSDRIVE } +} +"; + + /* wmi: "throw" | "silent" | a DriveType number (4 = network, 3 = local fixed disk). */ + var cases = new (string Wmi, string DisplayRoot, string Expected, string Because)[] + { + ("silent", @"\\fileserver\share", "mapped drive", + "WMI answered nothing on a restricted image and the letter maps to a share"), + ("throw", @"\\fileserver\share", "mapped drive", + "WMI threw and the letter maps to a share"), + ("silent", "", "", + "no evidence either way must stay not-network: the guard may not invent a refusal"), + ("4", "", "mapped drive", "WMI itself said network, no fallback needed"), + ("3", "throw", "", + "a definite local answer from WMI must not consult the fallback at all"), + ("0", @"\\fileserver\share", "mapped drive", + "DriveType 0 is unknown, not local - a partial WMI row must not short-circuit the fallback"), + }; + + for (var i = 0; i < cases.Length; i++) + { + var (wmi, displayRoot, expected, because) = cases[i]; + + var probe = new StringBuilder(); + probe.AppendLine($"$env:PM_WMI = '{wmi}'"); + probe.AppendLine($"$env:PM_PSDRIVE = '{displayRoot}'"); + probe.AppendLine(Shadows); + probe.AppendLine(ExtractFunction(InstallScript, "Get-NetworkPathKind")); + probe.AppendLine(@"$k = Get-NetworkPathKind 'Z:\PerformanceMonitorDarling'; " + + "if ($null -eq $k) { '' } else { $k }"); + + var answers = RunWindowsPowerShell(probe.ToString()); + + Assert.Single(answers); + /* Assert.True over Assert.Equal so the REASON travels with the failure: "expected mapped + drive, got " is not actionable on its own, and this test has six cases. */ + Assert.True(expected == answers[0], + $"case {i} (wmi={wmi}, DisplayRoot='{displayRoot}'): expected {expected}, got " + + $"{answers[0]} — {because}"); + } + } + + /// Runs under Windows PowerShell 5.1 and returns its non-empty output + /// lines. Written to a temp file rather than passed with -Command: the script under test is a whole + /// function body, and quoting it through a command line is a source of failures that have nothing to do + /// with what is being tested. + private static List RunWindowsPowerShell(string script) + { + var path = Path.Combine(Path.GetTempPath(), $"darling-2187-{Guid.NewGuid():N}.ps1"); + File.WriteAllText(path, script); + try + { + using var process = Process.Start(new ProcessStartInfo("powershell.exe", $"-NoProfile -ExecutionPolicy Bypass -File \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }); + Assert.NotNull(process); + + var stdout = process!.StandardOutput.ReadToEnd(); + var stderr = process.StandardError.ReadToEnd(); + process.WaitForExit(60_000); + + Assert.True(string.IsNullOrWhiteSpace(stderr), $"powershell.exe reported an error running the extracted function:\n{stderr}"); + + var lines = new List(); + foreach (var line in stdout.Split('\n')) + { + var trimmed = line.Trim(); + if (trimmed.Length > 0) { lines.Add(trimmed); } + } + + return lines; + } + finally + { + try { File.Delete(path); } catch (IOException) { /* best-effort */ } + } + } + + /// Returns the full function NAME(...) { ... } definition text from the script — + /// signature included, so the extracted copy takes its parameters the way the shipped one does. + private static string ExtractFunction(string script, string name) + { + var start = script.IndexOf("function " + name, StringComparison.Ordinal); + Assert.True(start >= 0, $"install-darling.ps1 no longer defines {name} (#2187)"); + + ExtractBracedBlockAt(script, script.IndexOf('{', start), out var end); + return script.Substring(start, end - start + 1); + } + + /// Returns the body of the first brace-balanced block introduced by — + /// the same idiom DarlingFirewallCheckTests uses. + private static string ExtractBracedBlock(string script, string header) + { + var start = script.IndexOf(header, StringComparison.Ordinal); + Assert.True(start >= 0, $"expected '{header}' in the script"); + return ExtractBracedBlockAt(script, script.IndexOf('{', start), out _); + } + + private static string ExtractBracedBlockAt(string script, int open, out int end) + { + Assert.True(open >= 0, "expected an opening brace"); + + var depth = 0; + for (var i = open; i < script.Length; i++) + { + if (script[i] == '{') { depth++; } + else if (script[i] == '}') + { + depth--; + if (depth == 0) + { + end = i; + return script.Substring(open + 1, i - open - 1); + } + } + } + + Assert.Fail("unbalanced braces while extracting a block from install-darling.ps1"); + end = -1; + return string.Empty; + } + + private static string ReadRepoFile(string relativePath) + { + var root = FindRepoRoot(); + Assert.NotNull(root); + var path = Path.Combine(root!, relativePath); + Assert.True(File.Exists(path), $"expected {path} to exist"); + return File.ReadAllText(path); + } + + /// Walks up from the test output directory to the repo root (the directory holding + /// PerformanceMonitor.sln) — the same idiom DarlingFileSecurityTests uses. + private static string? FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + for (var i = 0; i < 10 && directory is not null; i++) + { + if (File.Exists(Path.Combine(directory.FullName, "PerformanceMonitor.sln"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/Darling/Darling.Tests/DarlingManagedPostgresTests.cs b/Darling/Darling.Tests/DarlingManagedPostgresTests.cs index f273a033a..2ee0bf1a9 100644 --- a/Darling/Darling.Tests/DarlingManagedPostgresTests.cs +++ b/Darling/Darling.Tests/DarlingManagedPostgresTests.cs @@ -11,6 +11,7 @@ using System.IO; using System.Net; using System.Net.Sockets; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Threading; @@ -906,4 +907,147 @@ private static void TryDeleteRecursive(string path) } } } + + /* ============ #2186: the bootstrap's failure messages, in the operator's words ============ + These pin the SHIPPED strings, not the decoder — DarlingToolExitCodeTests owns the decode. + The distinction is the whole point: #1738 was a correct check that nothing invoked, and a + correct decoder no message calls would be the same defect wearing a new hat. */ + + private const int StatusDllNotFound = unchecked((int)0xC0000135); + private const string FieldBinDirectory = @"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin"; + private const string FieldDataDirectory = @"C:\ProgramData\PerformanceMonitorDarling\pg"; + + /// + /// The reported failure, rebuilt from the field's own numbers: exit -1073741515 and an empty capture. + /// Every clause the report was missing has to be present, and the clause it HAD has to survive — field + /// reports and the issue tracker are searchable by "initdb failed (exit code", so the fix must not + /// rename the thing operators paste into search. + /// + [Fact] + public void InitDbFailureMessage_TurnsTheFieldReportIntoADiagnosis() + { + var message = DarlingManagedPostgres.BuildInitDbFailureMessage( + -1073741515, Path.Combine(FieldBinDirectory, "initdb.exe"), FieldDataDirectory, string.Empty); + + Assert.StartsWith("initdb failed (exit code -1073741515", message, StringComparison.Ordinal); + Assert.Contains(FieldDataDirectory, message, StringComparison.Ordinal); + + /* What the number means, and that Windows rather than PostgreSQL set it. */ + Assert.Contains("0xC0000135", message, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", message, StringComparison.Ordinal); + + /* The empty field is stated as expected. The report's "Output:" trailing a blank line is what made + the whole thing read as missing data. */ + Assert.Contains("Output:", message, StringComparison.Ordinal); + Assert.DoesNotContain("Output:\n\n", message, StringComparison.Ordinal); + Assert.DoesNotContain("Output:\n(none)", message, StringComparison.Ordinal); + Assert.Contains("expected", message, StringComparison.OrdinalIgnoreCase); + + /* Both causes, and the directory the DLLs are supposed to be in. */ + Assert.Contains(FieldBinDirectory, message, StringComparison.Ordinal); + Assert.Contains("vcruntime140_1.dll", message, StringComparison.Ordinal); + Assert.Contains("NT SERVICE", message, StringComparison.Ordinal); + } + + /// An ordinary initdb failure is left to speak for itself: its own stderr is the diagnosis, + /// and it must not be pushed below a screen of loader boilerplate that does not apply. + [Fact] + public void InitDbFailureMessage_LeavesARealInitDbErrorAlone() + { + const string stderr = "initdb: error: directory \"C:\\pg\" exists but is not empty"; + + var message = DarlingManagedPostgres.BuildInitDbFailureMessage( + 1, Path.Combine(FieldBinDirectory, "initdb.exe"), FieldDataDirectory, stderr); + + Assert.Equal($"initdb failed (exit code 1) for {FieldDataDirectory}.\nOutput:\n{stderr}", message); + } + + /// + /// pg_ctl status's own exit 4 really does mean the data directory is unusable, and the message keeps + /// saying so. A Windows status means pg_ctl never ran — keeping the verdict there would point an + /// operator at deleting a healthy store to fix a missing DLL. + /// + [Fact] + public void StatusFailureMessage_BlamesTheDataDirectoryOnlyWhenPgCtlActuallySaidSo() + { + var pgCtl = Path.Combine(FieldBinDirectory, "pg_ctl.exe"); + + var pgCtlVerdict = DarlingManagedPostgres.BuildStatusFailureMessage(4, pgCtl, FieldDataDirectory, "pg_ctl: could not open ..."); + Assert.Contains("the data directory is not usable", pgCtlVerdict, StringComparison.Ordinal); + + var loaderFailure = DarlingManagedPostgres.BuildStatusFailureMessage(StatusDllNotFound, pgCtl, FieldDataDirectory, string.Empty); + Assert.DoesNotContain("the data directory is not usable", loaderFailure, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", loaderFailure, StringComparison.Ordinal); + } + + /// + /// The start failure's log tail has the initdb message's trap in another costume: on a loader status + /// pg_ctl never started a postmaster, so "(no server log written)" is true and useless. The diagnosis + /// has to arrive BEFORE the tail invites an operator to go read a log that was never going to exist. + /// + [Fact] + public void StartFailureMessage_DiagnosesBeforeItPointsAtAnEmptyServerLog() + { + var message = DarlingManagedPostgres.BuildStartFailureMessage( + StatusDllNotFound, Path.Combine(FieldBinDirectory, "pg_ctl.exe"), FieldDataDirectory, "(no server log written)"); + + var diagnosis = message.IndexOf("STATUS_DLL_NOT_FOUND", StringComparison.Ordinal); + var tail = message.IndexOf("Server log tail:", StringComparison.Ordinal); + + Assert.True(diagnosis >= 0, "the start failure must decode a Windows status"); + Assert.True(tail > diagnosis, "the loader diagnosis has to precede the server-log tail it explains"); + } + + /// + /// The wiring, pinned at the source: three correct builders that no throw site calls would leave the + /// shipped message exactly as it was reported. Behavioral coverage cannot reach these — reproducing + /// them needs a bundled Postgres that dies in the Windows loader, which is not something a CI runner + /// can be asked to arrange. + /// + [Fact] + public void TheBootstrapThrowSitesActuallyUseTheseMessages() + { + var source = ReadManagedPostgresSource(); + + Assert.Contains("BuildInitDbFailureMessage(exitCode, initDb, _dataDirectory, output, runtimeProbe));", source, StringComparison.Ordinal); + Assert.Contains("throw new InvalidOperationException(BuildStatusFailureMessage(exitCode, pgCtl, _dataDirectory, output)),", source, StringComparison.Ordinal); + Assert.Contains("BuildStartFailureMessage(exitCode, pgCtl, _dataDirectory, ReadServerLogTail())", source, StringComparison.Ordinal); + + /* And that no bootstrap failure went back to interpolating the bare code. */ + Assert.DoesNotContain("exit code {exitCode}", source, StringComparison.Ordinal); + } + + /// + /// #2185: the runtime probe is GATED on a loader status, pinned at the source because the gate is the + /// whole design and behavioral coverage cannot reach it. + /// + /// Two things would go wrong ungated. Every ordinary initdb failure — a non-empty data directory, + /// a bad locale, a permissions refusal — would launch two extra processes and then append a paragraph + /// about DLL loading to an error that has nothing to do with loading, which is worse than silence + /// because it sends the operator down the wrong path. And the probe only MEANS anything against a + /// loader status: "both binaries load fine" is a useful finding when Windows just refused to load one, + /// and noise otherwise. + /// + [Fact] + public void TheRuntimeProbeOnlyRunsForALoaderStatus() + { + var source = ReadManagedPostgresSource(); + + Assert.Contains("DarlingToolExitCode.IsLoaderStatus(exitCode)", source, StringComparison.Ordinal); + Assert.Contains("? await ProbeRuntimeBinariesAsync(binDirectory, cancellationToken)", source, StringComparison.Ordinal); + Assert.Contains(": string.Empty;", source, StringComparison.Ordinal); + } + + private static string ReadManagedPostgresSource([CallerFilePath] string thisFile = "") + { + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingManagedPostgres.cs"); + var dir = Path.GetDirectoryName(thisFile); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.False(dir is null, "could not locate the repo root from the test source path"); + return File.ReadAllText(Path.Combine(dir!, relative)); + } } diff --git a/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs b/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs index 85764d7cf..8ff14210b 100644 --- a/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs @@ -74,6 +74,19 @@ private static (string Name, bool Optional)[] McpParams(string toolName) .ToArray(); } + /// + /// Lite's parameter contract, pinned as a PREFIX rather than as the whole list (#2159). + /// + /// This used to assert the full parameter list, which was the same thing until Darling's incident + /// readers gained a trailing optional dedup_key that Lite does not have. The guarantee that actually + /// matters was never "the lists are identical" — it is that a client written against LITE's contract still + /// calls Darling correctly. A trailing optional parameter preserves exactly that, positionally and by name, + /// so the prefix is the honest form of the assertion and the full-list form was over-tight. + /// + /// Anything appended must therefore stay optional AND stay at the end. StartsWith semantics are + /// spelled out by comparing the first N rather than by trusting a substring of a joined string, so a + /// REORDERING that keeps the same names still fails. + /// [Theory] [InlineData("get_blocking", "server_name,hours_back,limit")] [InlineData("get_deadlocks", "server_name,hours_back,limit")] @@ -83,7 +96,55 @@ private static (string Name, bool Optional)[] McpParams(string toolName) [InlineData("get_deadlock_trend", "server_name,hours_back")] public void ParamContract_MatchesLite(string toolName, string expectedCsv) { - Assert.Equal(expectedCsv.Split(','), McpParams(toolName).Select(p => p.Name).ToArray()); + var expected = expectedCsv.Split(','); + var actual = McpParams(toolName).Select(p => p.Name).ToArray(); + + Assert.True(actual.Length >= expected.Length, + $"{toolName} dropped a parameter Lite has: [{string.Join(",", actual)}]"); + Assert.Equal(expected, actual.Take(expected.Length).ToArray()); + } + + /// + /// #2159's dedup_key, pinned as a TRAILING OPTIONAL parameter on exactly the three incident readers + /// that can resolve a fingerprint — and pinned as absent everywhere else. + /// + /// Optional and trailing is what keeps true, so it is asserted + /// here rather than left to review. Absent on the trend tools because a per-minute count series has no single + /// incident to resolve to, and absent on get_blocked_process_xml because it is reached FROM an incident + /// the operator has already identified rather than used to find one. + /// + [Fact] + public void ParamContract_DedupKeyIsATrailingOptionalOnTheIncidentReaders() + { + foreach (var tool in new[] { "get_blocking", "get_deadlocks", "get_deadlock_detail" }) + { + var ps = McpParams(tool); + Assert.Equal("dedup_key", ps[^1].Name); + Assert.True(ps[^1].Optional, $"{tool}.dedup_key must be optional so Lite-shaped calls still work"); + } + + foreach (var tool in new[] { "get_blocking_trend", "get_deadlock_trend", "get_blocked_process_xml" }) + Assert.DoesNotContain("dedup_key", McpParams(tool).Select(p => p.Name)); + } + + /// + /// The instructions have to ADVERTISE dedup_key, or the feature is unreachable in practice: an agent + /// picks tools and arguments from this text, and a parameter it never reads is one it never passes. Same + /// reason the store-metrics and AG tools pin their own mentions. + /// + /// Also pins the two caveats that turn an empty result into a diagnosable one — the display-name + /// scoping and that hours_back still bounds the search — because those are the failure modes an agent + /// would otherwise report as "no such incident". + /// + [Fact] + public void Instructions_AdvertiseDedupKeyAndItsScoping() + { + var text = DarlingMcpInstructions.Text; + + Assert.Contains("dedup_key", text, StringComparison.Ordinal); + Assert.Contains("Dedup Key", text, StringComparison.Ordinal); + Assert.Contains("DISPLAY name", text, StringComparison.Ordinal); + Assert.Contains("hours_back` still bounds the search", text, StringComparison.Ordinal); } [Fact] diff --git a/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs b/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs index 6688b459a..63ec2159a 100644 --- a/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs @@ -27,7 +27,8 @@ namespace Darling.Tests; /// /// Pins the config / trace-flag diagnostic-depth MCP slice — get_server_config_changes, -/// get_database_config_changes, get_trace_flag_changes, get_database_scoped_config over the Postgres store. +/// get_database_config_changes, get_trace_flag_changes, and the latest-snapshot pair +/// get_database_scoped_config + get_query_store_health (#2319) over the Postgres store. /// The Dashboard reads pre-materialized report.*_changes tables Darling does not have, so the change tools /// diff the store's append-only config snapshots IN C#; the bulk of these tests exercise that diff directly /// (no live PG). Also pins the tool surface, param contracts, snapshot-read SQL, the 27-setting database-config @@ -39,6 +40,7 @@ public sealed class DarlingMcpConfigHistoryToolsSurfaceAndSqlTests { "get_database_config_changes", "get_database_scoped_config", + "get_query_store_health", "get_server_config_changes", "get_trace_flag_changes", }; @@ -49,7 +51,7 @@ private static MethodInfo[] ToolMethods() => typeof(DarlingMcpConfigHistoryTools .ToArray(); [Fact] - public void ToolSurface_ExactlyTheFourConfigTools() + public void ToolSurface_ExactlyTheFiveConfigTools() { var toolMethods = ToolMethods(); var names = toolMethods @@ -77,6 +79,7 @@ private static (string Name, bool Optional)[] McpParams(string toolName) [InlineData("get_database_config_changes", "server_name,hours_back")] [InlineData("get_trace_flag_changes", "server_name,hours_back")] [InlineData("get_database_scoped_config", "server_name,database_name")] + [InlineData("get_query_store_health", "server_name,database_name")] public void ParamContract_MatchesContract(string toolName, string expectedCsv) { Assert.Equal(expectedCsv.Split(','), McpParams(toolName).Select(p => p.Name).ToArray()); @@ -130,11 +133,26 @@ public void DatabaseScopedConfigSql_LatestSnapshot() Assert.Contains("MAX(capture_time)", sql, StringComparison.Ordinal); } + /// The read must be the same latest-snapshot shape as the scoped-config sibling, over the + /// passthrough view, selecting the reader's ten ordinals in the collector's payload order. + [Fact] + public void QueryStoreHealthSql_LatestSnapshot_SelectsPayloadOrder() + { + var sql = Reader.QueryStoreHealthSql; + Assert.Contains("FROM v_query_store_health", sql, StringComparison.Ordinal); + Assert.Contains("MAX(capture_time)", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY database_name", sql, StringComparison.Ordinal); + Assert.Contains( + "database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes", + sql, StringComparison.Ordinal); + } + [Theory] [InlineData(nameof(Reader.ServerConfigSnapshotsSql))] [InlineData(nameof(Reader.DatabaseConfigSnapshotsSql))] [InlineData(nameof(Reader.TraceFlagSnapshotsSql))] [InlineData(nameof(Reader.DatabaseScopedConfigSql))] + [InlineData(nameof(Reader.QueryStoreHealthSql))] public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { var sql = sqlName switch @@ -142,7 +160,8 @@ public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) nameof(Reader.ServerConfigSnapshotsSql) => Reader.ServerConfigSnapshotsSql, nameof(Reader.DatabaseConfigSnapshotsSql) => Reader.DatabaseConfigSnapshotsSql, nameof(Reader.TraceFlagSnapshotsSql) => Reader.TraceFlagSnapshotsSql, - _ => Reader.DatabaseScopedConfigSql, + nameof(Reader.DatabaseScopedConfigSql) => Reader.DatabaseScopedConfigSql, + _ => Reader.QueryStoreHealthSql, }; var lower = sql.ToLowerInvariant(); Assert.DoesNotContain("getdate", lower); @@ -290,10 +309,10 @@ public void DatabaseConfigChanges_PerDatabase_UnchangedYieldsNothing() } [Fact] - public void AdvertisedSchema_IsGeminiClean_ForAllFourTools_NoRequiredParams() + public void AdvertisedSchema_IsGeminiClean_ForAllFiveTools_NoRequiredParams() { var tools = BuildToolSchemas(); - Assert.Equal(4, tools.Count); + Assert.Equal(5, tools.Count); var violations = tools.SelectMany(t => DarlingMcpSchemaAssert.Violations(t.Name, t.InputSchema)).ToList(); Assert.True(violations.Count == 0, "Gemini-incompatible schema keywords leaked:\n" + string.Join("\n", violations)); foreach (var t in tools) @@ -355,6 +374,13 @@ await DarlingMcpTestData.ExecAsync(connection, ct, VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", CollectionIdGenerator.Next(), newer, ServerId, ServerName, Db, "MAXDOP", "8", null); + /* Query Store health: the cap-hit shape the tool exists to surface — desired READ_WRITE, + actual READ_ONLY, readonly_reason 65536 (storage cap reached). */ + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO query_store_health (config_id, capture_time, server_id, server_name, database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)", + CollectionIdGenerator.Next(), newer, ServerId, ServerName, Db, "READ_ONLY", "READ_WRITE", 65536, 1000L, 1000L, "AUTO", 30L, 200L, 60L); + var serverChanges = await DarlingMcpConfigHistoryTools.GetServerConfigChanges(postgres, ServerName); DarlingMcpTestData.AssertEnvelope(serverChanges, ServerName, "changes"); Assert.Contains("max degree of parallelism", serverChanges, StringComparison.Ordinal); @@ -367,6 +393,12 @@ await DarlingMcpTestData.ExecAsync(connection, ct, DarlingMcpTestData.AssertEnvelope(scoped, ServerName, "databases"); Assert.Contains("MAXDOP", scoped, StringComparison.Ordinal); + var qsh = await DarlingMcpConfigHistoryTools.GetQueryStoreHealth(postgres, ServerName); + DarlingMcpTestData.AssertEnvelope(qsh, ServerName, "databases"); + Assert.Contains("\"state_matches_desired\": false", qsh, StringComparison.Ordinal); + Assert.Contains("storage cap reached", qsh, StringComparison.Ordinal); + Assert.Contains("\"pct_of_cap\": 100", qsh, StringComparison.Ordinal); + bodySucceeded = true; } finally @@ -378,7 +410,7 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = private static async Task DeleteRowsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) { - var sql = string.Join(" ", new[] { "server_config", "database_config", "trace_flags", "database_scoped_config" } + var sql = string.Join(" ", new[] { "server_config", "database_config", "trace_flags", "database_scoped_config", "query_store_health" } .Select(tbl => $"DELETE FROM {tbl} WHERE server_id = {ServerId};")) + $" DELETE FROM servers WHERE server_id = {ServerId};"; using var cleanup = new NpgsqlCommand(sql, connection); diff --git a/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs b/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs index 798b8a79a..297830c0b 100644 --- a/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs @@ -99,6 +99,16 @@ private static (string Name, bool Optional)[] McpParams(string toolName) .ToArray(); } + /// + /// Lite's parameter contract, pinned as a PREFIX rather than as the whole list (#2235). + /// + /// This asserted the full list, which was the same thing until get_top_queries_by_cpu gained a + /// trailing optional group_by that Lite does not have. The guarantee that matters was never "the + /// lists are identical" — it is that a client written against LITE's contract still calls Darling + /// correctly, which a trailing optional parameter preserves both positionally and by name. Anything + /// appended must therefore stay optional AND stay last; a REORDERING that keeps the same names still + /// fails, because the prefix is compared element-wise rather than as a joined substring. + /// [Theory] [InlineData("get_cpu_utilization", "server_name,hours_back")] [InlineData("get_wait_stats", "server_name,hours_back,limit")] @@ -118,7 +128,31 @@ public void ParamContract_MatchesLite(string toolName, string expectedCsv) { var expected = expectedCsv.Split(','); var actual = McpParams(toolName).Select(p => p.Name).ToArray(); - Assert.Equal(expected, actual); + + Assert.True(actual.Length >= expected.Length, + $"{toolName} dropped a parameter Lite has: [{string.Join(",", actual)}]"); + Assert.Equal(expected, actual.Take(expected.Length).ToArray()); + } + + /// + /// #2235's group_by, pinned as a TRAILING OPTIONAL on get_top_queries_by_cpu and absent from + /// its siblings. + /// + /// Trailing and optional is what keeps true, so it is asserted + /// rather than left to review. Absent on get_top_procedures_by_cpu because procedure stats are + /// ALREADY keyed on the object — the rollup would be a no-op there — and absent on + /// get_query_store_top because Query Store keys on query_id, which does not fragment the way a + /// shape hash does, so the same option would imply a grouping that surface cannot perform. + /// + [Fact] + public void ParamContract_GroupByIsATrailingOptionalOnTopQueriesOnly() + { + var ps = McpParams("get_top_queries_by_cpu"); + Assert.Equal("group_by", ps[^1].Name); + Assert.True(ps[^1].Optional, "group_by must be optional so Lite-shaped calls still work"); + + foreach (var tool in new[] { "get_top_procedures_by_cpu", "get_query_store_top" }) + Assert.DoesNotContain("group_by", McpParams(tool).Select(p => p.Name)); } [Fact] @@ -157,6 +191,21 @@ public void CpuSql_ReadsBaseTable_DeSkewsSampleTime_WindowsOnCollectionTime() Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); /* window on the reliable clock */ } + /// #2320: the attribution denominator windows on collection_time — the SAME bounds the + /// rankings use, so numerator and denominator share collection gaps — and aggregates rather than + /// pulling sample rows. + [Fact] + public void CpuWindowAggregateSql_WindowsOnCollectionTime_BothEdges() + { + var sql = DarlingDataReader.CpuWindowAggregateSql; + Assert.Contains("FROM cpu_utilization_stats", sql, StringComparison.Ordinal); + Assert.Contains("AVG(sqlserver_cpu_utilization)", sql, StringComparison.Ordinal); + Assert.Contains("MIN(collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("MAX(collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); + } + [Fact] public void WaitStatsSql_AggregatesDeltas_HeaviestFirst() { @@ -343,6 +392,7 @@ exactly like the viewer's UTC-offset read. */ [InlineData(nameof(DarlingDataReader.ServerListSql))] [InlineData(nameof(DarlingDataReader.CollectionHealthSql))] [InlineData(nameof(DarlingDataReader.LatestServerPropertiesSql))] + [InlineData(nameof(DarlingDataReader.CpuWindowAggregateSql))] public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { var sql = SqlByName(sqlName); @@ -371,6 +421,7 @@ public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) nameof(DarlingDataReader.QueryStoreTopSql) => DarlingDataReader.QueryStoreTopSql, nameof(DarlingDataReader.ServerListSql) => DarlingDataReader.ServerListSql, nameof(DarlingDataReader.CollectionHealthSql) => DarlingDataReader.CollectionHealthSql, + nameof(DarlingDataReader.CpuWindowAggregateSql) => DarlingDataReader.CpuWindowAggregateSql, _ => DarlingDataReader.LatestServerPropertiesSql, }; diff --git a/Darling/Darling.Tests/DarlingMcpToolsTests.cs b/Darling/Darling.Tests/DarlingMcpToolsTests.cs index e47ab4093..dd274a4c1 100644 --- a/Darling/Darling.Tests/DarlingMcpToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpToolsTests.cs @@ -328,15 +328,17 @@ including the #2000 occurrence stats. */ "finding_id", "analysis_time", "severity", "confidence", "category", "root_fact", "leaf_fact", "story_path", "story_path_hash", "fact_count", "incident_id", "occurrences", "first_seen", "last_seen", "peak_severity", - "co_fired", "time_range", "advice", "remediation_command" + "co_fired", "time_range", "advice", "remediation_command", "structured_remediation" }) { Assert.True(finding.TryGetProperty(field, out _), $"envelope field '{field}' missing"); } /* This planted finding has no persisted RemediationAction, so its copy-paste command - is present-but-null (JsonOptions does not ignore nulls). */ + and its #2138 machine-first projection are both present-but-null (JsonOptions does + not ignore nulls). */ Assert.Equal(JsonValueKind.Null, finding.GetProperty("remediation_command").ValueKind); + Assert.Equal(JsonValueKind.Null, finding.GetProperty("structured_remediation").ValueKind); Assert.Equal(TestStoryHash, finding.GetProperty("story_path_hash").GetString()); Assert.Equal(2.5, finding.GetProperty("severity").GetDouble()); diff --git a/Darling/Darling.Tests/DarlingObservabilityTests.cs b/Darling/Darling.Tests/DarlingObservabilityTests.cs index 59bd5aabb..42f4a1292 100644 --- a/Darling/Darling.Tests/DarlingObservabilityTests.cs +++ b/Darling/Darling.Tests/DarlingObservabilityTests.cs @@ -76,8 +76,10 @@ public void MigrationScripts_AreRegisteredInAscendingOrder_V34AgCollectors_V36Ag Assert.Equal(33, PgMigrations.Scripts[32].Version); /* The newest migration is asserted by identity rather than by ordinal: this ladder is walked by every stacked branch at once, and a positional pin turns each addition into a conflict for the next. */ - Assert.Equal(54, PgMigrations.Scripts[^1].Version); - Assert.Equal(54, StorageVersion.SchemaVersion); + /* The invariant the test name states, with no literal to go stale: the build's schema version IS + the newest registered rung. Three in-flight branches bumping versions made the literal form a + recurring multi-test failure (#2210 round, again here at V62). */ + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); /* V34 (#991) creates the two Availability Group collector tables. Schema-qualified collect.* and CREATE TABLE IF NOT EXISTS, per the file's additive-create idiom (V29): a no-op on a fresh store diff --git a/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs b/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs new file mode 100644 index 000000000..bbbef5507 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the autovacuum read: ranking by each table's own threshold ratio rather than by raw dead-tuple +/// count, the three-part per-database key, and a severity that separates autovacuum losing a race from +/// autovacuum not running. +/// +public class DarlingPgAutovacuumReaderTests +{ + private static string Sql => DarlingPgAutovacuumReader.PgAutovacuumSql; + + /// + /// The key is (database, schema, table), not table alone. This collector fans out per database, so two + /// databases on one server routinely hold same-named tables — keying on table alone would collapse + /// them and report one database's state as the other's. + /// + [Fact] + public void KeysOnDatabaseSchemaAndTable() + { + Assert.Contains("DISTINCT ON (database_name, schema_name, table_name)", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY database_name, schema_name, table_name, collection_time DESC", Sql, StringComparison.Ordinal); + } + + /// + /// The ranking crux. Ordering by raw dead_tuples would put the biggest tables on top permanently — + /// they always have the most dead tuples and are usually fine — and bury the small hot table that is + /// fifty times past its line. Dividing by the table's own threshold is what makes them comparable. + /// Asserted by STRUCTURE rather than by an exact substring: the previous version matched + /// "l.dead_tuples::numeric / NULLIF(l.vacuum_threshold, 0) DESC" verbatim and broke on a + /// reformat that left the behaviour intact, which is a test failing for the wrong reason. The ordering + /// SEMANTICS are what matter; they were hand-verified on live Aurora during the #2213 rounds + /// (no live test carries that name - an earlier draft of this comment asserted one into + /// existence). + /// + [Fact] + public void RanksByThresholdRatioNotRawDeadTupleCount() + { + var orderAt = Sql.IndexOf("ORDER BY", StringComparison.Ordinal); + Assert.True(orderAt > 0, "the read must have an ORDER BY at all"); + + var orderClause = Sql[orderAt..]; + + /* Both ratios divide by the table's OWN threshold, which is what makes tables comparable. */ + Assert.Contains("l.dead_tuples::numeric", orderClause, StringComparison.Ordinal); + Assert.Contains("NULLIF(l.vacuum_threshold, 0)", orderClause, StringComparison.Ordinal); + Assert.Contains("l.inserts_since_vacuum::numeric", orderClause, StringComparison.Ordinal); + Assert.Contains("l.insert_vacuum_threshold", orderClause, StringComparison.Ordinal); + + /* Both ratios must be considered TOGETHER — the worse one decides — rather than one after the other, + which would let a table with zero dead tuples sort below every table that has any. Pinned on + the RATIO GREATEST specifically: the raw tie-break below is itself GREATEST(dead, inserts), + so a bare "GREATEST(" token is present even after the exact regression this guards + (unwrapping the ratios into two sequential ORDER BY terms). */ + var collapsed = Regex.Replace(orderClause, @"\s+", ""); + Assert.Contains( + "GREATEST(l.dead_tuples::numeric/NULLIF(l.vacuum_threshold,0),", + collapsed, StringComparison.Ordinal); + + /* The disabled flag outranks everything, then the ratio, then the raw counts as a tie-break. */ + var disabledAt = orderClause.IndexOf("l.autovacuum_disabled DESC", StringComparison.Ordinal); + var ratioAt = orderClause.IndexOf("l.dead_tuples::numeric", StringComparison.Ordinal); + var rawAt = orderClause.LastIndexOf("GREATEST(l.dead_tuples, l.inserts_since_vacuum) DESC", StringComparison.Ordinal); + Assert.True( + disabledAt >= 0 && ratioAt > disabledAt && rawAt > ratioAt, + $"ORDER BY must be disabled, then ratio, then raw counts (got {disabledAt}/{ratioAt}/{rawAt})"); + } + + /// + /// A table with autovacuum switched off sorts to the top regardless of its count: it will never be + /// vacuumed no matter how bad it gets, which is a configuration finding rather than a workload one. + /// + [Fact] + public void SortsAutovacuumDisabledTablesFirst() + { + var orderAt = Sql.IndexOf("ORDER BY\n", StringComparison.Ordinal); + var disabledAt = Sql.IndexOf("l.autovacuum_disabled DESC", StringComparison.Ordinal); + var ratioAt = Sql.IndexOf("dead_tuples::numeric / NULLIF", StringComparison.Ordinal); + + Assert.True(orderAt < disabledAt && disabledAt < ratioAt); + } + + /// + /// NULLIF guards the never-analyzed table, whose threshold can be 0. Without it the division raises + /// division_by_zero and the whole read fails because of one table. + /// + [Fact] + public void GuardsDivisionByAZeroThreshold() + { + Assert.Contains("NULLIF(l.vacuum_threshold, 0)", Sql, StringComparison.Ordinal); + Assert.Contains("NULLS LAST", Sql, StringComparison.Ordinal); + } + + /// + /// Growth separates two conditions with the same count and different fixes: climbing means autovacuum + /// is losing a race, flat at ten times the threshold usually means it is blocked or switched off. + /// + [Fact] + public void CarriesTheEarliestDeadTupleCountForGrowthComparison() + { + Assert.Contains("ORDER BY database_name, schema_name, table_name, collection_time ASC", Sql, StringComparison.Ordinal); + Assert.Contains("first_dead_tuples", Sql, StringComparison.Ordinal); + Assert.Contains("first_seen_at", Sql, StringComparison.Ordinal); + } + + /// + /// The join must use IS NOT DISTINCT FROM: database_name / schema_name / table_name are nullable + /// columns, and a plain equality join silently drops every row where any of them is NULL. + /// + [Fact] + public void JoinsTheTwoBranchesNullSafely() + { + Assert.Contains("e.database_name IS NOT DISTINCT FROM l.database_name", Sql, StringComparison.Ordinal); + Assert.Contains("e.schema_name IS NOT DISTINCT FROM l.schema_name", Sql, StringComparison.Ordinal); + Assert.Contains("e.table_name IS NOT DISTINCT FROM l.table_name", Sql, StringComparison.Ordinal); + Assert.Equal(2, Sql.Split("server_id = $1").Length - 1); + Assert.Equal(2, Sql.Split("collection_time >= $2").Length - 1); + } + + [Fact] + public void ReadsTheAutovacuumTableAndBoundsTheRowCount() + { + Assert.Contains("FROM pg_autovacuum_stats", Sql, StringComparison.Ordinal); + Assert.Contains("LIMIT $4", Sql, StringComparison.Ordinal); + } + + /// + /// autovacuum_enabled = false outranks any ratio, including a healthy one: the table is excluded from + /// autovacuum entirely, so a low count today says nothing about tomorrow. + /// + [Fact] + public void DisabledAutovacuumOutranksEveryRatio() + { + Assert.Equal( + "critical_autovacuum_disabled_on_table", + DarlingMcpPgAutovacuumTools.Classify(true, 0.01, false)); + Assert.Equal( + "critical_autovacuum_disabled_on_table", + DarlingMcpPgAutovacuumTools.Classify(true, null, false)); + } + + /// + /// A missing threshold must read as unknown, never as healthy. The threshold is -1 (not applicable) or + /// 0 (never analyzed), and calling either "ok" would hide a table nothing is known about. + /// + [Fact] + public void MissingThresholdIsUnknownNotHealthy() + { + Assert.Equal("unknown_no_threshold", DarlingMcpPgAutovacuumTools.Classify(false, null, false)); + Assert.Equal("unknown_no_threshold", DarlingMcpPgAutovacuumTools.Classify(false, null, true)); + } + + /// The severity bands, and that growth only refines the middle one. + [Theory] + [InlineData(50.0, false, "critical_far_past_threshold")] + [InlineData(10.0, false, "critical_far_past_threshold")] + [InlineData(9.99, true, "warning_past_threshold_and_growing")] + [InlineData(2.0, true, "warning_past_threshold_and_growing")] + [InlineData(2.0, false, "warning_past_threshold")] + [InlineData(1.0, false, "info_at_threshold")] + [InlineData(1.99, false, "info_at_threshold")] + [InlineData(0.5, true, "ok")] + [InlineData(0.0, false, "ok")] + public void ClassifiesByRatioBand(double ratio, bool growing, string expected) + { + Assert.Equal(expected, DarlingMcpPgAutovacuumTools.Classify(false, ratio, growing)); + } + + /// + /// Ten times past the threshold is critical whether or not it is still growing — a pile that large and + /// static is the WORSE case, because it means nothing is draining it at all. + /// + [Fact] + public void FarPastThresholdIsCriticalEvenWhenFlat() + { + Assert.Equal("critical_far_past_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 25.0, false)); + Assert.Equal("critical_far_past_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 25.0, true)); + } + + /// Every severity is a distinct string, so a caller can switch on it. + [Fact] + public void SeveritiesAreDistinct() + { + var severities = new[] + { + DarlingMcpPgAutovacuumTools.Classify(true, 1.0, false), + DarlingMcpPgAutovacuumTools.Classify(false, null, false), + DarlingMcpPgAutovacuumTools.Classify(false, 20.0, false), + DarlingMcpPgAutovacuumTools.Classify(false, 3.0, true), + DarlingMcpPgAutovacuumTools.Classify(false, 3.0, false), + DarlingMcpPgAutovacuumTools.Classify(false, 1.2, false), + DarlingMcpPgAutovacuumTools.Classify(false, 0.1, false), + }; + + Assert.Equal(severities.Length, severities.Distinct().Count()); + } +} diff --git a/Darling/Darling.Tests/DarlingPgBlockingReaderOrdinalTests.cs b/Darling/Darling.Tests/DarlingPgBlockingReaderOrdinalTests.cs new file mode 100644 index 000000000..f5d803da8 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgBlockingReaderOrdinalTests.cs @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// EXECUTION coverage for the blocking reads' ordinal mapping, as opposed to the text pins in +/// . +/// +/// Why the text pins are not enough, which review had to point out. Every other test on this +/// reader asserts against the SQL STRING. The projection's column order and the mapper's +/// reader.GetX(n) ordinals are two independent lists that must agree, and no string assertion can see +/// a disagreement: swap root_username and root_application_name — adjacent, both +/// string?, semantically confusable — and every Assert.Contains still passes while the data +/// comes back transposed. +/// +/// The same hazard already had a pin on the probe/mapper pair +/// (StoreSchemaProbe_ColumnCount_MatchesTheMapArity) and on the collector side +/// (PgBlockingCollectorDefinitionTests.WritesEveryDeclaredPayloadColumn). The reader side had none, +/// and its projection was edited three times in one PR — a scalar became an array, a count became nullable, +/// a flag was appended — so the risk was live, not theoretical. +/// +/// Every sentinel is distinguishable, which is the whole method: same-typed neighbours get +/// different values, so a transposition changes an assertion rather than moving an identical value into an +/// identical slot. Testing this at all required a seam — the read methods take an +/// NpgsqlDataSource and build their own command — hence MapChainRow/MapCycleRow. +/// +public sealed class DarlingPgBlockingReaderOrdinalTests +{ + [Fact] + public void MapChainRow_LandsEveryColumnInItsOwnField() + { + var reader = new StubReader(new object[] + { + new DateTime(2026, 8, 13, 9, 15, 0, DateTimeKind.Unspecified), // 0 captured_at + 1_754_000_009_876_543L, // 1 root_backend_id + 9999, // 2 root_pid + new[] { "orders", "billing" }, // 3 databases + "root-user", // 4 root_username + "root-app", // 5 root_application_name + "idle in transaction", // 6 root_state + "SELECT 'root query'", // 7 root_query + true, // 8 root_is_idle_in_transaction + 240_111L, // 9 root_xact_duration_ms + 239_222L, // 10 root_query_duration_ms + 7, // 11 total_victims + 3, // 12 direct_victims + 4, // 13 max_depth + 8_444L, // 14 worst_victim_wait_ms + "UPDATE 'worst victim'", // 15 worst_victim_query + 5L, // 16 samples_as_root + true, // 17 query_text_may_be_truncated + false, // 18 chain_may_be_truncated + }); + + var row = DarlingPgBlockingReader.MapChainRow(reader); + + Assert.Equal(new DateTime(2026, 8, 13, 9, 15, 0, DateTimeKind.Unspecified), row.CapturedAt); + Assert.Equal(1_754_000_009_876_543L, row.RootBackendId); + Assert.Equal(9999, row.RootPid); + Assert.Equal(new[] { "orders", "billing" }, row.Databases); + + /* The transposition pair: distinct values, distinct fields. */ + Assert.Equal("root-user", row.RootUsername); + Assert.Equal("root-app", row.RootApplicationName); + + Assert.Equal("idle in transaction", row.RootState); + Assert.Equal("SELECT 'root query'", row.RootQuery); + Assert.True(row.RootIsIdleInTransaction); + + /* Two adjacent bigints that differ, so swapping them fails. */ + Assert.Equal(240_111L, row.RootXactDurationMs); + Assert.Equal(239_222L, row.RootQueryDurationMs); + + /* Three adjacent ints that differ, likewise. */ + Assert.Equal(7, row.TotalVictims); + Assert.Equal(3, row.DirectVictims); + Assert.Equal(4, row.MaxDepth); + + Assert.Equal(8_444L, row.WorstVictimWaitMs); + Assert.Equal("UPDATE 'worst victim'", row.WorstVictimQuery); + Assert.Equal(5L, row.SamplesAsRoot); + + /* The two booleans differ, so a swap is visible. */ + Assert.True(row.QueryTextMayBeTruncated); + Assert.False(row.ChainMayBeTruncated); + } + + /// + /// The nullable and sentinel semantics, at the ordinals that carry them: samples_as_root is NULL + /// when the root's own backend id did not resolve (recurrence genuinely unknown, distinct from "seen + /// once"), and the durations fall back to -1 rather than 0 because 0 reads as "started this instant". + /// + [Fact] + public void MapChainRow_PreservesNullRecurrenceAndTheDurationSentinels() + { + var reader = new StubReader(new object[] + { + new DateTime(2026, 8, 13, 9, 15, 0, DateTimeKind.Unspecified), + 0L, 4242, Array.Empty(), + DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, false, + DBNull.Value, DBNull.Value, // durations absent + 1, 1, 1, DBNull.Value, DBNull.Value, + DBNull.Value, // samples_as_root: the vanished-blocker case + false, false, + }); + + var row = DarlingPgBlockingReader.MapChainRow(reader); + + Assert.Null(row.SamplesAsRoot); + Assert.Equal(-1, row.RootXactDurationMs); + Assert.Equal(-1, row.RootQueryDurationMs); + Assert.Equal(-1, row.WorstVictimWaitMs); + Assert.Null(row.RootUsername); + Assert.Empty(row.Databases); + } + + [Fact] + public void MapCycleRow_LandsEveryColumnInItsOwnField() + { + var reader = new StubReader(new object[] + { + new DateTime(2026, 8, 13, 9, 16, 0, DateTimeKind.Unspecified), // 0 captured_at + 3, // 1 participant_count + new[] { 900, 901, 902 }, // 2 pids + "cycle-db", // 3 database_name + "cycle-app", // 4 application_name + 2, // 5 blocked_behind_count + new[] { 910, 911 }, // 6 blocked_behind_pids + }); + + var row = DarlingPgBlockingReader.MapCycleRow(reader); + + Assert.Equal(new DateTime(2026, 8, 13, 9, 16, 0, DateTimeKind.Unspecified), row.CapturedAt); + + /* participant_count and blocked_behind_count are both int and both counts — the pair most likely to + be transposed, so they differ here (3 vs 2), as do their pid arrays. */ + Assert.Equal(3, row.ParticipantCount); + Assert.Equal(new[] { 900, 901, 902 }, row.Pids); + Assert.Equal(2, row.BlockedBehindCount); + Assert.Equal(new[] { 910, 911 }, row.BlockedBehindPids); + + Assert.Equal("cycle-db", row.DatabaseName); + Assert.Equal("cycle-app", row.ApplicationName); + } + + /// A cycle with nothing queued behind it reports 0 and an empty array, never null. + [Fact] + public void MapCycleRow_ABareCycleReportsZeroBehind() + { + var reader = new StubReader(new object[] + { + new DateTime(2026, 8, 13, 9, 16, 0, DateTimeKind.Unspecified), + 2, new[] { 200, 201 }, DBNull.Value, DBNull.Value, 0, Array.Empty(), + }); + + var row = DarlingPgBlockingReader.MapCycleRow(reader); + + Assert.Equal(0, row.BlockedBehindCount); + Assert.Empty(row.BlockedBehindPids); + Assert.Null(row.DatabaseName); + } + + /// + /// The mapper arity must equal the projection's column count, so an appended column cannot be silently + /// ignored. Counted from the SQL by paren depth (the same technique + /// StoreSchemaProbe_ColumnCount_MatchesTheMapArity uses, for the same reason: nested parenthesised + /// expressions make token counting lie), and compared against the record's constructor arity, which is + /// what the mapper fills. + /// + [Theory] + [InlineData(nameof(DarlingPgBlockingReader.PgBlockingChainsSql))] + [InlineData(nameof(DarlingPgBlockingReader.PgBlockingCyclesSql))] + public void MapperArity_MatchesTheProjectionColumnCount(string sqlName) + { + var sql = sqlName == nameof(DarlingPgBlockingReader.PgBlockingChainsSql) + ? DarlingPgBlockingReader.PgBlockingChainsSql + : DarlingPgBlockingReader.PgBlockingCyclesSql; + var recordType = sqlName == nameof(DarlingPgBlockingReader.PgBlockingChainsSql) + ? typeof(DarlingPgBlockingReader.PgBlockingChainRow) + : typeof(DarlingPgBlockingReader.PgBlockingCycleRow); + + /* Strip SQL block comments FIRST. The projection carries explanatory comment blocks whose PROSE + contains commas at paren depth 0, and counting those as column separators over-reported the chains + projection by exactly 2 — caught by running this logic in a harness rather than leaving it to CI, + which is the only reason it is not now a pin that lies. */ + sql = Regex.Replace(sql, @"/\*.*?\*/", string.Empty, RegexOptions.Singleline); + + /* The final projection is the LAST bare "SELECT" line, and Last() is what makes that true — not + indentation. Trim() strips leading whitespace, so every CTE's own SELECT matches this predicate + too; the outer one wins only because it is textually last in the query. (An earlier version of + this comment claimed indentation was doing the work, which would have misled anyone reformatting + the query.) */ + var lines = sql.Split('\n'); + var start = Enumerable.Range(0, lines.Length).Last(i => lines[i].Trim() == "SELECT"); + + var columns = 1; + var depth = 0; + for (var i = start + 1; i < lines.Length; i++) + { + var line = lines[i]; + /* The projection ends at the first top-level FROM. */ + if (depth == 0 && line.TrimStart().StartsWith("FROM ", StringComparison.Ordinal)) + { + break; + } + + foreach (var c in line) + { + if (c == '(') depth++; + else if (c == ')') depth--; + else if (c == ',' && depth == 0) columns++; + } + } + + var arity = recordType.GetConstructors().Single().GetParameters().Length; + Assert.Equal(arity, columns); + } + + /// + /// A minimal over one row of boxed values. Only the members the mappers use + /// are implemented; anything else throws, so a mapper that starts reading by NAME (which would defeat the + /// point of an ordinal test) fails loudly instead of quietly passing. + /// + private sealed class StubReader : DbDataReader + { + private readonly object[] _values; + private int _row = -1; + + public StubReader(object[] values) => _values = values; + + private object Raw(int ordinal) => _values[ordinal]; + + public override bool IsDBNull(int ordinal) => Raw(ordinal) is DBNull; + public override DateTime GetDateTime(int ordinal) => (DateTime)Raw(ordinal); + public override long GetInt64(int ordinal) => (long)Raw(ordinal); + public override int GetInt32(int ordinal) => (int)Raw(ordinal); + public override bool GetBoolean(int ordinal) => (bool)Raw(ordinal); + public override string GetString(int ordinal) => (string)Raw(ordinal); + public override T GetFieldValue(int ordinal) => (T)Raw(ordinal); + public override object GetValue(int ordinal) => Raw(ordinal); + public override int FieldCount => _values.Length; + public override bool Read() => ++_row == 0; + public override bool HasRows => true; + + public override int Depth => 0; + public override bool IsClosed => false; + public override int RecordsAffected => 0; + public override object this[int ordinal] => Raw(ordinal); + public override object this[string name] => throw new NotSupportedException("ordinal access only"); + public override int GetOrdinal(string name) => throw new NotSupportedException("ordinal access only"); + public override string GetName(int ordinal) => throw new NotSupportedException("ordinal access only"); + public override bool NextResult() => false; + public override IEnumerator GetEnumerator() => throw new NotSupportedException(); + public override int GetValues(object[] values) => throw new NotSupportedException(); + public override string GetDataTypeName(int ordinal) => throw new NotSupportedException(); + public override Type GetFieldType(int ordinal) => Raw(ordinal).GetType(); + public override byte GetByte(int ordinal) => throw new NotSupportedException(); + public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) => throw new NotSupportedException(); + public override char GetChar(int ordinal) => throw new NotSupportedException(); + public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) => throw new NotSupportedException(); + public override decimal GetDecimal(int ordinal) => throw new NotSupportedException(); + public override double GetDouble(int ordinal) => throw new NotSupportedException(); + public override float GetFloat(int ordinal) => throw new NotSupportedException(); + public override Guid GetGuid(int ordinal) => throw new NotSupportedException(); + public override short GetInt16(int ordinal) => throw new NotSupportedException(); + } +} diff --git a/Darling/Darling.Tests/DarlingPgBlockingReaderTests.cs b/Darling/Darling.Tests/DarlingPgBlockingReaderTests.cs new file mode 100644 index 000000000..5e71e9485 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgBlockingReaderTests.cs @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the blocking read: chains assembled from the stored edge list, roots attributed, and the cycle +/// case that the chain query structurally cannot see. +/// +/// Two of these pin defects that live probing found and that no C# test could have caught on its own: +/// the missing WITH RECURSIVE (a runtime error on the first call, because PostgreSQL scopes the +/// keyword to the whole WITH clause) and the unaliased output columns. Both were correct-looking text. +/// +public class DarlingPgBlockingReaderTests +{ + private static string ChainsSql => DarlingPgBlockingReader.PgBlockingChainsSql; + + private static string CyclesSql => DarlingPgBlockingReader.PgBlockingCyclesSql; + + /// + /// WITH RECURSIVE, not WITH. PostgreSQL scopes RECURSIVE to the entire WITH clause + /// rather than to the one CTE that needs it, so a self-referencing CTE behind a plain WITH fails + /// with relation "chain" does not exist — at runtime, on the first call, never at build time. + /// Both queries here recurse. + /// + [Fact] + public void BothRecursiveQueriesDeclareWithRecursive() + { + Assert.StartsWith("WITH RECURSIVE", ChainsSql.TrimStart(), StringComparison.Ordinal); + Assert.StartsWith("WITH RECURSIVE", CyclesSql.TrimStart(), StringComparison.Ordinal); + } + + /// + /// A root is a backend that blocks something and is not itself blocked — found by absence, which is the + /// only way to find it and the reason the collector stores the whole edge set per capture rather than + /// only the pairs someone asked about. + /// + [Fact] + public void FindsRootsByAbsenceOfAnUpstreamBlocker() + { + Assert.Contains("WHERE NOT EXISTS (", ChainsSql, StringComparison.Ordinal); + Assert.Contains("upstream.blocked_pid = e.blocking_pid", ChainsSql, StringComparison.Ordinal); + } + + /// + /// BOTH recursions must be depth-capped. A cycle in the edge set would otherwise run an uncapped + /// recursive CTE until it exhausted memory — and cycles are reachable, since PostgreSQL only resolves + /// them after deadlock_timeout and a capture can land inside that window. + /// + [Fact] + public void BothRecursionsAreDepthCapped() + { + Assert.Contains("depth < 32", ChainsSql, StringComparison.Ordinal); + Assert.Contains("depth < 32", CyclesSql, StringComparison.Ordinal); + } + + /// + /// The CHAIN recursion must refuse to revisit a backend already on its walk. Without it, a cycle hanging + /// off an otherwise legitimate root is walked to the depth cap: root A blocks B while B/C/D cycle among + /// themselves, A still qualifies as a root, and the walk goes B → C → D → B → … until depth 32. The cap + /// stops the runaway, but the reader then sees max_depth = 32 and a worst victim drawn from + /// repeated revisits — indistinguishable from a genuine 32-deep chain. + /// Demonstrated against live Aurora on a fixture with exactly that shape: the unguarded recursion + /// reported max_depth = 32 where the guarded one reports 2. + /// + [Fact] + public void TheChainRecursionRefusesToRevisitABackend() + { + Assert.Contains("ARRAY[r.blocking_pid, e.blocked_pid] AS visited", ChainsSql, StringComparison.Ordinal); + Assert.Contains("e.blocked_pid <> ALL(c.visited)", ChainsSql, StringComparison.Ordinal); + } + + /// + /// Recurrence must exclude the vanished-blocker sentinel. The collector stores + /// coalesce(blocker.backend_id, 0), so every root whose own row had already left + /// pg_stat_activity lands on id 0 — and grouping those together counts unrelated one-off + /// incidents from different captures as repeat appearances of one backend. That is exactly the + /// conflation the synthetic backend id exists to prevent, arriving through the fallback rather than + /// through pid reuse. + /// Excluded rather than counted, so the outer LEFT JOIN yields NULL and the read reports + /// recurrence as UNKNOWN. "Seen once" and "cannot tell" are different claims and the tool says which. + /// + [Fact] + public void RecurrenceExcludesTheVanishedBlockerSentinel() + { + Assert.Contains("WHERE blocking_backend_id <> 0", ChainsSql, StringComparison.Ordinal); + /* And the projection must NOT coalesce the resulting NULL into a number. */ + Assert.DoesNotContain("coalesce(c.samples_as_root, 1)", ChainsSql, StringComparison.Ordinal); + Assert.Contains("c.samples_as_root", ChainsSql, StringComparison.Ordinal); + } + + /// + /// The cycle query must be scoped to its own participants and grouped per COMPONENT, not per capture. + /// A one-minute capture snapshots the whole instance, so one collection routinely holds several + /// unrelated situations — the case none of the original probe scenarios covered, since each was isolated. + /// Both failure modes were demonstrated live on a fixture holding a cycle in zz_cycle_db + /// beside an unrelated chain in aa_other_db, plus two disjoint cycles in one capture: the + /// unscoped join reported the deadlock's database as aa_other_db (alphabetically first across the + /// whole capture), and grouping on collection_id alone merged the two disjoint cycles into one + /// bogus four-participant component. + /// + [Fact] + public void TheCycleQueryIsScopedToItsOwnParticipantsAndGroupedPerComponent() + { + /* Scoped join: the edge rows feeding min(database_name) must belong to the cycle. */ + Assert.Contains("e.blocked_pid = ANY(c.members)", CyclesSql, StringComparison.Ordinal); + /* Per-component grouping, canonicalised so each participant's rotation collapses to one row. */ + Assert.Contains("array_agg(m ORDER BY m)", CyclesSql, StringComparison.Ordinal); + Assert.Contains("GROUP BY e.collection_id, e.collection_time, c.members", CyclesSql, StringComparison.Ordinal); + /* And the walk must not wander into a foreign cycle on the way. */ + Assert.Contains("e.blocking_pid <> ALL(w.members)", CyclesSql, StringComparison.Ordinal); + } + + /// + /// The cycle walk must also stop when it returns to where it started, not lean on the depth cap alone. + /// The cap bounds the damage; this is what makes the query correct rather than merely finite. + /// + [Fact] + public void TheCycleWalkStopsOnClosingTheLoop() + { + Assert.Contains("w.at_pid <> w.start_pid", CyclesSql, StringComparison.Ordinal); + /* And the detection itself: reachable from yourself. Unaliased — it lives in the `closed` CTE, + which selects straight from `walk`. */ + Assert.Contains("WHERE at_pid = start_pid", CyclesSql, StringComparison.Ordinal); + } + + /// + /// Recurrence is counted on the synthetic backend id, NOT the pid. That is the whole reason the collector + /// computes the id: a pid is reused, so a 30-day count keyed on it silently merges two different + /// backends, and "the same stuck session all afternoon" and "a succession of different ones" have + /// different remedies. + /// + [Fact] + public void RecurrenceIsKeyedOnTheBackendIdNotThePid() + { + Assert.Contains( + "count(DISTINCT collection_id) AS samples_as_root", ChainsSql, StringComparison.Ordinal); + Assert.Contains("GROUP BY blocking_backend_id", ChainsSql, StringComparison.Ordinal); + Assert.DoesNotContain("GROUP BY blocking_pid", ChainsSql, StringComparison.Ordinal); + } + + /// + /// Direct victims and total victims are different numbers and both are reported: one root blocking + /// thirty sessions directly is a different shape of problem from a thirty-deep chain, and a single + /// count cannot tell them apart. + /// + [Fact] + public void SeparatesDirectVictimsFromTotalVictims() + { + Assert.Contains( + "count(DISTINCT blocked_pid)::int AS total_victims", ChainsSql, StringComparison.Ordinal); + Assert.Contains("FILTER (WHERE depth = 1)", ChainsSql, StringComparison.Ordinal); + Assert.Contains("max(depth)::int AS max_depth", ChainsSql, StringComparison.Ordinal); + } + + /// + /// Ordered worst-first, not newest-first. Under a row limit a newest-first ordering answers a different + /// question than the one asked, and can omit the incident entirely. + /// + [Fact] + public void OrdersWorstFirstSoARowLimitCannotHideTheIncident() + { + Assert.Contains( + "ORDER BY s.total_victims DESC, s.max_depth DESC, d.collection_time DESC", + ChainsSql, + StringComparison.Ordinal); + } + + /// + /// Every output column of both queries carries an explicit alias. The C# readers are positional so this + /// changes no behaviour — but three unaliased coalesce() expressions came back as three columns + /// all named "coalesce", which makes the query unreadable in the one tool anyone debugging it will + /// actually reach for. + /// + [Fact] + public void EveryOutputColumnIsAliased() + { + foreach (var (name, sql) in new[] { ("chains", ChainsSql), ("cycles", CyclesSql) }) + { + /* The final SELECT is the last one in the text; its items are the output columns. Any that is a + bare function call with no AS is the hazard. */ + var lastSelect = sql.LastIndexOf("SELECT", StringComparison.Ordinal); + Assert.True(lastSelect > 0, $"the {name} query must have a final SELECT"); + var finalSelect = sql[lastSelect..]; + + Assert.False( + Regex.IsMatch( + finalSelect, + @"^\s+(?:coalesce|count|min|max|array_agg)\(.*\)\s*,?\s*$", + RegexOptions.Multiline), + $"the {name} query has an unaliased function in its output list — it comes back named after " + + "the function, and several such columns collide into identical headings"); + } + } + + /// + /// The capture-count read must come from collection_log, not from the edge table. The edge table + /// cannot tell "no blocking" from "not collected" — both are an absence of rows — and that distinction + /// is the difference between an all-clear and knowing nothing at all. + /// + [Fact] + public void CaptureCountsComeFromTheCollectionLog() + { + var sql = DarlingPgBlockingReader.PgBlockingCaptureCountsSql; + + Assert.Contains("FROM collection_log", sql, StringComparison.Ordinal); + Assert.Contains("l.collector_name = 'pg_blocking'", sql, StringComparison.Ordinal); + Assert.Contains("l.status = 'SUCCESS'", sql, StringComparison.Ordinal); + /* Both halves of the denominator: captures that found blocking, and captures at all. */ + Assert.Contains("count(*) FILTER (WHERE l.rows_collected > 0)", sql, StringComparison.Ordinal); + } + + /// + /// Every parameter is positional and bound, never interpolated — including the row limit, which was a + /// hardcoded LIMIT 50 in an earlier PostgreSQL reader and had to be corrected. + /// + [Fact] + public void TheRowLimitIsAParameter() + { + Assert.Contains("LIMIT $4", ChainsSql, StringComparison.Ordinal); + Assert.Contains("LIMIT $4", CyclesSql, StringComparison.Ordinal); + } + + /// + /// Both queries scope to one server and one window. A read that forgot either would silently mix + /// servers' pids together, and a pid is only unique within one instance. + /// + [Fact] + public void BothQueriesScopeToOneServerAndWindow() + { + foreach (var sql in new[] { ChainsSql, CyclesSql }) + { + Assert.Contains("WHERE server_id = $1", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); + } + } + + /// + /// Sessions queued BEHIND a cycle must be reported. Before this, a "lollipop" — X blocked by A where + /// A/B/C form a cycle — put X in neither read: chains cannot reach it because no cycle member + /// qualifies as a root, and the cycle walk cannot either because a walk starting at X's edge extends into + /// the cycle and is barred from closing on its own start, so it never lands in closed. A real, + /// captured blocking relationship appeared nowhere at all — the one outcome this collector's whole design + /// forbids. + /// Transitive, and members are excluded, so a session two hops behind the cycle still counts and a + /// participant is never double-reported as its own victim. Verified live: 910 and 911 appear as neither a + /// chain root nor a cycle participant, and surface only through this count. + /// + [Fact] + public void SessionsQueuedBehindACycleAreReported() + { + Assert.Contains("behind AS (", CyclesSql, StringComparison.Ordinal); + /* Transitive: the CTE must recurse on itself, not just take direct victims. */ + Assert.Contains("FROM behind AS b", CyclesSql, StringComparison.Ordinal); + /* Members are never counted as being behind their own cycle. */ + Assert.Contains("e.blocked_pid <> ALL(c.members)", CyclesSql, StringComparison.Ordinal); + Assert.Contains("blocked_behind_count", CyclesSql, StringComparison.Ordinal); + /* Zero, not NULL, for a cycle with nothing behind it — an absent count would read as unknown. */ + Assert.Contains("coalesce(max(b.blocked_behind_count), 0)", CyclesSql, StringComparison.Ordinal); + } + + /// + /// The two columns the collector computes per EDGE must be aggregated over all of the root's edges, never + /// taken from the one edge DISTINCT ON happens to pick. database_name is + /// coalesce(blocked.datname, blocker.datname) and query_text_may_be_truncated is an OR + /// across both queries, so an arbitrary pick attributes a victim's database, or a victim's clipped text, + /// to the root. Everything else on root_detail is genuinely backend-constant and the pick is fine. + /// + [Fact] + public void PerEdgeColumnsAreAggregatedNotArbitrarilyPicked() + { + Assert.Contains("root_edge_agg AS (", ChainsSql, StringComparison.Ordinal); + Assert.Contains("array_agg(DISTINCT e.database_name) AS databases", ChainsSql, StringComparison.Ordinal); + Assert.Contains("bool_or(e.query_text_may_be_truncated)", ChainsSql, StringComparison.Ordinal); + /* And root_detail must no longer carry either of them. */ + var rootDetail = ChainsSql[ChainsSql.IndexOf("root_detail AS (", StringComparison.Ordinal).. + ChainsSql.IndexOf("root_edge_agg AS (", StringComparison.Ordinal)]; + Assert.DoesNotContain("e.database_name", rootDetail, StringComparison.Ordinal); + Assert.DoesNotContain("e.query_text_may_be_truncated", rootDetail, StringComparison.Ordinal); + } + + /// + /// A chain that hit the walk cap must say so. With the revisit guard in place a max_depth of 32 is + /// no longer a masked cycle — it is a genuinely 32-level walk the cap stopped, which makes + /// total_victims and the worst victim FLOORS rather than totals, indistinguishable from a complete + /// answer. Implausible in practice and reported anyway, for the same reason the sampling caveat is: + /// a short answer must never pass for the whole picture. + /// + [Fact] + public void ATruncatedChainAnnouncesItself() + { + Assert.Contains("chain_may_be_truncated", ChainsSql, StringComparison.Ordinal); + Assert.Contains("(s.max_depth >= 32)", ChainsSql, StringComparison.Ordinal); + } + + /// + /// Every RemedyFor branch, pinned individually — the same treatment + /// DarlingMcpPgXminTools.RemedyFor already gets, and for the same reason: this text IS the + /// product's answer, and the branches are ordered, so reordering the aborted-state check relative to the + /// boolean flag would silently change which remedy a caller receives. + /// The distinctions are not cosmetic. Idle-in-transaction is an application defect, active is a + /// tuning problem, aborted is a client that ignored an error it already had, and a null state means the + /// chain resolved itself. Prescribing any one of those for another wastes the reader's time or loses + /// work. + /// + [Fact] + public void EveryRemedyBranchIsDistinctAndNamesItsOwnAction() + { + var idle = DarlingMcpPgBlockingTools.RemedyFor("idle in transaction", false, 240_000); + var idleByFlag = DarlingMcpPgBlockingTools.RemedyFor(null, true, 0); + var aborted = DarlingMcpPgBlockingTools.RemedyFor("idle in transaction (aborted)", false, 0); + var active = DarlingMcpPgBlockingTools.RemedyFor("active", false, 8_400); + var unknown = DarlingMcpPgBlockingTools.RemedyFor(null, false, 0); + var other = DarlingMcpPgBlockingTools.RemedyFor("fastpath function call", false, 0); + + Assert.Contains("IDLE IN TRANSACTION", idle, StringComparison.Ordinal); + Assert.Contains("idle_in_transaction_session_timeout", idle, StringComparison.Ordinal); + + /* The boolean flag alone must reach the same branch — the collector stamps it, and a root whose + state string was not captured can still be known to have been idle in transaction. */ + Assert.Equal(idle, idleByFlag); + + Assert.Contains("ABORTED", aborted, StringComparison.Ordinal); + Assert.Contains("ROLLBACK", aborted, StringComparison.Ordinal); + /* Ordering pin: 'idle in transaction (aborted)' must NOT fall into the plain idle branch, which its + prefix would match under a StartsWith or a reordered check. */ + Assert.NotEqual(idle, aborted); + + Assert.Contains("ACTIVE", active, StringComparison.Ordinal); + /* The duration is only mentioned when there is one to mention. */ + Assert.Contains("8400 ms", active, StringComparison.Ordinal); + Assert.DoesNotContain("ms", DarlingMcpPgBlockingTools.RemedyFor("active", false, 0), StringComparison.Ordinal); + + Assert.Contains("was not captured", unknown, StringComparison.Ordinal); + + Assert.Contains("fastpath function call", other, StringComparison.Ordinal); + + /* All six answers distinct — a collapsed branch is the failure this test exists for. */ + var all = new[] { idle, aborted, active, unknown, other }; + Assert.Equal(all.Length, all.Distinct(StringComparer.Ordinal).Count()); + } +} diff --git a/Darling/Darling.Tests/DarlingPgIoReaderTests.cs b/Darling/Darling.Tests/DarlingPgIoReaderTests.cs new file mode 100644 index 000000000..9175791bf --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgIoReaderTests.cs @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the pg_stat_io read: windowed differences clamped per interval, NULL surviving the arithmetic so +/// "not measured" stays distinguishable from "measured zero", and a distinct explanation per context. +/// +public class DarlingPgIoReaderTests +{ + private static string Sql => DarlingPgIoReader.PgIoSql; + + /// + /// The triple is the series identity, so the LAG must partition on all three. Partitioning on fewer + /// would difference unrelated series against each other — a client backend's reads against the + /// checkpointer's — and produce garbage intervals. + /// + [Fact] + public void DifferencesWithinTheFullBackendObjectContextTriple() + { + Assert.Contains("PARTITION BY backend_type, object_type, context", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY collection_time", Sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY backend_type, object_type, context", Sql, StringComparison.Ordinal); + } + + /// + /// Every differenced counter is clamped at zero per interval, so pg_stat_reset_shared('io') or a + /// restart drops one interval instead of producing a large negative figure. + /// + [Theory] + [InlineData("reads")] + [InlineData("read_time_ms")] + [InlineData("hits")] + [InlineData("extends")] + [InlineData("extend_time_ms")] + [InlineData("evictions")] + [InlineData("reuses")] + [InlineData("writes")] + [InlineData("write_time_ms")] + public void ClampsEveryDifferencedCounterAtZero(string column) + { + Assert.Contains($"GREATEST({column}", Sql, StringComparison.Ordinal); + Assert.Contains($"LAG({column})", Sql, StringComparison.Ordinal); + Assert.Contains($"SUM(d_{column})", Sql, StringComparison.Ordinal); + + /* No lifetime cumulative reading may reach the projection. */ + Assert.DoesNotContain($"MAX({column})", Sql, StringComparison.Ordinal); + } + + /// All nine clamps present and none left bare. + [Fact] + public void ClampCountMatchesTheDifferencedColumnCount() + { + Assert.Equal(9, Sql.Split("GREATEST(").Length - 1); + Assert.Equal(9, Sql.Split("OVER series, 0)").Length - 1); + } + + /// + /// The load-bearing distinction. On Aurora every write counter is NULL because backends there do not + /// write data files, so the read has to report whether writes are TRACKED separately from their value. + /// Without it a caller cannot tell "no writes happened" from "writes are not measured here", and would + /// read the second as the first. + /// + [Fact] + public void ReportsWhetherWriteCountersAreTrackedAtAll() + { + Assert.Contains("(writes IS NOT NULL) AS writes_tracked", Sql, StringComparison.Ordinal); + Assert.Contains("bool_or(writes_tracked)", Sql, StringComparison.Ordinal); + Assert.Contains("write_counters_tracked", Sql, StringComparison.Ordinal); + } + + /// + /// Only combinations that actually moved. An idle triple is not a finding and would crowd out the ones + /// that are — and the HAVING has to consider all four activity counters, not just reads, or a + /// write-only or extend-only combination would vanish. + /// + [Fact] + public void FiltersToCombinationsThatMovedUsingEveryActivityCounter() + { + var having = Sql[Sql.IndexOf("HAVING", StringComparison.Ordinal)..]; + + foreach (var counter in new[] { "d_reads", "d_writes", "d_extends", "d_hits" }) + { + Assert.Contains(counter, having, StringComparison.Ordinal); + } + } + + /// + /// Ordered by read TIME first, then read count. Time is what a user feels; a combination doing many + /// cheap reads matters less than one doing fewer expensive ones, and ordering by count would invert + /// exactly that. + /// + [Fact] + public void OrdersByReadTimeBeforeReadCount() + { + var order = Sql[Sql.IndexOf("ORDER BY coalesce", StringComparison.Ordinal)..]; + var timeAt = order.IndexOf("d_read_time_ms", StringComparison.Ordinal); + var countAt = order.IndexOf("d_reads", StringComparison.Ordinal); + + Assert.True(timeAt >= 0 && countAt >= 0 && timeAt < countAt); + } + + [Fact] + public void ReadsTheIoTableAndBoundsTheRowCount() + { + Assert.Contains("FROM pg_io_stats", Sql, StringComparison.Ordinal); + Assert.Contains("LIMIT $4", Sql, StringComparison.Ordinal); + Assert.Contains("server_id = $1", Sql, StringComparison.Ordinal); + } + + /// + /// Every context PostgreSQL and Aurora emit gets its own explanation, and the explanations are + /// genuinely different — the context is the dimension that changes the REMEDY, so a generic message + /// would defeat the purpose of surfacing it. + /// + [Theory] + [InlineData("normal", "shared_buffers")] + [InlineData("bulkread", "bypass")] + [InlineData("bulkwrite", "ring buffer")] + [InlineData("vacuum", "autovacuum")] + [InlineData("index", "Index")] + [InlineData("walreplay", "standby")] + public void EveryContextHasItsOwnMeaning(string context, string expectedFragment) + { + var meaning = DarlingMcpPgIoTools.ContextMeaning(context); + + Assert.Contains(expectedFragment, meaning, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("Unrecognized", meaning, StringComparison.Ordinal); + } + + /// + /// The bulkread explanation must say adding memory will NOT help. That is the single most common + /// misreading of this view: high bulkread volume looks like memory pressure and is not, because those + /// reads use a small ring buffer by design. + /// + [Fact] + public void BulkreadExplanationWarnsThatMoreMemoryWillNotHelp() + { + var meaning = DarlingMcpPgIoTools.ContextMeaning("bulkread"); + + Assert.Contains("will not", meaning, StringComparison.OrdinalIgnoreCase); + Assert.Contains("shared_buffers", meaning, StringComparison.Ordinal); + } + + [Fact] + public void ContextMeaningsAreDistinct() + { + var meanings = new[] { "normal", "bulkread", "bulkwrite", "vacuum", "index", "walreplay" } + .Select(DarlingMcpPgIoTools.ContextMeaning) + .ToArray(); + + Assert.Equal(meanings.Length, meanings.Distinct().Count()); + } + + [Fact] + public void UnknownContextIsReportedRatherThanGuessedAt() + { + Assert.Contains("Unrecognized", DarlingMcpPgIoTools.ContextMeaning("something_new"), StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs b/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs new file mode 100644 index 000000000..8a45d760b --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the replication-slot read: latest state per slot joined to its earliest reading in the window, +/// and a severity classification that turns on whether retained WAL is still growing rather than on how +/// large it currently is. +/// +public class DarlingPgSlotReaderTests +{ + private static string Sql => DarlingPgSlotReader.PgSlotsSql; + + /// Slot state is a level, so the current reading is the latest one, not an aggregate. + [Fact] + public void TakesTheLatestStatePerSlot() + { + Assert.Contains("DISTINCT ON (slot_name)", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY slot_name, collection_time DESC", Sql, StringComparison.Ordinal); + } + + /// + /// The earliest reading is what makes retained WAL actionable — without it a steady 45 GB and a 45 GB + /// that grew from 2 GB in an hour are indistinguishable, and only the second is an emergency. + /// + [Fact] + public void CarriesTheEarliestRetainedFigureForGrowthComparison() + { + Assert.Contains("ORDER BY slot_name, collection_time ASC", Sql, StringComparison.Ordinal); + Assert.Contains("first_retained_wal_bytes", Sql, StringComparison.Ordinal); + Assert.Contains("first_seen_at", Sql, StringComparison.Ordinal); + } + + /// + /// An inner join is safe only because both branches read the same window, so every slot in one is in + /// the other. Pinned because narrowing one branch's predicate later would silently drop slots. + /// + [Fact] + public void JoinsTheTwoBranchesOnSlotNameOverTheSameWindow() + { + Assert.Contains("JOIN earliest AS e ON e.slot_name = l.slot_name", Sql, StringComparison.Ordinal); + Assert.Equal(2, Sql.Split("server_id = $1").Length - 1); + Assert.Equal(2, Sql.Split("collection_time >= $2").Length - 1); + Assert.Equal(2, Sql.Split("collection_time <= $3").Length - 1); + } + + /// Biggest hole first, and both failure modes present in the projection. + [Fact] + public void OrdersByRetainedWalAndSelectsBothFailureModes() + { + Assert.Contains("ORDER BY l.retained_wal_bytes DESC", Sql, StringComparison.Ordinal); + Assert.Contains("l.wal_status", Sql, StringComparison.Ordinal); + Assert.Contains("l.catalog_xmin_age", Sql, StringComparison.Ordinal); + Assert.Contains("l.inactive_since", Sql, StringComparison.Ordinal); + } + + [Fact] + public void ReadsTheReplicationSlotsTable() + { + /* The STORE table, which is deliberately NOT named pg_replication_slots: that name belongs to + pg_catalog's view, which pg_catalog resolves first, so this read would have silently returned the + monitoring store's own (empty) slot list. Schema-qualified as well, to say which PostgreSQL the + query is aimed at. */ + Assert.Contains("FROM collect.pg_replication_slot_stats", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("FROM pg_replication_slots", Sql, StringComparison.Ordinal); + } + + /// + /// The two states where WAL is already gone or the slot is unusable are critical on their face — no + /// growth evidence needed, because the damage is done. + /// + [Theory] + [InlineData("lost", "critical_slot_lost")] + [InlineData("unreserved", "critical_wal_already_removed")] + public void TerminalWalStatesAreCriticalRegardlessOfActivityOrGrowth(string walStatus, string expected) + { + Assert.Equal(expected, DarlingMcpPgSlotTools.Classify(walStatus, isActive: true, retainedWalGrowing: false)); + Assert.Equal(expected, DarlingMcpPgSlotTools.Classify(walStatus, isActive: false, retainedWalGrowing: true)); + } + + /// + /// The disk bomb is the conjunction, not any one part: WAL retained BECAUSE of this slot, nobody + /// consuming it, and the pile still growing. Each part alone is a lesser finding. + /// + [Fact] + public void TheOrphanFillingDiskRequiresAllThreeConditions() + { + Assert.Equal( + "critical_orphan_filling_disk", + DarlingMcpPgSlotTools.Classify("extended", isActive: false, retainedWalGrowing: true)); + + /* Active consumer: it is behind, but someone is draining it. */ + Assert.Equal( + "warning_retaining_wal", + DarlingMcpPgSlotTools.Classify("extended", isActive: true, retainedWalGrowing: true)); + + /* Inactive but flat: a consumer between polls, not a volume filling. */ + Assert.Equal( + "warning_retaining_wal", + DarlingMcpPgSlotTools.Classify("extended", isActive: false, retainedWalGrowing: false)); + } + + /// + /// A healthy slot that has gone quiet and is accumulating still deserves a warning even while + /// wal_status says reserved — reserved only means the WAL is inside the configured floor, and that + /// floor is where "extended" starts, not where the risk does. + /// + [Fact] + public void InactiveAndGrowingWarnsEvenWhileReserved() + { + Assert.Equal( + "warning_inactive_and_growing", + DarlingMcpPgSlotTools.Classify("reserved", isActive: false, retainedWalGrowing: true)); + + Assert.Equal( + "info_inactive", + DarlingMcpPgSlotTools.Classify("reserved", isActive: false, retainedWalGrowing: false)); + } + + /// An active slot inside its reservation is the healthy default and must not cry wolf. + [Fact] + public void ActiveReservedSlotIsOk() + { + Assert.Equal("ok", DarlingMcpPgSlotTools.Classify("reserved", isActive: true, retainedWalGrowing: true)); + Assert.Equal("ok", DarlingMcpPgSlotTools.Classify("reserved", isActive: true, retainedWalGrowing: false)); + } + + /// + /// wal_status is NULL on a slot with no restart_lsn yet, and on Aurora it can be absent entirely. + /// That must fall through to the activity/growth branches rather than throwing or reading as healthy + /// when the slot is inactive and growing. + /// + [Fact] + public void NullWalStatusFallsThroughToActivityAndGrowth() + { + Assert.Equal("ok", DarlingMcpPgSlotTools.Classify(null, isActive: true, retainedWalGrowing: false)); + Assert.Equal("info_inactive", DarlingMcpPgSlotTools.Classify(null, isActive: false, retainedWalGrowing: false)); + Assert.Equal( + "warning_inactive_and_growing", + DarlingMcpPgSlotTools.Classify(null, isActive: false, retainedWalGrowing: true)); + } + + /// Every severity the classifier can emit is a distinct string, so callers can switch on it. + [Fact] + public void SeveritiesAreDistinct() + { + var severities = new[] + { + DarlingMcpPgSlotTools.Classify("lost", false, false), + DarlingMcpPgSlotTools.Classify("unreserved", false, false), + DarlingMcpPgSlotTools.Classify("extended", false, true), + DarlingMcpPgSlotTools.Classify("extended", true, false), + DarlingMcpPgSlotTools.Classify("reserved", false, true), + DarlingMcpPgSlotTools.Classify("reserved", false, false), + DarlingMcpPgSlotTools.Classify("reserved", true, false), + }; + + Assert.Equal(severities.Length, severities.Distinct().Count()); + } +} diff --git a/Darling/Darling.Tests/DarlingPgStatementReaderTests.cs b/Darling/Darling.Tests/DarlingPgStatementReaderTests.cs new file mode 100644 index 000000000..2ad6cdaf6 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgStatementReaderTests.cs @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the read side of pg_statement_stats: every counter covers the WINDOW — the rate columns from +/// stored deltas, the block/WAL columns differenced here — the difference is reset-safe, and the +/// grouping matches the collector's identity. +/// +public class DarlingPgStatementReaderTests +{ + private static string Sql => DarlingPgStatementReader.PgTopQueriesSql; + + /// + /// calls, total time and rows have per-interval deltas in the store, so they are SUMmed. Summing + /// their cumulative counterparts instead would multiply each query's whole lifetime by the number + /// of snapshots in the window. + /// + [Fact] + public void SumsTheDeltaColumnsForRateMetrics() + { + Assert.Contains("SUM(delta_calls)", Sql, StringComparison.Ordinal); + Assert.Contains("SUM(delta_total_exec_time_ms)", Sql, StringComparison.Ordinal); + Assert.Contains("SUM(delta_rows)", Sql, StringComparison.Ordinal); + + Assert.DoesNotContain("SUM(calls)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(rows_returned)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(total_exec_time_ms)", Sql, StringComparison.Ordinal); + } + + /// + /// The block and WAL columns keep no stored deltas, so they are differenced HERE rather than read as + /// the window's MAX. Reading them as MAX put a lifetime cumulative figure in the same row as a + /// windowed one, which a consumer cannot see and cannot correct for: it reads total_exec_time_ms for + /// the last hour beside shared_blks_read since the last counter reset, and any per-call ratio it + /// derives from the pair is nonsense. + /// + [Theory] + [InlineData("shared_blks_hit")] + [InlineData("shared_blks_read")] + [InlineData("storage_blks_read")] + [InlineData("orcache_blks_hit")] + [InlineData("temp_blks_read")] + [InlineData("temp_blks_written")] + [InlineData("wal_bytes")] + public void DifferencesTheCumulativeColumnsAcrossTheWindow(string column) + { + Assert.Contains($"LAG({column})", Sql, StringComparison.Ordinal); + Assert.Contains($"SUM(d_{column})", Sql, StringComparison.Ordinal); + + /* The lifetime reading must not survive anywhere in the projection. */ + Assert.DoesNotContain($"MAX({column})", Sql, StringComparison.Ordinal); + Assert.DoesNotContain($"SUM({column})", Sql, StringComparison.Ordinal); + } + + /// + /// GREATEST(..., 0) is what makes the differencing safe. A counter reset — an explicit + /// pg_stat_statements_reset(), an eviction and re-entry, or a major-version upgrade (queryid is not + /// stable across majors) — makes one interval negative, and an unclamped difference would report + /// that as a large negative figure. Clamping drops the reset interval and keeps the rest, the same + /// rule the stored delta machinery applies. + /// + [Fact] + public void ClampsEachIntervalAtZeroSoACounterResetCannotGoNegative() + { + var clamped = Sql.Split("GREATEST(").Length - 1; + + /* One per differenced column — seven of them, all clamped, none left bare. */ + Assert.Equal(7, clamped); + Assert.Equal(7, Sql.Split("OVER series, 0)").Length - 1); + } + + /// + /// The LAG partition must be the FULL series identity, matching how the stored deltas are keyed. The + /// same normalized statement run by another user or against another database is a separate + /// pg_stat_statements entry with its own counters, so differencing across those would interleave + /// unrelated series and produce garbage intervals. + /// + [Fact] + public void DifferencesWithinTheFullSeriesIdentityNotJustTheQueryId() + { + Assert.Contains("PARTITION BY queryid, database_id, user_id, toplevel", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY collection_time", Sql, StringComparison.Ordinal); + + /* Rolled up to the coarser grain a "top queries" answer wants, AFTER the differencing. */ + var windowAt = Sql.IndexOf("WINDOW series AS", StringComparison.Ordinal); + var groupAt = Sql.IndexOf("GROUP BY queryid, database_id", StringComparison.Ordinal); + Assert.True(windowAt < groupAt); + } + + /// + /// A series with one sample in the window has no measurable interval — its increment happened before + /// the window began — so it must report 0, not its lifetime total. coalesce on the SUM, never on the + /// cumulative column. + /// + [Fact] + public void ASingleSampleSeriesReportsZeroRatherThanItsLifetimeTotal() + { + Assert.Contains("coalesce(SUM(d_shared_blks_read), 0)", Sql, StringComparison.Ordinal); + Assert.Contains("coalesce(SUM(d_wal_bytes), 0)", Sql, StringComparison.Ordinal); + } + + /// + /// The two high-water marks are NOT counters and must stay MAX — differencing a high-water mark is + /// meaningless, and summing one would invent a total that never occurred. + /// + [Fact] + public void HighWaterMarksStayMax() + { + Assert.Contains("MAX(max_exec_time_ms)", Sql, StringComparison.Ordinal); + Assert.Contains("MAX(max_exec_peakmem_bytes)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("LAG(max_exec_peakmem_bytes)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("LAG(max_exec_time_ms)", Sql, StringComparison.Ordinal); + } + + /// + /// Grouped by the collector's identity, not by queryid alone: the same normalized statement against + /// a different database is a separate pg_stat_statements entry with its own counters. + /// + [Fact] + public void GroupsByQueryIdentityNotQueryIdAlone() + { + Assert.Contains("GROUP BY queryid, database_id", Sql, StringComparison.Ordinal); + } + + /// Total time is the currency: heaviest first, and bounded. + [Fact] + public void OrdersByTotalTimeAndBoundsTheResult() + { + Assert.Contains("ORDER BY SUM(delta_total_exec_time_ms) DESC", Sql, StringComparison.Ordinal); + Assert.Contains("LIMIT", Sql, StringComparison.Ordinal); + } + + /// + /// Shapes that did not execute in the window are excluded. pg_stat_statements retains an entry long + /// after its last execution, so without this the result is padded with idle shapes showing zero. + /// + [Fact] + public void ExcludesShapesThatDidNotRunInTheWindow() + { + Assert.Contains("HAVING SUM(delta_total_exec_time_ms) > 0", Sql, StringComparison.Ordinal); + } + + [Fact] + public void UsesTheStandardServerAndWindowParameters() + { + Assert.Contains("server_id = $1", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", Sql, StringComparison.Ordinal); + } + + [Fact] + public void ReadsThePostgresStatementTable() + { + Assert.Contains("FROM pg_statement_stats", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("v_query_stats", Sql, StringComparison.Ordinal); + } + + /// + /// The Aurora I/O split columns are selected — they are the reason this reads + /// aurora_stat_statements rather than the vanilla view, and dropping them would silently reduce + /// this to a worse version of the community query. + /// + [Fact] + public void SelectsTheAuroraIoSourceSplit() + { + Assert.Contains("storage_blks_read", Sql, StringComparison.Ordinal); + Assert.Contains("orcache_blks_hit", Sql, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/DarlingPgWaitReaderTests.cs b/Darling/Darling.Tests/DarlingPgWaitReaderTests.cs new file mode 100644 index 000000000..7ba2cd472 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgWaitReaderTests.cs @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the read side of pg_wait_stats: the aggregation reads deltas rather than raw counters, keeps +/// unnamed events visible, and converts the stored microseconds exactly once. +/// +public class DarlingPgWaitReaderTests +{ + private static string Sql => DarlingPgWaitReader.PgWaitStatsSql; + + /// + /// The single most important property of this query. The store holds CUMULATIVE counters plus + /// per-interval deltas; summing the cumulative columns across snapshots would multiply the entire + /// history by the number of snapshots in the window — a number that looks plausible and is wrong + /// by orders of magnitude. + /// + [Fact] + public void AggregatesDeltasNeverRawCumulativeCounters() + { + Assert.Contains("SUM(delta_waits)", Sql, StringComparison.Ordinal); + Assert.Contains("SUM(delta_wait_time_us)", Sql, StringComparison.Ordinal); + + Assert.DoesNotContain("SUM(waits)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(wait_time_us)", Sql, StringComparison.Ordinal); + } + + /// + /// wait_type and wait_event are nullable because their lookups are LEFT JOINed in the collector. An + /// event Aurora reports but does not name is the new-wait-type case an operator most wants to see, + /// so it gets a synthetic label from the numeric ids instead of being dropped by the GROUP BY. + /// + [Fact] + public void SurfacesUnnamedEventsInsteadOfDroppingThem() + { + Assert.Contains("COALESCE(wait_type", Sql, StringComparison.Ordinal); + Assert.Contains("COALESCE(wait_event", Sql, StringComparison.Ordinal); + Assert.Contains("unknown_type_", Sql, StringComparison.Ordinal); + Assert.Contains("unknown_event_", Sql, StringComparison.Ordinal); + + /* No WHERE clause filtering the nulls away, which would defeat the COALESCE. */ + Assert.DoesNotContain("wait_event IS NOT NULL", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("wait_type IS NOT NULL", Sql, StringComparison.Ordinal); + } + + /// + /// The store keeps microseconds because that is what Aurora reports. Conversion happens once, in + /// the read layer, so no consumer has to remember the unit — and the division is float, not + /// integer, so sub-millisecond events do not collapse to zero. + /// + [Fact] + public void ConvertsMicrosecondsToMillisecondsWithoutIntegerTruncation() + { + Assert.Contains("/ 1000.0", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("/ 1000 ", Sql, StringComparison.Ordinal); + } + + /// Guarded average: a window where an event accrued time but zero waits must not divide by zero. + [Fact] + public void GuardsTheAverageAgainstZeroWaits() + { + Assert.Contains("WHEN SUM(delta_waits) > 0", Sql, StringComparison.Ordinal); + Assert.Contains("ELSE 0", Sql, StringComparison.Ordinal); + } + + /// Same windowing contract as every other read in this store: $1 server, $2/$3 bounds. + [Fact] + public void UsesTheStandardServerAndWindowParameters() + { + Assert.Contains("server_id = $1", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", Sql, StringComparison.Ordinal); + } + + /// Heaviest first, and bounded — an operator reads the top of this list, not all of it. + [Fact] + public void OrdersByWeightAndBoundsTheResult() + { + Assert.Contains("ORDER BY SUM(delta_wait_time_us) DESC", Sql, StringComparison.Ordinal); + Assert.Contains("LIMIT", Sql, StringComparison.Ordinal); + } + + /// + /// Rows that accrued no time in the window are excluded. Without this the result is padded with + /// every event the instance has ever seen, each showing zero, burying the handful that moved. + /// + [Fact] + public void ExcludesEventsThatDidNotMoveInTheWindow() + { + Assert.Contains("HAVING SUM(delta_wait_time_us) > 0", Sql, StringComparison.Ordinal); + } + + /// Reads the collector's table, not the SQL Server one. + [Fact] + public void ReadsThePostgresWaitTable() + { + Assert.Contains("FROM pg_wait_stats", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("v_wait_stats", Sql, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/DarlingPgWraparoundReaderTests.cs b/Darling/Darling.Tests/DarlingPgWraparoundReaderTests.cs new file mode 100644 index 000000000..c7dfe475e --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgWraparoundReaderTests.cs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the freeze-headroom read: a level is read as the latest value plus the window peak, never +/// aggregated, and the severity ladder maps to real PostgreSQL behaviour changes. +/// +public class DarlingPgWraparoundReaderTests +{ + private static string Sql => DarlingPgWraparoundReader.PgWraparoundSql; + + /// + /// Freeze age is a level, not accumulated work. Averaging it would blur the only number that + /// matters and summing it would be meaningless — the opposite discipline from the rate collectors, + /// where summing deltas is mandatory. + /// + [Fact] + public void ReadsTheLatestValuePerDatabaseRatherThanAggregating() + { + Assert.Contains("DISTINCT ON (database_name)", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY database_name, collection_time DESC", Sql, StringComparison.Ordinal); + + Assert.DoesNotContain("SUM(frozen_xid_age)", Sql, StringComparison.Ordinal); + Assert.DoesNotContain("AVG(frozen_xid_age)", Sql, StringComparison.Ordinal); + } + + /// + /// The window peak is what distinguishes a healthy freezing sawtooth from a monotonic climb, so both + /// counters carry one. Computed as window functions so they see every row in the window, not only + /// the row DISTINCT ON keeps. + /// + [Fact] + public void CarriesAWindowPeakForBothCounters() + { + Assert.Contains("MAX(frozen_xid_age) OVER (PARTITION BY database_name)", Sql, StringComparison.Ordinal); + Assert.Contains("MAX(min_multixid_age) OVER (PARTITION BY database_name)", Sql, StringComparison.Ordinal); + } + + /// Both independent counters are read; reporting one would give false comfort. + [Fact] + public void ReadsBothTransactionIdAndMultiXactColumns() + { + foreach (var column in new[] + { + "frozen_xid_age", "min_multixid_age", + "pct_toward_wraparound", "pct_toward_multixact_wraparound", + "pct_toward_emergency_vacuum", "pct_toward_multixact_emergency", + "xids_remaining", "multixids_remaining", + }) + { + Assert.Contains(column, Sql, StringComparison.Ordinal); + } + } + + [Fact] + public void UsesTheStandardServerAndWindowParameters() + { + Assert.Contains("server_id = $1", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", Sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", Sql, StringComparison.Ordinal); + } + + [Fact] + public void ReadsTheWraparoundTable() + { + Assert.Contains("FROM pg_wraparound_stats", Sql, StringComparison.Ordinal); + } + + /// + /// Each severity boundary is a documented PostgreSQL behaviour change, not a round number: failsafe + /// mode around 1.6B ids, server warnings near 40M remaining. An "ok" that quietly spans the failsafe + /// range would be worse than no label at all. + /// + [Theory] + [InlineData(10.0, 20.0, "ok")] + [InlineData(10.0, 100.0, "info_anti_wraparound_vacuum_expected")] + [InlineData(60.0, 100.0, "warning")] + [InlineData(80.0, 100.0, "critical_failsafe_range")] + [InlineData(99.0, 100.0, "critical_wraparound_imminent")] + public void ClassifiesSeverityFromTheDocumentedLadder(double pctWrap, double pctEmergency, string expected) + { + Assert.Equal(expected, DarlingMcpPgWraparoundTools.Classify(pctWrap, pctEmergency)); + } + + /// + /// A database past its emergency-vacuum threshold but nowhere near the ceiling is INFO, not a + /// warning: a forced anti-wraparound vacuum is normal operation, and treating it as an alert is how + /// wraparound monitoring earns a reputation for crying wolf. + /// + [Fact] + public void AntiWraparoundVacuumAloneIsInformationalNotAWarning() + { + Assert.Equal("info_anti_wraparound_vacuum_expected", DarlingMcpPgWraparoundTools.Classify(9.3, 100.0)); + } +} diff --git a/Darling/Darling.Tests/DarlingPgXminReaderTests.cs b/Darling/Darling.Tests/DarlingPgXminReaderTests.cs new file mode 100644 index 000000000..05e7630a4 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgXminReaderTests.cs @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the xmin-horizon read: current holder per source joined to its persistence across the window, +/// and a distinct remedy for every source the collector can emit. +/// +public class DarlingPgXminReaderTests +{ + private static string Sql => DarlingPgXminReader.PgXminHorizonSql; + + /// Current state per source — a level, so latest rather than aggregated. + [Fact] + public void TakesTheLatestHolderPerSource() + { + Assert.Contains("DISTINCT ON (source)", Sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY source, collection_time DESC", Sql, StringComparison.Ordinal); + } + + /// + /// Persistence is what separates a chronic holder from a query that ran long, and that distinction + /// changes the response — so the window figures are not optional decoration. + /// + [Fact] + public void CarriesPersistenceFiguresAcrossTheWindow() + { + Assert.Contains("COUNT(*) FILTER (WHERE is_winner)", Sql, StringComparison.Ordinal); + Assert.Contains("MAX(xmin_age)", Sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY source", Sql, StringComparison.Ordinal); + } + + /// + /// An inner join is safe here only because both branches read the same window, so every source in one + /// is in the other. Worth pinning: widening one branch's predicate later without the other would + /// silently drop holders. + /// + [Fact] + public void JoinsTheTwoBranchesOnSource() + { + Assert.Contains("JOIN window_stats AS w ON w.source = l.source", Sql, StringComparison.Ordinal); + Assert.Equal(2, Sql.Split("server_id = $1").Length - 1); + Assert.Equal(2, Sql.Split("collection_time >= $2").Length - 1); + } + + /// Oldest holder first — that is the one setting the horizon. + [Fact] + public void OrdersByAgeDescending() + { + Assert.Contains("ORDER BY l.xmin_age DESC", Sql, StringComparison.Ordinal); + } + + [Fact] + public void ReadsTheXminHorizonTable() + { + Assert.Contains("FROM pg_xmin_horizon", Sql, StringComparison.Ordinal); + } + + /// + /// Every source the collector can emit needs its own remedy — a generic message would defeat the + /// point of attributing the cause in the first place. + /// + [Theory] + [InlineData("session", "idle in transaction")] + [InlineData("replication_slot", "dropped")] + [InlineData("replication_slot_catalog", "catalog_xmin")] + [InlineData("standby_feedback", "hot_standby_feedback")] + [InlineData("prepared_transaction", "ROLLBACK PREPARED")] + public void EverySourceHasItsOwnRemedy(string source, string expectedFragment) + { + var remedy = DarlingMcpPgXminTools.RemedyFor(source); + + Assert.Contains(expectedFragment, remedy, StringComparison.Ordinal); + Assert.DoesNotContain("Unrecognized", remedy, StringComparison.Ordinal); + } + + /// The five remedies are genuinely distinct, not one message with the noun swapped. + [Fact] + public void RemediesAreDistinctPerSource() + { + var remedies = new[] + { + DarlingMcpPgXminTools.RemedyFor("session"), + DarlingMcpPgXminTools.RemedyFor("replication_slot"), + DarlingMcpPgXminTools.RemedyFor("replication_slot_catalog"), + DarlingMcpPgXminTools.RemedyFor("standby_feedback"), + DarlingMcpPgXminTools.RemedyFor("prepared_transaction"), + }; + + Assert.Equal(remedies.Length, remedies.Distinct().Count()); + } + + [Fact] + public void UnknownSourceIsReportedRatherThanGuessedAt() + { + Assert.Contains("Unrecognized", DarlingMcpPgXminTools.RemedyFor("something_new"), StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/DarlingPlanCorrectionLiveMigrationTests.cs b/Darling/Darling.Tests/DarlingPlanCorrectionLiveMigrationTests.cs index 2df55cce4..988697118 100644 --- a/Darling/Darling.Tests/DarlingPlanCorrectionLiveMigrationTests.cs +++ b/Darling/Darling.Tests/DarlingPlanCorrectionLiveMigrationTests.cs @@ -93,12 +93,13 @@ public async Task FreshStore_AndUpgradedStore_BuildTheSamePlanCorrectionTable_Ag var applied = await PgMigrations.MigrateAsync(connection, cancellationToken); - /* Exactly the eight scripts above 44 ran — V46, V47, V48 (#1984), V49 (#1986 database-state alert), + /* Exactly the scripts above 44 ran — V46, V47, V48 (#1984), V49 (#1986 database-state alert), V50 (#2008 2a server-tag colour), V51 (#2012 stage 2 query-stats host object), V52 (#2060 - persisted finding drill-down), and V53 (#2068 store self-metrics). If the applier had stumbled - over the permanent V45 gap it would either re-run everything above 1 or nothing at all, and both - show up right here. */ - Assert.Equal(9, applied); + persisted finding drill-down), V53 (#2068 store self-metrics), V54 (#2069 plan-dim gzip), and + V55 (#2107 self-alert knobs). If the applier had stumbled over the permanent V45 gap it would + either re-run everything above 1 or nothing at all, and both show up right here. Version-agnostic + on purpose: the ladder-top pin lives in ScaffoldTests, and this count just tracks it. */ + Assert.Equal(PgMigrations.Scripts.Count(m => m.Version > 44), applied); Assert.Equal(StorageVersion.SchemaVersion, await CurrentVersionAsync(connection, cancellationToken)); var fromMigration = await ReadColumnsAsync(connection, cancellationToken); diff --git a/Darling/Darling.Tests/DarlingRetentionTests.cs b/Darling/Darling.Tests/DarlingRetentionTests.cs index 2a7949d75..db737ccfd 100644 --- a/Darling/Darling.Tests/DarlingRetentionTests.cs +++ b/Darling/Darling.Tests/DarlingRetentionTests.cs @@ -617,4 +617,64 @@ private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, Cance connection); await cleanup.ExecuteNonQueryAsync(ct); } + + /* ---------------- #2143 drop_chunks deadlock retry ---------------- */ + + private static PostgresException Deadlock() => + new("deadlock detected", "ERROR", "ERROR", PostgresErrorCodes.DeadlockDetected); + + [Fact] + public async Task DropChunksRetry_OneDeadlock_RetriesOnce_AndSucceeds() + { + /* The field case (#2143, caught by the nightly's purge e2e): the first attempt loses a deadlock + to a background job whose locks clear in milliseconds — the retry completes the purge instead + of wasting the cycle on the DELETE fallback. */ + var calls = 0; + var result = await DarlingRetention.ExecuteDropChunksWithDeadlockRetryAsync( + () => ++calls == 1 ? throw Deadlock() : Task.FromResult(3), + "collection_log", logger: null); + + Assert.Equal(2, calls); + Assert.Equal(3, result); + } + + [Fact] + public async Task DropChunksRetry_TwoDeadlocks_GivesUpToTheDeleteFallback() + { + /* A second deadlock in a row is STANDING contention — camping a retry loop on a lock queue is + worse than the DELETE fallback + next cycle. Exactly two attempts, then null. */ + var calls = 0; + var result = await DarlingRetention.ExecuteDropChunksWithDeadlockRetryAsync( + () => { calls++; throw Deadlock(); }, + "collection_log", logger: null); + + Assert.Equal(2, calls); + Assert.Null(result); + } + + [Fact] + public async Task DropChunksRetry_NonDeadlockFailure_DoesNotRetry() + { + /* Only 40P01 earns a retry — any other failure keeps the original single-shot posture (a missing + relation or a permission error does not get better by asking again). */ + var calls = 0; + var result = await DarlingRetention.ExecuteDropChunksWithDeadlockRetryAsync( + () => { calls++; throw new InvalidOperationException("not a deadlock"); }, + "collection_log", logger: null); + + Assert.Equal(1, calls); + Assert.Null(result); + } + + [Fact] + public async Task DropChunksRetry_CleanRun_IsSingleShot() + { + var calls = 0; + var result = await DarlingRetention.ExecuteDropChunksWithDeadlockRetryAsync( + () => Task.FromResult(++calls == 1 ? 7 : -1), + "collection_log", logger: null); + + Assert.Equal(1, calls); + Assert.Equal(7, result); + } } diff --git a/Darling/Darling.Tests/DarlingSecretsDecryptFailureTests.cs b/Darling/Darling.Tests/DarlingSecretsDecryptFailureTests.cs new file mode 100644 index 000000000..ae1fd8c2d --- /dev/null +++ b/Darling/Darling.Tests/DarlingSecretsDecryptFailureTests.cs @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2255: what the service says when it cannot DPAPI-decrypt a stored credential. +/// +/// The field report. After adding a server through a Viewer on a remote PC, the log filled with +/// [dcvs-bd01] Connect failed, retrying in 60s: Key not valid for use in specified state — once a minute, +/// forever. That is ProtectedData.Unprotect's own text, surfaced verbatim. It does not say DPAPI, does not +/// name what failed to decrypt, does not mention that a machine boundary is involved, and reads exactly like SQL +/// Server rejecting a login — which is where the operator looked. +/// +/// What is actually true, and why the message can be specific about it. Both +/// and the Viewer's ViewerServerSecret protect with +/// DataProtectionScope.LocalMachine and share the entropy string byte-for-byte. LocalMachine means ANY +/// user on the writing machine can decrypt, and NO other machine ever can. So this is never a service-account +/// permissions problem and never a user-boundary problem: it is a machine boundary, and the overwhelmingly likely +/// cause is a credential encrypted by a Viewer running somewhere else. Every remedy is therefore "produce the +/// blob on this host", which is what the message now lists. +/// +public sealed class DarlingSecretsDecryptFailureTests +{ + /// + /// The message has to carry the four things that turn a support round trip into a self-service fix: that it + /// is DPAPI on THIS host, that no credential reached the server, that the blob is machine-bound, and that a + /// remote Viewer is the usual cause. + /// + [Fact] + public void TheExplanationNamesDpapiTheMachineBoundaryAndTheLikelyCause() + { + var message = DarlingSecrets.DescribeDecryptFailure("the stored password for server 'dcvs-bd01'"); + + Assert.Contains("DPAPI-decrypt", message, StringComparison.Ordinal); + Assert.Contains("dcvs-bd01", message, StringComparison.Ordinal); + /* It must actively DENY the reading the raw text invited, or the operator keeps investigating the login. */ + Assert.Contains("not SQL Server", message, StringComparison.Ordinal); + Assert.Contains("LocalMachine", message, StringComparison.Ordinal); + Assert.Contains("only be decrypted on the machine that wrote them", message, StringComparison.Ordinal); + Assert.Contains("DIFFERENT PC", message, StringComparison.Ordinal); + } + + /// + /// And it must be ACTIONABLE — every remedy runs on the service host, because that is the only place a + /// decryptable blob can be produced. The env:/file: route is named too: it is the one option + /// that is not machine-bound at all, so it is the answer for anyone who genuinely cannot use the local + /// Viewer. + /// + [Fact] + public void TheExplanationListsRemediesThatAllRunOnTheServiceHost() + { + var message = DarlingSecrets.DescribeDecryptFailure("the store credential"); + + Assert.Contains("--add-server", message, StringComparison.Ordinal); + Assert.Contains("--encrypt-password", message, StringComparison.Ordinal); + Assert.Contains("env:", message, StringComparison.Ordinal); + Assert.Contains("not machine-bound", message, StringComparison.Ordinal); + } + + /// + /// THE BEHAVIORAL TEST: an undecryptable blob in encryptedPassword surfaces the explanation, not + /// CryptographicException's text — and keeps the original as the inner exception so nothing is lost + /// for a bug report. + /// + /// A random base64 blob is exactly the shape of the reported fault: well-formed base64 that this + /// machine's DPAPI cannot unprotect, which is what a blob from another machine looks like from here. Windows + /// only, because DPAPI is. + /// + [Fact] + public void AnUndecryptableServerPasswordExplainsItselfRatherThanLeakingTheRawCryptoError() + { + Assert.SkipUnless(OperatingSystem.IsWindows(), "DPAPI requires Windows."); + + var server = new MonitoredServer + { + Name = "dcvs-bd01", + Host = "dcvs-bd01", + Auth = "sql", + Username = "monitor", + /* Valid base64, not a valid DPAPI blob for this machine — the remote-Viewer case, locally. */ + EncryptedPassword = Convert.ToBase64String(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }), + }; + + var ex = Assert.Throws( + () => DarlingSecrets.ResolvePassword(server, out _)); + + Assert.Contains("DPAPI-decrypt", ex.Message, StringComparison.Ordinal); + Assert.Contains("dcvs-bd01", ex.Message, StringComparison.Ordinal); + Assert.Contains("encryptedPassword", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Key not valid for use in specified state", ex.Message, StringComparison.Ordinal); + /* The raw fault is preserved for a bug report, just not as the operator-facing text. */ + Assert.NotNull(ex.InnerException); + Assert.IsAssignableFrom(ex.InnerException); + } + + /// + /// A round-trip through this host's own DPAPI still works — the guard must not have turned a working + /// decrypt into a failure, and this is the only arm that proves the try/catch wraps rather than replaces. + /// + [Fact] + public void APasswordEncryptedOnThisHostStillResolves() + { + Assert.SkipUnless(OperatingSystem.IsWindows(), "DPAPI requires Windows."); + + var server = new MonitoredServer + { + Name = "local-01", + Host = "local-01", + Auth = "sql", + Username = "monitor", + EncryptedPassword = DarlingSecrets.Protect("correct horse battery staple"), + }; + + Assert.Equal("correct horse battery staple", DarlingSecrets.ResolvePassword(server, out var usedPlaintext)); + Assert.False(usedPlaintext); + } + + /// + /// #2255's second half, pinned at the source: the connect-retry log prints the full explanation ONCE per + /// distinct cause and one terse line while it persists, and clears the latch on a successful connect. + /// + /// Behavioral coverage cannot reach this — it needs a monitored server that fails to connect across + /// many sweeps. The failure it guards is the reported one: a permanent fault re-explaining itself 1,440 times + /// a day, which is how the log became unreadable. And the CLEAR matters as much as the dedup: without it a + /// cause that was fixed and then recurred would be silently swallowed as a repeat. + /// + [Fact] + public void TheConnectRetryLogExplainsOnceAndThenStaysQuiet() + { + var source = ReadWorkerSource(); + + Assert.Contains("LastConnectFailureLogged", source, StringComparison.Ordinal); + Assert.Contains("Connect still failing, retrying in 60s (same cause as logged above)", source, StringComparison.Ordinal); + /* A permanent credential fault is an Error, not a Warning — nothing about it clears on its own. */ + Assert.Contains("Connect failed and will keep failing until fixed", source, StringComparison.Ordinal); + Assert.Contains("server.LastConnectFailureLogged = null;", source, StringComparison.Ordinal); + } + + private static string ReadWorkerSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/DarlingSelfAlertTests.cs b/Darling/Darling.Tests/DarlingSelfAlertTests.cs index 2cb3ded6f..3c605a020 100644 --- a/Darling/Darling.Tests/DarlingSelfAlertTests.cs +++ b/Darling/Darling.Tests/DarlingSelfAlertTests.cs @@ -55,6 +55,7 @@ private sealed class FakeSettings : IAlertEngineSettings public bool FailedJobEnabled { get; set; } public bool PvsEnabled { get; set; } public bool DatabaseStateEnabled { get; set; } + public bool ForcePlanFailureEnabled { get; set; } = true; public int CpuThresholdPercent { get; set; } = 80; public int BlockingCountThreshold { get; set; } = 1; public int BlockingWaitSecondsThreshold { get; set; } @@ -70,6 +71,12 @@ private sealed class FakeSettings : IAlertEngineSettings public int TempDbSpaceThresholdPercent { get; set; } = 80; public int LowDiskThresholdPercent { get; set; } = 10; public int LowDiskThresholdGb { get; set; } = 5; + /* #2107: the previously-hardcoded knobs, at their shipped defaults. */ + public int DiskCriticalFreePercent { get; set; } = 3; + public int DiskCriticalFreeGb { get; set; } = 2; + public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int CollectionStaleMinutes { get; set; } = 30; + public int CollectionFailureThreshold { get; set; } = 10; public int PvsThresholdPercent { get; set; } = 40; public int PvsFloorGb { get; set; } = 1; public int LongRunningJobMultiplier { get; set; } = 3; @@ -160,6 +167,9 @@ private sealed class Harness /// #1696 (V37): AG disconnect re-fire minutes. Default 0 = off, the shipped behavior. public int AgDisconnectRefireMinutes { get; set; } + /// #2136 (V57): the Store Job Over Cadence warning percent. Default is the shipped 25. + public int StoreJobCadenceWarnPercent { get; set; } = 25; + public DateTime Now { get; set; } = new(2026, 7, 1, 12, 0, 0, DateTimeKind.Utc); /// #1681: captures what the evaluator writes to the service log, so the firing/recovery pair @@ -176,7 +186,8 @@ private sealed class Harness notifyAgHealth: () => NotifyAgHealth, agLagAlertSeconds: () => AgLagAlertSeconds, agRedoQueueAlertKb: () => AgRedoQueueAlertKb, - agDisconnectRefireMinutes: () => AgDisconnectRefireMinutes); + agDisconnectRefireMinutes: () => AgDisconnectRefireMinutes, + storeJobCadenceWarnPercent: () => StoreJobCadenceWarnPercent); } /* ---------------- #991 Availability Group fixtures ---------------- */ @@ -813,7 +824,7 @@ Pinned at the source so the invariant is visible where the number is produced. * /* ---------------- store disk pressure edge ---------------- */ [Fact] - public async Task DiskPressure_FiresOnce_ThenCooldownSuppresses_ThenReFires() + public async Task DiskPressure_FiresOnce_ThenStaysQuietAtUnchangedLevel_ReFiresOnlyOnWorsening() { var h = new Harness(); var e = h.Build(); @@ -829,9 +840,38 @@ public async Task DiskPressure_FiresOnce_ThenCooldownSuppresses_ThenReFires() await e.ApplyDiskPressureAsync(5 * Gib, 100 * Gib, null, Ct); Assert.Single(h.Deliverer.Outcomes); - /* After the cooldown the standing condition re-fires. */ + /* #2101: the cooldown elapsing is NOT enough — a standing breach at an UNCHANGED level stays + quiet (the field report: 7.3% free re-notified every 15 minutes for hours). */ h.Now = h.Now.AddMinutes(5); await e.ApplyDiskPressureAsync(5 * Gib, 100 * Gib, null, Ct); + Assert.Single(h.Deliverer.Outcomes); + + /* Worsened less than the 1pp margin (5.0% → 4.5%) — jitter, still quiet. */ + h.Now = h.Now.AddMinutes(5); + await e.ApplyDiskPressureAsync(45 * Gib, 1000 * Gib, null, Ct); + Assert.Single(h.Deliverer.Outcomes); + + /* Genuinely worsened (5.0% → 3.5%, past the margin) — re-fires, and re-anchors the watermark. */ + h.Now = h.Now.AddMinutes(5); + await e.ApplyDiskPressureAsync(35 * Gib, 1000 * Gib, null, Ct); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task DiskPressure_Recovery_ClearsTheWorseningWatermark_SoTheNextBreachIsFresh() + { + var h = new Harness(); + var e = h.Build(); + + await e.ApplyDiskPressureAsync(5 * Gib, 100 * Gib, null, Ct); /* breach at 5% */ + Assert.Single(h.Deliverer.Outcomes); + + await e.ApplyDiskPressureAsync(50 * Gib, 100 * Gib, null, Ct); /* recovered */ + + /* A NEW breach at the same 5% level after recovery must fire — the watermark died with the + old episode, or a volume that oscillates around the threshold would go permanently silent. */ + h.Now = h.Now.AddMinutes(6); + await e.ApplyDiskPressureAsync(5 * Gib, 100 * Gib, null, Ct); Assert.Equal(2, h.Deliverer.Outcomes.Count); } @@ -1988,6 +2028,9 @@ public Task GetAnomalousJobsAsync(string serverKey, int mul Task.FromResult(new AnomalousJobsResult(SnapshotIsFresh: true, new List())); public Task> GetDatabaseStatesAsync(string serverKey, CancellationToken cancellationToken = default) => Task.FromResult(new List()); + + public Task> GetForcePlanFailuresAsync(string serverKey, CancellationToken cancellationToken = default) => + Task.FromResult(new List()); } private sealed class StubStateStore : IAlertStateStore @@ -1996,6 +2039,16 @@ private sealed class StubStateStore : IAlertStateStore public Task SaveEdgeTriggerWatermarkAsync(string serverKey, string metricName, int watermark) => Task.CompletedTask; public Task LoadFailedJobWatermarkAsync(string serverKey) => Task.FromResult(null); public Task SaveFailedJobWatermarkAsync(string serverKey, DateTime watermark) => Task.CompletedTask; + public Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState) => Task.CompletedTask; + public Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName) => Task.CompletedTask; + + /* #2216: the self-alert paths carry no fingerprintable incidents, so the engine never accumulates + against this stub — it exists to satisfy the seam. */ + public Task> LoadIncidentOccurrencesAsync(string serverKey, string metricName) => + Task.FromResult>( + new Dictionary(StringComparer.Ordinal)); + + public Task SaveIncidentOccurrencesAsync(string serverKey, string metricName, IReadOnlyDictionary states) => Task.CompletedTask; } /* ---------------- live collection_log reads (gated on DARLING_TEST_PG) ---------------- */ @@ -2281,4 +2334,108 @@ public async Task CompressionStuck_Firing_IsLoggedEvenWhenMuted() Assert.Contains(warnings, x => x.Message.Contains("[muted]", StringComparison.Ordinal)); } + + /* ---------------- #2136 Store Job Over Cadence ---------------- */ + + private static StoreJobCadenceReading CadenceJob( + long id = 1028, long? durMs = 900_000, long schedMs = 3_600_000, + string name = "policy_compression query_store_stats") => + new(id, name, durMs, schedMs); + + [Fact] + public async Task JobOverCadence_WarningTier_FiresAtTheKnobPercent() + { + var h = new Harness(); + var e = h.Build(); + + /* 900s of 3600s = exactly 25%, the shipped default — the boundary is inclusive. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob() }, Ct); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(DarlingSelfAlertEvaluator.JobCadenceMetric, fired.MetricName); + Assert.Equal(AlertSeverityLevel.Warning, fired.Severity); + Assert.Equal("storejob:1028", fired.ServerKey); /* prefixed so it never parses as a server_id */ + Assert.Contains("25% of its schedule interval", fired.ShortMessage, StringComparison.Ordinal); + } + + [Fact] + public async Task JobOverCadence_UnderTheKnob_StaysSilent() + { + var h = new Harness(); + var e = h.Build(); + + /* 249s of 3600s ≈ 7% — the production store's worst job today. Must not fire at the default 25. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob(durMs: 249_000) }, Ct); + + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.History.Records); + } + + [Fact] + public async Task JobOverCadence_At100Percent_EscalatesToCritical() + { + var h = new Harness(); + var e = h.Build(); + + /* 3700s of 3600s — the job outruns its own cadence; runs back up behind each other. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob(durMs: 3_700_000) }, Ct); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + } + + [Fact] + public async Task JobOverCadence_HonorsTheLiveKnob() + { + var h = new Harness { StoreJobCadenceWarnPercent = 50 }; + var e = h.Build(); + + /* 30% breaches the default 25 but not the configured 50 — the seam is read live. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob(durMs: 1_080_000) }, Ct); + + Assert.Empty(h.Deliverer.Outcomes); + } + + [Fact] + public async Task JobOverCadence_NoScheduleOrNoRun_IsSkippedWithoutJudging() + { + var h = new Harness(); + var e = h.Build(); + + /* A one-shot job (no interval) and a job with no completed run have no cadence to breach. */ + await e.ApplyStoreJobCadenceAsync(new[] + { + CadenceJob(id: 1, schedMs: 0), + CadenceJob(id: 2, durMs: null), + }, Ct); + + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.History.Records); + } + + [Fact] + public async Task JobOverCadence_IsAStandingCondition_ReFiresOnlyOnCooldown_AndWritesOneRecoveryRow() + { + var h = new Harness(); + var e = h.Build(); + + /* Breach: fires once. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob() }, Ct); + Assert.Single(h.Deliverer.Outcomes); + + /* Still breaching one minute later — inside the 5-minute cooldown, no re-fire. */ + h.Now = h.Now.AddMinutes(1); + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob() }, Ct); + Assert.Single(h.Deliverer.Outcomes); + + /* Still breaching past the cooldown — re-fires under the SAME metric name. */ + h.Now = h.Now.AddMinutes(10); + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob() }, Ct); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + + /* A later run comes back under: exactly one recovery audit row, and a fresh breach fires again. */ + await e.ApplyStoreJobCadenceAsync(new[] { CadenceJob(durMs: 200_000) }, Ct); + var recovered = Assert.Single(h.History.Records); + Assert.Equal("Store Job Cadence Recovered", recovered.MetricName); + } } diff --git a/Darling/Darling.Tests/DarlingToolExitCodeTests.cs b/Darling/Darling.Tests/DarlingToolExitCodeTests.cs new file mode 100644 index 000000000..d0ee2a379 --- /dev/null +++ b/Darling/Darling.Tests/DarlingToolExitCodeTests.cs @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// The bundled-Postgres tool exit-code decoder (#2186). The bug it exists for was reported twice from +/// the field: a managed bootstrap failed with initdb failed (exit code -1073741515) ... Output: +/// and nothing else — a signed Win32 integer plus an empty field, on the one class of failure where the +/// empty field is guaranteed rather than informative, because the process was killed in the LOADER +/// before it could write a line. The operator's attention then went to the follow-on missing-credential +/// message and darling.json, neither of which was the fault. +/// +/// These pin the decode itself. The pins that the SHIPPED messages carry it live in +/// — a correct decoder nothing calls is exactly the shape of +/// the defect #1738 already was. +/// +public sealed class DarlingToolExitCodeTests +{ + /* The four loader statuses, as .NET reports them from Process.ExitCode (signed). */ + private const int StatusDllNotFound = unchecked((int)0xC0000135); + private const int StatusEntryPointNotFound = unchecked((int)0xC0000139); + private const int StatusInvalidImageFormat = unchecked((int)0xC000007B); + private const int StatusDllInitFailed = unchecked((int)0xC0000142); + private const int StatusAccessDenied = unchecked((int)0xC0000022); + private const int StatusAccessViolation = unchecked((int)0xC0000005); + private const int StatusNoMemory = unchecked((int)0xC0000017); + + private const string InitDb = @"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin\initdb.exe"; + + /// + /// The literal number from the field report decodes. Written against the DECIMAL the operator + /// actually saw rather than a hex constant, so this test fails if the two ever stop being the same + /// number — that equality is the entire premise of the fix. + /// + [Fact] + public void Describe_DecodesTheExactCodeTheFieldReported() + { + Assert.Equal(StatusDllNotFound, -1073741515); + + var described = DarlingToolExitCode.Describe(-1073741515); + + Assert.Contains("-1073741515", described, StringComparison.Ordinal); + Assert.Contains("0xC0000135", described, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", described, StringComparison.Ordinal); + } + + [Theory] + [InlineData(StatusEntryPointNotFound, "0xC0000139", "STATUS_ENTRYPOINT_NOT_FOUND")] + [InlineData(StatusInvalidImageFormat, "0xC000007B", "STATUS_INVALID_IMAGE_FORMAT")] + [InlineData(StatusDllInitFailed, "0xC0000142", "STATUS_DLL_INIT_FAILED")] + [InlineData(StatusAccessDenied, "0xC0000022", "STATUS_ACCESS_DENIED")] + [InlineData(StatusAccessViolation, "0xC0000005", "STATUS_ACCESS_VIOLATION")] + public void Describe_NamesTheOtherStatusesWorthNaming(int exitCode, string hex, string name) + { + var described = DarlingToolExitCode.Describe(exitCode); + + Assert.Contains(hex, described, StringComparison.Ordinal); + Assert.Contains(name, described, StringComparison.Ordinal); + } + + /// + /// An NTSTATUS this decoder has no name for still gets the half that matters most: that the number + /// is a WINDOWS status, not the program's own exit code, plus the hex to search for. Decoding only + /// the codes on a list would leave the next unfamiliar one exactly as opaque as -1073741515 was. + /// + [Fact] + public void Describe_StillSaysWindowsKilledItForAnUnlistedStatus() + { + var described = DarlingToolExitCode.Describe(StatusNoMemory); + + Assert.Contains("0xC0000017", described, StringComparison.Ordinal); + Assert.Contains("Windows", described, StringComparison.Ordinal); + } + + /// + /// A tool's OWN exit code is left completely alone. initdb exits 1 on a bad option and pg_ctl status + /// exits 3 for "not running"; dressing those up as Windows statuses would be a new lie in place of + /// the old one. + /// + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(3)] + [InlineData(4)] + [InlineData(127)] + public void Describe_LeavesAnOrdinaryExitCodeAsTheBareNumber(int exitCode) + { + Assert.Equal(exitCode.ToString(System.Globalization.CultureInfo.InvariantCulture), DarlingToolExitCode.Describe(exitCode)); + } + + /// + /// The diagnosis names BOTH causes the issue asks for and both checks that separate them, and it + /// points at the directory the DLLs must be in rather than at "the install" in the abstract. + /// + [Fact] + public void Diagnose_NamesBothCausesAndBothChecks() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusDllNotFound, InitDb); + + /* Cause 1: the bundled MSVC runtime, in the directory it is bundled into. */ + Assert.Contains(@"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin", diagnosis, StringComparison.Ordinal); + Assert.Contains("vcruntime140.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("vcruntime140_1.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("msvcp140.dll", diagnosis, StringComparison.Ordinal); + + /* Cause 2: the service account cannot read the install tree. */ + Assert.Contains("NT SERVICE", diagnosis, StringComparison.Ordinal); + + /* Check 1: run the binary by hand. Check 2: the Windows log that names the module. */ + Assert.Contains("--version", diagnosis, StringComparison.Ordinal); + Assert.Contains("Event Viewer", diagnosis, StringComparison.Ordinal); + } + + /// + /// The empty-output half of the report: the diagnosis has to say the blank field is EXPECTED for a + /// loader failure. Leaving it unexplained is what made the field report read as "no information + /// available" instead of "this is a load failure". + /// + [Fact] + public void Diagnose_SaysAnEmptyOutputIsExpectedNotMissing() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusDllNotFound, InitDb); + + Assert.Contains("expected", diagnosis, StringComparison.OrdinalIgnoreCase); + Assert.Contains("loader", diagnosis, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(StatusEntryPointNotFound)] + [InlineData(StatusInvalidImageFormat)] + [InlineData(StatusDllInitFailed)] + [InlineData(StatusAccessDenied)] + public void Diagnose_CoversEveryLoaderStatusNotJustTheReportedOne(int exitCode) + { + var diagnosis = DarlingToolExitCode.Diagnose(exitCode, InitDb); + + Assert.Contains("vcruntime140.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("NT SERVICE", diagnosis, StringComparison.Ordinal); + } + + /// + /// A CRASH is not a load failure, and telling an operator to go looking for missing DLLs after an + /// access violation would be the same wrong-direction error the raw number caused — just pointed + /// somewhere new. The process ran; its output is real; the DLL advice must not appear. + /// + [Fact] + public void Diagnose_DoesNotSendACrashLookingForMissingDlls() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusAccessViolation, InitDb); + + Assert.DoesNotContain("vcruntime140", diagnosis, StringComparison.Ordinal); + Assert.Contains("crash", diagnosis, StringComparison.OrdinalIgnoreCase); + } + + /// An ordinary non-zero exit gets no paragraph at all — initdb exiting 1 with a real error + /// message on stderr needs no help from here, and burying that message under boilerplate would make + /// the common failure worse to read. + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(4)] + public void Diagnose_IsSilentForAnOrdinaryExitCode(int exitCode) + { + Assert.Equal(string.Empty, DarlingToolExitCode.Diagnose(exitCode, InitDb)); + } + + /// + /// The Output: field itself. On a Windows status an empty capture is stated as expected; on an + /// ordinary exit it is just absent, with no loader story attached to it. + /// + [Fact] + public void FormatOutput_ExplainsABlankCaptureOnlyWhenWindowsKilledIt() + { + var loader = DarlingToolExitCode.FormatOutput(string.Empty, StatusDllNotFound); + Assert.Contains("expected", loader, StringComparison.OrdinalIgnoreCase); + + var ordinary = DarlingToolExitCode.FormatOutput(" ", 1); + Assert.DoesNotContain("expected", ordinary, StringComparison.OrdinalIgnoreCase); + Assert.False(string.IsNullOrWhiteSpace(ordinary), "a blank capture still has to render as something an operator can read"); + } + + [Fact] + public void FormatOutput_PassesRealOutputThroughUntouched() + { + const string real = "initdb: error: directory \"C:\\pg\" exists but is not empty"; + + Assert.Equal(real, DarlingToolExitCode.FormatOutput(real, 1)); + Assert.Equal(real, DarlingToolExitCode.FormatOutput(real, StatusDllNotFound)); + } + + /// + /// The empirical half, and the reason this test uses a real process rather than a constant: it pins + /// that a Windows status really does arrive at as + /// the signed number this decoder is built around, with an EMPTY capture — the two facts the whole + /// fix rests on. cmd /c exit sets the same exit status the loader would; it is the status's + /// journey through the runner that is under test, not how it was produced. + /// + [Fact] + public async Task RunTool_SurfacesAWindowsStatusAsTheSignedFieldValueWithNoOutput() + { + Assert.SkipUnless(OperatingSystem.IsWindows(), "cmd.exe carries the Windows status; on other platforms this reports skipped, not vacuously passed."); + + var cmd = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "cmd.exe"); + + var (exitCode, output) = await DarlingManagedPostgres.RunToolAsync( + cmd, "/c exit 3221225781", TimeSpan.FromSeconds(30), CancellationToken.None); + + Assert.Equal(-1073741515, exitCode); + Assert.Equal(string.Empty, output); + Assert.Contains("STATUS_DLL_NOT_FOUND", DarlingToolExitCode.Describe(exitCode), StringComparison.Ordinal); + } + + /// + /// #2185: the loader predicate, which decides whether gathering more evidence is worth two process + /// launches. Every status the table calls Loader qualifies; a crash and a plain code do not. + /// + [Theory] + [InlineData(StatusDllNotFound, true)] + [InlineData(StatusEntryPointNotFound, true)] + [InlineData(StatusInvalidImageFormat, true)] + [InlineData(StatusDllInitFailed, true)] + [InlineData(StatusAccessDenied, true)] + [InlineData(StatusAccessViolation, false)] + [InlineData(StatusNoMemory, false)] + [InlineData(1, false)] + [InlineData(0, false)] + public void OnlyLoaderStatusesAreWorthProbingFor(int exitCode, bool expected) + { + Assert.Equal(expected, DarlingToolExitCode.IsLoaderStatus(exitCode)); + } + + /// + /// THE #2185 CASE, and the reason this decoder grew a probe at all. + /// + /// The field report survived four exchanges undiagnosed. The reporter had moved the install out of + /// their user profile, confirmed all three bundled MSVC DLLs present — the two causes the loader + /// diagnosis suggests — and then ran initdb --version by hand, which printed + /// initdb (PostgreSQL) 18.4. A binary that dies in the loader cannot print its own version, so + /// that one observation killed both hypotheses, and it existed only in their shell. + /// + /// The asymmetry it points at: a real initdb run spawns postgres.exe in bootstrap + /// mode and propagates its status, so a dependency missing only for postgres.exe produces exactly + /// the reported shape. The probe must name that binary, and must contradict the generic diagnosis rather + /// than quietly sitting next to it. + /// + [Fact] + public void WhenOnlyPostgresCannotLoad_TheProbeNamesPostgresAndTheBootstrapSpawn() + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(initDbExitCode: 0, postgresExitCode: StatusDllNotFound); + + Assert.Contains("postgres.exe did NOT", probe, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", probe, StringComparison.Ordinal); + Assert.Contains("bootstrap mode", probe, StringComparison.Ordinal); + /* And that it explicitly clears the two things the operator already ruled out, so the message does + not read as agreeing with the diagnosis printed directly above it. */ + Assert.Contains("not the install location or the service account", probe, StringComparison.Ordinal); + } + + /// + /// Both dead is a DIFFERENT fix — the shared runtime rather than one binary — so it must not read as the + /// postgres-specific case. Ambiguity here costs the operator the same days the original report did. + /// + [Fact] + public void WhenNeitherLoads_TheProbeBlamesTheSharedRuntimeInstead() + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(StatusDllNotFound, StatusDllNotFound); + + Assert.Contains("NEITHER", probe, StringComparison.Ordinal); + Assert.Contains("MSVC runtime", probe, StringComparison.Ordinal); + Assert.DoesNotContain("postgres.exe did NOT", probe, StringComparison.Ordinal); + } + + /// + /// Both alive is a finding, not a shrug: it rules out a permanently missing dependency and hands the + /// operator the one source that names the offending module. Returning nothing here would leave the + /// generic diagnosis standing after the probe had already disproved it — the original bug. + /// + [Fact] + public void WhenBothLoad_TheProbeSaysSoAndRedirectsToTheEventLog() + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(0, 0); + + Assert.Contains("BOTH", probe, StringComparison.Ordinal); + Assert.Contains("Event Viewer", probe, StringComparison.Ordinal); + Assert.NotEqual(string.Empty, probe); + } + + /// + /// The inverted case is real (a single damaged file) and gets its own fix, so it may not silently fall + /// through to one of the other three arms. + /// + [Fact] + public void WhenOnlyInitDbCannotLoad_TheProbeBlamesInitDbItself() + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(StatusDllNotFound, 0); + + Assert.Contains("initdb.exe did not", probe, StringComparison.Ordinal); + Assert.Contains("re-extract", probe, StringComparison.Ordinal); + Assert.DoesNotContain("NEITHER", probe, StringComparison.Ordinal); + } + + /// + /// A NON-loader exit code from a probed binary means it loaded — it ran and then failed for its own + /// reasons — so it must not be reported as a load failure. Guards the obvious over-read of "exit code + /// != 0 means broken", which would blame postgres.exe on every probe that merely returned non-zero. + /// + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(StatusAccessViolation)] + public void ANonLoaderExitCodeStillCountsAsLoaded(int postgresExitCode) + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(0, postgresExitCode); + + Assert.Contains("BOTH", probe, StringComparison.Ordinal); + } + + /// + /// Every arm leads with the same Runtime probe: label on its own line. It is appended to a message + /// the field greps and the tracker searches, so it has to be findable and must not run onto the end of + /// the loader diagnosis's last sentence. + /// + [Theory] + [InlineData(0, 0)] + [InlineData(0, StatusDllNotFound)] + [InlineData(StatusDllNotFound, 0)] + [InlineData(StatusDllNotFound, StatusDllNotFound)] + public void EveryProbeVerdictIsLabelledAndStartsItsOwnLine(int initDbExitCode, int postgresExitCode) + { + var probe = DarlingToolExitCode.DescribeRuntimeProbe(initDbExitCode, postgresExitCode); + + Assert.StartsWith("\nRuntime probe: ", probe, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/DatabaseMismatchTripwireTests.cs b/Darling/Darling.Tests/DatabaseMismatchTripwireTests.cs new file mode 100644 index 000000000..dd9f9964e --- /dev/null +++ b/Darling/Darling.Tests/DatabaseMismatchTripwireTests.cs @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2228: the tripwire that notices a registration is connected to a database it does not name. +/// +/// The defect. Identity is registration-derived and was never checked against the connection, so a +/// registration whose Initial Catalog is absent, misspelled or overridden lands in a different database and every +/// collected row is stored under that registration's identity while describing somewhere else — indefinitely, and +/// with nothing anywhere saying so. When a sibling registration names that same database, both collect it and its +/// history exists twice under two identities: #2220's field report of byte-identical deadlock graphs under six +/// server_ids, one real incident alerting six times. +/// +/// Why a connect-time check and not a registry rule. #2158 established that identity must be +/// assigned rather than derived, which fixes the re-key class but cannot touch this one: the two registrations +/// here genuinely differ in configuration, so no amount of care in hashing config can tell that they resolve to +/// one database. Only the server can answer what a connection actually reached, which makes the connect the one +/// place this is knowable. +/// +public sealed class DatabaseMismatchTripwireTests +{ + /// THE CASE: registered for one database, landed in another. + [Fact] + public void ARegistrationConnectedElsewhereIsReported() + { + var message = DarlingServerConnector.DescribeDatabaseMismatch("Sibling-A", "Source-DB", "Sibling-A"); + + Assert.NotNull(message); + Assert.Contains("Sibling-A", message, StringComparison.Ordinal); + Assert.Contains("Source-DB", message, StringComparison.Ordinal); + /* It must say what is actually at stake — mis-attributed rows, and duplication when a sibling names the + same database — or it reads as a cosmetic naming complaint and gets ignored. */ + Assert.Contains("two identities", message, StringComparison.Ordinal); + /* And where to fix it, on both surfaces that can set it. */ + Assert.Contains("Initial Catalog", message, StringComparison.Ordinal); + Assert.Contains("darling.json", message, StringComparison.Ordinal); + } + + /// + /// Agreement is silent. The tripwire runs on every connect of every server, so a false positive is not a + /// cosmetic problem — it is what trains an operator past the one line that matters. + /// + [Theory] + [InlineData("SalesDB", "SalesDB")] + [InlineData("SalesDB", "salesdb")] + [InlineData("salesdb", "SALESDB")] + [InlineData(" SalesDB ", "SalesDB")] + public void AMatchIsSilent_CaseAndWhitespaceInsensitive(string registered, string connected) + { + Assert.Null(DarlingServerConnector.DescribeDatabaseMismatch(registered, connected, "srv")); + } + + /// + /// A registration that names NO database is server-scoped by design — it is meant to land wherever the + /// login defaults and enumerate from there. Comparing it against whatever that default turned out to be + /// would fire on every correctly-configured server-scoped registration in the fleet, which is the single + /// most likely way to make this feature useless. + /// + [Theory] + [InlineData(null, "master")] + [InlineData("", "master")] + [InlineData(" ", "SalesDB")] + public void AServerScopedRegistrationIsNeverATripwire(string? registered, string connected) + { + Assert.Null(DarlingServerConnector.DescribeDatabaseMismatch(registered, connected, "srv")); + } + + /// + /// An absent probe answer is silent too: a null connected_database means the probe did not report it + /// (an older build's row, or a column that came back null), and "unknown" must not be rendered as + /// "mismatched". Guessing here would fire on every server the moment anything upstream changed shape. + /// + [Theory] + [InlineData("SalesDB", null)] + [InlineData("SalesDB", "")] + public void AnUnknownConnectedDatabaseIsNotAMismatch(string registered, string? connected) + { + Assert.Null(DarlingServerConnector.DescribeDatabaseMismatch(registered, connected, "srv")); + } + + /// + /// Both probes ASK for it, and appended so every existing positional read keeps its ordinal — the readers + /// index by number, so inserting a column mid-list silently shifts five other fields onto the wrong values. + /// + [Fact] + public void BothEngineProbesAskWhichDatabaseTheyLandedIn() + { + Assert.Contains("DB_NAME() AS connected_database", DarlingServerConnector.DetectionQueryText, StringComparison.Ordinal); + Assert.Contains("current_database() AS connected_database", DarlingServerConnector.PostgresDetectionQueryText, StringComparison.Ordinal); + + /* Last in each list. */ + Assert.EndsWith("connected_database", DarlingServerConnector.DetectionQueryText.TrimEnd(), StringComparison.Ordinal); + Assert.EndsWith("connected_database", DarlingServerConnector.PostgresDetectionQueryText.TrimEnd(), StringComparison.Ordinal); + + /* No DMV: the SQL Server probe deliberately avoids sys.dm_os_sys_info because an Azure SQL DB + monitoring login often lacks VIEW DATABASE STATE (#1535), and DB_NAME() keeps that property. */ + Assert.DoesNotContain("sys.dm_os_sys_info", DarlingServerConnector.DetectionQueryText, StringComparison.Ordinal); + } + + /// + /// The worker fires on the TRANSITION, at Error, and reports the recovery too. + /// + /// Pinned at the source because reproducing it needs a live server whose connection lands in a + /// different database than its registration names. Two things would go wrong silently. Logging per connect + /// instead of per transition buries a standing misconfiguration — it persists until an operator edits the + /// registration, so it would repeat on every reconnect forever, which is how a tripwire gets trained past. + /// And omitting the recovery line means an operator who fixes it has only silence as confirmation. + /// + [Fact] + public void TheWorkerFiresOnTheTransitionAndReportsTheRecovery() + { + var source = ReadWorkerSource(); + + Assert.Contains("DarlingServerConnector.DescribeDatabaseMismatch(", source, StringComparison.Ordinal); + Assert.Contains("LastDatabaseMismatchLogged", source, StringComparison.Ordinal); + /* Error, because nothing clears it on its own and every sweep meanwhile stores mis-attributed rows. */ + Assert.Contains("_logger.LogError(\"[{Server}] {Mismatch}\"", source, StringComparison.Ordinal); + Assert.Contains("is resolved.", source, StringComparison.Ordinal); + } + + private static string ReadWorkerSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/DatabaseStateAlertMemoryLiveTests.cs b/Darling/Darling.Tests/DatabaseStateAlertMemoryLiveTests.cs new file mode 100644 index 000000000..4ad271c43 --- /dev/null +++ b/Darling/Darling.Tests/DatabaseStateAlertMemoryLiveTests.cs @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Gated-live contracts for the maintenance statements the database-state alert runs against the store +/// beside its deviation read — the parts of this alert whose correctness lives in SQL rather than in the +/// engine: (#2166, the +/// store-derived half of the edge trigger), +/// and (#2189, what a baseline is +/// allowed to be learned from and what un-learns a stale one). +/// +/// Why these have to be LIVE rather than harness tests: each guards a bug that only a store can +/// exhibit. #2166's clear depended on the engine's in-memory active set, so a restart between an alert and +/// the recovery left last_alerted_state sticky forever — a test that drives the engine can only prove +/// the path a running process takes, and the restart gap is invisible to it by construction. #2189's pair +/// decide what rows EXIST, which no amount of engine stubbing observes. +/// +/// Runs against a real Postgres gated on DARLING_TEST_PG, on the serialized "live-postgres" +/// collection, against a negative sentinel server_id, cleaning up in finally — the house pattern. +/// +[Collection("live-postgres")] +public sealed class DatabaseStateAlertMemoryLiveTests +{ + private const int LiveServerId = -915758; + private const string Name = "DBSTATE-MEMORY-SRV"; + + [Fact] + public async Task ClearRecovered_ForgetsOnlyDatabasesBackAtExpected_WithNothingHeldInMemory() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live database-state memory clear."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + var bodySucceeded = false; + try + { + var newest = new DateTime(2026, 08, 11, 9, 0, 0, DateTimeKind.Unspecified); + + /* Recovered: alerted OFFLINE, now back ONLINE == expected. MUST be cleared — this is the + restart-gap case, where no process ever witnessed the falling edge. */ + await StateAsync(connection, ct, "BackOnline", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "BackOnline", expected: "ONLINE", lastAlerted: "OFFLINE"); + + /* Still deviating: alerted OFFLINE and still OFFLINE. MUST be kept, or the repetition this alert + went quiet about starts over on the next cycle. */ + await StateAsync(connection, ct, "StillParked", "OFFLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "StillParked", expected: "ONLINE", lastAlerted: "OFFLINE"); + + /* Deviating DIFFERENTLY: alerted OFFLINE, now SUSPECT. Not at expected, so the memory stays — + the engine's own state comparison is what fires this one again, not a cleared memory. */ + await StateAsync(connection, ct, "TurnedSuspect", "SUSPECT", standby: false, at: newest); + await ExpectedAsync(connection, ct, "TurnedSuspect", expected: "ONLINE", lastAlerted: "OFFLINE"); + + /* Standby: expected STANDBY and currently in standby, which the effective-state CASE resolves to + STANDBY rather than the raw state_desc. Pins that this statement reads the same effective + state the deviation query does — comparing against state_desc would leave it uncleared. */ + await StateAsync(connection, ct, "LogShipped", "RESTORING", standby: true, at: newest); + await ExpectedAsync(connection, ct, "LogShipped", expected: "STANDBY", lastAlerted: "RESTORING"); + + /* The (ignore) sentinel: an operator silenced it, so a memory must not outlive the silence. */ + await StateAsync(connection, ct, "Silenced", "OFFLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "Silenced", expected: "(ignore)", lastAlerted: "OFFLINE"); + + using (var clear = new NpgsqlCommand(DarlingAlertReadAdapter.ClearRecoveredDatabaseStateAlertsSql, connection)) + { + clear.Parameters.AddWithValue(LiveServerId); + await clear.ExecuteNonQueryAsync(ct); + } + + Assert.Null(await MemoryAsync(connection, ct, "BackOnline")); + Assert.Null(await MemoryAsync(connection, ct, "LogShipped")); + Assert.Null(await MemoryAsync(connection, ct, "Silenced")); + + Assert.Equal("OFFLINE", await MemoryAsync(connection, ct, "StillParked")); + Assert.Equal("OFFLINE", await MemoryAsync(connection, ct, "TurnedSuspect")); + + /* Idempotent: a second sweep over an already-clear store is a no-op, which matters because this + runs on EVERY evaluation of every server. */ + using (var again = new NpgsqlCommand(DarlingAlertReadAdapter.ClearRecoveredDatabaseStateAlertsSql, connection)) + { + again.Parameters.AddWithValue(LiveServerId); + Assert.Equal(0, await again.ExecuteNonQueryAsync(ct)); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task Seed_RefusesToLearnATransientState_SoAMidRestoreOnboardingStaysPending() + { + /* #2189: the seed's exclusion list is what decides a database's accepted normal FOREVER, and it used + to exclude only the integrity states. A database observed mid-restore therefore learned RESTORING + as expected and deviated by being healthy ever after. RESTORING and RECOVERING are databases in + the middle of an operation, not steady states, and are now refused the same way SUSPECT is. */ + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live database-state seed pins."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + var bodySucceeded = false; + try + { + var newest = new DateTime(2026, 08, 11, 10, 0, 0, DateTimeKind.Unspecified); + + await StateAsync(connection, ct, "MidRestore", "RESTORING", standby: false, at: newest); + await StateAsync(connection, ct, "ComingUp", "RECOVERING", standby: false, at: newest); + await StateAsync(connection, ct, "Corrupt", "SUSPECT", standby: false, at: newest); + await StateAsync(connection, ct, "Healthy", "ONLINE", standby: false, at: newest); + await StateAsync(connection, ct, "Parked", "OFFLINE", standby: false, at: newest); + /* A standby secondary's effective state is the synthetic STANDBY, which IS stable by construction + and so is exactly the kind of state worth learning — the raw RESTORING underneath it is not. */ + await StateAsync(connection, ct, "LogShipped", "RESTORING", standby: true, at: newest); + + using (var seed = new NpgsqlCommand(DarlingAlertReadAdapter.SeedDatabaseStateExpectedSql, connection)) + { + seed.Parameters.AddWithValue(LiveServerId); + await seed.ExecuteNonQueryAsync(ct); + } + + Assert.Null(await ExpectedStateAsync(connection, ct, "MidRestore")); + Assert.Null(await ExpectedStateAsync(connection, ct, "ComingUp")); + Assert.Null(await ExpectedStateAsync(connection, ct, "Corrupt")); + + Assert.Equal("ONLINE", await ExpectedStateAsync(connection, ct, "Healthy")); + Assert.Equal("OFFLINE", await ExpectedStateAsync(connection, ct, "Parked")); + Assert.Equal("STANDBY", await ExpectedStateAsync(connection, ct, "LogShipped")); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task Heal_RelearnsOnline_OnlyForInferredBaselinesTheSeedWouldHaveRefused() + { + /* #2189's other half: the widened seed governs rows that do not exist yet, and cannot touch the ones + already written — five databases on the reporting fleet were baselined RESTORING and then alerted + ~127 times each in 24 hours for being ONLINE. The heal applies the seed's own rule after the fact: + a baseline recording a state the seed would REFUSE to learn is not a baseline anyone chose, so once + the database is demonstrably healthy the steady state is learned instead. + + Every row below the first is one that must NOT be healed. The failure mode on this side is silence, + and there are more ways to cause it than to fix it. */ + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live database-state baseline heal."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + var bodySucceeded = false; + try + { + var newest = new DateTime(2026, 08, 11, 11, 0, 0, DateTimeKind.Unspecified); + + /* The reported row: baselined mid-restore, restore finished, now permanently "deviating" by being + healthy. Healed — and its alerted-state memory goes with the baseline it described. */ + await StateAsync(connection, ct, "Poisoned", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "Poisoned", expected: "RESTORING", lastAlerted: "ONLINE"); + + /* An operator's declaration. #2166's composition contract says a database parked at expected + OFFLINE stays quiet while parked and still alerts the moment it comes back ONLINE — which only + works if the heal leaves overrides alone. Rewriting this to ONLINE would silently delete the + operator's intent AND the alert they set it up to get. */ + await StateAsync(connection, ct, "Parked", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "Parked", expected: "OFFLINE", lastAlerted: "", isOverride: true); + + /* The trap: a standby secondary reports state_desc = 'ONLINE' with is_in_standby set. Matching the + RAW column would re-baseline every log-shipping secondary from STANDBY to ONLINE and then alert + it forever for being STANDBY — #2189 recreated for the family #1986 works hardest to keep quiet. */ + await StateAsync(connection, ct, "LogShipped", "ONLINE", standby: true, at: newest); + await ExpectedAsync(connection, ct, "LogShipped", expected: "STANDBY", lastAlerted: ""); + + /* Not ONLINE yet: a database that moved from one un-settled state to another has not settled, so + its illegitimate baseline stays exactly as illegitimate as it was. */ + await StateAsync(connection, ct, "StillDown", "OFFLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "StillDown", expected: "RESTORING", lastAlerted: ""); + + /* A STANDBY secondary that turns up TRULY online (is_in_standby now 0) has been recovered out of + standby: log shipping is broken, and that deviation is the alert's entire job. Healing it would + replace the alert with silence and then fire when the operator put standby BACK. STANDBY is a + steady state the seed learns on purpose, so it is not on the heal's list. */ + await StateAsync(connection, ct, "RecoveredSecondary", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "RecoveredSecondary", expected: "STANDBY", lastAlerted: ""); + + /* Same reasoning for the other steady state. An auto-baselined OFFLINE database brought up for an + hour of maintenance must not have its baseline rewritten, or re-parking it leaves it deviating + forever against a baseline it never had - this bug, inverted, by the fix for it. */ + await StateAsync(connection, ct, "WasParked", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "WasParked", expected: "OFFLINE", lastAlerted: ""); + + /* The sentinel. Not on the list either, so un-ignoring a database by accident cannot happen even + if some future path writes "(ignore)" without the override flag. */ + await StateAsync(connection, ct, "Silenced", "ONLINE", standby: false, at: newest); + await ExpectedAsync(connection, ct, "Silenced", expected: "(ignore)", lastAlerted: ""); + + using (var heal = new NpgsqlCommand(DarlingAlertReadAdapter.HealDatabaseStateBaselineToOnlineSql, connection)) + { + heal.Parameters.AddWithValue(LiveServerId); + Assert.Equal(1, await heal.ExecuteNonQueryAsync(ct)); + } + + Assert.Equal("ONLINE", await ExpectedStateAsync(connection, ct, "Poisoned")); + Assert.Null(await MemoryAsync(connection, ct, "Poisoned")); + + Assert.Equal("OFFLINE", await ExpectedStateAsync(connection, ct, "Parked")); + Assert.Equal("STANDBY", await ExpectedStateAsync(connection, ct, "LogShipped")); + Assert.Equal("RESTORING", await ExpectedStateAsync(connection, ct, "StillDown")); + Assert.Equal("STANDBY", await ExpectedStateAsync(connection, ct, "RecoveredSecondary")); + Assert.Equal("OFFLINE", await ExpectedStateAsync(connection, ct, "WasParked")); + Assert.Equal("(ignore)", await ExpectedStateAsync(connection, ct, "Silenced")); + + /* Idempotent, which matters because this runs on EVERY evaluation of every server: a store with + nothing left to heal must cost zero writes, not rewrite the same rows and churn updated_at. */ + using (var again = new NpgsqlCommand(DarlingAlertReadAdapter.HealDatabaseStateBaselineToOnlineSql, connection)) + { + again.Parameters.AddWithValue(LiveServerId); + Assert.Equal(0, await again.ExecuteNonQueryAsync(ct)); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + private static async Task StateAsync( + NpgsqlConnection connection, CancellationToken ct, string database, string stateDesc, bool standby, DateTime at) + { + using var command = new NpgsqlCommand(@" +INSERT INTO collect.database_states (collection_id, collection_time, server_id, server_name, database_name, database_id, state_desc, is_in_standby) +VALUES (0, $1, $2, $3, $4, 5, $5, $6)", connection); + command.Parameters.AddWithValue(at); + command.Parameters.AddWithValue(LiveServerId); + command.Parameters.AddWithValue(Name); + command.Parameters.AddWithValue(database); + command.Parameters.AddWithValue(stateDesc); + command.Parameters.AddWithValue(standby); + await command.ExecuteNonQueryAsync(ct); + } + + /// + /// Plants an expectation row. is the load-bearing one for #2189: it is the + /// only thing separating a baseline the machine INFERRED (heal-able) from one an operator DECLARED + /// (never second-guessed), so a test that cannot set it cannot tell the two apart. + /// + private static async Task ExpectedAsync( + NpgsqlConnection connection, CancellationToken ct, string database, string expected, string lastAlerted, + bool isOverride = false) + { + using var command = new NpgsqlCommand(@" +INSERT INTO config.database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at, last_alerted_state, last_alerted_at) +VALUES ($1, $2, $3, $5, (now() AT TIME ZONE 'UTC'), $4, (now() AT TIME ZONE 'UTC'))", connection); + command.Parameters.AddWithValue(LiveServerId); + command.Parameters.AddWithValue(database); + command.Parameters.AddWithValue(expected); + command.Parameters.AddWithValue(lastAlerted); + command.Parameters.AddWithValue(isOverride); + await command.ExecuteNonQueryAsync(ct); + } + + /// The stored expected state, or null when no row exists (a database still pending a baseline). + private static async Task ExpectedStateAsync(NpgsqlConnection connection, CancellationToken ct, string database) + { + using var command = new NpgsqlCommand( + "SELECT expected_state FROM config.database_state_expected WHERE server_id = $1 AND database_name = $2", + connection); + command.Parameters.AddWithValue(LiveServerId); + command.Parameters.AddWithValue(database); + var value = await command.ExecuteScalarAsync(ct); + return value is DBNull or null ? null : (string)value; + } + + private static async Task MemoryAsync(NpgsqlConnection connection, CancellationToken ct, string database) + { + using var command = new NpgsqlCommand( + "SELECT last_alerted_state FROM config.database_state_expected WHERE server_id = $1 AND database_name = $2", + connection); + command.Parameters.AddWithValue(LiveServerId); + command.Parameters.AddWithValue(database); + var value = await command.ExecuteScalarAsync(ct); + return value is DBNull or null ? null : (string)value; + } + + private static async Task DeleteLiveRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + var id = LiveServerId.ToString(CultureInfo.InvariantCulture); + using var cleanup = new NpgsqlCommand( + $"DELETE FROM collect.database_states WHERE server_id = {id};" + + $"DELETE FROM config.database_state_expected WHERE server_id = {id};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/DeltaGapPolicyTests.cs b/Darling/Darling.Tests/DeltaGapPolicyTests.cs new file mode 100644 index 000000000..d8783c4cb --- /dev/null +++ b/Darling/Darling.Tests/DeltaGapPolicyTests.cs @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2233 / #2234: the gap policy every delta collector shares, and the invariant that makes a stored +/// zero readable. +/// +/// The old policy rejected any baseline older than 300 s and returned 0. Measured against the +/// fleet that threshold sat on the MEDIAN sweep gap — 99,717 consecutive perfmon gaps over 52 servers +/// and 7 days: p50 299 s, p90 580 s, p99 830 s, max 2,514 s — so it fired 50.0% of the time during +/// ordinary operation instead of after the restarts it was written for, and each firing stored a 0 that +/// is indistinguishable from a genuinely idle interval. +/// +/// The pins below are behavioral, not textual: the 600 s case is the one that flips with the +/// threshold (0 before, a real delta after), and the reset case pins the invariant +/// interval == 0 <=> no delta was knowable that the whole (delta, interval) reading rests +/// on. +/// +public sealed class DeltaGapPolicyTests +{ + private const int ServerId = 1; + private const string Collector = "perfmon"; + private const string Key = "SQLServer:SQL Statistics|Batch Requests/sec|"; + + private static DateTime T0 => new(2026, 8, 13, 12, 0, 0, DateTimeKind.Unspecified); + + /// The measured fleet median. A gap this size is ORDINARY, and the old 300 s policy + /// rejected it — this is the 50% of output that was a fabricated zero. + [Fact] + public void AGapAtTheFleetMedian_YieldsARealDelta_NotAFabricatedZero() + { + var calc = new CollectorDeltaCalculator(); + + var first = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 1_000, out var firstInterval, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + Assert.Equal(0, first); /* first sighting: baseline only */ + Assert.Equal(0, firstInterval); + + /* 299 s — the measured p50. Under the old 300 s policy this squeaked through; at p90 (580 s) + it did not, which is why half the fleet's points were zeros. */ + var second = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 1_500, out var secondInterval, + collectionTime: T0.AddSeconds(299), maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + Assert.Equal(500, second); + Assert.Equal(299, secondInterval); + } + + /// The pin that flips with the fix: 600 s is past the OLD 300 s policy and inside the new + /// one. 8.3% of real fleet gaps land here. + [Fact] + public void AGapPastTheOldPolicyButInsideTheNewOne_YieldsARealDelta() + { + var calc = new CollectorDeltaCalculator(); + calc.CalculateDelta(ServerId, Collector, Key, 20_000_000, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + var delta = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 20_055_000, out var interval, + collectionTime: T0.AddSeconds(600), maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + /* Exactly the shape #2234 reported from production: the cumulative counter advanced ~55,000 + while delta_value came back 0. It must now report the advance and the span it covered. */ + Assert.Equal(55_000, delta); + Assert.Equal(600, interval); + } + + /// The guard still guards: past an hour the baseline is treated as too stale to subtract + /// from, and the interval says so rather than implying an idle hour. + [Fact] + public void AGapPastTheNewPolicy_YieldsZeroAndReportsNoInterval() + { + var calc = new CollectorDeltaCalculator(); + calc.CalculateDelta(ServerId, Collector, Key, 1_000, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + var delta = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 9_999, out var interval, + collectionTime: T0.AddSeconds(CollectorDeltaCalculator.DefaultMaxGapSeconds + 1), + maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// The invariant, and the case that used to break it: a counter reset makes the delta + /// unknowable, so the interval must be 0 too. Reporting 0 work over a REAL interval is a claim + /// that nothing happened for that long — the one place that claim would be false. + [Fact] + public void ACounterReset_YieldsZeroAndReportsNoInterval_SoItCannotReadAsIdle() + { + var calc = new CollectorDeltaCalculator(); + calc.CalculateDelta(ServerId, Collector, Key, 5_000, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + /* Counter went backwards (instance restart / plan cache eviction). */ + var delta = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 12, out var interval, + collectionTime: T0.AddSeconds(120), maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// A normal sweep with real work: both halves non-zero, which is the only combination a + /// consumer may read as a rate. + [Fact] + public void AnOrdinarySweep_ReportsBothTheDeltaAndTheSpanItCovered() + { + var calc = new CollectorDeltaCalculator(); + calc.CalculateDelta(ServerId, Collector, Key, 100, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + var delta = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 700, out var interval, + collectionTime: T0.AddSeconds(60), maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + Assert.Equal(600, delta); + Assert.Equal(60, interval); + Assert.Equal(10.0, delta / (double)interval); /* the rate a caller derives */ + } + + /// A genuinely idle interval is the one case that legitimately stores a 0 delta — and it + /// keeps a real interval, which is exactly what distinguishes it from the three unknowns. + [Fact] + public void AGenuinelyIdleInterval_KeepsItsRealInterval() + { + var calc = new CollectorDeltaCalculator(); + calc.CalculateDelta(ServerId, Collector, Key, 4_242, + collectionTime: T0, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + var delta = calc.CalculateDeltaWithInterval(ServerId, Collector, Key, 4_242, out var interval, + collectionTime: T0.AddSeconds(180), maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + Assert.Equal(0, delta); + Assert.Equal(180, interval); + } + + /// No collector may reintroduce a hard-coded gap. The literal is what drifted for 41 call + /// sites and five doc comments, so the pin is on the literal, not on any one collector. + [Fact] + public void NoCollectorPassesAHardCodedGap() + { + var collectors = Path.Combine(RepoRoot(), "PerformanceMonitor.Collectors"); + Assert.True(Directory.Exists(collectors), $"collectors directory not found at {collectors}"); + + var offenders = Directory.EnumerateFiles(collectors, "*.cs", SearchOption.AllDirectories) + .Select(path => (path, text: File.ReadAllText(path))) + .Where(f => System.Text.RegularExpressions.Regex.IsMatch(f.text, @"maxGapSeconds: *[0-9]")) + .Select(f => Path.GetFileName(f.path)) + .ToList(); + + Assert.Empty(offenders); + } + + /// The perfmon collector must MEASURE its interval. It wrote a literal 60 — the configured + /// cadence — while the real median gap was 299 s, so every rate derived from it was up to 5x high. + /// + [Fact] + public void ThePerfmonCollectorMeasuresItsInterval_RatherThanAssertingTheCadence() + { + var source = File.ReadAllText(Path.Combine( + RepoRoot(), "PerformanceMonitor.Collectors", "PerfmonStatsCollector.cs")); + + Assert.Contains("CalculateDeltaWithInterval", source, StringComparison.Ordinal); + Assert.Contains(".Value(sampleIntervalSeconds)", source, StringComparison.Ordinal); + /* The literal it used to write, as the payload value. */ + Assert.DoesNotContain(".Value(60)", source, StringComparison.Ordinal); + } + + /// The read has to carry the denominator, or the distinction dies at the API boundary — + /// which is the half of #2234 that made a fabricated zero unfalsifiable from outside. + /// And it must AGGREGATE it correctly: the value and delta are additive across a counter's + /// instance rows, the interval is not — it is one measured gap repeated per instance. Fleet-measured, + /// Transactions/sec carries a median of 12 (max 17) rows per collection_time, so SUM would report a + /// rate 12-17x too low. Pinning the aggregate by name because a review caught exactly that. + /// + [Fact] + public void ThePerfmonTrendReadProjectsTheInterval_AggregatedAsMaxNotSum() + { + var sql = PerformanceMonitor.Darling.Service.Mcp.DarlingTrendReader.PerfmonTrendSql; + + Assert.Contains("MAX(sample_interval_seconds)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(sample_interval_seconds)", sql, StringComparison.Ordinal); + /* The additive pair must stay additive — this guards the fix from being over-applied. */ + Assert.Contains("SUM(cntr_value)", sql, StringComparison.Ordinal); + Assert.Contains("SUM(delta_cntr_value)", sql, StringComparison.Ordinal); + } + + private static string RepoRoot() + { + var dir = AppContext.BaseDirectory; + while (dir != null && !File.Exists(Path.Combine(dir, "PerformanceMonitor.sln"))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return dir!; + } +} diff --git a/Darling/Darling.Tests/DeltaSeriesAgeTests.cs b/Darling/Darling.Tests/DeltaSeriesAgeTests.cs new file mode 100644 index 000000000..bc85d91c3 --- /dev/null +++ b/Darling/Darling.Tests/DeltaSeriesAgeTests.cs @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2235: telling a key that is new TO US apart from a counter that is new to the WORLD. +/// +/// The defect. query_stats keys its deltas on the full row identity, which includes +/// plan_handle — and plan_handle changes on every recompile. So a plan-churning statement +/// presents a fresh key on nearly every sighting, and a first sighting reports 0. On a production +/// replica that discarded most of the instance's CPU: a query Datadog measured at ~43% of an 8-vCPU box +/// read through these collectors as 18 executions and 2,824 ms over 168 hours, and the top-25 procedures +/// accounted for ~49M ms of roughly 498M core-ms available. +/// +/// Why it was invisible, which is the worse half. The calculator already had an honest path +/// for cache churn — the counter-reset branch reports interval = 0 precisely so a reader can tell +/// a fabricated zero from an idle one (#2234, the same invariant +/// pins). But that branch needs the SAME key to reappear with a lower value, and a recompile never +/// does: it arrives under a new key and takes the baseline path instead. Same class of harm as the +/// 300-second gap policy #2233 replaced — it did not merely lose data, it invented quiet. +/// +/// The rule. The caller passes how old the counter series is; the calculator combines that +/// with its own record of when it last looked. If the series began since the previous pass, the whole +/// counter accrued inside that window and its baseline was 0 — so the delta is the full value, reported +/// with a real interval because it IS knowable. Otherwise nothing changes. +/// +/// The age is an age and not a timestamp on purpose: a DMV creation_time is in the +/// monitored server's local time while collection times are UTC, so comparing them client-side is a +/// timezone bug on every server that is not UTC. +/// +public sealed class DeltaSeriesAgeTests +{ + private const int ServerId = 1; + private const string Collector = "query_stats_worker"; + private const int Gap = CollectorDeltaCalculator.DefaultMaxGapSeconds; + + private static DateTime T0 => new(2026, 8, 15, 12, 0, 0, DateTimeKind.Unspecified); + private static DateTime T1 => T0.AddSeconds(60); + + /// + /// THE FIX: a recompiled plan's counter is credited instead of silently reporting nothing. + /// + [Fact] + public void ASeriesThatBeganSinceTheLastPassIsCreditedInFull() + { + var deltas = new CollectorDeltaCalculator(); + + /* Pass one establishes when we last looked. */ + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "sql:0:99:planA", 500, 10, out _, T0, Gap); + + /* Pass two: the statement recompiled, so this is a DIFFERENT plan_handle and therefore a key we + have never seen — carrying 900 us accrued by a plan compiled 20 s ago, i.e. inside the 60 s + since our last look. */ + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, Collector, "sql:0:99:planB", 900, seriesAgeSeconds: 20, out var interval, T1, Gap); + + Assert.Equal(900, delta); + /* A real interval, because this delta is knowable — (0, 0) stays reserved for the cases that + genuinely are not, which is what makes a stored zero readable at all. */ + Assert.Equal(60, interval); + } + + /// + /// A series OLDER than the gap is still refused. Most of that counter accrued before we were + /// looking, so crediting it would invent work in this interval rather than merely lose some. + /// + [Fact] + public void ASeriesOlderThanTheGapStaysUnknown() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 500, 10, out _, T0, Gap); + + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, Collector, "old", 999_999, seriesAgeSeconds: 3_600, out var interval, T1, Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// + /// The gap policy still bounds it. A plan compiled during a two-hour outage is not two hours of + /// work in the next minute — this is the inflated-spike guard, and the new path must not route + /// around it. + /// + [Fact] + public void AGapBeyondThePolicyRefusesToCreditAWholeCounter() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 500, 10, out _, T0, Gap); + + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, Collector, "fresh", 777_777, seriesAgeSeconds: 30, out var interval, T0.AddHours(2), Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// + /// The first pass ever credits nothing: with no previous look there is no window to attribute a + /// counter to, and a cold start must not dump every cached plan's lifetime into one interval. + /// + [Fact] + public void TheFirstPassEverBaselinesRatherThanCrediting() + { + var deltas = new CollectorDeltaCalculator(); + + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, Collector, "k", 12_345, seriesAgeSeconds: 5, out var interval, T0, Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// + /// EVERY row of one pass measures against the same previous pass. + /// + /// The trap this pins: a collector calls in once per row, so if the first row rolled the pass + /// window forward, every later row in that same pass would compare against its own pass, see a zero + /// gap, and be credited nothing. The fix would then work for exactly one row per cycle and look like + /// it worked. + /// + [Fact] + public void EveryRowOfOnePassSeesTheSamePreviousPass() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "seed", 1, 1, out _, T0, Gap); + + var first = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "r1", 100, 5, out _, T1, Gap); + var second = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "r2", 200, 5, out _, T1, Gap); + var third = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "r3", 300, 5, out _, T1, Gap); + + Assert.Equal(100, first); + Assert.Equal(200, second); + Assert.Equal(300, third); + } + + /// + /// Passing no age is exactly the old behaviour, which is what lets the other forty-odd delta call + /// sites stay untouched. + /// + [Fact] + public void WithoutAnAgeTheBehaviourIsUnchanged() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, "perfmon", "k", 500, null, out _, T0, Gap); + + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, "perfmon", "newkey", 900, seriesAgeSeconds: null, out var interval, T1, Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + + /* And through the original entry point, which must not have changed at all. */ + var legacy = new CollectorDeltaCalculator(); + legacy.CalculateDeltaWithInterval(ServerId, "perfmon", "k", 500, out _, T0, Gap); + var legacyDelta = legacy.CalculateDeltaWithInterval(ServerId, "perfmon", "newkey", 900, out var legacyInterval, T1, Gap); + + Assert.Equal(0, legacyDelta); + Assert.Equal(0, legacyInterval); + } + + /// Ordinary same-key subtraction is untouched, age supplied or not. + [Fact] + public void AnExistingKeyStillSubtracts() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 1_000, 5, out _, T0, Gap); + + var delta = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 1_750, 65, out var interval, T1, Gap); + + Assert.Equal(750, delta); + Assert.Equal(60, interval); + } + + /// + /// A counter reset on the SAME key is still the honest (0, 0) — the age must not be read as + /// permission to treat a decrease as a fresh series and credit the post-reset value. + /// + [Fact] + public void AResetOnTheSameKeyIsStillUnknowable() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 1_000, 100, out _, T0, Gap); + + var delta = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 5, seriesAgeSeconds: 5, out var interval, T1, Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// + /// ClearServer drops the pass window along with the baselines. Left behind, a re-added + /// server's first pass would measure a series age against a look from before it was removed and + /// credit a full counter to an interval that never happened. + /// + [Fact] + public void ClearServerAlsoForgetsWhenWeLastLooked() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 100, 5, out _, T0, Gap); + + deltas.ClearServer(ServerId); + + var delta = deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k2", 999, 5, out var interval, T1, Gap); + + Assert.Equal(0, delta); + Assert.Equal(0, interval); + } + + /// + /// The pass window is per server AND per collector, so one server's sweep cannot make another + /// server's first pass look warm. + /// + [Fact] + public void ThePassWindowIsPerServerAndPerCollector() + { + var deltas = new CollectorDeltaCalculator(); + deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 1, 1, out _, T0, Gap); + + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(2, Collector, "k", 500, 5, out _, T1, Gap)); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, "other_collector", "k", 500, 5, out _, T1, Gap)); + } + + /// + /// An implementer that never opted in keeps compiling and keeps its old behaviour, which is the + /// reason the interface method is default-implemented rather than abstract. + /// + [Fact] + public void ADefaultImplementerIgnoresTheAge() + { + ICollectorDeltaCalculator legacy = new PreSeriesAgeCalculator(); + + var delta = legacy.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 900, 5, out var interval, T1, Gap); + + Assert.Equal(-1, delta); + Assert.Equal(-1, interval); + } + + /// A pre-#2235 implementer: only the two original methods exist on it. + private sealed class PreSeriesAgeCalculator : ICollectorDeltaCalculator + { + public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, + DateTime? collectionTime = null, int maxGapSeconds = 0) => -1; + + public long CalculateDeltaWithInterval(int serverId, string collectorName, string key, long currentValue, + out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + { + intervalSeconds = -1; + return -1; + } + } +} diff --git a/Darling/Darling.Tests/DistinctTextsLiveTests.cs b/Darling/Darling.Tests/DistinctTextsLiveTests.cs index 9b5f7b0dc..f5b4213f6 100644 --- a/Darling/Darling.Tests/DistinctTextsLiveTests.cs +++ b/Darling/Darling.Tests/DistinctTextsLiveTests.cs @@ -70,7 +70,7 @@ public async Task TopQueries_CountDistinctTexts_AndAnnotateBlendedGroups_Against /* ---- reader: the blend counts 2, the single-text hash 1, the legacy hash 0. */ var rows = await DarlingDataReader.GetTopQueriesByCpuAsync( - postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 10, databaseName: null, ct); + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 10, databaseName: null, cancellationToken: ct); Assert.Equal(2, rows.Single(r => r.QueryHash == "0xINSEXECHASH").DistinctTexts); Assert.Equal(1, rows.Single(r => r.QueryHash == "0xSINGLEHASH").DistinctTexts); diff --git a/Darling/Darling.Tests/DocCommentHygieneTests.cs b/Darling/Darling.Tests/DocCommentHygieneTests.cs index 6a892e2c5..ecfc75daf 100644 --- a/Darling/Darling.Tests/DocCommentHygieneTests.cs +++ b/Darling/Darling.Tests/DocCommentHygieneTests.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using Xunit; @@ -31,6 +32,15 @@ namespace Darling.Tests; /// superseded duplicate safe to simply delete. A blind "remove the extra summary" sweep would have destroyed /// documentation at seven of the eight. /// +/// Counted by openings, not by closing tags (#2190). The first version of this rule keyed off a +/// </summary> immediately followed by a reopening, so it saw a stacked pair only when the FIRST +/// block was closed. Two instances sat on dev unseen: a duplicated opening tag, and a doc block that an +/// insertion had split, stranding its head on the following member with no closing tag near it. Both are +/// caught by counting <summary> OPENINGS inside each contiguous run of /// lines — a run +/// documents exactly one member, so two openings in one run means two summaries, whether or not either is +/// closed and whether they are written single-line or spread over many. The mixed form is the one a +/// closing-tag matcher cannot see at all: a single-line summary followed by a multi-line one. +/// /// Coverage limit, stated rather than assumed. CI path filters are per-project, so this runs on /// any pull request that trips the darling or core filter, and on every nightly and release /// build — but a change touching ONLY Lite or Installer will not run it, and would be caught on the next @@ -41,12 +51,13 @@ namespace Darling.Tests; public sealed class DocCommentHygieneTests { /// - /// A </summary> closed and immediately reopened. Deliberately narrow: it matches only the - /// stacked-block shape, never a legitimate <summary> followed by <param>, - /// <returns> or <remarks>. + /// One <summary> OPENING tag. Counting these per doc run, rather than pairing them against a + /// closing tag, is what lets the rule see an unclosed first block. Still deliberately narrow: a summary + /// followed by <param>, <returns>, <remarks> or any number of + /// <para> blocks is one opening and never matches twice, and an escaped mention in prose + /// (&lt;summary&gt;, as used throughout this very file) is not an opening at all. /// - private static readonly Regex StackedSummary = - new(@"\s*\r?\n\s*///\s*", RegexOptions.Compiled); + private static readonly Regex SummaryOpening = new(@"", RegexOptions.Compiled); [Fact] public void NoMemberCarriesTwoStackedSummaryBlocks() @@ -70,12 +81,13 @@ public void NoMemberCarriesTwoStackedSummaryBlocks() continue; } - var text = File.ReadAllText(file); - foreach (Match match in StackedSummary.Matches(text)) + foreach (var run in StackedSummaryRuns(File.ReadAllLines(file))) { - /* Line number of the match start, so the failure names somewhere you can actually open. */ - var line = text.AsSpan(0, match.Index).Count('\n') + 1; - offenders.Add($"{Path.GetRelativePath(root!, file)}:{line}"); + /* Name the run's first line — somewhere you can actually open — and every opening in it, since + the second one is usually the insertion point that caused the stacking. */ + offenders.Add( + $"{Path.GetRelativePath(root!, file)}:{run.Start} " + + $"( openings at lines {string.Join(", ", run.Openings)})"); } } @@ -86,9 +98,89 @@ public void NoMemberCarriesTwoStackedSummaryBlocks() "an insertion pushed it away from. Seven of the eight found in #1745 were displaced doc blocks " + "whose real member had been left undocumented, and deleting them would have lost the documentation " + "rather than deduplicating it.\n\n" + + "Where an insertion split a block, also check whether the member it came from has since been " + + "re-documented in place. If it has, the stray text is a stranded HEAD rather than the whole block, " + + "and moving it back would create the very duplicate this rule forbids — confirm sentence by " + + "sentence that nothing is lost, then delete it (#2190).\n\n" + string.Join("\n", offenders)); } + /// + /// The rule's own self-test. #2190 was a blind spot in the DETECTOR rather than in anyone's reading of the + /// tree, and it was a synthetic case that exposed it — so the shapes this must catch, and the ones it must + /// leave alone, are pinned here instead of being left to whatever the tree happens to contain. Every + /// true case below is a real shape that has appeared in this repo. + /// + [Theory] + /* Closed and immediately reopened: the only shape the pre-#2190 rule could see. */ + [InlineData(true, "/// \n/// A.\n/// \n/// \n/// B.\n/// \nvoid M();")] + /* A duplicated opening tag, first block never closed. */ + [InlineData(true, "/// \n/// \n/// A.\n/// \nvoid M();")] + /* An insertion split a block, stranding its unclosed head above the next member's whole block. */ + [InlineData(true, "/// \n/// A.\n///\n/// \n/// B.\n/// \nvoid M();")] + /* Single-line followed by multi-line — invisible to a closing-tag matcher, since the reopening does not + follow a on its own line. */ + [InlineData(true, "/// A.\n/// \n/// B.\n/// \nvoid M();")] + /* A stacked run reaching end of file with no member under it. Not valid C#, but it pins the scan's + one-past-the-end step: a run that never meets a non-doc line has to be closed, not dropped. */ + [InlineData(true, "/// \n/// A.\n/// \n/// B.")] + /* One summary plus the other doc tags that legitimately follow it. */ + [InlineData(false, "/// \n/// A.\n/// \n/// X.\n/// Y.\nvoid M(int x);")] + /* One summary carrying several blocks, as most of this repo's docs do. */ + [InlineData(false, "/// \n/// A.\n///\n/// B.\n///\n/// C.\n/// \nvoid M();")] + /* Two members, one summary each: the declarations between them end each run. */ + [InlineData(false, "/// A.\nint A;\n/// B.\nint B;")] + /* Escaped mentions in prose are not openings — this very file is full of them. */ + [InlineData(false, "/// \n/// Two <summary> mentions in one <summary> block.\n/// \nvoid M();")] + public void DetectorCountsSummaryOpeningsPerDocRun(bool stacked, string source) + { + var runs = StackedSummaryRuns(source.Split('\n')); + + Assert.True( + runs.Count == (stacked ? 1 : 0), + $"Expected {(stacked ? "one stacked run" : "no stacked run")}, found {runs.Count}, in:\n{source}"); + } + + /// + /// Every contiguous run of /// lines carrying more than one <summary> opening, as the + /// run's first line and the line of each opening. A run ends at the first line that is not a doc comment, + /// which is what ties it to exactly one member: the declaration itself terminates it. + /// + private static List<(int Start, List Openings)> StackedSummaryRuns(string[] lines) + { + var runs = new List<(int Start, List Openings)>(); + + /* 0 means "not currently inside a run"; line numbers reported to a human are 1-based. Iterating one + past the end closes a run that reaches EOF rather than dropping it. */ + var start = 0; + var openings = new List(); + + for (var i = 0; i <= lines.Length; i++) + { + if (i < lines.Length && lines[i].TrimStart().StartsWith("///", StringComparison.Ordinal)) + { + if (start == 0) + { + start = i + 1; + openings = new List(); + } + + openings.AddRange(Enumerable.Repeat(i + 1, SummaryOpening.Matches(lines[i]).Count)); + } + else if (start != 0) + { + if (openings.Count > 1) + { + runs.Add((start, openings)); + } + + start = 0; + } + } + + return runs; + } + /// /// Walks up from the test output directory to the repo root — the directory holding /// PerformanceMonitor.sln. Same walk-up idiom as ThemeParityTests.FindRepoRoot. diff --git a/Darling/Darling.Tests/FileOnlyServerCauseSplitTests.cs b/Darling/Darling.Tests/FileOnlyServerCauseSplitTests.cs new file mode 100644 index 000000000..e8118f684 --- /dev/null +++ b/Darling/Darling.Tests/FileOnlyServerCauseSplitTests.cs @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2258: telling "never registered" from "deliberately removed" in the darling.json reconciliation. +/// +/// The ambiguity. A server named in the file but absent from the registry has two indistinguishable +/// causes: it was added to the file after the first seed and never registered (the #2252 field report — the +/// operator expects monitoring and is not getting it), or it was registered once and then removed via the Viewer, +/// which is a CORRECT state. The reconciliation therefore reported at Information and worded itself for both — +/// honest, but it could neither warn about the first nor stay calm about the second. +/// +/// No tombstone table was needed, which is the interesting part. #2258 proposed a +/// config_removed_servers table or an is_removed flag, plus a migration rung. But the fact it wanted +/// is already recorded: collect.servers — the OBSERVED registry — gets a row on every successful connect, +/// the Viewer's Remove deletes only from config_monitored_servers (the DESIRED config), and nothing purges +/// the observed registry because it is a registry rather than a time series. So the distinction is answerable +/// today, with no schema change and no second piece of state that could disagree with the first. +/// +/// An is_removed flag was worth rejecting explicitly rather than merely not choosing: +/// is_enabled = FALSE already means "registered but paused", so a second flag makes +/// (is_enabled, is_removed) a four-state space with two meaningless combinations, and every existing reader +/// of that table would have to learn the new flag or silently start including removed servers — the same seam +/// failure #2280 had to fix in the dedupe gate. +/// +public sealed class FileOnlyServerCauseSplitTests +{ + private static HashSet Observed(params string[] names) => + new(names, StringComparer.OrdinalIgnoreCase); + + /// + /// THE FIELD REPORT (#2252): in the file, never observed. That is the case where something the operator wants + /// is not happening, so it must be the one that warns. + /// + [Fact] + public void AServerNeverObservedIsReportedAsNeverRegistered() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + new[] { "new-server" }, Observed("some-other-server")); + + Assert.Equal(new[] { "new-server" }, never); + Assert.Empty(removed); + } + + /// + /// THE CORRECT STATE: in the file, and the observed registry remembers it. The operator removed it on purpose + /// and left the file alone — advising them to re-add it would be wrong advice, repeated at every startup + /// forever, which is exactly what the old single message risked. + /// + [Fact] + public void AServerTheStoreHasObservedIsReportedAsDeliberatelyRemoved() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + new[] { "retired-server" }, Observed("retired-server")); + + Assert.Empty(never); + Assert.Equal(new[] { "retired-server" }, removed); + } + + /// + /// Both causes in one reconciliation, which is the normal case on a fleet that has been running a while — + /// and the reason the two are reported as separate lines with different severities rather than one blended + /// list. A single list would force the sharper cause to inherit the calmer wording. + /// + [Fact] + public void BothCausesAreSeparatedInOnePass() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + new[] { "new-a", "retired-b", "new-c", "retired-d" }, + Observed("retired-b", "retired-d", "unrelated")); + + Assert.Equal(new[] { "new-a", "new-c" }, never); + Assert.Equal(new[] { "retired-b", "retired-d" }, removed); + } + + /// + /// Matched case-insensitively, because the observed registry's names come from a different write path than + /// the file's and neither normalizes case. A case difference is not a different server, and treating it as + /// one would warn about a deliberately-removed server forever. + /// + [Theory] + [InlineData("Retired-Server", "retired-server")] + [InlineData("retired-server", "RETIRED-SERVER")] + public void NamesMatchCaseInsensitively(string fileName, string observedName) + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + new[] { fileName }, Observed(observedName)); + + Assert.Empty(never); + Assert.Single(removed); + } + + /// + /// An EMPTY observed registry sends everything to the warning arm. That is the fresh-store case — nothing has + /// connected yet — and it is the safe direction: on a store where the file genuinely is the declared intent, + /// a warning that these are not yet monitored is correct, whereas silence would hide it. + /// + [Fact] + public void AnEmptyObservedRegistryWarnsRatherThanGoingQuiet() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + new[] { "a", "b" }, Observed()); + + Assert.Equal(new[] { "a", "b" }, never); + Assert.Empty(removed); + } + + /// Nothing file-only means nothing to report, in either arm. + [Fact] + public void NoFileOnlyServersProducesNothing() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored( + Array.Empty(), Observed("anything")); + + Assert.Empty(never); + Assert.Empty(removed); + } + + /// Null inputs are not a crash on a startup diagnostic path. + [Fact] + public void NullInputsAreTolerated() + { + var (never, removed) = StoreConfigProvider.SplitByEverMonitored(null!, Observed("x")); + Assert.Empty(never); + Assert.Empty(removed); + + var (never2, removed2) = StoreConfigProvider.SplitByEverMonitored(new[] { "a" }, null!); + Assert.Equal(new[] { "a" }, never2); + Assert.Empty(removed2); + } + + /// + /// The two arms carry the severities their causes deserve, and the observed registry is what they read — + /// pinned at the source because the log calls need a live store to exercise. + /// + /// The severity split IS the feature: the old single Information line could not warn about a server + /// the operator wanted monitored, and a naive fix that warned about both would have told them to re-add a + /// server they deliberately dropped, at every startup. + /// + [Fact] + public void TheWarningAndTheInformationArmsAreSplitByCause() + { + var source = ReadProviderSource(); + + Assert.Contains("SELECT display_name, server_name FROM collect.servers", source, StringComparison.Ordinal); + Assert.Contains("NOT monitored and never have been", source, StringComparison.Ordinal); + Assert.Contains("LogWarning(", source, StringComparison.Ordinal); + Assert.Contains("were monitored and have since been removed", source, StringComparison.Ordinal); + /* The removed arm must not advise anything — it is a correct state. */ + Assert.Contains("That is expected", source, StringComparison.Ordinal); + /* And it must still say their history is kept, since "removed" reads as "deleted" to most people. */ + Assert.Contains("collected history is", source, StringComparison.Ordinal); + } + + private static string ReadProviderSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/FileOnlyServerWarningTests.cs b/Darling/Darling.Tests/FileOnlyServerWarningTests.cs new file mode 100644 index 000000000..05a168041 --- /dev/null +++ b/Darling/Darling.Tests/FileOnlyServerWarningTests.cs @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2254: the diagnostic for servers that exist in darling.json but not in the store. +/// +/// The field report. An operator added a second server to the file, ran +/// --test-connection (which validated BOTH as PASS, because it reads the file), restarted the service, +/// and the server never appeared. The seed only runs while config_monitored_servers is empty, so after +/// the first start a file edit is a permanent no-op — and nothing said so. Two correct outputs answering +/// different questions, with no way to see the disagreement: config edit, restart, support round trip. +/// +/// Why the comparison is on server_id. That is the identity the collectors and the registry key +/// on. Comparing by NAME would report a file entry as present when its host or read-only intent differs from +/// the stored row — which is the same identity confusion behind the split-history class in #2158. The +/// same-name-different-host case below is the one that would pass a naive name match. +/// +public sealed class FileOnlyServerWarningTests +{ + private static MonitoredServer Server(string name, string host) => + new() { Name = name, Host = host }; + + private static int IdOf(MonitoredServer server) => + ServerIdHelper.GetDeterministicHashCode(server.StorageName); + + /// The steady state: everything in the file is registered, so there is nothing to warn about and + /// the log stays quiet on every start. + [Fact] + public void WhenTheStoreHasEveryFileServer_NothingIsReported() + { + var a = Server("plvs-itebd", "plvs-itebd"); + var b = Server("dcvs-bd01", "dcvs-bd01"); + + var missing = StoreConfigProvider.ServersOnlyInFile( + new[] { a, b }, + new HashSet { IdOf(a), IdOf(b) }); + + Assert.Empty(missing); + } + + /// The reported case exactly: two in the file, one seeded, and the second silently ignored. + [Fact] + public void TheServerAddedAfterTheSeed_IsReportedByName() + { + var seeded = Server("plvs-itebd", "plvs-itebd"); + var added = Server("dcvs-bd01", "dcvs-bd01"); + + var missing = StoreConfigProvider.ServersOnlyInFile( + new[] { seeded, added }, + new HashSet { IdOf(seeded) }); + + Assert.Equal(new[] { "dcvs-bd01" }, missing); + } + + /// + /// The case a name comparison gets wrong. Same display name, different host — so a different + /// server_id, a different registry row, and a server that is genuinely not being monitored under + /// the identity the file describes. + /// + [Fact] + public void SameNameButADifferentHost_IsStillReportedAsAbsent() + { + var inFile = Server("reporting", "reporting-new-host"); + var inStore = Server("reporting", "reporting-old-host"); + + Assert.NotEqual(IdOf(inFile), IdOf(inStore)); + + var missing = StoreConfigProvider.ServersOnlyInFile( + new[] { inFile }, + new HashSet { IdOf(inStore) }); + + Assert.Equal(new[] { "reporting" }, missing); + } + + /// An empty store reports everything. This is the shape a seed that failed leaves behind, and it + /// should be loud rather than silent. + [Fact] + public void AnEmptyStoreReportsEveryFileServer() + { + var missing = StoreConfigProvider.ServersOnlyInFile( + new[] { Server("a", "host-a"), Server("b", "host-b") }, + new HashSet()); + + Assert.Equal(new[] { "a", "b" }, missing); + } + + /// A config with no servers must not throw on the startup path — this runs before anything is + /// monitored, so a null-reference here would take the service down over a diagnostic. + [Fact] + public void ANullOrEmptyFileListIsNotAnError() + { + Assert.Empty(StoreConfigProvider.ServersOnlyInFile(null!, new HashSet())); + Assert.Empty(StoreConfigProvider.ServersOnlyInFile(new List(), new HashSet { 1 })); + } +} diff --git a/Darling/Darling.Tests/Fixtures/migration-ladder-v3.3.0.sql b/Darling/Darling.Tests/Fixtures/migration-ladder-v3.3.0.sql new file mode 100644 index 000000000..9d595b783 --- /dev/null +++ b/Darling/Darling.Tests/Fixtures/migration-ladder-v3.3.0.sql @@ -0,0 +1,1967 @@ +-- Migration ladder as RESOLVED by the release this fixture is named for — every rung's +-- SQL frozen as that era's code (including its generators) emitted it, plus the same +-- version-table DDL and stamps MigrateLockedAsync writes. Generated by tools/generate- +-- ladder-fixture; regenerate at each release cut from the release tag. DO NOT HAND-EDIT. +-- ===BATCH=== bootstrap +SET search_path = collect, config, public; +CREATE TABLE IF NOT EXISTS darling_schema_version ( + version integer NOT NULL PRIMARY KEY, + name text NOT NULL, + applied_at timestamp NOT NULL +); +-- ===BATCH=== V1 collector-tables +/* Darling collector tables — generated from PerformanceMonitor.Collectors definitions. + Column names and types mirror Lite's DuckDB schema (see PgSchemaGenerator remarks). */ + +CREATE TABLE IF NOT EXISTS wait_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + wait_type text, + waiting_tasks_count bigint, + wait_time_ms bigint, + signal_wait_time_ms bigint, + delta_waiting_tasks bigint, + delta_wait_time_ms bigint, + delta_signal_wait_time_ms bigint +); +CREATE INDEX IF NOT EXISTS idx_wait_stats_time ON wait_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS latch_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + latch_class text, + waiting_requests_count bigint, + wait_time_ms bigint, + max_wait_time_ms bigint, + delta_waiting_requests_count bigint, + delta_wait_time_ms bigint, + delta_max_wait_time_ms bigint +); +CREATE INDEX IF NOT EXISTS idx_latch_stats_time ON latch_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS spinlock_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + spinlock_name text, + collisions bigint, + spins bigint, + spins_per_collision double precision, + sleep_time bigint, + backoffs bigint, + delta_collisions bigint, + delta_spins bigint, + delta_sleep_time bigint, + delta_backoffs bigint +); +CREATE INDEX IF NOT EXISTS idx_spinlock_stats_time ON spinlock_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS cpu_scheduler_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + max_workers_count integer, + scheduler_count integer, + cpu_count integer, + total_runnable_tasks_count integer, + total_work_queue_count bigint, + total_current_workers_count integer, + avg_runnable_tasks_count numeric(38,2), + total_active_request_count integer, + total_queued_request_count integer, + total_blocked_task_count integer, + total_active_parallel_thread_count bigint, + runnable_request_count integer, + total_request_count integer, + runnable_percent numeric(38,2), + worker_thread_exhaustion_warning boolean, + runnable_tasks_warning boolean, + blocked_tasks_warning boolean, + queued_requests_warning boolean, + total_physical_memory_kb bigint, + available_physical_memory_kb bigint, + system_memory_state_desc text, + physical_memory_pressure_warning boolean, + total_node_count integer, + nodes_online_count integer, + offline_cpu_count integer, + offline_cpu_warning boolean +); +CREATE INDEX IF NOT EXISTS idx_cpu_scheduler_stats_time ON cpu_scheduler_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS plan_cache_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + cacheobjtype text, + objtype text, + total_plans integer, + total_size_mb integer, + single_use_plans integer, + single_use_size_mb integer, + multi_use_plans integer, + multi_use_size_mb integer, + avg_use_count numeric(38,2), + avg_size_kb integer, + oldest_plan_create_time timestamp +); +CREATE INDEX IF NOT EXISTS idx_plan_cache_stats_time ON plan_cache_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS tempdb_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + user_object_reserved_mb numeric(18,2), + internal_object_reserved_mb numeric(18,2), + version_store_reserved_mb numeric(18,2), + total_reserved_mb numeric(18,2), + unallocated_mb numeric(18,2), + total_sessions_using_tempdb bigint, + top_session_id integer, + top_session_tempdb_mb numeric(18,2) +); +CREATE INDEX IF NOT EXISTS idx_tempdb_stats_time ON tempdb_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS memory_grant_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + resource_semaphore_id smallint, + pool_id integer, + target_memory_mb numeric(18,2), + max_target_memory_mb numeric(18,2), + total_memory_mb numeric(18,2), + available_memory_mb numeric(18,2), + granted_memory_mb numeric(18,2), + used_memory_mb numeric(18,2), + grantee_count integer, + waiter_count integer, + timeout_error_count bigint, + forced_grant_count bigint, + timeout_error_count_delta bigint, + forced_grant_count_delta bigint +); +CREATE INDEX IF NOT EXISTS idx_memory_grant_stats_time ON memory_grant_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS cpu_utilization_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + sample_time timestamp, + sqlserver_cpu_utilization integer, + other_process_cpu_utilization integer +); +CREATE INDEX IF NOT EXISTS idx_cpu_utilization_stats_time ON cpu_utilization_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS memory_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + total_physical_memory_mb numeric(18,2), + available_physical_memory_mb numeric(18,2), + total_page_file_mb numeric(18,2), + available_page_file_mb numeric(18,2), + system_memory_state text, + sql_memory_model text, + target_server_memory_mb numeric(18,2), + total_server_memory_mb numeric(18,2), + buffer_pool_mb numeric(18,2), + plan_cache_mb numeric(18,2), + max_workers_count integer, + current_workers_count integer +); +CREATE INDEX IF NOT EXISTS idx_memory_stats_time ON memory_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS memory_clerks ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + clerk_type text, + memory_mb numeric(18,2) +); +CREATE INDEX IF NOT EXISTS idx_memory_clerks_time ON memory_clerks(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS memory_pressure_events ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + sample_time timestamp, + memory_notification text, + memory_indicators_process integer, + memory_indicators_system integer +); +CREATE INDEX IF NOT EXISTS idx_memory_pressure_events_time ON memory_pressure_events(server_id, sample_time); + +CREATE TABLE IF NOT EXISTS file_io_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + file_name text, + file_type text, + physical_name text, + size_mb numeric(18,2), + num_of_reads bigint, + num_of_writes bigint, + read_bytes bigint, + write_bytes bigint, + io_stall_read_ms bigint, + io_stall_write_ms bigint, + io_stall_queued_read_ms bigint, + io_stall_queued_write_ms bigint, + delta_reads bigint, + delta_writes bigint, + delta_read_bytes bigint, + delta_write_bytes bigint, + delta_stall_read_ms bigint, + delta_stall_write_ms bigint, + delta_stall_queued_read_ms bigint, + delta_stall_queued_write_ms bigint +); +CREATE INDEX IF NOT EXISTS idx_file_io_stats_time ON file_io_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS server_properties ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + edition text, + product_version text, + product_level text, + product_update_level text, + engine_edition integer, + cpu_count integer, + hyperthread_ratio integer, + physical_memory_mb bigint, + socket_count integer, + cores_per_socket integer, + is_hadr_enabled boolean, + is_clustered boolean, + enterprise_features text, + service_objective text, + vcore_count integer, + lock_pages_in_memory boolean, + instant_file_initialization_enabled boolean, + memory_dump_count integer, + sqlserver_start_time timestamp, + host_os_version text, + ag_replica_role text, + utc_offset_minutes integer +); +CREATE INDEX IF NOT EXISTS idx_server_properties_time ON server_properties(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS server_config ( + config_id bigint NOT NULL, + capture_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + configuration_name text, + value_configured bigint, + value_in_use bigint, + is_dynamic boolean, + is_advanced boolean +); + +CREATE TABLE IF NOT EXISTS database_config ( + config_id bigint NOT NULL, + capture_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + state_desc text, + compatibility_level integer, + collation_name text, + recovery_model text, + is_read_only boolean, + is_auto_close_on boolean, + is_auto_shrink_on boolean, + is_auto_create_stats_on boolean, + is_auto_update_stats_on boolean, + is_auto_update_stats_async_on boolean, + is_read_committed_snapshot_on boolean, + snapshot_isolation_state text, + is_parameterization_forced boolean, + is_query_store_on boolean, + is_encrypted boolean, + is_trustworthy_on boolean, + is_db_chaining_on boolean, + is_broker_enabled boolean, + is_cdc_enabled boolean, + is_mixed_page_allocation_on boolean, + log_reuse_wait_desc text, + page_verify_option text, + target_recovery_time_seconds integer, + delayed_durability text, + is_accelerated_database_recovery_on boolean, + is_memory_optimized_enabled boolean, + is_optimized_locking_on boolean +); + +CREATE TABLE IF NOT EXISTS trace_flags ( + config_id bigint NOT NULL, + capture_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + trace_flag integer, + status boolean, + is_global boolean, + is_session boolean +); +CREATE INDEX IF NOT EXISTS idx_trace_flags_time ON trace_flags(server_id, capture_time); + +CREATE TABLE IF NOT EXISTS database_scoped_config ( + config_id bigint NOT NULL, + capture_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + configuration_name text, + value text, + value_for_secondary text +); +CREATE INDEX IF NOT EXISTS idx_database_scoped_config_time ON database_scoped_config(server_id, capture_time); + +CREATE TABLE IF NOT EXISTS session_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + program_name text, + connection_count bigint, + running_count integer, + sleeping_count integer, + dormant_count integer, + total_cpu_time_ms bigint, + total_reads bigint, + total_writes bigint, + total_logical_reads bigint +); +CREATE INDEX IF NOT EXISTS idx_session_stats_time ON session_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS session_summary_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + total_sessions integer, + running_sessions integer, + sleeping_sessions integer, + background_sessions integer, + dormant_sessions integer, + idle_sessions_over_30min integer, + sessions_waiting_for_memory integer, + databases_with_connections integer, + top_application_name text, + top_application_connections integer, + top_host_name text, + top_host_connections integer +); +CREATE INDEX IF NOT EXISTS idx_session_summary_stats_time ON session_summary_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS waiting_tasks ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + session_id integer, + wait_type text, + wait_duration_ms bigint, + blocking_session_id integer, + resource_description text, + database_name text +); +CREATE INDEX IF NOT EXISTS idx_waiting_tasks_time ON waiting_tasks(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS procedure_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + schema_name text, + object_name text, + object_type text, + cached_time timestamp, + last_execution_time timestamp, + execution_count bigint, + total_worker_time bigint, + total_elapsed_time bigint, + total_logical_reads bigint, + total_physical_reads bigint, + total_logical_writes bigint, + min_worker_time bigint, + max_worker_time bigint, + min_elapsed_time bigint, + max_elapsed_time bigint, + min_logical_reads bigint, + max_logical_reads bigint, + min_physical_reads bigint, + max_physical_reads bigint, + min_logical_writes bigint, + max_logical_writes bigint, + total_spills bigint, + min_spills bigint, + max_spills bigint, + sql_handle text, + plan_handle text, + delta_execution_count bigint, + delta_worker_time bigint, + delta_elapsed_time bigint, + delta_logical_reads bigint, + delta_logical_writes bigint, + delta_physical_reads bigint, + delta_spills bigint, + query_plan_xml text +); +CREATE INDEX IF NOT EXISTS idx_procedure_stats_time ON procedure_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS running_jobs ( + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + job_name text, + job_id text, + job_enabled boolean, + start_time timestamp, + current_duration_seconds bigint, + avg_duration_seconds bigint, + p95_duration_seconds bigint, + successful_run_count bigint, + is_running_long boolean, + percent_of_average numeric(10,1) +); +CREATE INDEX IF NOT EXISTS idx_running_jobs_time ON running_jobs(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS perfmon_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + object_name text, + counter_name text, + instance_name text, + cntr_value bigint, + delta_cntr_value bigint, + sample_interval_seconds integer +); +CREATE INDEX IF NOT EXISTS idx_perfmon_stats_time ON perfmon_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS dmv_blocking_snapshots ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + monitor_loop integer, + event_time timestamp, + database_name text, + blocked_spid integer, + blocked_ecid integer, + blocked_last_tran_started timestamp, + blocking_spid integer, + blocking_ecid integer, + blocking_last_tran_started timestamp, + wait_time_ms bigint, + lock_mode text, + blocking_status text, + contentious_object text, + blocked_sql_text text, + blocking_sql_text text, + blocked_login_name text, + blocked_host_name text, + blocked_client_app text, + blocking_login_name text, + blocking_host_name text, + blocking_client_app text +); +CREATE INDEX IF NOT EXISTS idx_dmv_blocking_snapshots_time ON dmv_blocking_snapshots(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS database_size_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + database_id integer, + file_id integer, + file_type_desc text, + file_name text, + physical_name text, + total_size_mb numeric(19,2), + used_size_mb numeric(19,2), + auto_growth_mb numeric(19,2), + max_size_mb numeric(19,2), + recovery_model_desc text, + compatibility_level integer, + state_desc text, + volume_mount_point text, + volume_total_mb numeric(19,2), + volume_free_mb numeric(19,2), + is_percent_growth boolean, + growth_pct integer, + vlf_count integer +); +CREATE INDEX IF NOT EXISTS idx_database_size_stats_time ON database_size_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS index_object_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + sqlserver_start_time timestamp, + database_name text, + database_id integer, + schema_name text, + object_id integer, + table_name text, + index_id integer, + index_name text, + index_type_desc text, + is_unique boolean, + is_primary_key boolean, + is_filtered boolean, + partition_count integer, + reserved_mb numeric(19,2), + used_mb numeric(19,2), + in_row_data_mb numeric(19,2), + lob_data_mb numeric(19,2), + row_overflow_mb numeric(19,2), + total_rows bigint, + user_seeks bigint, + user_scans bigint, + user_lookups bigint, + user_updates bigint, + last_user_seek timestamp, + last_user_scan timestamp, + last_user_lookup timestamp, + last_user_update timestamp, + leaf_insert_count bigint, + leaf_update_count bigint, + leaf_delete_count bigint, + range_scan_count bigint, + singleton_lookup_count bigint, + row_lock_count bigint, + row_lock_wait_count bigint, + row_lock_wait_in_ms bigint, + page_lock_count bigint, + page_lock_wait_count bigint, + page_lock_wait_in_ms bigint, + index_lock_promotion_attempt_count bigint, + index_lock_promotion_count bigint, + page_latch_wait_count bigint, + page_latch_wait_in_ms bigint, + page_io_latch_wait_count bigint, + page_io_latch_wait_in_ms bigint, + key_columns text, + included_columns text, + filter_definition text, + is_unique_constraint boolean, + is_foreign_key boolean, + is_foreign_key_reference boolean, + is_disabled boolean, + data_compression_desc text, + optimize_for_sequential_key boolean, + fill_factor smallint, + is_padded boolean, + allow_page_locks boolean, + allow_row_locks boolean, + is_indexed_view boolean +); +CREATE INDEX IF NOT EXISTS idx_index_object_stats_object ON index_object_stats(server_id, database_name, object_id, index_id, collection_time); + +CREATE TABLE IF NOT EXISTS query_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + query_hash text, + query_plan_hash text, + creation_time timestamp, + last_execution_time timestamp, + execution_count bigint, + total_worker_time bigint, + total_elapsed_time bigint, + total_logical_reads bigint, + total_logical_writes bigint, + total_physical_reads bigint, + total_clr_time bigint, + total_rows bigint, + total_spills bigint, + min_worker_time bigint, + max_worker_time bigint, + min_elapsed_time bigint, + max_elapsed_time bigint, + min_physical_reads bigint, + max_physical_reads bigint, + min_rows bigint, + max_rows bigint, + min_dop bigint, + max_dop bigint, + min_grant_kb bigint, + max_grant_kb bigint, + min_used_grant_kb bigint, + max_used_grant_kb bigint, + min_ideal_grant_kb bigint, + max_ideal_grant_kb bigint, + min_reserved_threads bigint, + max_reserved_threads bigint, + min_used_threads bigint, + max_used_threads bigint, + min_spills bigint, + max_spills bigint, + query_text text, + query_plan_xml text, + sql_handle text, + plan_handle text, + delta_execution_count bigint, + delta_worker_time bigint, + delta_elapsed_time bigint, + delta_logical_reads bigint, + delta_logical_writes bigint, + delta_physical_reads bigint, + delta_rows bigint, + delta_spills bigint, + plan_generation_num bigint, + sample_interval_seconds integer +); +CREATE INDEX IF NOT EXISTS idx_query_stats_time ON query_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS query_snapshots ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + session_id integer, + database_name text, + elapsed_time_formatted text, + query_text text, + query_plan text, + live_query_plan text, + status text, + blocking_session_id integer, + wait_type text, + wait_time_ms bigint, + wait_resource text, + cpu_time_ms bigint, + total_elapsed_time_ms bigint, + reads bigint, + writes bigint, + logical_reads bigint, + granted_query_memory_gb numeric(18,2), + transaction_isolation_level text, + dop integer, + parallel_worker_count integer, + login_name text, + host_name text, + program_name text, + open_transaction_count integer, + percent_complete numeric(5,2), + is_cdc_capture boolean, + query_hash text, + requested_memory_mb double precision, + used_memory_mb double precision, + max_used_memory_mb double precision, + tempdb_current_mb double precision, + tempdb_allocations_mb double precision, + tran_log_used_mb double precision, + tran_start_time timestamp, + request_id integer +); +CREATE INDEX IF NOT EXISTS idx_query_snapshots_time ON query_snapshots(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS query_store_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + query_id bigint, + plan_id bigint, + execution_type_desc text, + first_execution_time timestamp, + last_execution_time timestamp, + module_name text, + query_text text, + query_hash text, + execution_count bigint, + avg_duration_us bigint, + min_duration_us bigint, + max_duration_us bigint, + avg_cpu_time_us bigint, + min_cpu_time_us bigint, + max_cpu_time_us bigint, + avg_logical_io_reads bigint, + min_logical_io_reads bigint, + max_logical_io_reads bigint, + avg_logical_io_writes bigint, + min_logical_io_writes bigint, + max_logical_io_writes bigint, + avg_physical_io_reads bigint, + min_physical_io_reads bigint, + max_physical_io_reads bigint, + avg_clr_time_us bigint, + min_clr_time_us bigint, + max_clr_time_us bigint, + min_dop bigint, + max_dop bigint, + avg_query_max_used_memory bigint, + min_query_max_used_memory bigint, + max_query_max_used_memory bigint, + avg_rowcount bigint, + min_rowcount bigint, + max_rowcount bigint, + avg_num_physical_io_reads bigint, + min_num_physical_io_reads bigint, + max_num_physical_io_reads bigint, + avg_log_bytes_used bigint, + min_log_bytes_used bigint, + max_log_bytes_used bigint, + avg_tempdb_space_used bigint, + min_tempdb_space_used bigint, + max_tempdb_space_used bigint, + plan_type text, + plan_forcing_type text, + is_forced_plan boolean, + force_failure_count bigint, + last_force_failure_reason text, + compatibility_level integer, + query_plan_text text, + query_plan_hash text, + replica_role text +); +CREATE INDEX IF NOT EXISTS idx_query_store_stats_time ON query_store_stats(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS deadlocks ( + deadlock_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + deadlock_time timestamp, + victim_process_id text, + victim_sql_text text, + deadlock_graph_xml text, + victim_query_plan_xml text, + database_name text +); +CREATE INDEX IF NOT EXISTS idx_deadlocks_time ON deadlocks(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS blocked_process_reports ( + blocked_report_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + database_name text, + blocked_spid integer, + blocked_ecid integer, + blocking_spid integer, + blocking_ecid integer, + wait_time_ms bigint, + wait_resource text, + lock_mode text, + blocked_status text, + blocked_isolation_level text, + blocked_log_used bigint, + blocked_transaction_count integer, + blocked_client_app text, + blocked_host_name text, + blocked_login_name text, + blocked_sql_text text, + blocking_status text, + blocking_isolation_level text, + blocking_client_app text, + blocking_host_name text, + blocking_login_name text, + blocking_sql_text text, + blocked_transaction_name text, + blocking_transaction_name text, + blocked_last_tran_started timestamp, + blocking_last_tran_started timestamp, + blocked_last_batch_started timestamp, + blocking_last_batch_started timestamp, + blocked_last_batch_completed timestamp, + blocking_last_batch_completed timestamp, + blocked_priority integer, + blocking_priority integer, + blocked_process_report_xml text, + object_id integer, + database_id integer, + contentious_object text, + monitor_loop integer, + blocked_query_plan_xml text, + blocking_query_plan_xml text +); +CREATE INDEX IF NOT EXISTS idx_blocked_process_reports_time ON blocked_process_reports(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS long_query_completions ( + long_query_completion_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_type text, + database_id integer, + database_name text, + session_id integer, + client_app_name text, + client_pid integer, + nt_username text, + server_principal_name text, + query_hash text, + event_sequence bigint, + duration_microseconds bigint, + cpu_time_microseconds bigint, + physical_reads bigint, + logical_reads bigint, + writes bigint, + row_count bigint, + result text, + statement_text text, + object_name text +); +CREATE INDEX IF NOT EXISTS idx_long_query_completions_time ON long_query_completions(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS system_health_events ( + system_health_event_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_type text, + event_xml text +); +CREATE INDEX IF NOT EXISTS idx_system_health_events_time ON system_health_events(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS default_trace_events ( + default_trace_event_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_name text, + event_class integer, + spid integer, + database_name text, + database_id integer, + login_name text, + host_name text, + application_name text, + object_name text, + filename text, + integer_data bigint, + integer_data_2 bigint, + text_data text, + session_login_name text, + error_number integer, + severity integer, + state integer, + event_sequence bigint, + duration_us bigint, + end_time timestamp +); +CREATE INDEX IF NOT EXISTS idx_default_trace_events_time ON default_trace_events(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS job_history ( + job_history_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + instance_id bigint, + job_id text, + job_name text, + job_enabled boolean, + category_name text, + step_id integer, + step_name text, + run_status integer, + run_status_desc text, + run_datetime timestamp, + run_duration_seconds bigint, + retries_attempted integer, + message text +); +CREATE INDEX IF NOT EXISTS idx_job_history_time ON job_history(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS agent_status ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + agent_running boolean, + agent_status_desc text, + agent_startup_desc text, + next_scheduled_run timestamp +); +CREATE INDEX IF NOT EXISTS idx_agent_status_time ON agent_status(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS ag_replica_states ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + ag_name text, + replica_server_name text, + role_desc text, + operational_state_desc text, + connected_state_desc text, + recovery_health_desc text, + synchronization_health_desc text, + availability_mode_desc text, + failover_mode_desc text, + endpoint_url text, + is_local boolean +); +CREATE INDEX IF NOT EXISTS idx_ag_replica_states_time ON ag_replica_states(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS ag_database_replica_states ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + ag_name text, + database_name text, + replica_server_name text, + is_local boolean, + synchronization_state_desc text, + last_hardened_lsn text, + last_commit_lsn text, + log_send_queue_size bigint, + redo_queue_size bigint, + log_send_rate bigint, + redo_rate bigint, + is_suspended boolean, + suspend_reason_desc text, + availability_mode_desc text, + secondary_lag_seconds bigint, + last_commit_time timestamp, + last_hardened_time timestamp, + last_redone_time timestamp, + last_received_time timestamp, + est_redo_completion_time_min double precision, + est_send_drain_time_min double precision +); +CREATE INDEX IF NOT EXISTS idx_ag_database_replica_states_time ON ag_database_replica_states(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (1, 'collector-tables', now()::timestamp); +-- ===BATCH=== V2 server-registry-and-collection-log + +CREATE TABLE IF NOT EXISTS servers ( + server_id integer NOT NULL PRIMARY KEY, + server_name text NOT NULL, + display_name text, + is_enabled boolean NOT NULL DEFAULT TRUE, + sql_engine_edition integer, + sql_major_version integer, + created_date timestamp, + modified_date timestamp +); + +CREATE TABLE IF NOT EXISTS collection_log ( + log_id bigint NOT NULL, + server_id integer NOT NULL, + server_name text, + collector_name text NOT NULL, + collection_time timestamp NOT NULL, + duration_ms integer, + status text NOT NULL, + error_message text, + rows_collected integer, + sql_duration_ms integer, + duckdb_duration_ms integer +); + +CREATE INDEX IF NOT EXISTS idx_collection_log_time ON collection_log(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (2, 'server-registry-and-collection-log', now()::timestamp); +-- ===BATCH=== V3 alerting-stores + +CREATE TABLE IF NOT EXISTS config_alert_log ( + alert_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + metric_name text NOT NULL, + current_value double precision NOT NULL, + threshold_value double precision NOT NULL, + alert_sent boolean NOT NULL DEFAULT FALSE, + notification_type text NOT NULL DEFAULT 'tray', + send_error text, + dismissed boolean NOT NULL DEFAULT FALSE, + muted boolean NOT NULL DEFAULT FALSE, + detail_text text, + context_json text +); + +CREATE INDEX IF NOT EXISTS idx_config_alert_log_time ON config_alert_log(server_id, metric_name, alert_time); + +CREATE TABLE IF NOT EXISTS config_edge_trigger_watermarks ( + server_id integer NOT NULL, + metric_name text NOT NULL, + watermark integer NOT NULL, + watermark_time timestamp, + updated_at timestamp NOT NULL, + PRIMARY KEY (server_id, metric_name) +); + +CREATE TABLE IF NOT EXISTS config_mute_rules ( + id text NOT NULL PRIMARY KEY, + enabled boolean NOT NULL DEFAULT TRUE, + created_at_utc timestamp NOT NULL, + expires_at_utc timestamp, + reason text, + server_name text, + metric_name text, + database_pattern text, + query_text_pattern text, + wait_type_pattern text, + job_name_pattern text +); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (3, 'alerting-stores', now()::timestamp); +-- ===BATCH=== V4 analysis-tables + +CREATE TABLE IF NOT EXISTS analysis_findings ( + finding_id bigint NOT NULL, + analysis_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + time_range_start timestamp, + time_range_end timestamp, + severity double precision NOT NULL, + confidence double precision NOT NULL, + category text NOT NULL, + story_path text NOT NULL, + story_path_hash text NOT NULL, + story_text text NOT NULL, + root_fact_key text NOT NULL, + root_fact_value double precision, + leaf_fact_key text, + leaf_fact_value double precision, + fact_count integer NOT NULL, + incident_id text, + remediation_action_json text +); + +CREATE INDEX IF NOT EXISTS idx_analysis_findings_time ON analysis_findings(server_id, analysis_time); +CREATE INDEX IF NOT EXISTS idx_analysis_findings_hash ON analysis_findings(story_path_hash); + +CREATE TABLE IF NOT EXISTS analysis_muted ( + mute_id bigint NOT NULL PRIMARY KEY, + server_id integer, + database_name text, + story_path_hash text NOT NULL, + story_path text NOT NULL, + muted_date timestamp NOT NULL, + reason text +); + +CREATE INDEX IF NOT EXISTS idx_analysis_muted_hash ON analysis_muted(story_path_hash); + +CREATE OR REPLACE VIEW v_wait_stats AS SELECT * FROM wait_stats; +CREATE OR REPLACE VIEW v_query_stats AS SELECT * FROM query_stats; +CREATE OR REPLACE VIEW v_query_store_stats AS SELECT * FROM query_store_stats; +CREATE OR REPLACE VIEW v_cpu_utilization_stats AS SELECT * FROM cpu_utilization_stats; +CREATE OR REPLACE VIEW v_memory_grant_stats AS SELECT * FROM memory_grant_stats; +CREATE OR REPLACE VIEW v_memory_stats AS SELECT * FROM memory_stats; +CREATE OR REPLACE VIEW v_perfmon_stats AS SELECT * FROM perfmon_stats; +CREATE OR REPLACE VIEW v_session_stats AS SELECT * FROM session_stats; +CREATE OR REPLACE VIEW v_file_io_stats AS SELECT * FROM file_io_stats; +CREATE OR REPLACE VIEW v_blocked_process_reports AS SELECT * FROM blocked_process_reports; +CREATE OR REPLACE VIEW v_deadlocks AS SELECT * FROM deadlocks; +CREATE OR REPLACE VIEW v_dmv_blocking_snapshots AS SELECT * FROM dmv_blocking_snapshots; +CREATE OR REPLACE VIEW v_index_object_stats AS SELECT * FROM index_object_stats; +CREATE OR REPLACE VIEW v_database_size_stats AS SELECT * FROM database_size_stats; +CREATE OR REPLACE VIEW v_tempdb_stats AS SELECT * FROM tempdb_stats; +CREATE OR REPLACE VIEW v_query_snapshots AS SELECT * FROM query_snapshots; +CREATE OR REPLACE VIEW v_database_config AS SELECT * FROM database_config; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (4, 'analysis-tables', now()::timestamp); +-- ===BATCH=== V5 viewer-passthrough-views + +CREATE OR REPLACE VIEW v_running_jobs AS SELECT * FROM running_jobs; +CREATE OR REPLACE VIEW v_server_config AS SELECT * FROM server_config; +CREATE OR REPLACE VIEW v_database_scoped_config AS SELECT * FROM database_scoped_config; +CREATE OR REPLACE VIEW v_trace_flags AS SELECT * FROM trace_flags; +CREATE OR REPLACE VIEW v_collection_log AS SELECT * FROM collection_log; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (5, 'viewer-passthrough-views', now()::timestamp); +-- ===BATCH=== V6 memory-tab-passthrough-views + +CREATE OR REPLACE VIEW v_memory_clerks AS SELECT * FROM memory_clerks; +CREATE OR REPLACE VIEW v_memory_pressure_events AS SELECT * FROM memory_pressure_events; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (6, 'memory-tab-passthrough-views', now()::timestamp); +-- ===BATCH=== V7 viewer-plan-capture-columns + +ALTER TABLE procedure_stats ADD COLUMN IF NOT EXISTS query_plan_xml text; +ALTER TABLE blocked_process_reports ADD COLUMN IF NOT EXISTS blocked_query_plan_xml text; +ALTER TABLE blocked_process_reports ADD COLUMN IF NOT EXISTS blocking_query_plan_xml text; +ALTER TABLE deadlocks ADD COLUMN IF NOT EXISTS victim_query_plan_xml text; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (7, 'viewer-plan-capture-columns', now()::timestamp); +-- ===BATCH=== V8 schema-split-collect-config +/* V8: split public into collect/config (Darling security hardening, #1262). + Table NAMES are unchanged; search_path = collect, config, public resolves the bare + references every SQL site already uses, so no query is re-qualified. */ + +CREATE SCHEMA IF NOT EXISTS collect AUTHORIZATION darling; +CREATE SCHEMA IF NOT EXISTS config AUTHORIZATION darling; + +/* collect: all collector tables (from the catalog) + registry/metadata + the V4-V6 views */ +ALTER TABLE IF EXISTS public.wait_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.latch_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.spinlock_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.cpu_scheduler_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.plan_cache_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.tempdb_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.memory_grant_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.cpu_utilization_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.memory_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.memory_clerks SET SCHEMA collect; +ALTER TABLE IF EXISTS public.memory_pressure_events SET SCHEMA collect; +ALTER TABLE IF EXISTS public.file_io_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.server_properties SET SCHEMA collect; +ALTER TABLE IF EXISTS public.server_config SET SCHEMA collect; +ALTER TABLE IF EXISTS public.database_config SET SCHEMA collect; +ALTER TABLE IF EXISTS public.trace_flags SET SCHEMA collect; +ALTER TABLE IF EXISTS public.database_scoped_config SET SCHEMA collect; +ALTER TABLE IF EXISTS public.session_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.session_summary_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.waiting_tasks SET SCHEMA collect; +ALTER TABLE IF EXISTS public.procedure_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.running_jobs SET SCHEMA collect; +ALTER TABLE IF EXISTS public.perfmon_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.dmv_blocking_snapshots SET SCHEMA collect; +ALTER TABLE IF EXISTS public.database_size_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.index_object_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.query_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.query_snapshots SET SCHEMA collect; +ALTER TABLE IF EXISTS public.query_store_stats SET SCHEMA collect; +ALTER TABLE IF EXISTS public.deadlocks SET SCHEMA collect; +ALTER TABLE IF EXISTS public.blocked_process_reports SET SCHEMA collect; +ALTER TABLE IF EXISTS public.long_query_completions SET SCHEMA collect; +ALTER TABLE IF EXISTS public.system_health_events SET SCHEMA collect; +ALTER TABLE IF EXISTS public.default_trace_events SET SCHEMA collect; +ALTER TABLE IF EXISTS public.job_history SET SCHEMA collect; +ALTER TABLE IF EXISTS public.agent_status SET SCHEMA collect; +ALTER TABLE IF EXISTS public.ag_replica_states SET SCHEMA collect; +ALTER TABLE IF EXISTS public.ag_database_replica_states SET SCHEMA collect; +ALTER TABLE IF EXISTS public.servers SET SCHEMA collect; +ALTER TABLE IF EXISTS public.collection_log SET SCHEMA collect; +ALTER TABLE IF EXISTS public.analysis_findings SET SCHEMA collect; +ALTER TABLE IF EXISTS public.darling_schema_version SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_wait_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_query_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_query_store_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_cpu_utilization_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_memory_grant_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_memory_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_perfmon_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_session_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_file_io_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_blocked_process_reports SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_deadlocks SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_dmv_blocking_snapshots SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_index_object_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_database_size_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_tempdb_stats SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_query_snapshots SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_database_config SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_running_jobs SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_server_config SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_database_scoped_config SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_trace_flags SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_collection_log SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_memory_clerks SET SCHEMA collect; +ALTER VIEW IF EXISTS public.v_memory_pressure_events SET SCHEMA collect; + +/* config: the operator-writable coordination + analysis-mute tables */ +ALTER TABLE IF EXISTS public.config_alert_log SET SCHEMA config; +ALTER TABLE IF EXISTS public.config_edge_trigger_watermarks SET SCHEMA config; +ALTER TABLE IF EXISTS public.config_mute_rules SET SCHEMA config; +ALTER TABLE IF EXISTS public.analysis_muted SET SCHEMA config; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (8, 'schema-split-collect-config', now()::timestamp); +-- ===BATCH=== V9 server-inventory-cost-fields + +ALTER TABLE server_properties ADD COLUMN IF NOT EXISTS sqlserver_start_time timestamp; +ALTER TABLE server_properties ADD COLUMN IF NOT EXISTS host_os_version text; +ALTER TABLE server_properties ADD COLUMN IF NOT EXISTS ag_replica_role text; +ALTER TABLE servers ADD COLUMN IF NOT EXISTS monthly_cost_usd numeric; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (9, 'server-inventory-cost-fields', now()::timestamp); +-- ===BATCH=== V10 latch-spinlock-collectors +/* V10: latch_stats + spinlock_stats collectors (Dashboard->Darling parity, #1262). + Generated from the collector definitions so the tables match the fresh V1 shape. */ + +CREATE TABLE IF NOT EXISTS latch_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + latch_class text, + waiting_requests_count bigint, + wait_time_ms bigint, + max_wait_time_ms bigint, + delta_waiting_requests_count bigint, + delta_wait_time_ms bigint, + delta_max_wait_time_ms bigint +); +CREATE INDEX IF NOT EXISTS idx_latch_stats_time ON latch_stats(server_id, collection_time); +CREATE OR REPLACE VIEW v_latch_stats AS SELECT * FROM latch_stats; + +CREATE TABLE IF NOT EXISTS spinlock_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + spinlock_name text, + collisions bigint, + spins bigint, + spins_per_collision double precision, + sleep_time bigint, + backoffs bigint, + delta_collisions bigint, + delta_spins bigint, + delta_sleep_time bigint, + delta_backoffs bigint +); +CREATE INDEX IF NOT EXISTS idx_spinlock_stats_time ON spinlock_stats(server_id, collection_time); +CREATE OR REPLACE VIEW v_spinlock_stats AS SELECT * FROM spinlock_stats; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (10, 'latch-spinlock-collectors', now()::timestamp); +-- ===BATCH=== V11 cpu-scheduler-plan-cache-collectors +/* V11: cpu_scheduler_stats + plan_cache_stats collectors (Dashboard->Darling parity, #1262). + Generated from the collector definitions so the tables match the fresh V1 shape. */ + +CREATE TABLE IF NOT EXISTS cpu_scheduler_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + max_workers_count integer, + scheduler_count integer, + cpu_count integer, + total_runnable_tasks_count integer, + total_work_queue_count bigint, + total_current_workers_count integer, + avg_runnable_tasks_count numeric(38,2), + total_active_request_count integer, + total_queued_request_count integer, + total_blocked_task_count integer, + total_active_parallel_thread_count bigint, + runnable_request_count integer, + total_request_count integer, + runnable_percent numeric(38,2), + worker_thread_exhaustion_warning boolean, + runnable_tasks_warning boolean, + blocked_tasks_warning boolean, + queued_requests_warning boolean, + total_physical_memory_kb bigint, + available_physical_memory_kb bigint, + system_memory_state_desc text, + physical_memory_pressure_warning boolean, + total_node_count integer, + nodes_online_count integer, + offline_cpu_count integer, + offline_cpu_warning boolean +); +CREATE INDEX IF NOT EXISTS idx_cpu_scheduler_stats_time ON cpu_scheduler_stats(server_id, collection_time); +CREATE OR REPLACE VIEW v_cpu_scheduler_stats AS SELECT * FROM cpu_scheduler_stats; + +CREATE TABLE IF NOT EXISTS plan_cache_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + cacheobjtype text, + objtype text, + total_plans integer, + total_size_mb integer, + single_use_plans integer, + single_use_size_mb integer, + multi_use_plans integer, + multi_use_size_mb integer, + avg_use_count numeric(38,2), + avg_size_kb integer, + oldest_plan_create_time timestamp +); +CREATE INDEX IF NOT EXISTS idx_plan_cache_stats_time ON plan_cache_stats(server_id, collection_time); +CREATE OR REPLACE VIEW v_plan_cache_stats AS SELECT * FROM plan_cache_stats; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (11, 'cpu-scheduler-plan-cache-collectors', now()::timestamp); +-- ===BATCH=== V12 session-summary-collector +/* V12: session_summary_stats collector (Dashboard->Darling connection-leak / idle parity, #1262). + Generated from the collector definition so the table matches the fresh V1 shape. */ + +CREATE TABLE IF NOT EXISTS session_summary_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + total_sessions integer, + running_sessions integer, + sleeping_sessions integer, + background_sessions integer, + dormant_sessions integer, + idle_sessions_over_30min integer, + sessions_waiting_for_memory integer, + databases_with_connections integer, + top_application_name text, + top_application_connections integer, + top_host_name text, + top_host_connections integer +); +CREATE INDEX IF NOT EXISTS idx_session_summary_stats_time ON session_summary_stats(server_id, collection_time); +CREATE OR REPLACE VIEW v_session_summary_stats AS SELECT * FROM session_summary_stats; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (12, 'session-summary-collector', now()::timestamp); +-- ===BATCH=== V13 system-health-events-collector +/* V13: system_health_events collector (Dashboard->Darling health-parser capture parity, #1262). + Generated from the collector definition so the table matches the fresh V1 shape. */ + +CREATE TABLE IF NOT EXISTS system_health_events ( + system_health_event_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_type text, + event_xml text +); +CREATE INDEX IF NOT EXISTS idx_system_health_events_time ON system_health_events(server_id, collection_time); +CREATE OR REPLACE VIEW v_system_health_events AS SELECT * FROM system_health_events; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (13, 'system-health-events-collector', now()::timestamp); +-- ===BATCH=== V14 refresh-passthrough-views +/* V14: refresh every v_* passthrough view's pinned SELECT * column list (#1262). + Postgres freezes SELECT * at CREATE, so an upgraded store's views omit columns + ADDed later (V7 plan columns, V9 inventory); CREATE OR REPLACE re-expands them. + Append-only ADD COLUMNs => the refresh only adds columns at the end (always legal). */ + +CREATE OR REPLACE VIEW v_wait_stats AS SELECT * FROM wait_stats; +CREATE OR REPLACE VIEW v_query_store_stats AS SELECT * FROM query_store_stats; +CREATE OR REPLACE VIEW v_cpu_utilization_stats AS SELECT * FROM cpu_utilization_stats; +CREATE OR REPLACE VIEW v_memory_grant_stats AS SELECT * FROM memory_grant_stats; +CREATE OR REPLACE VIEW v_memory_stats AS SELECT * FROM memory_stats; +CREATE OR REPLACE VIEW v_perfmon_stats AS SELECT * FROM perfmon_stats; +CREATE OR REPLACE VIEW v_session_stats AS SELECT * FROM session_stats; +CREATE OR REPLACE VIEW v_file_io_stats AS SELECT * FROM file_io_stats; +CREATE OR REPLACE VIEW v_blocked_process_reports AS SELECT * FROM blocked_process_reports; +CREATE OR REPLACE VIEW v_deadlocks AS SELECT * FROM deadlocks; +CREATE OR REPLACE VIEW v_dmv_blocking_snapshots AS SELECT * FROM dmv_blocking_snapshots; +CREATE OR REPLACE VIEW v_index_object_stats AS SELECT * FROM index_object_stats; +CREATE OR REPLACE VIEW v_database_size_stats AS SELECT * FROM database_size_stats; +CREATE OR REPLACE VIEW v_tempdb_stats AS SELECT * FROM tempdb_stats; +CREATE OR REPLACE VIEW v_query_snapshots AS SELECT * FROM query_snapshots; +CREATE OR REPLACE VIEW v_database_config AS SELECT * FROM database_config; +CREATE OR REPLACE VIEW v_running_jobs AS SELECT * FROM running_jobs; +CREATE OR REPLACE VIEW v_server_config AS SELECT * FROM server_config; +CREATE OR REPLACE VIEW v_database_scoped_config AS SELECT * FROM database_scoped_config; +CREATE OR REPLACE VIEW v_trace_flags AS SELECT * FROM trace_flags; +CREATE OR REPLACE VIEW v_collection_log AS SELECT * FROM collection_log; +CREATE OR REPLACE VIEW v_memory_clerks AS SELECT * FROM memory_clerks; +CREATE OR REPLACE VIEW v_memory_pressure_events AS SELECT * FROM memory_pressure_events; +CREATE OR REPLACE VIEW v_latch_stats AS SELECT * FROM latch_stats; +CREATE OR REPLACE VIEW v_spinlock_stats AS SELECT * FROM spinlock_stats; +CREATE OR REPLACE VIEW v_cpu_scheduler_stats AS SELECT * FROM cpu_scheduler_stats; +CREATE OR REPLACE VIEW v_plan_cache_stats AS SELECT * FROM plan_cache_stats; +CREATE OR REPLACE VIEW v_session_summary_stats AS SELECT * FROM session_summary_stats; +CREATE OR REPLACE VIEW v_system_health_events AS SELECT * FROM system_health_events; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (14, 'refresh-passthrough-views', now()::timestamp); +-- ===BATCH=== V15 index-metadata-columns + +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS key_columns text; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS included_columns text; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS filter_definition text; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_unique_constraint boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_foreign_key boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_foreign_key_reference boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_disabled boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS data_compression_desc text; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS optimize_for_sequential_key boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS fill_factor smallint; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_padded boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS allow_page_locks boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS allow_row_locks boolean; +ALTER TABLE index_object_stats ADD COLUMN IF NOT EXISTS is_indexed_view boolean; + +CREATE OR REPLACE VIEW v_index_object_stats AS SELECT * FROM index_object_stats; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (15, 'index-metadata-columns', now()::timestamp); +-- ===BATCH=== V16 server-utc-offset + +ALTER TABLE server_properties ADD COLUMN IF NOT EXISTS utc_offset_minutes integer; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (16, 'server-utc-offset', now()::timestamp); +-- ===BATCH=== V17 config-control-plane + +/* V17: store<->service control plane (Stage 1). EVERY object is schema-qualified config.* — + the migrate session's search_path resolves bare names to collect (wrong schema/ACL). */ + +/* --- A. Config plane: the Viewer writes desired state, the service reads + honors it. --- */ + +/* 1. config_monitored_servers — the desired-state twin of the collect.servers observed registry. + server_id = ServerIdHelper.GetDeterministicHashCode(BuildStorageName(host,database,ro)), the + SAME identity the collectors stamp, so it JOINs collected data. is_enabled drives collection; + the connection fields reconstruct a MonitoredServer for the service's connect path. */ +CREATE TABLE IF NOT EXISTS config.config_monitored_servers ( + server_id integer NOT NULL PRIMARY KEY, + name text NOT NULL, + host text NOT NULL, + database text, + auth text NOT NULL DEFAULT 'integrated', + username text, + encrypted_password text, + encrypt_mode text NOT NULL DEFAULT 'Mandatory', + trust_server_certificate boolean NOT NULL DEFAULT FALSE, + read_only_intent boolean NOT NULL DEFAULT FALSE, + multi_subnet_failover boolean NOT NULL DEFAULT FALSE, + excluded_databases text[] NOT NULL DEFAULT '{}'::text[], + monthly_cost_usd numeric NOT NULL DEFAULT 0, + capture_plans boolean, + is_enabled boolean NOT NULL DEFAULT TRUE, + created_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + modified_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC') +); + +/* 2. config_alert_settings — single row (id=1), one column per AlertsConfig field + the analysis + cadence knobs (analysis_enabled / interval / notifications_enabled / notify_severity). */ +CREATE TABLE IF NOT EXISTS config.config_alert_settings ( + id smallint NOT NULL PRIMARY KEY DEFAULT 1 CHECK (id = 1), + enabled boolean NOT NULL DEFAULT TRUE, + cpu_enabled boolean NOT NULL DEFAULT TRUE, + cpu_threshold_percent integer NOT NULL DEFAULT 80, + cpu_mode text NOT NULL DEFAULT 'total', + blocking_enabled boolean NOT NULL DEFAULT TRUE, + blocking_count_threshold integer NOT NULL DEFAULT 1, + deadlock_enabled boolean NOT NULL DEFAULT TRUE, + deadlock_count_threshold integer NOT NULL DEFAULT 1, + poison_wait_enabled boolean NOT NULL DEFAULT TRUE, + poison_wait_threshold_ms integer NOT NULL DEFAULT 500, + long_running_query_enabled boolean NOT NULL DEFAULT TRUE, + long_running_query_threshold_minutes integer NOT NULL DEFAULT 30, + tempdb_space_enabled boolean NOT NULL DEFAULT TRUE, + tempdb_space_threshold_percent integer NOT NULL DEFAULT 80, + low_disk_enabled boolean NOT NULL DEFAULT TRUE, + low_disk_threshold_percent integer NOT NULL DEFAULT 10, + low_disk_threshold_gb integer NOT NULL DEFAULT 5, + long_running_job_enabled boolean NOT NULL DEFAULT TRUE, + long_running_job_multiplier integer NOT NULL DEFAULT 3, + failed_job_enabled boolean NOT NULL DEFAULT TRUE, + failed_job_lookback_minutes integer NOT NULL DEFAULT 60, + cooldown_minutes integer NOT NULL DEFAULT 5, + excluded_databases text[] NOT NULL DEFAULT '{}'::text[], + analysis_enabled boolean NOT NULL DEFAULT TRUE, + analysis_interval_minutes integer NOT NULL DEFAULT 30, + analysis_notifications_enabled boolean NOT NULL DEFAULT TRUE, + analysis_notify_severity double precision NOT NULL DEFAULT 1.5, + modified_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC') +); + +/* 3. config_notification — single row: SMTP + webhook delivery. Non-secret fields plus the SMTP + DPAPI blob (smtp_encrypted_password); webhook URLs/proxies carry as darling.json holds them. */ +CREATE TABLE IF NOT EXISTS config.config_notification ( + id smallint NOT NULL PRIMARY KEY DEFAULT 1 CHECK (id = 1), + smtp_host text NOT NULL DEFAULT '', + smtp_port integer NOT NULL DEFAULT 587, + smtp_use_ssl boolean NOT NULL DEFAULT TRUE, + smtp_username text, + smtp_encrypted_password text, + smtp_from_address text NOT NULL DEFAULT '', + smtp_recipients text NOT NULL DEFAULT '', + email_cooldown_minutes integer NOT NULL DEFAULT 15, + teams_url text NOT NULL DEFAULT '', + teams_proxy text NOT NULL DEFAULT '', + slack_url text NOT NULL DEFAULT '', + slack_proxy text NOT NULL DEFAULT '', + modified_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC') +); + +/* 4. config_collector_schedules — SPARSE per-collector overrides layered on CollectorScheduleDefaults. + Absent row / NULL column = code default; server_id NULL = fleet-wide. A PRIMARY KEY cannot span a + nullable server_id, so two partial-unique indexes enforce one fleet-wide row + one per-server row + per collector. */ +CREATE TABLE IF NOT EXISTS config.config_collector_schedules ( + server_id integer, + collector_name text NOT NULL, + frequency_minutes integer CHECK (frequency_minutes >= 0), + retention_days integer CHECK (retention_days >= 1), + enabled boolean NOT NULL DEFAULT TRUE +); +CREATE UNIQUE INDEX IF NOT EXISTS ux_config_collector_schedules_fleet + ON config.config_collector_schedules (collector_name) WHERE server_id IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS ux_config_collector_schedules_server + ON config.config_collector_schedules (server_id, collector_name) WHERE server_id IS NOT NULL; + +/* 5. config_service — single row: the service-wide flags + the config_version reload beacon. */ +CREATE TABLE IF NOT EXISTS config.config_service ( + id smallint NOT NULL PRIMARY KEY DEFAULT 1 CHECK (id = 1), + paused boolean NOT NULL DEFAULT FALSE, + capture_plans boolean NOT NULL DEFAULT TRUE, + mcp_enabled boolean NOT NULL DEFAULT FALSE, + mcp_port integer NOT NULL DEFAULT 5152, + config_version bigint NOT NULL DEFAULT 0, + updated_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + updated_by text +); + +/* --- B. Command plane: the Viewer enqueues, the service (Stage 2) claims/executes/reports. --- */ + +/* 6. config_command — the imperative queue. GENERATED ALWAYS AS IDENTITY (a natural queue key that + needs no sequence USAGE grant for admin INSERTs, unlike serial). */ +CREATE TABLE IF NOT EXISTS config.config_command ( + command_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + created_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + requested_by text, + command_type text NOT NULL, + target_server_id integer, + args_json jsonb, + status text NOT NULL DEFAULT 'pending', + claimed_at timestamp, + completed_at timestamp, + result_status text, + result_json jsonb, + service_instance text +); +CREATE INDEX IF NOT EXISTS idx_config_command_status ON config.config_command (status); + +/* --- C. The config_version reload beacon (bump triggers). --- */ + +/* Statement-level bump on the four desired-state tables: any write increments the beacon so the + Viewer can never forget to signal. Targets config_service by qualified name (SECURITY INVOKER — + both writers, the owner during seed and admin via the Viewer, hold UPDATE on config_service). */ +CREATE OR REPLACE FUNCTION config.config_bump_version() RETURNS trigger +LANGUAGE plpgsql AS $bump$ +BEGIN + UPDATE config.config_service + SET config_version = config_version + 1, + updated_at = (now() AT TIME ZONE 'UTC') + WHERE id = 1; + RETURN NULL; +END; +$bump$; + +/* Direct writes to config_service (pause/capture/mcp) self-bump the beacon without recursion: the + BEFORE-UPDATE trigger increments NEW.config_version only when the writer did not already change it + (so the config_bump_version UPDATE above, which sets config_version explicitly, is not doubled). */ +CREATE OR REPLACE FUNCTION config.config_service_bump() RETURNS trigger +LANGUAGE plpgsql AS $svc$ +BEGIN + IF NEW.config_version = OLD.config_version THEN + NEW.config_version := OLD.config_version + 1; + END IF; + NEW.updated_at := (now() AT TIME ZONE 'UTC'); + RETURN NEW; +END; +$svc$; + +DROP TRIGGER IF EXISTS trg_bump_monitored_servers ON config.config_monitored_servers; +CREATE TRIGGER trg_bump_monitored_servers + AFTER INSERT OR UPDATE OR DELETE ON config.config_monitored_servers + FOR EACH STATEMENT EXECUTE FUNCTION config.config_bump_version(); + +DROP TRIGGER IF EXISTS trg_bump_alert_settings ON config.config_alert_settings; +CREATE TRIGGER trg_bump_alert_settings + AFTER INSERT OR UPDATE OR DELETE ON config.config_alert_settings + FOR EACH STATEMENT EXECUTE FUNCTION config.config_bump_version(); + +DROP TRIGGER IF EXISTS trg_bump_notification ON config.config_notification; +CREATE TRIGGER trg_bump_notification + AFTER INSERT OR UPDATE OR DELETE ON config.config_notification + FOR EACH STATEMENT EXECUTE FUNCTION config.config_bump_version(); + +DROP TRIGGER IF EXISTS trg_bump_collector_schedules ON config.config_collector_schedules; +CREATE TRIGGER trg_bump_collector_schedules + AFTER INSERT OR UPDATE OR DELETE ON config.config_collector_schedules + FOR EACH STATEMENT EXECUTE FUNCTION config.config_bump_version(); + +DROP TRIGGER IF EXISTS trg_service_self_bump ON config.config_service; +CREATE TRIGGER trg_service_self_bump + BEFORE UPDATE ON config.config_service + FOR EACH ROW EXECUTE FUNCTION config.config_service_bump(); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (17, 'config-control-plane', now()::timestamp); +-- ===BATCH=== V18 alert-delivery-mode + +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS delivery_mode text NOT NULL DEFAULT 'Summary'; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS per_event_max integer NOT NULL DEFAULT 5; +ALTER TABLE config.config_monitored_servers ADD COLUMN IF NOT EXISTS alert_delivery_mode_override text; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (18, 'alert-delivery-mode', now()::timestamp); +-- ===BATCH=== V19 analysis-state-marker + +CREATE TABLE IF NOT EXISTS collect.analysis_state ( + server_id integer NOT NULL PRIMARY KEY, + insufficient_data boolean NOT NULL DEFAULT FALSE, + message text, + analysis_time timestamp NOT NULL +); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (19, 'analysis-state-marker', now()::timestamp); +-- ===BATCH=== V20 alert-tuning-knobs + +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_max_results integer NOT NULL DEFAULT 5; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_exclude_sp_server_diagnostics boolean NOT NULL DEFAULT TRUE; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_exclude_wait_for boolean NOT NULL DEFAULT TRUE; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_exclude_backups boolean NOT NULL DEFAULT TRUE; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_exclude_misc_waits boolean NOT NULL DEFAULT TRUE; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS long_running_query_exclude_cdc boolean NOT NULL DEFAULT TRUE; +ALTER TABLE config.config_alert_settings ADD COLUMN IF NOT EXISTS notify_connection_changes boolean NOT NULL DEFAULT TRUE; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (20, 'alert-tuning-knobs', now()::timestamp); +-- ===BATCH=== V21 default-trace-events-collector + +CREATE TABLE IF NOT EXISTS collect.default_trace_events ( + default_trace_event_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_name text, + event_class integer, + spid integer, + database_name text, + database_id integer, + login_name text, + host_name text, + application_name text, + object_name text, + filename text, + integer_data bigint, + integer_data_2 bigint, + text_data text, + session_login_name text, + error_number integer, + severity integer, + state integer, + event_sequence bigint, + duration_us bigint, + end_time timestamp +); + +CREATE INDEX IF NOT EXISTS idx_default_trace_events_time ON collect.default_trace_events(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (21, 'default-trace-events-collector', now()::timestamp); +-- ===BATCH=== V22 index-object-stats-latest-index + +CREATE INDEX IF NOT EXISTS idx_index_object_stats_latest ON collect.index_object_stats (server_id, database_id, object_id, index_id, collection_time DESC); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (22, 'index-object-stats-latest-index', now()::timestamp); +-- ===BATCH=== V23 collection-log-hypertable + +/* V23: best-effort UPGRADE fast-path that converts collect.collection_log to a TimescaleDB hypertable + compresses + it, mirroring the collector hypertables. GUARDED on the extension (plain PostgreSQL skips it, keeping the table a + heap with batched-DELETE retention) and wrapped in EXCEPTION WHEN OTHERS so it can NEVER abort the startup-critical + migration. The AUTHORITATIVE conversion is TimescaleSupport.EnsureCollectionLogHypertableAsync at runtime (after + CREATE EXTENSION), which is the proven-live path and self-heals a fresh store this guard skipped. collection_log is + NOT in the collector catalog, so the runtime catalog loops never touch it. */ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') THEN + PERFORM create_hypertable('collect.collection_log', by_range('collection_time', INTERVAL '1 days'), if_not_exists => true, migrate_data => true); + ALTER TABLE collect.collection_log SET (timescaledb.compress, timescaledb.compress_segmentby = 'server_id'); + PERFORM add_compression_policy('collect.collection_log', compress_after => INTERVAL '1 days', if_not_exists => true); + END IF; +EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'V23: deferred collection_log hypertable conversion to the runtime path (%): %', SQLSTATE, SQLERRM; +END +$$; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (23, 'collection-log-hypertable', now()::timestamp); +-- ===BATCH=== V24 job-history-collector + +CREATE TABLE IF NOT EXISTS collect.job_history ( + job_history_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + instance_id bigint, + job_id text, + job_name text, + job_enabled boolean, + category_name text, + step_id integer, + step_name text, + run_status integer, + run_status_desc text, + run_datetime timestamp, + run_duration_seconds bigint, + retries_attempted integer, + message text +); + +CREATE INDEX IF NOT EXISTS idx_job_history_time ON collect.job_history(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (24, 'job-history-collector', now()::timestamp); +-- ===BATCH=== V25 agent-status-collector + +CREATE TABLE IF NOT EXISTS collect.agent_status ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + agent_running boolean, + agent_status_desc text, + agent_startup_desc text, + next_scheduled_run timestamp +); + +CREATE INDEX IF NOT EXISTS idx_agent_status_time ON collect.agent_status(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (25, 'agent-status-collector', now()::timestamp); +-- ===BATCH=== V26 generic-webhook-channel + +ALTER TABLE config.config_notification ADD COLUMN IF NOT EXISTS generic_url text NOT NULL DEFAULT ''; +ALTER TABLE config.config_notification ADD COLUMN IF NOT EXISTS generic_headers text NOT NULL DEFAULT ''; +ALTER TABLE config.config_notification ADD COLUMN IF NOT EXISTS generic_body_template text NOT NULL DEFAULT ''; +ALTER TABLE config.config_notification ADD COLUMN IF NOT EXISTS generic_proxy text NOT NULL DEFAULT ''; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (26, 'generic-webhook-channel', now()::timestamp); +-- ===BATCH=== V27 deadlocks-database-name + +ALTER TABLE deadlocks ADD COLUMN IF NOT EXISTS database_name text; +CREATE OR REPLACE VIEW v_deadlocks AS SELECT * FROM deadlocks; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (27, 'deadlocks-database-name', now()::timestamp); +-- ===BATCH=== V28 query-store-replica-role + +ALTER TABLE query_store_stats ADD COLUMN IF NOT EXISTS replica_role text; +CREATE OR REPLACE VIEW v_query_store_stats AS SELECT * FROM query_store_stats; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (28, 'query-store-replica-role', now()::timestamp); +-- ===BATCH=== V29 long-query-completions-collector + +CREATE TABLE IF NOT EXISTS collect.long_query_completions ( + long_query_completion_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + event_time timestamp, + event_type text, + database_id integer, + database_name text, + session_id integer, + client_app_name text, + client_pid integer, + nt_username text, + server_principal_name text, + query_hash text, + event_sequence bigint, + duration_microseconds bigint, + cpu_time_microseconds bigint, + physical_reads bigint, + logical_reads bigint, + writes bigint, + row_count bigint, + result text, + statement_text text, + object_name text +); + +CREATE INDEX IF NOT EXISTS idx_long_query_completions_time ON collect.long_query_completions(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (29, 'long-query-completions-collector', now()::timestamp); +-- ===BATCH=== V30 web-dashboard-config + +ALTER TABLE config.config_service ADD COLUMN IF NOT EXISTS web_enabled boolean NOT NULL DEFAULT FALSE; +ALTER TABLE config.config_service ADD COLUMN IF NOT EXISTS web_port integer NOT NULL DEFAULT 5153; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (30, 'web-dashboard-config', now()::timestamp); +-- ===BATCH=== V31 custom-views-table + +CREATE TABLE IF NOT EXISTS config.custom_views ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name text NOT NULL UNIQUE, + definition jsonb NOT NULL, + description text, + version integer NOT NULL DEFAULT 1, + created_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + updated_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + updated_by text +); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (31, 'custom-views-table', now()::timestamp); +-- ===BATCH=== V32 server-tags + +CREATE TABLE IF NOT EXISTS config.server_tags ( + id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name text NOT NULL, + parent_id integer REFERENCES config.server_tags(id) ON DELETE CASCADE, + sort_order integer NOT NULL DEFAULT 0, + created_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC') +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_server_tags_parent_name + ON config.server_tags (COALESCE(parent_id, 0), lower(name)); + +CREATE TABLE IF NOT EXISTS config.server_tag_map ( + server_id integer NOT NULL, + tag_id integer NOT NULL REFERENCES config.server_tags(id) ON DELETE CASCADE, + PRIMARY KEY (server_id, tag_id) +); + +CREATE INDEX IF NOT EXISTS idx_server_tag_map_tag + ON config.server_tag_map (tag_id); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (32, 'server-tags', now()::timestamp); +-- ===BATCH=== V33 connection-alert-refire + +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS notify_connection_down_at_startup boolean NOT NULL DEFAULT false, + ADD COLUMN IF NOT EXISTS connection_refire_minutes integer NOT NULL DEFAULT 0; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (33, 'connection-alert-refire', now()::timestamp); +-- ===BATCH=== V34 availability-group-collectors + +CREATE TABLE IF NOT EXISTS collect.ag_replica_states ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + ag_name text, + replica_server_name text, + role_desc text, + operational_state_desc text, + connected_state_desc text, + recovery_health_desc text, + synchronization_health_desc text, + availability_mode_desc text, + failover_mode_desc text, + endpoint_url text +); + +CREATE INDEX IF NOT EXISTS idx_ag_replica_states_time ON collect.ag_replica_states(server_id, collection_time); + +CREATE TABLE IF NOT EXISTS collect.ag_database_replica_states ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + ag_name text, + database_name text, + replica_server_name text, + is_local boolean, + synchronization_state_desc text, + last_hardened_lsn text, + last_commit_lsn text, + log_send_queue_size bigint, + redo_queue_size bigint, + log_send_rate bigint, + redo_rate bigint, + is_suspended boolean, + suspend_reason_desc text, + availability_mode_desc text, + secondary_lag_seconds bigint +); + +CREATE INDEX IF NOT EXISTS idx_ag_database_replica_states_time ON collect.ag_database_replica_states(server_id, collection_time); +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (34, 'availability-group-collectors', now()::timestamp); +-- ===BATCH=== V35 availability-group-alerts + +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS notify_ag_health boolean NOT NULL DEFAULT true, + ADD COLUMN IF NOT EXISTS ag_lag_alert_seconds integer NOT NULL DEFAULT 300, + ADD COLUMN IF NOT EXISTS ag_redo_queue_alert_kb bigint NOT NULL DEFAULT 0; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (35, 'availability-group-alerts', now()::timestamp); +-- ===BATCH=== V36 ag-latency-columns + +ALTER TABLE collect.ag_database_replica_states + ADD COLUMN IF NOT EXISTS last_commit_time timestamp, + ADD COLUMN IF NOT EXISTS last_hardened_time timestamp, + ADD COLUMN IF NOT EXISTS last_redone_time timestamp, + ADD COLUMN IF NOT EXISTS last_received_time timestamp, + ADD COLUMN IF NOT EXISTS est_redo_completion_time_min double precision, + ADD COLUMN IF NOT EXISTS est_send_drain_time_min double precision; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (36, 'ag-latency-columns', now()::timestamp); +-- ===BATCH=== V37 ag-local-replica-and-disconnect-refire + +ALTER TABLE collect.ag_replica_states + ADD COLUMN IF NOT EXISTS is_local boolean; + +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS ag_disconnect_refire_minutes integer NOT NULL DEFAULT 0; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (37, 'ag-local-replica-and-disconnect-refire', now()::timestamp); +-- ===BATCH=== V38 query-payload-dimensions +/* V38: hash-keyed dimension tables for query_stats / procedure_stats payloads (#1767). + query_text/query_plan_xml stored inline per row were 94% of a 250 GB field store; + a measured 1-hour window held 3,166 MB of payload across 23 MB of distinct content. + ZERO-REWRITE: the inline columns stay, new rows leave them NULL, readers coalesce, + and raw retention ages the inline copies out on its own. */ + +CREATE TABLE IF NOT EXISTS query_text_dim ( + digest bytea NOT NULL PRIMARY KEY, + query_text text NOT NULL, + last_seen timestamp NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_query_text_dim_last_seen ON query_text_dim(last_seen); + +CREATE TABLE IF NOT EXISTS query_plan_dim ( + digest bytea NOT NULL PRIMARY KEY, + query_plan_xml text NOT NULL, + last_seen timestamp NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_query_plan_dim_last_seen ON query_plan_dim(last_seen); + +ALTER TABLE query_stats ADD COLUMN IF NOT EXISTS query_text_digest bytea; +ALTER TABLE query_stats ADD COLUMN IF NOT EXISTS query_plan_digest bytea; +ALTER TABLE procedure_stats ADD COLUMN IF NOT EXISTS query_plan_digest bytea; + +/* v_query_stats resolves each row's payload: the inline column for rows written before + the dimension tables existed, the dimension row for every row written since (#1767). + Column list generated from the collector definition so it can never go stale. */ +CREATE OR REPLACE VIEW v_query_stats AS +SELECT + f.collection_id, + f.collection_time, + f.server_id, + f.server_name, + f.database_name, + f.query_hash, + f.query_plan_hash, + f.creation_time, + f.last_execution_time, + f.execution_count, + f.total_worker_time, + f.total_elapsed_time, + f.total_logical_reads, + f.total_logical_writes, + f.total_physical_reads, + f.total_clr_time, + f.total_rows, + f.total_spills, + f.min_worker_time, + f.max_worker_time, + f.min_elapsed_time, + f.max_elapsed_time, + f.min_physical_reads, + f.max_physical_reads, + f.min_rows, + f.max_rows, + f.min_dop, + f.max_dop, + f.min_grant_kb, + f.max_grant_kb, + f.min_used_grant_kb, + f.max_used_grant_kb, + f.min_ideal_grant_kb, + f.max_ideal_grant_kb, + f.min_reserved_threads, + f.max_reserved_threads, + f.min_used_threads, + f.max_used_threads, + f.min_spills, + f.max_spills, + COALESCE(f.query_text, qtd.query_text) AS query_text, + COALESCE(f.query_plan_xml, qpd.query_plan_xml) AS query_plan_xml, + f.sql_handle, + f.plan_handle, + f.delta_execution_count, + f.delta_worker_time, + f.delta_elapsed_time, + f.delta_logical_reads, + f.delta_logical_writes, + f.delta_physical_reads, + f.delta_rows, + f.delta_spills, + f.plan_generation_num, + f.sample_interval_seconds, + f.query_text_digest, + f.query_plan_digest +FROM query_stats AS f +LEFT JOIN query_text_dim AS qtd ON qtd.digest = f.query_text_digest +LEFT JOIN query_plan_dim AS qpd ON qpd.digest = f.query_plan_digest +; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (38, 'query-payload-dimensions', now()::timestamp); +-- ===BATCH=== V39 dim-feeding-fact-floor-indexes + +CREATE INDEX IF NOT EXISTS ix_query_stats_digest_floor + ON query_stats (collection_time) + WHERE query_text_digest IS NOT NULL OR query_plan_digest IS NOT NULL; + +CREATE INDEX IF NOT EXISTS ix_procedure_stats_digest_floor + ON procedure_stats (collection_time) + WHERE query_plan_digest IS NOT NULL; +INSERT INTO darling_schema_version (version, name, applied_at) VALUES (39, 'dim-feeding-fact-floor-indexes', now()::timestamp); diff --git a/Darling/Darling.Tests/HostObjectRollupLiveTests.cs b/Darling/Darling.Tests/HostObjectRollupLiveTests.cs new file mode 100644 index 000000000..713d7c650 --- /dev/null +++ b/Darling/Darling.Tests/HostObjectRollupLiveTests.cs @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Darling.Tests; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace PerformanceMonitor.Darling.Tests; + +/// +/// #2235: group_by: "host_object" — rolling a procedure's dynamic-SQL fragments into one row. +/// +/// The defect. query_hash is a SHAPE hash, so dynamic SQL built with per-value literals +/// fragments one logical statement across as many hashes as there are literal sets — measured at 21 for a single +/// API.GetInventoryWithLabsV5 statement on prod-pos-use2-apex-01. Ranking by hash therefore +/// STRUCTURALLY cannot surface it: two fragments together were 58-65% of the instance's worker_time in every +/// window sampled, while the hash never entered the 168-hour top 20 and the ranking as a whole accounted for +/// roughly a tenth of the box's CPU. Nothing in the output said so. +/// +/// Why this is a LIVE test and not an SQL-text pin. The whole change is a GROUP BY — an +/// Assert.Contains on the clause would restate the code rather than test it. The two properties that +/// matter are what the grouping DOES to rows, and one of them (ad-hoc rows must not pool) is a silent +/// mis-attribution rather than an error, so it has to be observed on real rows through real Postgres grouping +/// semantics. +/// +[Collection("live-postgres")] +public sealed class HostObjectRollupLiveTests +{ + private const string ServerName = "darling-host-rollup-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private const string Db = "apex"; + + [Fact] + public async Task HostObjectRollup_CollapsesProcFragments_ButNeverPoolsAdHoc_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live host-object rollup test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await CleanupAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var succeeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var now = DarlingMcpTestData.Naive(DateTime.UtcNow); + + /* The reported shape: one proc's statement fragmented across three literal variants, each too + small to rank, but the largest consumer once summed. */ + await PlantAsync(connection, ct, now.AddMinutes(-9), "0xFRAG1", "API.GetInventoryWithLabsV5", + "insert #result select ... where LocId = 101", 100L); + await PlantAsync(connection, ct, now.AddMinutes(-8), "0xFRAG2", "API.GetInventoryWithLabsV5", + "insert #result select ... where LocId = 102", 100L); + await PlantAsync(connection, ct, now.AddMinutes(-7), "0xFRAG3", "API.GetInventoryWithLabsV5", + "insert #result select ... where LocId = 103", 100L); + + /* A second proc, so the rollup is proven to group per host object rather than per database. */ + await PlantAsync(connection, ct, now.AddMinutes(-6), "0xOTHER1", "dbo.SomethingElse", + "select 1 from dbo.Other where Id = 1", 40L); + + /* THE TRAP ROWS: ad-hoc statements, host_object_name NULL. A bare GROUP BY host_object_name + would pool these two unrelated statements into one row — a worse attribution bug than the one + being fixed. They must stay one row each. */ + await PlantAsync(connection, ct, now.AddMinutes(-5), "0xADHOC1", null, "SELECT AdHocOne FROM T1", 250L); + await PlantAsync(connection, ct, now.AddMinutes(-4), "0xADHOC2", null, "SELECT AdHocTwo FROM T2", 240L); + + /* ---- default grouping is UNCHANGED: every fragment is its own row and none of them wins. */ + var perHash = await DarlingDataReader.GetTopQueriesByCpuAsync( + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 20, databaseName: null, + rollUpByHostObject: false, cancellationToken: ct); + + Assert.Equal(3, perHash.Count(r => r.HostObjectName == "API.GetInventoryWithLabsV5")); + Assert.All(perHash, r => Assert.Equal(1, r.DistinctQueryHashes)); + /* The reported failure, reproduced: the fragmented statement does NOT rank first per-hash. */ + Assert.NotEqual("API.GetInventoryWithLabsV5", perHash[0].HostObjectName); + + /* ---- rollup: the three fragments become ONE row that now outranks everything. */ + var rolled = await DarlingDataReader.GetTopQueriesByCpuAsync( + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 20, databaseName: null, + rollUpByHostObject: true, cancellationToken: ct); + + var proc = Assert.Single(rolled, r => r.HostObjectName == "API.GetInventoryWithLabsV5"); + Assert.Equal(3, proc.DistinctQueryHashes); + Assert.Equal(300L, proc.TotalExecutions); + /* THE POINT: summed, it is the top consumer — which is what a per-hash ranking could not show. */ + Assert.Equal("API.GetInventoryWithLabsV5", rolled[0].HostObjectName); + + /* The other proc stays its own row: grouped per host object, not per database. */ + var other = Assert.Single(rolled, r => r.HostObjectName == "dbo.SomethingElse"); + Assert.Equal(1, other.DistinctQueryHashes); + + /* ---- THE TRAP, pinned: ad-hoc rows are STILL one per hash, not pooled into a NULL-host row. */ + var adHoc = rolled.Where(r => r.HostObjectName is null).ToList(); + Assert.Equal(2, adHoc.Count); + Assert.All(adHoc, r => Assert.Equal(1, r.DistinctQueryHashes)); + Assert.Contains(adHoc, r => r.QueryHash == "0xADHOC1"); + Assert.Contains(adHoc, r => r.QueryHash == "0xADHOC2"); + /* And their text is their OWN, not a neighbour's — the LATERAL must still key on hash when the + host is null, or an ad-hoc row would borrow an unrelated statement's text. */ + Assert.Equal("SELECT AdHocOne FROM T1", adHoc.Single(r => r.QueryHash == "0xADHOC1").QueryText); + Assert.Equal("SELECT AdHocTwo FROM T2", adHoc.Single(r => r.QueryHash == "0xADHOC2").QueryText); + + /* ---- totals are conserved: a rollup must redistribute CPU, never invent or lose it. */ + Assert.Equal(perHash.Sum(r => r.TotalCpuUs), rolled.Sum(r => r.TotalCpuUs)); + Assert.Equal(perHash.Sum(r => r.TotalExecutions), rolled.Sum(r => r.TotalExecutions)); + + succeeded = true; + } + finally + { + /* #1902: teardown on its OWN connection, never the body's. A finally that cleans up on the + body's connection throws from the finally and REPLACES the body's exception with the + teardown's — and it is the body's failure that closed the connection, so the teardown fails + because of the very thing it then hides. Enforced by + LiveCleanupConversionRatchetTests.NoLiveTestCleansUpOnItsOwnBodysConnection. */ + await LiveStoreCleanup.RunAsync(connectionString!, succeeded, async (cleanup, cleanupCt) => + await CleanupAsync(cleanup, cleanupCt)); + } + } + + private static async Task PlantAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, + string queryHash, string? hostObject, string queryText, long weight) + { + var sqlHandle = "0xSQLH" + Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(queryText)))[..12]; + var digest = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(queryText)); + + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, + query_hash, query_plan_hash, sql_handle, plan_handle, query_text, + query_text_digest, host_object_name, delta_execution_count, + delta_worker_time, delta_elapsed_time, delta_logical_reads, min_dop, max_dop) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)", + CollectionIdGenerator.Next(), at, ServerId, ServerName, Db, + queryHash, "0xPLANHASH", sqlHandle, "0xPLANH", queryText, + digest, (object?)hostObject ?? DBNull.Value, + weight, weight * 1000L, weight * 2000L, weight * 10L, 1, 1); + } + + private static async Task CleanupAsync(NpgsqlConnection connection, CancellationToken ct) => + await DarlingMcpTestData.ExecAsync(connection, ct, + $"DELETE FROM query_stats WHERE server_id = {ServerId}; DELETE FROM servers WHERE server_id = {ServerId}"); +} diff --git a/Darling/Darling.Tests/IncidentOccurrenceAccumulatorTests.cs b/Darling/Darling.Tests/IncidentOccurrenceAccumulatorTests.cs new file mode 100644 index 000000000..1e59e7df6 --- /dev/null +++ b/Darling/Darling.Tests/IncidentOccurrenceAccumulatorTests.cs @@ -0,0 +1,437 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// Coverage for and the V61 store surface it needs (#2216). +/// +/// The property under test is the one the feature exists for: the number an alert reports must be +/// recoverable by a consumer that only sees THROTTLED deliveries. That makes the interesting cases the ones +/// where the underlying gauge is uninformative — it held steady while events arrived and aged out, it fell +/// and rose back to the same level, or the process died and came back — because those are exactly where a +/// naive "remember the last count" implementation reports a number that is wrong in the confident +/// direction. +/// +public sealed class IncidentOccurrenceAccumulatorTests +{ + private static readonly DateTime T0 = new(2026, 8, 12, 14, 0, 0, DateTimeKind.Utc); + private static readonly TimeSpan Window = TimeSpan.FromHours(1); + + private const string KeyA = "aaaa1111"; + private const string KeyB = "bbbb2222"; + + private static AlertIncident Incident(string dedupKey, int windowCount) => + new(dedupKey, new[] { "dbo.Users" }, windowCount); + + private static IncidentOccurrenceAccumulator.Result Accumulate( + IReadOnlyList incidents, + IReadOnlyDictionary? persisted, + DateTime now) => + IncidentOccurrenceAccumulator.Accumulate(incidents, persisted, now, Window); + + [Fact] + public void FirstContact_CountsTheWholeWindow_AndStampsTheIncidentStart() + { + /* With no persisted state the accumulator cannot know which of the events already in the window it + would have counted before, so the total starts at the window count — the same first-read behavior + the gauge has always had. The stamp is what lets a consumer see that this is a NEW incident. */ + var result = Accumulate(new[] { Incident(KeyA, 3) }, persisted: null, T0); + + Assert.Equal(3L, result.Incidents[0].TotalOccurrences); + Assert.Equal(T0, result.Incidents[0].IncidentStartedUtc); + Assert.True(result.Changed); + + var state = result.States[KeyA]; + Assert.Equal(3L, state.TotalOccurrences); + Assert.Equal(3, state.ObservedWindowCount); + Assert.Equal(T0, state.IncidentStartedUtc); + Assert.Equal(T0, state.LastObservedUtc); + } + + [Fact] + public void OccurrencesArrivingDuringTheCooldown_AreCountedOnTheNextDelivery() + { + /* THE REPORTED CASE. Two deadlocks are delivered, three more happen while the per-fingerprint + cooldown suppresses delivery, and the next delivery's gauge reads 5. The total must be 5 — not 2 + (only what was delivered) and not 7 (the two deliveries added together). */ + var first = Accumulate(new[] { Incident(KeyA, 2) }, persisted: null, T0); + var second = Accumulate(new[] { Incident(KeyA, 5) }, first.States, T0.AddMinutes(5)); + + Assert.Equal(2L, first.Incidents[0].TotalOccurrences); + Assert.Equal(5L, second.Incidents[0].TotalOccurrences); + + /* Same incident throughout — the start time did not move, which is how the consumer knows the 5 is + a continuation of the 2 rather than a fresh incident that happens to read 5. */ + Assert.Equal(T0, second.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void AgingOut_ThenRisingBackToTheSameLevel_CountsTheNewEventsOnce() + { + /* The case a single watermark cannot express. The gauge goes 5 -> 3 (two aged out) -> 5 (two new + arrived). A mark that did not decay would see 5 then 5 and count nothing; a mark that decayed but + re-counted from zero would count 5 again. The answer is 7: five, then two more. */ + var a = Accumulate(new[] { Incident(KeyA, 5) }, persisted: null, T0); + var b = Accumulate(new[] { Incident(KeyA, 3) }, a.States, T0.AddMinutes(10)); + var c = Accumulate(new[] { Incident(KeyA, 5) }, b.States, T0.AddMinutes(20)); + + Assert.Equal(5L, a.Incidents[0].TotalOccurrences); + Assert.Equal(5L, b.Incidents[0].TotalOccurrences); /* aging out is not an occurrence */ + Assert.Equal(7L, c.Incidents[0].TotalOccurrences); + Assert.Equal(5, c.States[KeyA].ObservedWindowCount); + } + + [Fact] + public void TheTotal_NeverDecreases_AcrossAFallingGauge() + { + /* Monotonicity is the whole contract: a consumer that SETs a gauge from this field must never see it + walk backwards while one incident is in flight. */ + var states = (IReadOnlyDictionary?)null; + long previous = 0; + var now = T0; + + foreach (var windowCount in new[] { 4, 6, 2, 1, 5, 3, 9 }) + { + var result = Accumulate(new[] { Incident(KeyA, windowCount) }, states, now); + var total = result.Incidents[0].TotalOccurrences!.Value; + + Assert.True(total >= previous, + $"total went backwards: {previous} -> {total} at window count {windowCount}"); + previous = total; + states = result.States; + now = now.AddMinutes(5); + } + + /* 4, then +2 (6), then nothing while it falls to 2 and 1, then +4 (5), nothing at 3, then +6 (9). */ + Assert.Equal(16L, previous); + } + + [Fact] + public void ObservingOnlyAtDeliveryTime_Undercounts_WhichIsWhyTheEngineObservesEverySweep() + { + /* PR #2221's review found this: with observation only at delivery time, an event the window RETIRES + during a cooldown cancels an arrival in the gauge and the arrival becomes invisible. + + Delivery A sees 10. Before delivery B, three age out (window 7) and four arrive (window 11). Seen + only at the two deliveries, the rise from 10 to 11 counts ONE — the correct answer is four. Seen on + the sweeps in between, both movements are observed and the total is exact. + + This is the pin on the reason the engine calls Accumulate on every sweep rather than inside the + Fire branch. The pure function was always capable of the right answer; the first cut of the wiring + did not call it often enough to get it. */ + var deliveryA = Accumulate(new[] { Incident(KeyA, 10) }, persisted: null, T0); + + var deliveryOnly = Accumulate(new[] { Incident(KeyA, 11) }, deliveryA.States, T0.AddMinutes(5)); + Assert.Equal(11L, deliveryOnly.Incidents[0].TotalOccurrences); + + var sweep1 = Accumulate(new[] { Incident(KeyA, 7) }, deliveryA.States, T0.AddMinutes(2)); + var sweep2 = Accumulate(new[] { Incident(KeyA, 11) }, sweep1.States, T0.AddMinutes(4)); + Assert.Equal(14L, sweep2.Incidents[0].TotalOccurrences); + + /* One incident throughout, either way. */ + Assert.Equal(T0, sweep2.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void ObservingEverySweep_DoesNotMeanWritingEverySweep() + { + /* Per-sweep observation would otherwise put a store round trip on every metric of every server on + every sweep. A flat gauge is not a write; a moved total is; and the heartbeat fires at half the + horizon so a live incident whose gauge never moves cannot sit long enough to judge ITSELF stale. */ + var seeded = Accumulate(new[] { Incident(KeyA, 4) }, persisted: null, T0); + Assert.True(seeded.Changed); + + Assert.False(Accumulate(new[] { Incident(KeyA, 4) }, seeded.States, T0.AddMinutes(1)).Changed); + Assert.True(Accumulate(new[] { Incident(KeyA, 6) }, seeded.States, T0.AddMinutes(1)).Changed); + + /* Half of the one-hour horizon. */ + Assert.True(Accumulate(new[] { Incident(KeyA, 4) }, seeded.States, T0.AddMinutes(31)).Changed); + + /* A fingerprint leaving the window is a write even when the survivor held steady — the replace-the-set + contract has to record the removal. */ + var two = Accumulate(new[] { Incident(KeyA, 4), Incident(KeyB, 2) }, persisted: null, T0); + Assert.True(Accumulate(new[] { Incident(KeyA, 4) }, two.States, T0.AddMinutes(1)).Changed); + } + + [Fact] + public void EveryFingerprintKeepsState_NotJustTheOnesTheAlertRenders() + { + /* The other half of PR #2221's review: the blocking context renders at most 10 incidents, and when + the accumulation rode inside that capped list, an 11th concurrent fingerprint had its row dropped + by the replace-the-set write and restarted from scratch the next time it surfaced. The engine now + accumulates over the UNCAPPED grouping, so the render budget cannot evict occurrence state. */ + var many = new List(); + for (int n = 0; n < 14; n++) + { + many.Add(Incident($"fp{n:00}", 1)); + } + + var first = Accumulate(many, persisted: null, T0); + Assert.Equal(14, first.States.Count); + + var second = Accumulate(many, first.States, T0.AddMinutes(1)); + Assert.Equal(T0, second.Incidents[13].IncidentStartedUtc); + Assert.Equal(1L, second.Incidents[13].TotalOccurrences); + } + + [Fact] + public void DistinctFingerprints_KeepSeparateTotals() + { + /* Why the key is the fingerprint and not (server, metric): a deadlock on one object set and a + deadlock on another are different incidents, and pooling them would report each one's total as + the sum of both. */ + var first = Accumulate(new[] { Incident(KeyA, 2), Incident(KeyB, 1) }, persisted: null, T0); + var second = Accumulate(new[] { Incident(KeyA, 2), Incident(KeyB, 4) }, first.States, T0.AddMinutes(5)); + + Assert.Equal(2L, second.Incidents[0].TotalOccurrences); + Assert.Equal(4L, second.Incidents[1].TotalOccurrences); + } + + [Fact] + public void AFingerprintRepeatedWithinOneObservation_IsNotCountedTwice() + { + /* The groupers collapse by fingerprint, so this should not arrive — but if it does, accumulating the + second appearance against the state the first one just wrote (rather than against the stale + persisted mark) is what keeps the total from doubling. */ + var result = Accumulate(new[] { Incident(KeyA, 3), Incident(KeyA, 3) }, persisted: null, T0); + + Assert.Equal(3L, result.States[KeyA].TotalOccurrences); + Assert.All(result.Incidents, i => Assert.Equal(3L, i.TotalOccurrences)); + } + + [Fact] + public void AnEmptyObservation_ClearsTheSet_ButOnlyWritesWhenThereWasState() + { + /* The falling edge. An empty result with Changed=true is the delete; with no persisted state there is + nothing to write, and saying so lets the caller skip the store entirely. */ + var seeded = Accumulate(new[] { Incident(KeyA, 2) }, persisted: null, T0); + + var cleared = Accumulate(Array.Empty(), seeded.States, T0.AddMinutes(5)); + Assert.Empty(cleared.States); + Assert.True(cleared.Changed); + + var quiet = IncidentOccurrenceAccumulator.Accumulate(null, null, T0, Window); + Assert.Empty(quiet.States); + Assert.False(quiet.Changed); + } + + [Fact] + public void AStaleRow_IsTreatedAsAbsent_SoARecurrenceIsANewIncident() + { + /* The crash case, and the reason LastObservedUtc is stored at all. A row stranded by a host that + died mid-incident must not be trusted weeks later: its high observed-mark would decay to the new + window count, the recurrence would read as nothing new, and the alert would report a stale total + under a stale start time — an undercount delivered with a confident timestamp. */ + var stranded = new Dictionary(StringComparer.Ordinal) + { + [KeyA] = new(TotalOccurrences: 40, ObservedWindowCount: 40, IncidentStartedUtc: T0, LastObservedUtc: T0), + }; + + var recurrence = Accumulate(new[] { Incident(KeyA, 2) }, stranded, T0.AddHours(3)); + + Assert.Equal(2L, recurrence.Incidents[0].TotalOccurrences); + Assert.Equal(T0.AddHours(3), recurrence.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void ARowInsideTheWindow_IsStillTrusted() + { + /* The boundary the horizon is chosen for: inside the read window a persisted row is describing the + same events the gauge is still counting, so it must be trusted or every delivery inside an + incident would restart the total. */ + var recent = new Dictionary(StringComparer.Ordinal) + { + [KeyA] = new(TotalOccurrences: 6, ObservedWindowCount: 6, IncidentStartedUtc: T0, LastObservedUtc: T0.AddMinutes(30)), + }; + + var next = Accumulate(new[] { Incident(KeyA, 8) }, recent, T0.AddMinutes(70)); + + Assert.Equal(8L, next.Incidents[0].TotalOccurrences); + Assert.Equal(T0, next.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void ARowStampedInTheFuture_IsTrustedRatherThanJudgedStale() + { + /* A clock that stepped backwards (NTP correction, host migration) must not reset a live incident's + total and start time. Only genuine age discards a row. */ + var future = new Dictionary(StringComparer.Ordinal) + { + [KeyA] = new(TotalOccurrences: 5, ObservedWindowCount: 5, IncidentStartedUtc: T0, LastObservedUtc: T0.AddHours(2)), + }; + + var next = Accumulate(new[] { Incident(KeyA, 7) }, future, T0); + + Assert.Equal(7L, next.Incidents[0].TotalOccurrences); + Assert.Equal(T0, next.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void AnIncidentWithNoFingerprint_PassesThroughWithNoTotal() + { + /* A blank dedup key cannot be keyed, and inventing one would pool unrelated incidents under a single + total. It travels unchanged, with TotalOccurrences null — "no total available" rather than 0. */ + var blank = new AlertIncident(string.Empty, new[] { "dbo.Users" }, 4); + + var result = Accumulate(new[] { blank }, persisted: null, T0); + + Assert.Null(result.Incidents[0].TotalOccurrences); + Assert.Null(result.Incidents[0].IncidentStartedUtc); + Assert.Empty(result.States); + Assert.False(result.Changed); + } + + [Fact] + public void ADisabledHorizon_TrustsEveryRow() + { + /* TimeSpan.Zero turns the staleness rule off — for tests that want the unguarded behavior, not for + hosts. Pinned so the escape hatch cannot rot into "any non-positive value means one hour". */ + var ancient = new Dictionary(StringComparer.Ordinal) + { + [KeyA] = new(TotalOccurrences: 11, ObservedWindowCount: 3, IncidentStartedUtc: T0, LastObservedUtc: T0), + }; + + var result = IncidentOccurrenceAccumulator.Accumulate( + new[] { Incident(KeyA, 5) }, ancient, T0.AddDays(30), TimeSpan.Zero); + + Assert.Equal(13L, result.Incidents[0].TotalOccurrences); + Assert.Equal(T0, result.Incidents[0].IncidentStartedUtc); + } + + [Fact] + public void TheHorizonMatchesTheEnginesReadWindow() + { + /* The horizon is only the right answer because it equals the window the gauge is computed over. If + someone widens the read window, this is the pin that says the horizon has to move with it. */ + Assert.Equal(1, AlertEngine.RollingCountWindowHours); + } + + /* ---------------- what a consumer actually reads ---------------- */ + + [Fact] + public void TheRenderer_EmitsTheTotalAndTheIncidentStart_AsTheirOwnFacts() + { + /* Downstream automation keys on fact NAMES, so the total is a new fact rather than a redefinition of + "Occurrences" — that one still means the window gauge, and changing its meaning under consumers + who already read it would be the silent kind of break. */ + var incident = new AlertIncident( + KeyA, new[] { "dbo.Users" }, OccurrenceCount: 3, WaitRange: null, DetailFields: null, + TotalOccurrences: 12, IncidentStartedUtc: T0); + + var item = AlertIncidentRenderer.BuildItem(incident, "Deadlock", includeDetailFields: false); + + Assert.Equal("3", item.Fields.Single(f => f.Label == "Occurrences").Value); + Assert.Equal("12", item.Fields.Single(f => f.Label == "Total Occurrences").Value); + Assert.Equal("2026-08-12 14:00:00Z", item.Fields.Single(f => f.Label == "Incident Since").Value); + } + + [Fact] + public void TheRenderer_EmitsTheTotalEvenWhenItEqualsTheWindowCount() + { + /* An incident's FIRST delivery has total == window. The field must still appear, or a consumer + polling for it sees it wink in and out depending on how far along the incident is. */ + var incident = new AlertIncident( + KeyA, new[] { "dbo.Users" }, OccurrenceCount: 1, WaitRange: null, DetailFields: null, + TotalOccurrences: 1, IncidentStartedUtc: T0); + + var item = AlertIncidentRenderer.BuildItem(incident, "Deadlock", includeDetailFields: false); + + Assert.Contains(item.Fields, f => f.Label == "Total Occurrences" && f.Value == "1"); + /* "Occurrences" keeps its > 1 condition — unchanged behavior for the gauge. */ + Assert.DoesNotContain(item.Fields, f => f.Label == "Occurrences"); + } + + [Fact] + public void AnIncidentWithNoTotal_RendersNeitherNewFact() + { + var incident = new AlertIncident(KeyA, new[] { "dbo.Users" }, OccurrenceCount: 4); + + var item = AlertIncidentRenderer.BuildItem(incident, "Deadlock", includeDetailFields: false); + + Assert.DoesNotContain(item.Fields, f => f.Label == "Total Occurrences"); + Assert.DoesNotContain(item.Fields, f => f.Label == "Incident Since"); + } + + [Fact] + public void TheContextJson_RoundTripsBothNewMembers() + { + /* The alert-history row has to carry them: the in-app dialog rehydrates the context, and a total that + survived delivery but not persistence would read as "no total" the moment anyone looked at it + later. */ + var context = new AlertContext(); + AlertIncidentRenderer.Apply(context, new[] + { + new AlertIncident(KeyA, new[] { "dbo.Users" }, OccurrenceCount: 3, WaitRange: null, + DetailFields: null, TotalOccurrences: 12, IncidentStartedUtc: T0), + }); + + var json = AlertContextSerializer.Serialize(context); + Assert.True(AlertContextSerializer.TryDeserialize(json, out var rehydrated)); + + var incident = Assert.Single(rehydrated!.Incidents!); + Assert.Equal(12L, incident.TotalOccurrences); + Assert.Equal(T0, incident.IncidentStartedUtc); + } + + /* ---------------- the V61 store surface ---------------- */ + + [Fact] + public void V61_MigrationIdentity_AndStorageVersionTracksTheNewestRung() + { + var v61 = PgMigrations.Scripts.Single(m => m.Version == 61); + + Assert.Equal("incident-occurrence-counters", v61.Name); + /* Invariant form, no literal to go stale: the build's schema version IS the newest + registered rung (the recurring in-flight-branch failure; converted on the V62 merge). */ + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); + + /* config.-qualified per the V17 rule — an unqualified CREATE resolves into collect (first on the + migrate session's search_path), which is the wrong schema and the wrong ACL. */ + Assert.Contains("CREATE TABLE IF NOT EXISTS config.incident_occurrences (", v61.Sql, StringComparison.Ordinal); + + /* Keyed per FINGERPRINT, not per metric — the distinction the feature rests on. */ + Assert.Contains("PRIMARY KEY (server_id, metric_name, dedup_key)", v61.Sql, StringComparison.Ordinal); + + /* bigint: a long-running incident's total is not bounded by anything the gauge is bounded by. */ + Assert.Contains("total_occurrences bigint NOT NULL", v61.Sql, StringComparison.Ordinal); + + /* The staleness stamp — without it a crash-stranded row silently undercounts the next incident. */ + Assert.Contains("last_observed_at timestamp NOT NULL", v61.Sql, StringComparison.Ordinal); + } + + [Fact] + public void ViewerSchemaGate_KnowsV61_SoAFullyMigratedStoreIsNotRefused() + { + /* The trap a StorageVersion bump sets: a probe that cannot SEE the newest migration maps every + healthy store below RequiredStoreSchemaVersion and the connect-time gate refuses it permanently. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + Assert.Contains("table_name = 'incident_occurrences'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + } + + [Fact] + public void TheCounterTable_IsNotFoldedIntoTheWatermarkTable() + { + /* Two independent reasons, one pin. The key is wrong (watermarks are per (server, metric), these are + per fingerprint), and Lite writes the watermark row with INSERT OR REPLACE over a PARTIAL column + list — so a counter column living there would be reset to its default on every fired alert, which + is precisely when it is read. If someone later "simplifies" these into one table, this fails. */ + var v61 = PgMigrations.Scripts.Single(m => m.Version == 61); + + Assert.DoesNotContain("config_edge_trigger_watermarks", v61.Sql, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/McpConfigReadAvoidsSecretColumnsTests.cs b/Darling/Darling.Tests/McpConfigReadAvoidsSecretColumnsTests.cs new file mode 100644 index 000000000..9d0c4fe9a --- /dev/null +++ b/Darling/Darling.Tests/McpConfigReadAvoidsSecretColumnsTests.cs @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// The MCP host must not ask the store for columns its own role is denied. +/// +/// Observed on the dogfood box, 2026-08-16. Startup logged +/// 42501: permission denied for table config_notification followed by "MCP could not read the +/// monitored-server registry — live plan fetch will use darling.json". Two separate defects met: +/// +/// 1. deliberately REVOKEs table-wide +/// SELECT on config_notification from BOTH viewer and mcp and re-grants only the +/// non-secret columns — the SMTP password and username and the Teams/Slack/generic/PagerDuty bearer URLs +/// stay unreadable. That carve is correct and is not what changed. The MCP host was simply asking for the +/// whole row, and a column-level denial answers for the TABLE. +/// +/// 2. Worse, and the reason one password cost the registry: every section of +/// LoadViewAsync shares ONE try/catch, so the failed notification read discarded the four reads that +/// had already succeeded. MCP therefore lost the monitored-server registry and fell back to +/// darling.json for live plan fetches — a silent capability loss whose cause named a table MCP does +/// not use. +/// +/// The fix is for the host to stop re-reading as mcp what the process already loaded +/// privileged. The first cut (#2293) merely skipped the notification row — and the failure moved to the +/// next denied column — so #2298 removed the host's own config-view read entirely: the plan-fetch resolver +/// serves the worker-published registry state. These pins hold that agreement so it cannot drift back. +/// +public sealed class McpConfigReadAvoidsSecretColumnsTests +{ + /// + /// THE FIX, as revised by #2298: the MCP host performs NO config-view read of its own at all. + /// + /// The first cut (#2293) skipped the notification row — and the failure simply moved to the next + /// denied column, because ReadMonitoredServersAsync selects encrypted_password, which the + /// section-6 secret ACL SELECT-carves from mcp. Skipping rows one 42501 at a time was chasing the + /// carve. The durable agreement with the boundary is that the host does not re-read as mcp what + /// the process already loaded privileged: the plan-fetch resolver reads the worker-published + /// MonitoredServerRegistryState instead. + /// + /// Pinned textually because reproducing it needs a live store provisioned with the least-privilege + /// roles and a connection as mcp — and the failure is invisible to every other test, because a + /// host that reads secrets it never uses works perfectly as the owner. + /// + [Fact] + public void TheMcpHostReadsNoConfigViewOfItsOwn() + { + var source = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpHostService.cs")); + + Assert.DoesNotContain("LoadViewAsync", source, StringComparison.Ordinal); + Assert.Contains("_registryState.Read()", source, StringComparison.Ordinal); + } + + /// + /// The MCP host has no use for what it was asking for. If this ever fails, MCP grew an alert-delivery + /// path and the skip above needs revisiting rather than silently starving it. + /// + [Fact] + public void TheMcpSurfaceUsesNeitherSmtpNorWebhooks() + { + var mcpDir = Path.Combine(RepoRoot(), "Darling", "PerformanceMonitor.Darling.Service", "Mcp"); + var offenders = Directory.GetFiles(mcpDir, "*.cs", SearchOption.AllDirectories) + .Where(f => + { + var text = File.ReadAllText(f); + /* The comment explaining the skip names both types, so only real USES count. */ + var code = string.Join("\n", text.Split('\n').Where(l => !l.TrimStart().StartsWith("/", StringComparison.Ordinal) && !l.TrimStart().StartsWith("*", StringComparison.Ordinal))); + return code.Contains(".Smtp", StringComparison.Ordinal) || code.Contains(".Webhooks", StringComparison.Ordinal); + }) + .Select(Path.GetFileName) + .ToArray(); + + Assert.Empty(offenders); + } + + /// + /// The reason the skip is necessary, asserted rather than remembered: the notification SELECT really does + /// name columns the carve revokes. If someone narrows that SELECT to non-secret columns only, this fails + /// and tells them the skip has become unnecessary — which is the useful direction for a guard to fail in. + /// + [Fact] + public void TheNotificationReadStillNamesCarvedSecretColumns() + { + var provider = ReadSource(Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs")); + + var start = provider.IndexOf("FROM config_notification", StringComparison.Ordinal); + Assert.True(start > 0, "the notification read moved — re-point this guard"); + var selectStart = provider.LastIndexOf("SELECT", start, StringComparison.Ordinal); + var readSql = provider[selectStart..start]; + + var carved = DarlingManagedRoles.ViewerRestrictedConfigTables + .Single(t => string.Equals(t.Table, "config_notification", StringComparison.Ordinal)) + .SecretColumns; + + var namedSecrets = carved.Where(c => readSql.Contains(c, StringComparison.Ordinal)).ToArray(); + Assert.NotEmpty(namedSecrets); + } + + private static string RepoRoot([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + while (dir is not null && !File.Exists(Path.Combine(dir, "PerformanceMonitor.sln")) && !Directory.Exists(Path.Combine(dir, ".git"))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return dir!; + } + + private static string ReadSource(string relative) => File.ReadAllText(Path.Combine(RepoRoot(), relative)); +} diff --git a/Darling/Darling.Tests/MigrationLadderPins.cs b/Darling/Darling.Tests/MigrationLadderPins.cs new file mode 100644 index 000000000..5be35a1cd --- /dev/null +++ b/Darling/Darling.Tests/MigrationLadderPins.cs @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2119: pins against the generated-rung replay hazard. Several migration rungs are assembled at +/// RUNTIME from the live schema generators (14, 38, 51, 54 today), so their SQL is whatever the +/// generator emits on the CURRENT build — not what it emitted when the rung shipped. When a later +/// rung teaches a generator a new column, every earlier generator-built rung silently re-emits SQL +/// referencing it, and a store old enough to replay that rung fails the whole ladder against a +/// table that does not have the column yet. The field failure: rung 51 re-emitted the resolving +/// view with V54's query_plan_gz, and every 3.3.0→3.4.0 upgrade died with 42703 at service +/// start. The dogfood box never sees this class — it walks each rung in the era it shipped — so +/// only a pin can. +/// +public sealed class MigrationLadderPins +{ + [Fact] + public void EveryRungReferencingTheGzColumn_PreAddsItBeforeTheViewUsesIt() + { + var column = PayloadDimensions.CompressedContentColumn; + var offenders = PgMigrations.Scripts + .Where(m => m.Sql.Contains(column, StringComparison.Ordinal)) + .Where(m => + { + /* Both guard forms establish the column ahead of use: the ALTER pre-add + ("ADD COLUMN IF NOT EXISTS query_plan_gz bytea") and V38's generated CREATE + ("query_plan_gz bytea NULL,"). The resolving view's reference is qualified + ("qpd.query_plan_gz"), so the guard index is found by the bare " bytea" + shape and must come FIRST. */ + var guard = m.Sql.IndexOf(column + " bytea", StringComparison.Ordinal); + var use = m.Sql.IndexOf("." + column, StringComparison.Ordinal); + return use >= 0 && (guard < 0 || guard > use); + }) + .Select(m => $"V{m.Version} ({m.Name})") + .ToList(); + + Assert.True(offenders.Count == 0, + $"Migration rung(s) reference {column} before anything establishes it — a store replaying " + + "that rung on current code fails the whole ladder with 42703 (#2119's field failure, which " + + "broke every 3.3.0→3.4.0 upgrade). Pre-add the column in the rung (prepend V54Sql) or move " + + $"the generated SQL to a later rung:\n{string.Join("\n", offenders)}"); + } + + [Fact] + public void TheLadder_IsStrictlyOrdered_WithNoDuplicates() + { + /* The replay math above only holds when versions are strictly increasing and unique — a + duplicated or out-of-order version would let two rungs disagree about what "already ran" + means. The historical V45 hole stays sanctioned (a gap NOBODY fills is harmless — the + stamp comparison is >, not sequence arithmetic); new gaps are the next pin's job. Cheap + to pin, catastrophic to debug from a half-migrated field store. */ + var versions = PgMigrations.Scripts.Select(m => m.Version).ToList(); + Assert.Equal(versions.OrderBy(v => v).ToList(), versions); + Assert.Equal(versions.Count, versions.Distinct().Count()); + } + + [Fact] + public void TheLadder_IsDenseAboveTheHistoricalGap() + { + /* #2226: a NEW gap is a rung some other branch intends to fill later — and the applier + ascends with `version <= currentVersion ? skip`, so a rung filled AFTER a store stamped + past it is skipped silently and forever: its objects never exist, readers of them fail + permanently, and no upgrade can repair the store. That exact window nearly opened between + two in-flight branches (one at V61, one at V62 while dev's MAX was 60; nightlies ship + from dev, so the window is real). Density makes the hazard fail at AUTHORING time, in the + author's own test run: every branch must take max(dev)+1, and two branches that both do + so collide loudly at rebase instead of coexisting into a field incident. V45 is the one + sanctioned hole, vacant for many releases and filled by nobody. */ + var above = PgMigrations.Scripts.Select(m => m.Version).Where(v => v > 45).OrderBy(v => v).ToList(); + Assert.Equal(Enumerable.Range(above[0], above.Count).ToList(), above); + } +} diff --git a/Darling/Darling.Tests/MigrationUpgradeLadderLiveTests.cs b/Darling/Darling.Tests/MigrationUpgradeLadderLiveTests.cs new file mode 100644 index 000000000..6a1acec7f --- /dev/null +++ b/Darling/Darling.Tests/MigrationUpgradeLadderLiveTests.cs @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2119: the upgrade-path gate the 3.4.0 release lacked. Every pre-release validation ran either a +/// FRESH store (the full generator, no ladder) or the dogfood box (which walks each rung in the era +/// it ships) — nobody pointed new binaries at a store a RELEASED build had made, which is the only +/// path that REPLAYS old rungs with current-code generator output, and the only path users take. +/// The field failure: rung 51 replayed on a 3.3.0 store referenced V54's query_plan_gz three +/// rungs early and every 3.3.0→3.4.0 upgrade died with 42703 at service start. +/// +/// The fixture is the previous release's ladder EXACTLY as that release resolved it — every +/// rung's SQL frozen from the v3.3.0 tag's own code (its generators included: its V38 dim genuinely +/// lacks the gz column), plus the same version-table DDL and stamps its MigrateLockedAsync +/// writes. This test builds that store on scratch Postgres and runs the CURRENT ladder over it — +/// red on the day a generator learns a column an old rung will replay, green when the rung +/// pre-adds it. Regenerated at each release cut (Darling/tools/generate-ladder-fixture). +/// Verified two-sided at birth: against the pre-#2120 build it fails with the exact field 42703; +/// against the fixed build it climbs V39→V54 clean, including from a mid-failure retry. +/// +[Collection("live-postgres")] +public sealed class MigrationUpgradeLadderLiveTests +{ + private const string SkipReason = + "DARLING_TEST_PG not set — this test needs a scratch PostgreSQL (with the timescaledb " + + "extension available) it may create roles and databases on."; + + private const string FixtureRelativePath = "Darling/Darling.Tests/Fixtures/migration-ladder-v3.3.0.sql"; + + private const string ScratchDatabase = "darling_upgrade_ladder_test"; + + [Fact] + public async Task PreviousReleaseStore_ClimbsTheCurrentLadder_ToTheTop() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), SkipReason); + + var root = FindRepoRoot(); + Assert.True(root is not null, + "Could not locate the repository root (walked up from the test binary looking for " + + "PerformanceMonitor.sln) — the fixture lives in the source tree."); + var fixturePath = Path.Combine(root!, FixtureRelativePath.Replace('/', Path.DirectorySeparatorChar)); + Assert.True(File.Exists(fixturePath), $"Previous-release ladder fixture missing: {fixturePath}"); + + /* Scratch database, dropped and recreated per run — the fixture creates schemas, hypertables, + roles-adjacent grants, and the version table, none of which may leak between runs. The + darling role is cluster-level and idempotently ensured (rung SQL grants to it). */ + await using (var admin = new NpgsqlConnection(baseConnectionString)) + { + await admin.OpenAsync(); + await using (var role = new NpgsqlCommand( + "DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'darling') THEN CREATE ROLE darling LOGIN; END IF; END $$", + admin)) + { + await role.ExecuteNonQueryAsync(); + } + + await using (var drop = new NpgsqlCommand( + $"DROP DATABASE IF EXISTS {ScratchDatabase} WITH (FORCE)", admin)) + { + await drop.ExecuteNonQueryAsync(); + } + + await using (var create = new NpgsqlCommand($"CREATE DATABASE {ScratchDatabase}", admin)) + { + await create.ExecuteNonQueryAsync(); + } + } + + var scratch = new NpgsqlConnectionStringBuilder(baseConnectionString) { Database = ScratchDatabase }; + + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(); + + /* Batch-execute the fixture the way psql -f does: autocommit per batch, one session (the + leading SET search_path must hold for every later batch). NOT wrapped in transactions — + TimescaleDB continuous aggregates refuse to be created inside one, and per-batch + atomicity is meaningless for a fixture that either loads fully or fails the test. */ + var batches = File.ReadAllText(fixturePath) + .Split("-- ===BATCH===", StringSplitOptions.RemoveEmptyEntries) + /* Each split element leads with the marker's own label text (" bootstrap", " V40 …") up + to its newline — drop that line; what follows is the batch's SQL. Element 0 is the + file header comment, which has no newline-led SQL and trims to comment-only. */ + .Select(b => b.IndexOf('\n', StringComparison.Ordinal) is var nl && nl >= 0 ? b[(nl + 1)..].Trim() : "") + /* Keep any batch with at least one non-comment line — a rung's SQL may legitimately + OPEN with a comment, so a StartsWith filter would silently drop a whole rung. Only + the file header (all-comment) and empty tails fall out. */ + .Where(b => b.Split('\n').Any(line => line.Trim().Length > 0 && !line.TrimStart().StartsWith("--", StringComparison.Ordinal))); + + foreach (var batch in batches) + { + await using var apply = new NpgsqlCommand(batch, connection) { CommandTimeout = 300 }; + await apply.ExecuteNonQueryAsync(); + } + + /* The previous release's store now exists exactly as its own code built it. Run the CURRENT + ladder over it — the operator's upgrade, the path #2119 broke. */ + var applied = await PgMigrations.MigrateAsync(connection, null); + Assert.True(applied > 0, + "The current ladder applied nothing over the previous-release fixture — either the fixture " + + "is stale (regenerate it from the release tag) or the ladder top never moved this cycle."); + + await using (var top = new NpgsqlCommand( + "SELECT MAX(version) FROM collect.darling_schema_version", connection)) + { + Assert.Equal(PgMigrations.Scripts.Max(m => m.Version), Convert.ToInt32(await top.ExecuteScalarAsync())); + } + + /* The #2119 column specifically: the fixture's V38-era dim genuinely lacks it, so its + presence proves the replayed rungs both passed and did their work. */ + await using (var gz = new NpgsqlCommand( + "SELECT count(*) FROM information_schema.columns WHERE table_name = 'query_plan_dim' AND column_name = '" + + PayloadDimensions.CompressedContentColumn + "'", connection)) + { + Assert.Equal(1L, await gz.ExecuteScalarAsync()); + } + } + + /// Same walk-up idiom as DocCommentHygieneTests.FindRepoRoot. + private static string? FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + for (var i = 0; i < 10 && directory is not null; i++) + { + if (File.Exists(Path.Combine(directory.FullName, "PerformanceMonitor.sln"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/Darling/Darling.Tests/MonitoredServerRegistryStateTests.cs b/Darling/Darling.Tests/MonitoredServerRegistryStateTests.cs new file mode 100644 index 000000000..f644373a7 --- /dev/null +++ b/Darling/Darling.Tests/MonitoredServerRegistryStateTests.cs @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// The #2298 worker→MCP registry seam. The behavioral contract is small and every piece is load-bearing: +/// null-before-first-publish is what routes the MCP host to its darling.json fallback, the snapshot swap is +/// what lets the per-fetch resolver heal without a restart, and first-wins on a duplicate id mirrors the +/// resolver map this state replaced (and the worker's FirstOrDefault over runtimes) — last-wins would make +/// the MCP host resolve a different server than the worker collects from. +/// +public sealed class MonitoredServerRegistryStateTests +{ + private static MonitoredServer Server(string name) => new() { Name = name, Host = name }; + + [Fact] + public void NullBeforeFirstPublish_SoTheReaderTakesItsFileFallback() + { + var state = new MonitoredServerRegistryState(); + + Assert.Null(state.Read()); + } + + [Fact] + public void PublishedSnapshotCarriesTheSetAndItsIdMap() + { + var state = new MonitoredServerRegistryState(); + var alpha = Server("alpha"); + var bravo = Server("bravo"); + + state.Publish(new List { alpha, bravo }); + + var snapshot = state.Read(); + Assert.NotNull(snapshot); + Assert.Equal(2, snapshot!.Servers.Count); + Assert.Same(alpha, snapshot.ById[alpha.ServerId]); + Assert.Same(bravo, snapshot.ById[bravo.ServerId]); + } + + [Fact] + public void RepublishSwapsTheWholeSnapshot_SoAResolverSeesTheNewSetOnItsNextRead() + { + var state = new MonitoredServerRegistryState(); + var original = Server("original"); + var added = Server("added-through-add-servers"); + + state.Publish(new List { original }); + var before = state.Read(); + + state.Publish(new List { original, added }); + var after = state.Read(); + + Assert.NotSame(before, after); + Assert.False(before!.ById.ContainsKey(added.ServerId)); + Assert.True(after!.ById.ContainsKey(added.ServerId)); + } + + [Fact] + public void FirstEntryWinsOnADuplicateServerId() + { + var state = new MonitoredServerRegistryState(); + /* Same name + host → the same derived ServerId, the duplicate the resolver map deduped first-wins. */ + var first = Server("twin"); + var second = Server("twin"); + Assert.Equal(first.ServerId, second.ServerId); + + state.Publish(new List { first, second }); + + Assert.Same(first, state.Read()!.ById[first.ServerId]); + } +} diff --git a/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs b/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs new file mode 100644 index 000000000..bbf7c745d --- /dev/null +++ b/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2117 end-to-end: a REAL at SSL Mode=VerifyFull;Root +/// Certificate=… against an in-test TLS listener that speaks exactly enough of the postgres +/// wire protocol to reach the handshake (read the 8-byte SSLRequest, answer 'S', then TLS). The +/// deciding signal is whether the SERVER's handshake completes: Npgsql's certificate validation +/// runs inside the client's handshake, so a rejection aborts it server-side — precisely the +/// "SSL error: unexpected eof while reading" the field report's postgres log showed. This is the +/// arbiter a bare mirror turned out not to be: the first cut of these pins +/// mirrored CustomRootTrust by hand and PASSED the legacy shape on Windows CI, proving the mirror +/// wasn't the whole of what Npgsql does — only the real driver on the real platforms answers. +/// +public sealed class NpgsqlRootCertificateValidationTests +{ + [Fact] + public async Task ChainShape_VerifyFullWithPrintedRoot_CompletesTheHandshake_OnEveryPlatform() + { + var generated = StoreTlsCertificates.Create("localhost", IPAddress.Loopback, validityYears: 2); + + var completed = await HandshakeCompletesAsync(generated.ServerCertChainPem, generated.ServerKeyPem, generated.RootCertPem); + + Assert.True(completed, + "VerifyFull with the printed root must survive Npgsql's certificate validation on this platform — " + + "this is the exact remote-viewer path #2117 exists to fix."); + } + + [Fact] + public async Task LegacySelfSignedShape_VerifyFullWithItselfAsRoot_TheFieldConfiguration() + { + /* The pre-#2117 single self-signed end-entity shape, with itself as the Root Certificate — + the exact configuration --print-viewer-connection used to emit. The field report (Windows, + same Npgsql version this build ships) shows it failing; this test records what the CI + platforms do with it. If it COMPLETES here, the field failure is environmental rather than + shape-intrinsic — still worth fixing via the chain (which passes everywhere and matches + what every other TLS client expects), but the issue text should say so honestly. */ + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=localhost", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var san = new SubjectAlternativeNameBuilder(); + san.AddIpAddress(IPAddress.Loopback); + san.AddDnsName("localhost"); + request.CertificateExtensions.Add(san.Build()); + request.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, true)); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension(new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") }, false)); + using var legacy = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddYears(2)); + + var pem = legacy.ExportCertificatePem(); + var completed = await HandshakeCompletesAsync(pem, rsa.ExportPkcs8PrivateKeyPem(), pem); + + /* Recorded, not required: the CHAIN shape's test above is the guarantee. The dynamic skip + puts the platform fact in every CI log without inventing a requirement that the legacy + shape fail — the first cut asserted that and Windows CI refuted it. */ + Assert.Skip($"legacy self-signed shape at VerifyFull: handshake completed = {completed} on {Environment.OSVersion.Platform}"); + } + + /// Runs the fake server + a VerifyFull Npgsql connect; true when the server-side TLS + /// handshake completed (the client accepted the certificate). + private static async Task HandshakeCompletesAsync(string serverCertChainPem, string serverKeyPem, string rootPem) + { + var rootPath = Path.Combine(Path.GetTempPath(), $"darling-test-root-{Guid.NewGuid():N}.crt"); + await File.WriteAllTextAsync(rootPath, rootPem); + + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + + var handshakeCompleted = false; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + + var serverTask = Task.Run(async () => + { + using var client = await listener.AcceptTcpClientAsync(cts.Token); + var stream = client.GetStream(); + + /* The 8-byte SSLRequest (length 8, code 80877103) — answer 'S' to start TLS. */ + var request = new byte[8]; + await stream.ReadExactlyAsync(request, cts.Token); + await stream.WriteAsync(new[] { (byte)'S' }, cts.Token); + + /* Serve the WHOLE chain like postgres does with a multi-cert ssl_cert_file. */ + var chain = new X509Certificate2Collection(); + chain.ImportFromPem(serverCertChainPem); + using var keyRsa = RSA.Create(); + keyRsa.ImportFromPem(serverKeyPem); + /* Windows SChannel cannot serve TLS from an EPHEMERAL private key — CopyWithPrivateKey + alone makes AuthenticateAsServer fail server-side before the client validates anything, + poisoning both shapes' verdicts (the first CI round's lesson: BOTH shapes reported + completed=false on Windows while passing on macOS). The PFX round-trip persists the + key where SChannel can use it; a no-op on the other platforms. */ + using var ephemeral = chain[0].CopyWithPrivateKey(keyRsa); + using var serving = X509CertificateLoader.LoadPkcs12( + ephemeral.Export(X509ContentType.Pkcs12), password: null, + keyStorageFlags: X509KeyStorageFlags.DefaultKeySet); + var extras = new X509Certificate2Collection(); + for (var i = 1; i < chain.Count; i++) + { + extras.Add(chain[i]); + } + + using var ssl = new SslStream(stream); + await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions + { + ServerCertificateContext = SslStreamCertificateContext.Create(serving, extras, offline: true), + }, cts.Token); + + handshakeCompleted = true; + + /* Past the handshake the client sends its startup message; just swallow a little and + close — the failure Npgsql then reports is a protocol error, not a certificate one. + + CA2022 (inexact read) is suppressed rather than "fixed", because the fix it asks for would + break this. The byte COUNT is deliberately meaningless here: the read exists only to let the + client finish its startup write before the server drops the connection, and any number of + bytes serves that. ReadExactlyAsync — the usual remedy, and the one #2193 suggested — would + block until a full 256 bytes arrived, which this client never sends, hanging the test until + its timeout. The warning is right that the result is unused; it is wrong that this code + depends on a complete read. */ + var scratch = new byte[256]; +#pragma warning disable CA2022 // Avoid inexact read: any count satisfies this drain, see above. + try { await ssl.ReadAsync(scratch, cts.Token); } catch { /* client may bail first */ } +#pragma warning restore CA2022 + }, cts.Token); + + var builder = new NpgsqlConnectionStringBuilder + { + Host = "localhost", + Port = port, + Username = "test", + Password = "test", + Database = "test", + SslMode = SslMode.VerifyFull, + RootCertificate = rootPath, + Timeout = 10, + }; + + try + { + await using var connection = new NpgsqlConnection(builder.ConnectionString); + await connection.OpenAsync(cts.Token); + } + catch + { + /* Always throws — the fake server speaks no postgres past the handshake. The verdict + is handshakeCompleted, not the exception. */ + } + + try { await serverTask; } catch { /* aborted handshakes land here; the flag says enough */ } + try { File.Delete(rootPath); } catch { /* temp file, best-effort */ } + + return handshakeCompleted; + } +} diff --git a/Darling/Darling.Tests/PayloadDimensionLiveTests.cs b/Darling/Darling.Tests/PayloadDimensionLiveTests.cs index 6fb15d8d4..a6eeddfbe 100644 --- a/Darling/Darling.Tests/PayloadDimensionLiveTests.cs +++ b/Darling/Darling.Tests/PayloadDimensionLiveTests.cs @@ -107,7 +107,8 @@ private static async Task WriteQueryStatsBatchUncommittedAsyn string serverName, DateTime collectionTime, IReadOnlyList rows, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool compressPlanContent = true) { var definition = QueryStatsCollector.Instance; var context = new CollectorContext @@ -150,7 +151,7 @@ private static async Task WriteQueryStatsBatchUncommittedAsyn await importer.CompleteAsync(cancellationToken); } - await PayloadDimensionWriter.FlushAsync(connection, transaction, dimensions, stored, cancellationToken); + await PayloadDimensionWriter.FlushAsync(connection, transaction, dimensions, stored, cancellationToken, compressPlanContent); return transaction; } catch @@ -160,6 +161,134 @@ private static async Task WriteQueryStatsBatchUncommittedAsyn } } + /// + /// #2171: plan_xml_compression = 'none' - the dim writer stores PLAIN TEXT in query_plan_xml and + /// leaves query_plan_gz NULL, so a direct-SQL consumer reads the plan bare, and the resolving + /// view (text-first) returns it unchanged. The digest is codec-independent, so a later gzip-mode + /// batch carrying the SAME plan is a conflict no-op: the text row STAYS text - flipping the knob + /// never rewrites existing content in either direction. + /// + [Fact] + public async Task WritePath_PlanXmlCompressionNone_StoresText_AndAGzipBatchLater_LeavesItText() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), SkipReason); + + var ct = TestContext.Current.CancellationToken; + var (serverId, serverName) = NewServer(); + var queryText = $"SELECT 'pm2171-{serverName}' AS marker;"; + var planXml = $""; + var planDigest = PayloadDimensions.Digest(planXml); + + await using var connection = await OpenMigratedStoreAsync(connectionString!, ct); + var bodySucceeded = false; + try + { + await WriteQueryStatsBatchAsync( + connection, serverId, serverName, DateTime.UtcNow, + new[] { NewRow("0x2171A", queryText, planXml) }, ct, compressPlanContent: false); + + await using (var check = new NpgsqlCommand( + "SELECT query_plan_xml, query_plan_gz FROM query_plan_dim WHERE digest = $1", connection)) + { + check.Parameters.AddWithValue(planDigest); + await using var reader = await check.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), "the plan dim row must exist"); + Assert.Equal(planXml, reader.GetString(0)); + Assert.True(reader.IsDBNull(1), "'none' mode must leave query_plan_gz NULL - that is the whole contract"); + } + + /* A gzip-mode batch with the SAME plan (hours later, past the last_seen guard) must not + convert the row - the conflict arm only refreshes last_seen. */ + await WriteQueryStatsBatchAsync( + connection, serverId, serverName, DateTime.UtcNow.AddHours(2), + new[] { NewRow("0x2171B", queryText, planXml) }, ct, compressPlanContent: true); + + await using (var still = new NpgsqlCommand( + "SELECT query_plan_xml IS NOT NULL, query_plan_gz IS NULL FROM query_plan_dim WHERE digest = $1", connection)) + { + still.Parameters.AddWithValue(planDigest); + await using var reader = await still.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + Assert.True(reader.GetBoolean(0), "the text content must survive a later gzip-mode batch"); + Assert.True(reader.GetBoolean(1), "the gz column must stay NULL - mode flips never rewrite existing rows"); + } + + /* And the reader contract holds with no new arm: the resolving view returns the plan. */ + await using (var resolved = new NpgsqlCommand( + "SELECT query_plan_xml FROM v_query_stats WHERE server_id = $1 AND query_hash = '0x2171A'", connection)) + { + resolved.Parameters.AddWithValue(serverId); + Assert.Equal(planXml, (string?)await resolved.ExecuteScalarAsync(ct)); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteServerRowsAsync(cleanup, serverId, cleanupCt)); + } + } + + /// + /// #2171: the recompression verb's guard - a store set to plan_xml_compression = 'none' refuses + /// recompression (the live writer keeps producing text; converting would fight it forever), and + /// 'gzip' proceeds. Tested at the extracted seam against a real migrated store so the SQL and the + /// V62 column are exercised, not mocked. + /// + [Fact] + public async Task RecompressGuard_RefusesOnNone_ProceedsOnGzip() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), SkipReason); + + var ct = TestContext.Current.CancellationToken; + await using var connection = await OpenMigratedStoreAsync(connectionString!, ct); + var bodySucceeded = false; + try + { + /* A migrated-but-never-served store has no config_service row — the SERVICE seeds it, not + the migrations (measured here: the bare UPDATE hit zero rows). Materialize it the way + the sibling MCP tests do; every column defaults. */ + await using (var seed = new NpgsqlCommand( + "INSERT INTO config_service (id) VALUES (1) ON CONFLICT (id) DO NOTHING", connection)) + { + await seed.ExecuteNonQueryAsync(ct); + } + + await using (var setNone = new NpgsqlCommand( + "UPDATE config_service SET plan_xml_compression = 'none' WHERE id = 1", connection)) + { + await setNone.ExecuteNonQueryAsync(ct); + } + + Assert.True(await DarlingCliCommands.StoreIsSetToPlainTextPlansAsync(connection, ct), + "a store set to 'none' must be recognized - the verb refusing is the whole guard"); + + await using (var setGzip = new NpgsqlCommand( + "UPDATE config_service SET plan_xml_compression = 'gzip' WHERE id = 1", connection)) + { + await setGzip.ExecuteNonQueryAsync(ct); + } + + Assert.False(await DarlingCliCommands.StoreIsSetToPlainTextPlansAsync(connection, ct), + "the default mode must not trip the guard - recompression is the supported path there"); + + bodySucceeded = true; + } + finally + { + /* The setting is store-global state shared with sibling tests - always restore the default. */ + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, static async (cleanup, cleanupCt) => + { + await using var restore = new NpgsqlCommand( + "UPDATE config_service SET plan_xml_compression = 'gzip' WHERE id = 1", cleanup); + await restore.ExecuteNonQueryAsync(cleanupCt); + }); + } + } + /// The committing wrapper — what every test that just needs the batch landed uses. private static async Task WriteQueryStatsBatchAsync( NpgsqlConnection connection, @@ -167,10 +296,11 @@ private static async Task WriteQueryStatsBatchAsync( string serverName, DateTime collectionTime, IReadOnlyList rows, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool compressPlanContent = true) { await using var transaction = await WriteQueryStatsBatchUncommittedAsync( - connection, serverId, serverName, collectionTime, rows, cancellationToken); + connection, serverId, serverName, collectionTime, rows, cancellationToken, compressPlanContent); await transaction.CommitAsync(cancellationToken); } diff --git a/Darling/Darling.Tests/PayloadDimensionTests.cs b/Darling/Darling.Tests/PayloadDimensionTests.cs index 9a7b44bd4..18a9a2328 100644 --- a/Darling/Darling.Tests/PayloadDimensionTests.cs +++ b/Darling/Darling.Tests/PayloadDimensionTests.cs @@ -456,6 +456,22 @@ public void UpsertSql_IsOneUnnestStatement_WithTheOnConflictStalenessGuard() "WHERE query_plan_dim.last_seen < EXCLUDED.last_seen - INTERVAL '1 hour'", PayloadDimensions.UpsertSql(PayloadDimensions.QueryPlanDimTable)); + /* #2171: compressContent false routes the plan dim through the TEXT branch - query_plan_xml + written, query_plan_gz untouched (stays NULL on new rows) - which is plan_xml_compression = + 'none'. Same statement shape, same conflict semantics, text payload array. */ + Assert.Equal( + "INSERT INTO query_plan_dim (digest, query_plan_xml, last_seen)\n" + + "SELECT u.digest, u.payload, $3\n" + + "FROM unnest($1::bytea[], $2::text[]) AS u(digest, payload)\n" + + "ON CONFLICT (digest) DO UPDATE SET last_seen = EXCLUDED.last_seen\n" + + "WHERE query_plan_dim.last_seen < EXCLUDED.last_seen - INTERVAL '1 hour'", + PayloadDimensions.UpsertSql(PayloadDimensions.QueryPlanDimTable, compressContent: false)); + + /* Explicit true is byte-identical to the default - the parameter cannot drift the gzip shape. */ + Assert.Equal( + PayloadDimensions.UpsertSql(PayloadDimensions.QueryPlanDimTable), + PayloadDimensions.UpsertSql(PayloadDimensions.QueryPlanDimTable, compressContent: true)); + Assert.Throws(() => PayloadDimensions.UpsertSql("not_a_dim")); } diff --git a/Darling/Darling.Tests/PgReadKindDisciplineTests.cs b/Darling/Darling.Tests/PgReadKindDisciplineTests.cs new file mode 100644 index 000000000..ae5ef790a --- /dev/null +++ b/Darling/Darling.Tests/PgReadKindDisciplineTests.cs @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using Xunit; + +namespace Darling.Tests; + +/// +/// The naive-UTC bind discipline for the PostgreSQL read family (#2213 round 2). The store's timestamp +/// columns are timestamp without time zone; a Kind=Utc DateTime makes Npgsql infer +/// timestamptz, and PostgreSQL resolves the mixed comparison by converting the NAIVE side at the store +/// session's TimeZone — which initdb takes from the host OS. East of UTC every fresh row falls outside the +/// freshness window and the three Tier 0 outage predictors silently never fire; west of UTC the window +/// stretches and alerts grade stale data. No error is raised anywhere, and every UTC-hosted test store +/// hides it, which is why this is a SOURCE pin rather than a live assertion: the live store that would +/// catch it is exactly the one nobody runs. +/// +/// The established adapter documents the same convention (DarlingAlertReadAdapter's NaiveUtcNow and +/// DarlingDataReader's AddTimestamp); this pin holds the NEW family to it. It scans for the hazard — +/// binding a window parameter or a raw DateTime.UtcNow without stripping Kind — rather than for the +/// idiom, so a refactor that binds safely through a different helper still passes. +/// +public sealed class PgReadKindDisciplineTests +{ + [Fact] + public void NoPgReaderBindsAKindUtcTimestamp() + { + var offenders = new List(); + + foreach (var path in PgReadFamilyFiles()) + { + var source = File.ReadAllText(path); + var name = Path.GetFileName(path); + + /* Bare window-parameter binds: any *Utc-named identifier bound raw. The callers hand these + down from DateTime.UtcNow, so an unwrapped bind ships Kind=Utc — and the name-suffix match + survives a rename to fooUtc where a literal startUtc/endUtc list would not. */ + foreach (Match match in Regex.Matches( + source, @"AddWithValue\(\s*\w*[Uu]tc\s*\)")) + { + offenders.Add(name + ": " + match.Value); + } + + /* Direct now-arithmetic binds: AddWithValue(DateTime.UtcNow ...) — the alert adapter's + original form. */ + foreach (Match match in Regex.Matches( + source, @"AddWithValue\(\s*DateTime\.UtcNow")) + { + offenders.Add(name + ": " + match.Value); + } + } + + Assert.True( + offenders.Count == 0, + "PostgreSQL read binds a Kind=Utc timestamp against the store's naive columns: " + + string.Join(", ", offenders) + + ". Wrap the value in DateTime.SpecifyKind(..., DateTimeKind.Unspecified) at the bind (or a " + + "NaiveUtcNow helper) — Kind=Utc infers timestamptz, the session zone shifts the window, and " + + "east of UTC the Tier 0 alerts silently never fire."); + } + + [Fact] + public void TheAdapterHelper_ActuallyStripsKind() + { + /* The three adapter binds route through NaiveUtcNow(), so the scan above cannot see them revert + if the HELPER quietly becomes DateTime.UtcNow again. Pin its body: the SpecifyKind call is the + whole point of the function. */ + var adapter = PgReadFamilyFiles().Last(); + var source = File.ReadAllText(adapter); + var body = Regex.Match( + source, + @"private static DateTime NaiveUtcNow\(\)\s*=>\s*(?[^;]+);", + RegexOptions.Singleline); + + Assert.True(body.Success, "DarlingPostgresAlertReadAdapter must carry the NaiveUtcNow helper."); + Assert.Contains("SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified)", body.Groups["body"].Value, StringComparison.Ordinal); + } + + private static IEnumerable PgReadFamilyFiles([CallerFilePath] string thisFile = "") + { + var testsDir = Path.GetDirectoryName(thisFile)!; + var service = Path.GetFullPath(Path.Combine(testsDir, "..", "PerformanceMonitor.Darling.Service")); + + foreach (var reader in Directory.EnumerateFiles(Path.Combine(service, "Mcp"), "DarlingPg*Reader.cs")) + { + yield return reader; + } + + /* The Tools siblings create the DateTime.UtcNow values the readers bind — a Tools file that + starts binding directly is the same hazard one hop up. */ + foreach (var tools in Directory.EnumerateFiles(Path.Combine(service, "Mcp"), "DarlingMcpPg*Tools.cs")) + { + yield return tools; + } + + yield return Path.Combine(service, "DarlingPostgresAlertReadAdapter.cs"); + } +} diff --git a/Darling/Darling.Tests/PgSchemaGeneratorTests.cs b/Darling/Darling.Tests/PgSchemaGeneratorTests.cs index 44c790f4e..65faf3177 100644 --- a/Darling/Darling.Tests/PgSchemaGeneratorTests.cs +++ b/Darling/Darling.Tests/PgSchemaGeneratorTests.cs @@ -30,10 +30,24 @@ public void Catalog_CoversAllCollectors_WithUniqueTablesAndNames() /* 35 through agent_status + long_query_completions (#1496 long-query trace) = 36, plus the two Availability Group collectors (#991) = 38, plus plan_correction (#1952 automatic plan correction) = 39, plus pvs_stats (#1951 ADR version store) = 40, plus database_states - (baseline-deviation database-state alert) = 41. */ - Assert.Equal(41, CollectorCatalog.All.Count); - Assert.Equal(41, CollectorCatalog.All.Select(s => s.TargetTable).Distinct().Count()); - Assert.Equal(41, CollectorCatalog.All.Select(s => s.Name).Distinct().Count()); + (baseline-deviation database-state alert) = 41, plus pg_wait_stats (the first PostgreSQL + collector) = 42, plus pg_statement_stats = 43, plus pg_wraparound_stats = 44, plus pg_xmin_horizon = 45, + plus pg_replication_slot_stats = 46, plus pg_autovacuum_stats (the first per-database PostgreSQL + collector) = 47, plus pg_io_stats = 48, plus pg_blocking = 49, plus query_store_health + (#2319) = 50. The catalog is deliberately + engine-mixed: the schema generator walks it to + create tables and one store can hold both engines' data, so splitting it per engine would + fragment DDL generation. Dispatch is gated separately, by engine, in + CollectorCatalog.AppliesTo(definition, target). */ + Assert.Equal(50, CollectorCatalog.All.Count); + + /* Uniqueness is asserted AGAINST THE COUNT rather than against a second literal. The literals here + had drifted to 45 while the real figure tracked the count, so the test that exists to catch a + duplicate table or name was itself failing for an unrelated reason — and could not say so, because + this project does not execute on the machine the collectors were written on. Two collectors sharing + a TargetTable would still fail this, which is the point. */ + Assert.Equal(CollectorCatalog.All.Count, CollectorCatalog.All.Select(s => s.TargetTable).Distinct().Count()); + Assert.Equal(CollectorCatalog.All.Count, CollectorCatalog.All.Select(s => s.Name).Distinct().Count()); } [Fact] @@ -423,6 +437,14 @@ static string CollectQualified(ICollectorSchemaInfo schema) Assert.Contains(CollectQualified(AgentStatusCollector.Instance), v25, StringComparison.Ordinal); Assert.Contains("CREATE INDEX IF NOT EXISTS idx_agent_status_time ON collect.agent_status(server_id, collection_time);", v25, StringComparison.Ordinal); + /* V76 (#2319) creates query_store_health for an already-existing store — same contract as the + blocks below. */ + var v76 = Lf(PgMigrations.Scripts.Single(m => m.Version == 76).Sql); + + Assert.Contains(CollectQualified(QueryStoreHealthCollector.Instance), v76, StringComparison.Ordinal); + Assert.Contains("CREATE INDEX IF NOT EXISTS idx_query_store_health_time ON collect.query_store_health(server_id, capture_time);", v76, StringComparison.Ordinal); + Assert.Contains("CREATE OR REPLACE VIEW v_query_store_health AS SELECT * FROM query_store_health;", v76, StringComparison.Ordinal); + /* V47 (#1951) creates pvs_stats for an already-existing store; a fresh store gets it from V1's GenerateFullSchema. Same contract as V24/V25 — and it is the ONLY thing standing between a hand-typed 26-column CREATE TABLE and a silent fresh-vs-upgraded shape fork. */ @@ -552,18 +574,88 @@ public void CreateIndex_MirrorsLiteIndexColumns() Assert.Null(PgSchemaGenerator.CreateIndex(DatabaseConfigCollector.Instance)); } + /// + /// THE LADDER-GENERATOR DIFF. Every PostgreSQL collector rung's hand-written DDL must be + /// column-for-column identical to what emits for that collector. + /// + /// Why this invariant matters more than it looks. There are two populations of store and + /// they get their tables from different places: a FRESH store's tables come from V1's generated schema + /// (walked from the collector catalog), while an ALREADY-EXISTING store's come from these rungs. Nothing + /// forces the two texts to agree. Let one column's type drift and the divergence is permanent and + /// invisible — every read works against one population and fails against the other, and which one you + /// have depends on when the store was created. + /// + /// Whitespace is normalised because the rungs wrap their CREATE INDEX across two lines for + /// readability and the generator emits it on one; the schema qualification is normalised because the + /// rungs name collect. explicitly while V1 runs with search_path already set. Those two are + /// the only differences that are allowed to exist, so they are the only two normalised away — anything + /// else, including a reordered column, fails. + /// + /// Written after the fact for all eight rungs at once, because adding the eighth collector was the + /// first time anyone checked: the suite asserted the generator emits every table and that each rung is + /// well-formed, but never that they said the SAME thing. + /// + [Fact] + public void EveryPostgresRung_IsIdenticalToTheGeneratedSchema() + { + var rungs = new (int Version, ICollectorSchemaInfo Collector)[] + { + (63, PgWaitStatsCollector.Instance), + (64, PgStatementStatsCollector.Instance), + (65, PgWraparoundStatsCollector.Instance), + (66, PgXminHorizonCollector.Instance), + (67, PgReplicationSlotsCollector.Instance), + (68, PgAutovacuumStatsCollector.Instance), + (69, PgIoStatsCollector.Instance), + (71, PgBlockingCollector.Instance), + }; + + /* Every PostgreSQL collector must appear above. A ninth added without a rung listed here would + otherwise pass this test by simply not being checked. */ + Assert.Equal( + CollectorCatalog.All.Count(c => c.TargetEngine == CollectorTargetEngine.PostgreSql), + rungs.Length); + + foreach (var (version, collector) in rungs) + { + var rung = NormalizeDdl(PgMigrations.Scripts.Single(m => m.Version == version).Sql); + + var generated = NormalizeDdl( + PgSchemaGenerator.CreateTable(collector) + + "\n\n" + + PgSchemaGenerator.CreateIndex(collector)) + .Replace( + $"CREATE TABLE IF NOT EXISTS {collector.TargetTable} (", + $"CREATE TABLE IF NOT EXISTS collect.{collector.TargetTable} (", + StringComparison.Ordinal) + .Replace( + $"ON {collector.TargetTable}(", + $"ON collect.{collector.TargetTable}(", + StringComparison.Ordinal); + + Assert.Equal(generated, rung); + } + } + + private static string NormalizeDdl(string sql) => + System.Text.RegularExpressions.Regex.Replace(sql, @"\s+", " ").Trim(); + [Fact] public void GenerateFullSchema_EmitsEveryTableAndIndex() { var script = PgSchemaGenerator.GenerateFullSchema(); + /* EVERY table, asserted against the catalog count rather than a literal — the test's name is the + invariant, and a literal here silently became a subset check (it read 46 of 48) the moment the + catalog grew. A collector whose table the generator skips still fails this. */ var tableCount = CollectorCatalog.All.Count(s => script.Contains($"CREATE TABLE IF NOT EXISTS {s.TargetTable} (", StringComparison.Ordinal)); - Assert.Equal(41, tableCount); + Assert.Equal(CollectorCatalog.All.Count, tableCount); - /* 41 tables minus the two index-less config tables (server_config, database_config) = 39 indexes - (database_states is a time-series collector and gets the default retrieval index). */ + /* Every table gets a retrieval index except the two index-less config tables (server_config, + database_config), which CreateIndex returns null for — so this tracks the catalog minus exactly + those two, not a literal that has to be remembered per collector. */ var indexCount = script.Split("CREATE INDEX IF NOT EXISTS").Length - 1; - Assert.Equal(39, indexCount); + Assert.Equal(CollectorCatalog.All.Count - 2, indexCount); /* The precision guard can never regress silently. */ Assert.DoesNotContain("numeric(0,0)", script, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/PgStatementTextTests.cs b/Darling/Darling.Tests/PgStatementTextTests.cs new file mode 100644 index 000000000..ba3f577dd --- /dev/null +++ b/Darling/Darling.Tests/PgStatementTextTests.cs @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2219: PostgreSQL statement text, stored once per (server_id, queryid). +/// +/// The gap. pg_statement_stats identifies queries by queryid and stores no text — +/// aurora_stat_statements's showtext is a real per-collection cost and normalized text is highly +/// repetitive. But queryid is NOT stable across a major version upgrade, so afterwards the stored history +/// joins to nothing readable: a list of integers that used to be your slowest queries, and unrecoverable, because +/// the live view no longer holds the old ids and anything else on the instance may have reset +/// pg_stat_statements out from under us. +/// +/// Why inline text rather than the query_text_dim digest V64's comment promised. The dimension +/// route is blocked and expensive to unblock — V38 is generated from PayloadDimensions.All, so registering +/// the fact table makes V38 ALTER a table it has not created yet, and it would break V64's own ladder diff. +/// More importantly the dimension needs the liveness interlock documents at +/// length, whose failure mode is SILENTLY missing text. Inline cannot dangle. The cost is cross-server dedup, a +/// few hundred MB on the measured fleet against a store whose Query Store plan XML alone was 43 GB. +/// +public sealed class PgStatementTextTests +{ + /// + /// The rung and the helper's own DDL must agree, or a fresh store and an upgraded one get different tables — + /// the same discipline the ladder diff enforces for collector tables, applied by hand here because a + /// non-collector table is outside that generator. + /// + [Fact] + public void TheRungMatchesTheHelpersCreateTableSql() + { + var rung = PgMigrations.Scripts.Single(s => s.Version == 73); + + Assert.Equal("pg-statement-text", rung.Name); + Assert.Equal(Normalize(PgStatementText.CreateTableSql), Normalize(rung.Sql)); + } + + /// + /// The ladder's own invariants, restated for this rung: it is the top, it is dense, and the build's schema + /// version tracks it. A gap is skipped SILENTLY on every upgraded store, so the objects would never exist and + /// no later upgrade would repair it. + /// + [Fact] + public void TheRungIsTheTopOfADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + /* #2150 added V74, so this rung is no longer the top — the "I am the top" claim moves to the newest + rung's own test (QueryStoreTextStoreTests) and this one keeps the invariants that stay true + forever: the rung is PRESENT, the ladder is ordered and dense, and the build's schema version + tracks the maximum. A gap is skipped SILENTLY on every upgraded store, so the objects would never + exist and no later upgrade would repair it. */ + Assert.Contains(73, versions); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + + /* Dense above the one sanctioned historical hole at V45. */ + var above = versions.Where(v => v > 45).OrderBy(v => v).ToList(); + Assert.Equal(Enumerable.Range(above[0], above.Count), above); + } + + /// + /// The table is keyed on (server_id, queryid) — one row per statement per server, which is what makes + /// "stored once" true and the upsert idempotent. Without the primary key the upsert has nothing to conflict + /// on and every refresh would append a copy, which is precisely the per-snapshot duplication being avoided. + /// + [Fact] + public void TheTableIsKeyedOnServerAndQueryId() + { + Assert.Contains("PRIMARY KEY (server_id, queryid)", PgStatementText.CreateTableSql, StringComparison.Ordinal); + Assert.Contains("ON CONFLICT (server_id, queryid)", PgStatementText.UpsertSql, StringComparison.Ordinal); + /* Pruned on last_seen, so it needs the index the prune scans. */ + Assert.Contains("idx_pg_statement_text_last_seen", PgStatementText.CreateTableSql, StringComparison.Ordinal); + } + + /// + /// first_seen is PRESERVED on conflict and query_text is advanced. + /// + /// That asymmetry is the point: first_seen records when this statement shape was first seen on + /// this server, which is the one fact that survives a major-version re-key and cannot be reconstructed + /// afterwards — overwriting it would quietly turn every row's age into "since the last refresh". The text, by + /// contrast, should track what the server says now. + /// + [Fact] + public void TheUpsertKeepsFirstSeenAndAdvancesTheText() + { + var setClause = PgStatementText.UpsertSql[PgStatementText.UpsertSql.IndexOf("DO UPDATE SET", StringComparison.Ordinal)..]; + + Assert.Contains("query_text = EXCLUDED.query_text", setClause, StringComparison.Ordinal); + Assert.Contains("last_seen = EXCLUDED.last_seen", setClause, StringComparison.Ordinal); + Assert.DoesNotContain("first_seen = EXCLUDED", setClause, StringComparison.Ordinal); + } + + /// + /// The upsert is ORDERED by the conflict key. Concurrent batch upserts that take row locks in different + /// relative orders deadlock (#1801), and this runs per server across a fleet — the same reason + /// carries its ORDER BY. Cheap to keep, expensive to rediscover. + /// + [Fact] + public void TheUpsertIsOrderedByTheConflictKey() + { + var beforeConflict = PgStatementText.UpsertSql[..PgStatementText.UpsertSql.IndexOf("ON CONFLICT", StringComparison.Ordinal)]; + Assert.Contains("ORDER BY server_id, queryid", beforeConflict, StringComparison.Ordinal); + } + + /// + /// The fetch asks for text (showtext = true) — which is the entire reason it is a separate query from + /// pg_statement_stats', which passes false every minute and must keep doing so. + /// + /// Capped and ordered by total execution time, so a catalog larger than the cap keeps the text for the + /// queries anyone would look at rather than an arbitrary slice; and parameterized rather than a hardcoded + /// LIMIT, which the repo has had to correct once before. + /// + [Fact] + public void TheFetchAsksForTextAndIsBoundedByRank() + { + Assert.Contains("aurora_stat_statements(true)", PgStatementText.FetchSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY s.total_exec_time DESC", PgStatementText.FetchSql, StringComparison.Ordinal); + Assert.Contains("LIMIT $1", PgStatementText.FetchSql, StringComparison.Ordinal); + + /* Every output column aliased, per the house rule — an unaliased expression comes back named after the + function and the query stops being debuggable in psql. */ + Assert.Contains("AS queryid", PgStatementText.FetchSql, StringComparison.Ordinal); + Assert.Contains("AS query_text", PgStatementText.FetchSql, StringComparison.Ordinal); + } + + /// + /// Due-ness is asked of the STORE, not remembered in the service — so a restart cannot re-fetch the fleet and + /// two hosts writing one store cannot disagree about when text was last written. COALESCE(..., TRUE) is + /// what makes a server with no rows yet due rather than never due. + /// + [Fact] + public void DuenessComesFromTheStoreAndAFirstFetchIsDue() + { + Assert.Contains("max(last_seen)", PgStatementText.IsDueSql, StringComparison.Ordinal); + Assert.Contains("WHERE server_id = $1", PgStatementText.IsDueSql, StringComparison.Ordinal); + Assert.Contains("COALESCE(", PgStatementText.IsDueSql, StringComparison.Ordinal); + Assert.Contains("TRUE)", PgStatementText.IsDueSql, StringComparison.Ordinal); + /* The caller's clock, not the store's — the same value stamps the rows, so the cadence cannot drift + against its own timestamps. */ + Assert.Contains("$2::timestamp", PgStatementText.IsDueSql, StringComparison.Ordinal); + Assert.DoesNotContain("now()", PgStatementText.IsDueSql, StringComparison.Ordinal); + } + + /// + /// The prune margin makes text OUTLIVE the statistics that reference it, which is the opposite direction from + /// the plan map's — and the asymmetry is the reason. Text kept past its facts is a few dead bytes; facts kept + /// past their text is a top-queries answer that reads as a list of integers, the exact failure this table + /// exists to fix. + /// + [Fact] + public void ThePruneMarginKeepsTextAliveLongerThanItsFacts() + { + Assert.True(PgStatementText.PruneMarginDays > 0); + + var sql = PgStatementText.PruneSql(7); + Assert.Contains("DELETE FROM collect.pg_statement_text", sql, StringComparison.Ordinal); + Assert.Contains("last_seen < $1", sql, StringComparison.Ordinal); + /* Time-sliced like every sibling purge, so one sweep cannot take an unbounded lock. */ + Assert.Contains("INTERVAL '7 days'", sql, StringComparison.Ordinal); + /* Timestamp-driven, never an anti-join against the fact table — that is the cost this shape avoids. */ + Assert.DoesNotContain("pg_statement_stats", sql, StringComparison.Ordinal); + } + + /// + /// strips the Kind without shifting the value. The #1969 trap is silent: + /// Npgsql infers timestamptz from a Utc Kind, PostgreSQL converts into the session zone on the way into + /// a naive column, and last_seen lands at the wrong hour — ageing text out ahead of the facts that + /// reference it, which is this design's own failure mode arrived at through a timezone. + /// + [Fact] + public void NaiveStripsTheKindWithoutShiftingTheValue() + { + var utc = new DateTime(2026, 8, 15, 4, 30, 0, DateTimeKind.Utc); + var naive = PgStatementText.Naive(utc); + + Assert.Equal(DateTimeKind.Unspecified, naive.Kind); + Assert.Equal(utc.Ticks, naive.Ticks); + } + + /// + /// The reader joins the text on (server_id, queryid) and LEFT joins it, so a statement whose text has + /// not been captured yet still ranks — it simply reads as null. An inner join would silently drop exactly the + /// newest and most interesting statements. + /// + [Fact] + public void TheReaderLeftJoinsTheTextSoUncapturedStatementsStillRank() + { + var sql = DarlingPgStatementReader.PgTopQueriesSql; + + Assert.Contains("LEFT JOIN collect.pg_statement_text", sql, StringComparison.Ordinal); + Assert.Contains("t.server_id = $1", sql, StringComparison.Ordinal); + Assert.Contains("t.queryid = differenced.queryid", sql, StringComparison.Ordinal); + /* MAX, because the read's grain is (queryid, database_id) while text is keyed on queryid alone — + one text per group by construction, so this picks it without widening the GROUP BY. */ + Assert.Contains("MAX(t.query_text) AS query_text", sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY queryid, database_id", sql, StringComparison.Ordinal); + } + + /// + /// The text refresh is hung off the statement-stats collector's success, keyed on the collector's OWN declared + /// name so renaming it cannot silently unhook the text path — and it is best-effort, because losing text is a + /// degraded read while losing a collection is lost data. + /// + [Fact] + public void TheRefreshRidesTheStatsCollectorAndCannotCostACollection() + { + var source = ReadWorkerSource(); + + Assert.Contains("PgStatementStatsCollector.Instance.Name", source, StringComparison.Ordinal); + Assert.Contains("TryRefreshPgStatementTextAsync", source, StringComparison.Ordinal); + /* Gated on the engine as well, rather than trusting the collector's own gate. */ + Assert.Contains("runtime.Target.Engine != CollectorTargetEngine.PostgreSql", source, StringComparison.Ordinal); + /* A failure warns and leaves the statistics alone. */ + Assert.Contains("statistics are unaffected", source, StringComparison.Ordinal); + } + + private static string Normalize(string sql) => string.Join(" ", sql.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + + private static string ReadWorkerSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/PgTableTuningTests.cs b/Darling/Darling.Tests/PgTableTuningTests.cs index c785d3f8f..0910e2247 100644 --- a/Darling/Darling.Tests/PgTableTuningTests.cs +++ b/Darling/Darling.Tests/PgTableTuningTests.cs @@ -42,12 +42,16 @@ LATERAL probes (server_id, sql_handle, newest-first — bounded by raw retention Assert.Equal(7, CountOccurrences(sql, "CREATE INDEX IF NOT EXISTS")); /* +1: the #1981 handle index */ Assert.DoesNotContain("CREATE INDEX ON", sql, StringComparison.Ordinal); - /* Per-table autovacuum-insert override on exactly the three growing tables (NOT a global GUC change). */ + /* Per-table autovacuum-insert override on exactly the FOUR high-rate insert tables (NOT a global GUC + change). pg_statement_stats joined them: it is query_stats' per-minute PostgreSQL twin, same shape + and cadence and the same pure-insert hypertable chunks, so the stock 0.2 scale factor leaves the + day's hot chunk stale before the TimescaleDB rollover exactly as it did for the other three. */ Assert.Contains("ALTER TABLE collect.procedure_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", sql, StringComparison.Ordinal); Assert.Contains("ALTER TABLE collect.query_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", sql, StringComparison.Ordinal); Assert.Contains("ALTER TABLE collect.query_store_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", sql, StringComparison.Ordinal); + Assert.Contains("ALTER TABLE collect.pg_statement_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", sql, StringComparison.Ordinal); - Assert.Equal(10, PgTableTuning.Statements.Count); /* +1: the #1981 query_stats handle index */ + Assert.Equal(11, PgTableTuning.Statements.Count); /* +1 #1981 query_stats handle index, +1 pg_statement_stats */ } private static int CountOccurrences(string haystack, string needle) diff --git a/Darling/Darling.Tests/PlanContentRetentionTests.cs b/Darling/Darling.Tests/PlanContentRetentionTests.cs new file mode 100644 index 000000000..d30342feb --- /dev/null +++ b/Darling/Darling.Tests/PlanContentRetentionTests.cs @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// The V75 plan-content retention knob (#2316). The payload dimensions' GC horizon is coupled to the +/// widest dim-feeding fact retention so a raised override can never orphan a reader — which also means a +/// store YOUNGER than that horizon has an unbounded plan dimension: measured on the dogfood fleet, +/// query_plan_dim reached 127 GB (63% of the store) in its first 22 days of parameter-sniffing +/// recompile churn (65 distinct XMLs per plan shape per day), with the coupled GC unable to delete a +/// single row until ~a month AFTER the projected disk-full. The knob gives plan CONTENT its own horizon; +/// facts keep theirs. These facts pin the rung, the knob's clamps, the cutoff arithmetic in both +/// directions (bounding when enabled, byte-identical old behavior when disabled), and the viewer probe. +/// +public sealed class PlanContentRetentionTests +{ + private static readonly DateTime Now = new(2026, 8, 18, 12, 0, 0, DateTimeKind.Unspecified); + + /* ---------------- the rung ---------------- */ + + [Fact] + public void TheRungIsTheTopOfADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + /* #2319 added V76, so this rung is no longer the top — the "I am the top" claim moves to the + newest rung's own test (QueryStoreHealthStoreTests) and this one keeps the invariants that + stay true forever: the rung is PRESENT, the ladder is ordered and dense, and the build's + schema version tracks the maximum. */ + Assert.Contains(75, versions); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + + /* Dense above the one sanctioned historical hole at V45. */ + var above = versions.Where(v => v > 45).OrderBy(v => v).ToList(); + Assert.Equal(Enumerable.Range(above[0], above.Count), above); + } + + [Fact] + public void TheRungAddsTheKnobWithThe21DayDefault() + { + var rung = PgMigrations.Scripts.Single(s => s.Version == 75); + + Assert.Equal("plan-content-retention-knob", rung.Name); + Assert.Contains("config.config_service", rung.Sql, StringComparison.Ordinal); + Assert.Contains("plan_content_retention_days integer NOT NULL DEFAULT 21", rung.Sql, StringComparison.Ordinal); + } + + /* ---------------- the clamps ---------------- */ + + /// + /// 0 and below mean DISABLED (the fact-coupled horizon stands alone); an enabled value clamps to + /// [7,365] — a sub-week horizon would age plan XML out from under the viewer's default history + /// windows, and clamping a bad stored value beats failing the config load (the V59 knobs' posture). + /// + [Theory] + [InlineData(0, 0)] + [InlineData(-5, 0)] + [InlineData(1, 7)] + [InlineData(6, 7)] + [InlineData(7, 7)] + [InlineData(21, 21)] + [InlineData(365, 365)] + [InlineData(9999, 365)] + public void TheClampDisablesAtZeroAndBoundsEnabledValues(int stored, int effective) + => Assert.Equal(effective, StoreConfigProvider.ClampPlanContentRetentionDays(stored)); + + /* ---------------- the cutoff arithmetic ---------------- */ + + /// + /// Disabled (0) must be BYTE-IDENTICAL to the pre-knob behavior — every existing caller and test + /// passes nothing and must observe no change. + /// + [Fact] + public void Disabled_ReproducesTheFactCoupledCutoffExactly() + { + var withoutKnob = DarlingRetention.ComputeDimensionCutoff(Now, 90, Now.AddDays(-40)); + var withZero = DarlingRetention.ComputeDimensionCutoff(Now, 90, Now.AddDays(-40), planContentRetentionDays: 0); + + Assert.Equal(withoutKnob, withZero); + } + + /// + /// The knob's entire point: on a store younger than the fact horizon, the dedicated cutoff is NEWER + /// than the coupled one and must win — with the same one-day margin as the measured side, covering + /// the hourly last_seen refresh guard. + /// + [Fact] + public void Enabled_FloorsTheCutoffAtTheDedicatedHorizon() + { + /* The dogfood shape: 90-day fact retention, a dim only ~3 weeks old. Coupled cutoff sits ~92 + days back (nothing ever eligible); a 21-day knob must pull it to now - 22. */ + var cutoff = DarlingRetention.ComputeDimensionCutoff(Now, 90, Now.AddDays(-22), planContentRetentionDays: 21); + + Assert.Equal(Now.AddDays(-22), cutoff); + } + + /// + /// A knob WIDER than the coupled horizon must not widen retention — the coupled cutoff (which is + /// newer in that case) still governs, so raising the knob past the fact retention is a no-op rather + /// than a way to keep plan XML no fact can reference. + /// + [Fact] + public void Enabled_NeverWidensPastTheCoupledCutoff() + { + var coupledOnly = DarlingRetention.ComputeDimensionCutoff(Now, 30, oldestSurvivingDigestFact: null); + var withWideKnob = DarlingRetention.ComputeDimensionCutoff(Now, 30, oldestSurvivingDigestFact: null, planContentRetentionDays: 365); + + Assert.Equal(coupledOnly, withWideKnob); + } + + /// The measured-floor clamp still applies underneath the knob: when held facts reach further + /// back than the assumed horizon, the knob (being newer still) wins over both — held history must not + /// re-unbound the dimension. + [Fact] + public void Enabled_WinsOverTheMeasuredFloorToo() + { + var cutoff = DarlingRetention.ComputeDimensionCutoff(Now, 90, Now.AddDays(-200), planContentRetentionDays: 21); + + Assert.Equal(Now.AddDays(-22), cutoff); + } + + /* ---------------- the scoping router (review catch) ---------------- */ + + /// + /// The dedicated horizon governs the PLAN dimension only — query text keeps the fact-coupled cutoff, + /// or the knob would quietly break "text stays analyzable for the facts' full retention", which is + /// half its own justification (and buy ~40 MB for the damage). + /// + [Fact] + public void TheRouterScopesTheKnobToThePlanDimensionOnly() + { + var coupled = Now.AddDays(-92); + var dedicated = Now.AddDays(-22); + + Assert.Equal(dedicated, DarlingRetention.ComputeDimTableCutoff(PayloadDimensions.QueryPlanDimTable, coupled, dedicated)); + Assert.Equal(coupled, DarlingRetention.ComputeDimTableCutoff(PayloadDimensions.QueryTextDimTable, coupled, dedicated)); + } + + /* ---------------- the map ordering (review catch) ---------------- */ + + /// + /// The invariant the review caught this PR breaking: the DIMENSION must outlive the MAP under every + /// knob value, or a live query_store_plan_map row resolves to deleted content — the + /// silent-missing-plans failure the margin ordering exists to prevent. Swept across every age from + /// inside retention to well past both horizons, for the shipped default, the clamp edges, disabled, + /// and a knob wider than the fact horizon. + /// + [Theory] + [InlineData(30, 0)] + [InlineData(30, 7)] + [InlineData(30, 21)] + [InlineData(90, 21)] + [InlineData(90, 365)] + [InlineData(7, 21)] + [InlineData(1, 7)] + public void NeitherPruneOrder_CanLeaveAMapRowResolvingToAnAbsentDigest_UnderTheKnob(int factRetentionDays, int knobDays) + { + var dimCutoff = DarlingRetention.ComputeDimensionCutoff(Now, factRetentionDays, oldestSurvivingDigestFact: null, planContentRetentionDays: knobDays); + var mapCutoff = DarlingRetention.ComputeMapCutoff(Now, factRetentionDays, knobDays); + + /* The dim must outlive the map: strictly older cutoff. */ + Assert.True(dimCutoff < mapCutoff, + $"the dim GC would take content the map still points at: dim cutoff {dimCutoff:o} is not earlier " + + $"than map cutoff {mapCutoff:o} at {factRetentionDays}d retention / knob {knobDays}"); + + /* And the both-orders sweep: no age where the dim row is takeable while its map row survives. */ + for (var age = 0; age <= factRetentionDays + 370; age++) + { + var lastSeen = Now.AddDays(-age); + var mapEligible = lastSeen < mapCutoff; + var dimEligible = lastSeen < dimCutoff; + + Assert.False(dimEligible && !mapEligible, + $"at {age}d the dim row is prunable while its map row survives (retention {factRetentionDays}d, knob {knobDays})"); + } + } + + /// Disabled must reproduce the pre-knob map cutoff exactly, like the dim's disabled path. + [Fact] + public void MapCutoff_Disabled_ReproducesTheOldBehaviorExactly() + { + var old = Now.AddDays(-(30 + QueryStorePlanMap.PruneMarginDays)); + + Assert.Equal(old, DarlingRetention.ComputeMapCutoff(Now, 30)); + Assert.Equal(old, DarlingRetention.ComputeMapCutoff(Now, 30, planContentRetentionDays: 0)); + } + + /// + /// The destructive-sink clamp (review catch, twice: the first "fix" commit lost the edit to a + /// failed batch-script assertion and shipped only its comment). PurgeAsync cannot be executed here + /// without a live store, so this pins the clamp the way the repo pins other unexecutable seams — + /// at the source: the sink must re-clamp before first use, because on a store-unreachable boot the + /// worker passes darling.json's RAW value and a file value of 1-6 would prune plan content below + /// the [7,365] contract. Proven to fail against the unclamped code before the fix landed. + /// + [Fact] + public void PurgeAsyncClampsTheKnobAtTheDestructiveSink() + { + var source = ReadRetentionSource(); + var body = source[source.IndexOf("public static async Task PurgeAsync", StringComparison.Ordinal)..]; + + var clampAt = body.IndexOf("planContentRetentionDays = StoreConfigProvider.ClampPlanContentRetentionDays(planContentRetentionDays);", StringComparison.Ordinal); + Assert.True(clampAt > 0, "PurgeAsync no longer clamps planContentRetentionDays at the destructive sink"); + + /* And the clamp must come BEFORE the first use — both cutoff computations. */ + var firstUse = body.IndexOf("ComputeDimensionCutoff(", StringComparison.Ordinal); + var mapUse = body.IndexOf("ComputeMapCutoff(", StringComparison.Ordinal); + Assert.True(clampAt < firstUse, "the clamp sits after the dimension cutoff computation"); + Assert.True(clampAt < mapUse, "the clamp sits after the map cutoff computation"); + } + + /* ---------------- the viewer probe ---------------- */ + + [Fact] + public void TheProbeMapsAFullyMigratedStoreTo75() + { + /* #2319: no longer the top (that claim lives in QueryStoreHealthStoreTests) — this fact keeps + pinning that a store at exactly 75 maps to 75 and one at 74 maps to 74, forever. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + /* 50 positional sentinels then the V75 one by name — the map takes 51 parameters. Present => 75, + newest-first; absent => the previous arm still answers 74 rather than falling through. */ + var all = Enumerable.Repeat(true, 50).Cast().ToArray(); + + Assert.Equal(75, InvokeMap(all, hasPlanContentRetentionKnob: true)); + Assert.Equal(74, InvokeMap(all, hasPlanContentRetentionKnob: false)); + } + + [Fact] + public void TheProbeAsksForTheColumn_AndTheThreePlacesAgree() + { + Assert.Contains("column_name = 'plan_content_retention_days'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + + var mapParameters = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .GetParameters().Length; + + var viewerSource = ReadViewerSource(); + + /* The reader must hand over exactly one argument per map parameter: ordinals are 0-based, so the + highest is Count - 1, and the next one up must NOT appear. */ + Assert.Contains($"reader.GetBoolean({mapParameters - 1})", viewerSource, StringComparison.Ordinal); + Assert.DoesNotContain($"reader.GetBoolean({mapParameters})", viewerSource, StringComparison.Ordinal); + } + + /* ---------------- helpers ---------------- */ + + private static int InvokeMap(object[] leading, bool hasPlanContentRetentionKnob) + { + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + + /* #2319 appended hasQueryStoreHealth after this rung's parameter — pass it FALSE so these + facts keep exercising the V75/V74 arms rather than the newer one. */ + var args = leading.Concat(new object[] { hasPlanContentRetentionKnob, false }).ToArray(); + Assert.Equal(method.GetParameters().Length, args.Length); + + return (int)method.Invoke(null, args)!; + } + + private static string ReadViewerSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } + + private static string ReadRetentionSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingRetention.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs b/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs new file mode 100644 index 000000000..be287c3cf --- /dev/null +++ b/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the PostgreSQL alert thresholds. These fire pages, so the tests are about the boundaries and the +/// cases that must NOT fire — a predictor that cries wolf gets muted, and a muted outage predictor is worse +/// than none. +/// +public class PostgresAlertEvaluatorTests +{ + private const long StockFreezeMaxAge = 200_000_000; + + /// PostgreSQL's own default for the MultiXact counter — TWICE the XID default, which is the + /// whole reason grading one against the other's setting was wrong. + private const long StockMultixactFreezeMaxAge = 400_000_000; + + private static PostgresWraparoundAlertInfo Wrap( + long xid, + long multi = 0, + long freezeMax = StockFreezeMaxAge, + long multixactFreezeMax = StockMultixactFreezeMaxAge) + => new("appdb", xid, multi, freezeMax, multixactFreezeMax); + + /// + /// Thresholds scale to the SERVER's own autovacuum_freeze_max_age, not to a constant. A cluster tuned to + /// 1.5 billion is in a different place at 400 million than a stock one, and a fixed threshold would + /// either never fire for the first or constantly for the second. + /// + [Fact] + public void WraparoundThresholdsScaleToTheServersOwnFreezeMaxAge() + { + /* 400M is CRITICAL on a stock 200M server (2x)... */ + Assert.Equal( + AlertSeverityLevel.Critical, + PostgresAlertEvaluator.EvaluateWraparound(Wrap(400_000_000))!.Severity); + + /* ...and does not fire at all on one tuned to 1.5 billion, where 90% of the setting is 1.35B. */ + Assert.Null(PostgresAlertEvaluator.EvaluateWraparound(Wrap(400_000_000, freezeMax: 1_500_000_000))); + } + + [Theory] + [InlineData(179_999_999, null)] // just under 90% of 200M + [InlineData(180_000_000, "Warning")] // exactly 90% + [InlineData(399_999_999, "Warning")] // just under 2x + [InlineData(400_000_000, "Critical")] // exactly 2x + [InlineData(1_900_000_000, "Critical")] + public void WraparoundGradesAtTheDocumentedBoundaries(long age, string? expected) + { + var finding = PostgresAlertEvaluator.EvaluateWraparound(Wrap(age)); + + if (expected is null) + { + Assert.Null(finding); + return; + } + + Assert.Equal(Enum.Parse(expected), finding!.Severity); + } + + /// + /// MultiXact exhaustion stops writes exactly as XID exhaustion does, and is the less familiar of the + /// two, so the worse counter has to win AND be named — the remedies differ. + /// + [Fact] + public void MultiXactAgeCanBeTheWorseCounterAndIsNamedAsSuch() + { + var finding = PostgresAlertEvaluator.EvaluateWraparound(Wrap(1_000, multi: 500_000_000)); + + Assert.NotNull(finding); + Assert.Contains("MultiXact", finding!.CurrentValue, StringComparison.Ordinal); + } + + /// + /// Each counter is graded against ITS OWN setting. This assertion changed with the fix and the change is + /// the point: 500M MultiXacts used to grade Critical because it was measured against + /// autovacuum_freeze_max_age (200M, so 2.5x), when the governing setting is + /// autovacuum_multixact_freeze_max_age — 400M by default, making 500M a 1.25x Warning. The old behaviour + /// fired Critical 2.2x premature on every MultiXact-heavy workload. + /// + [Fact] + public void MultiXactIsGradedAgainstItsOwnSettingNotTheXidOne() + { + var finding = PostgresAlertEvaluator.EvaluateWraparound(Wrap(1_000, multi: 500_000_000)); + + Assert.Equal(AlertSeverityLevel.Warning, finding!.Severity); + + /* And the body must quote the setting it actually judged against — it used to print + "autovacuum_freeze_max_age N" beside a counter name saying MultiXact, contradicting itself. */ + Assert.Contains("autovacuum_multixact_freeze_max_age", finding.ThresholdValue, StringComparison.Ordinal); + Assert.Contains("400,000,000", finding.ThresholdValue, StringComparison.Ordinal); + } + + /// + /// Crossing twice the MultiXact setting IS Critical — the fix moves the line, it does not remove it. + /// + [Fact] + public void MultiXactGoesCriticalAtTwiceItsOwnSetting() + { + Assert.Equal( + AlertSeverityLevel.Critical, + PostgresAlertEvaluator.EvaluateWraparound(Wrap(1_000, multi: 800_000_000))!.Severity); + } + + /// + /// The absolute arm. On a cluster tuned near the top of the range, 2 x setting lands BEYOND the + /// 2^31 wall, so the relative Critical was unreachable — on exactly the clusters closest to a write + /// outage. vacuum_failsafe_age (~74.5% of the space) is the floor that makes it reachable. + /// + [Fact] + public void CriticalIsReachableOnAClusterTunedPastHalfTheWall() + { + const long Tuned = 1_500_000_000; + + /* Relative critical would be 3B — past the 2^31 wall, i.e. unreachable. */ + Assert.True(Tuned * 2 > PostgresAlertEvaluator.WraparoundCeiling); + + /* 1.7B is 79% of the space and past vacuum_failsafe_age, so it must be Critical anyway. */ + var finding = PostgresAlertEvaluator.EvaluateWraparound(Wrap(1_700_000_000, freezeMax: Tuned)); + + Assert.Equal(AlertSeverityLevel.Critical, finding!.Severity); + Assert.Contains("wraparound space", finding.ThresholdValue, StringComparison.Ordinal); + Assert.Contains("vacuum_failsafe_age", finding.ShortMessage, StringComparison.Ordinal); + } + + /// + /// One counter having an unusable setting must not silence the other. A server can have a sane + /// autovacuum_freeze_max_age and a nonsensical multixact one. + /// + [Fact] + public void AnUnjudgeableCounterDoesNotSilenceTheJudgeableOne() + { + var finding = PostgresAlertEvaluator.EvaluateWraparound( + Wrap(190_000_000, multi: 900_000_000, multixactFreezeMax: 0)); + + Assert.NotNull(finding); + Assert.Contains("XID", finding!.CurrentValue, StringComparison.Ordinal); + } + + /// + /// A missing or nonsensical setting makes every derived threshold zero, which would fire on every + /// database on every sweep forever. "Cannot judge" must mean silence, not criticality. + /// + [Theory] + [InlineData(0L)] + [InlineData(-1L)] + public void AMissingFreezeMaxAgeSettingSilencesRatherThanFiringOnEverything(long freezeMax) + { + /* Both settings unusable = cannot judge either counter = silence. */ + Assert.Null(PostgresAlertEvaluator.EvaluateWraparound( + Wrap(1_000_000_000, freezeMax: freezeMax, multixactFreezeMax: freezeMax))); + } + + /// + /// The persistence gate is what keeps this from firing on every long-running report. Same age, same + /// holder — only the persistence differs, and only the chronic one alerts. + /// + [Fact] + public void XminFiresOnlyForAChronicHolderNotALongQuery() + { + var chronic = new PostgresXminHorizonAlertInfo("session", "12345", 100_000_000, 30, 40, "idle in transaction"); + var transient = new PostgresXminHorizonAlertInfo("session", "12345", 100_000_000, 2, 40, "running"); + + Assert.NotNull(PostgresAlertEvaluator.EvaluateXmin(chronic)); + Assert.Null(PostgresAlertEvaluator.EvaluateXmin(transient)); + } + + [Fact] + public void XminBelowTheAgeThresholdNeverFiresHoweverPersistent() + { + Assert.Null(PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo("session", "1", 1_000_000, 40, 40, null))); + } + + /// A zero denominator must not divide — it means nothing was observed, so nothing fires. + [Fact] + public void XminWithNoObservationsDoesNotFireOrThrow() + { + Assert.Null(PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo("session", "1", 900_000_000, 0, 0, null))); + } + + /// + /// Each of the five causes is indistinguishable from the others by symptom and needs a different fix, so + /// the alert body must carry that fix rather than making the reader work out which one it is. + /// + [Theory] + [InlineData("session", "idle in transaction")] + [InlineData("replication_slot", "dropped")] + [InlineData("replication_slot_catalog", "catalog_xmin")] + [InlineData("standby_feedback", "hot_standby_feedback")] + [InlineData("prepared_transaction", "PREPARED")] + public void XminMessageCarriesTheRemedyForItsCause(string source, string fragment) + { + var finding = PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo(source, "x", 100_000_000, 10, 10, null)); + + Assert.Contains(fragment, finding!.ShortMessage, StringComparison.OrdinalIgnoreCase); + } + + /// + /// lost and unreserved are failures that have already happened, so they fire at ANY size — a byte + /// threshold would let a small-but-broken slot pass silently. + /// + [Theory] + [InlineData("lost")] + [InlineData("unreserved")] + public void TerminalSlotStatesFireCriticalAtAnySize(string walStatus) + { + var finding = PostgresAlertEvaluator.EvaluateSlot( + new PostgresSlotAlertInfo("s1", walStatus, IsActive: true, RetainedWalBytes: 1024, + RetainedWalGrowthBytes: 0, InactiveSince: null)); + + Assert.NotNull(finding); + Assert.Equal(AlertSeverityLevel.Critical, finding!.Severity); + } + + /// + /// The disk-fill emergency is the conjunction: over the line, nobody consuming, still growing. Each part + /// alone is a lesser finding, and grading them all critical would make the critical meaningless. + /// + [Fact] + public void TheOrphanFillingDiskIsCriticalButItsPartsAloneAreNot() + { + const long OverLine = 20L * 1024 * 1024 * 1024; + + Assert.Equal(AlertSeverityLevel.Critical, PostgresAlertEvaluator.EvaluateSlot( + new PostgresSlotAlertInfo("s", "extended", false, OverLine, 5L * 1024 * 1024 * 1024, null))!.Severity); + + /* Active consumer: behind, but someone is draining it. */ + Assert.Equal(AlertSeverityLevel.Warning, PostgresAlertEvaluator.EvaluateSlot( + new PostgresSlotAlertInfo("s", "extended", true, OverLine, 5L * 1024 * 1024 * 1024, null))!.Severity); + + /* Inactive but flat: a consumer between polls, not a volume filling. */ + Assert.Equal(AlertSeverityLevel.Warning, PostgresAlertEvaluator.EvaluateSlot( + new PostgresSlotAlertInfo("s", "extended", false, OverLine, 0, null))!.Severity); + } + + /// A healthy slot under the byte line is silent, growing or not. + [Fact] + public void ASlotUnderTheByteLineDoesNotFire() + { + Assert.Null(PostgresAlertEvaluator.EvaluateSlot( + new PostgresSlotAlertInfo("s", "reserved", false, 1024, 512, null))); + } + + /// The healthy fleet case: nothing fires, and an empty list is not an error. + [Fact] + public void AHealthyServerProducesNoFindings() + { + var findings = PostgresAlertEvaluator.Evaluate( + new[] { Wrap(1_000_000) }, + new PostgresXminHorizonAlertInfo("session", "1", 1_000, 40, 40, null), + new[] { new PostgresSlotAlertInfo("s", "reserved", true, 0, 0, null) }); + + Assert.Empty(findings); + } + + /// Nulls throughout (nothing collected yet) must be silent rather than throwing mid-sweep. + [Fact] + public void NoDataIsSilentRatherThanThrowing() + { + Assert.Empty(PostgresAlertEvaluator.Evaluate(null, null, null)); + } + + /// Worst-first, so a host that caps delivery keeps the ones that matter. + [Fact] + public void FindingsAreOrderedWorstFirst() + { + var findings = PostgresAlertEvaluator.Evaluate( + new[] { Wrap(190_000_000) }, // Warning + null, + new[] { new PostgresSlotAlertInfo("s", "lost", false, 1, 0, null) }); // Critical + + Assert.Equal(AlertSeverityLevel.Critical, findings[0].Severity); + Assert.True(findings.Count > 1); + } + + /// + /// Metric names are the key mute rules and history filtering match on, so they are part of the contract + /// and must not drift casually. + /// + [Fact] + public void MetricNamesArePinnedAndDistinct() + { + var names = new[] + { + PostgresAlertEvaluator.WraparoundMetric, + PostgresAlertEvaluator.XminHorizonMetric, + PostgresAlertEvaluator.SlotRetentionMetric, + }; + + Assert.Equal(names.Length, names.Distinct().Count()); + Assert.All(names, n => Assert.StartsWith("PostgreSQL ", n, StringComparison.Ordinal)); + } + + /// + /// Every finding carries a subject, because the host builds its dedup fingerprint from it — without one, + /// two databases breaching at once would collapse into a single alert. + /// + [Fact] + public void EveryFindingIdentifiesItsSubject() + { + var findings = PostgresAlertEvaluator.Evaluate( + new[] { new PostgresWraparoundAlertInfo("db_a", 500_000_000, 0, StockFreezeMaxAge, StockMultixactFreezeMaxAge), + new PostgresWraparoundAlertInfo("db_b", 500_000_000, 0, StockFreezeMaxAge, StockMultixactFreezeMaxAge) }, + null, + null); + + Assert.Equal(2, findings.Count); + Assert.Equal(2, findings.Select(f => f.Subject).Distinct().Count()); + Assert.All(findings, f => Assert.False(string.IsNullOrWhiteSpace(f.Subject))); + } +} diff --git a/Darling/Darling.Tests/PostgresEngineGateBehaviorTests.cs b/Darling/Darling.Tests/PostgresEngineGateBehaviorTests.cs new file mode 100644 index 000000000..3fb4ebed3 --- /dev/null +++ b/Darling/Darling.Tests/PostgresEngineGateBehaviorTests.cs @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// BEHAVIOURAL coverage for the PostgreSQL engine gate on the analyze_now operator door (#2230). +/// +/// What was missing. The three gates added by #2213's round-3 fix were covered by +/// source-scanning pins (TheScheduledAnalysisPassIsGatedByEngine greps the call site) and by a live +/// rig run, but nothing drove a PostgreSQL runtime through a gate and asserted the short-circuit. A source +/// scan cannot tell a gate that returns early from one that falls through and happens to write the same +/// text. +/// +/// All three doors. The analyze_now gate's observable is a PRESENCE — a row in +/// analysis_state carrying the engine tombstone — so it is asserted directly. The reconcile gate +/// looked like it needed a counting seam to observe an absence, and that was wrong: the belt gate inside +/// is public and its own precondition, +/// so an ungated call THROWS and a gated one returns — a difference an assertion can see with no seam, no +/// live store, and no network. snapshot_now needed neither trick: its dispatch loop already reports how many +/// collectors it ran, and every run it makes writes itself to collection_log, so the gate's effect is +/// a count and a set of rows rather than something that has to be inferred. +/// +/// The regression it guards is specific and was real. Clicking "Generate now" against a +/// PostgreSQL target used to run the full SQL-Server-shaped pass, find nothing, and persist the GENERIC +/// insufficient_data message — OVERWRITING the honest engine tombstone the scheduled arm had already +/// written. The Recommendations tab regressed from "does not apply, use the PG reads" back to "still +/// collecting" the moment an operator pressed the button. So the assertion that matters is not just +/// "insufficient_data is true", it is that the MESSAGE is the engine one. +/// +/// Live-store gated on DARLING_TEST_PG, which CI's "Darling PostgreSQL tests" job sets — the +/// gate's whole effect is a write through _postgres, so there is nothing to observe without one. +/// +[Collection("live-postgres")] +public sealed class PostgresEngineGateBehaviorTests +{ + /// + /// The HOST is what identity derives from, which is the trap this test tripped over first: + /// MonitoredServer.StorageName is BuildStorageName(Host, Database, ReadOnlyIntent, Engine, Port) + /// (#2218 added the last two) — NOT + /// Name — and RunAnalyzeNowAsync finds a server by hashing that. A server_id hashed from + /// anything else simply is not found, and the gate then returns "server not monitored" rather than the + /// arm under test. Unique hosts so neither case can collide with a real server's analysis_state row. + /// + private const string PgHost = "pg-engine-gate-behavior-2230.invalid"; + + private const string SqlHost = "sql-engine-gate-behavior-2230.invalid"; + + /* snapshot_now writes to collection_log rather than analysis_state, and it asserts on EMPTINESS — so it + needs hosts of its own, or the analyze_now cases sharing a server_id would make the absence assertion + depend on test ordering. */ + private const string PgSnapshotHost = "pg-snapshot-gate-behavior-2230.invalid"; + + private const string SqlSnapshotHost = "sql-snapshot-gate-behavior-2230.invalid"; + + /// The one collector both snapshot arms leave enabled: SQL-Server-only, so the engine gate is + /// the ONLY thing that can decide whether it is dispatched. + private const string SnapshotCollector = "wait_stats"; + + /// + /// Derived through the SAME helper the worker uses, so the test cannot drift from the lookup. + /// + /// #2218 is that drift happening: the ENGINE joined the derivation, and because this helper hashed + /// without it, a PostgreSQL host produced an id the product never uses — so the gate returned "server not + /// monitored" rather than the arm under test, which is the same trap the class comment above describes for + /// Name vs Host. The engine is defaulted to null so the SQL Server call sites stay unchanged, + /// exactly as the shared helper does it. + /// + private static int ServerIdFor(string host, string? engine = null) => + ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(host, null, false, engine, 0)); + + [Fact] + public async Task AnalyzeNow_AgainstAPostgresTarget_WritesTheEngineTombstone_AndDoesNotRunThePass() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the analyze_now engine-gate test."); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var serverId = ServerIdFor(PgHost, "postgres"); + + /* Fabricated worker, the CollectorMemoryKnobTests.SweepGate idiom: the real ctor wants a host's worth + of dependencies, and the gate under test reads exactly three fields. Reflection because pinning the + BEHAVIOUR beats widening the surface just to observe it. */ + var worker = (DarlingWorker)System.Runtime.CompilerServices.RuntimeHelpers + .GetUninitializedObject(typeof(DarlingWorker)); + SetField(worker, "_serversLock", new object()); + SetField(worker, "_logger", NullLogger.Instance); + SetField(worker, "_postgres", postgres); + + var server = PostgresLoopState(serverId); + var servers = NewLoopStateList(server); + + var bodySucceeded = false; + try + { + var outcome = await InvokeAnalyzeNowAsync(worker, servers, serverId); + + /* 1. The gate returned the success shape, not a failure and not the analysis result. */ + /* Assert the STATUS first: if the lookup missed, the status is "server not monitored" and says + so, where a bare Assert.True on Success only reports Expected/Actual booleans. */ + Assert.Equal("analysis not applicable", GetOutcomeStatus(outcome)); + Assert.True(GetOutcomeSuccess(outcome)); + + /* 2. The once-latch is set, so the scheduled tick will not re-write what this just wrote — + the two arms share the tombstone rather than racing to overwrite it. */ + Assert.True(AnalysisStateWritten(server)); + + /* 3. THE REGRESSION GUARD: the persisted message is the ENGINE tombstone, not the generic + insufficient-data text the SQL-Server-shaped pass would have left. */ + var state = await ReadAnalysisStateAsync(postgres, serverId); + var (found, insufficient, message) = (state.Found, state.Insufficient, state.Message); + Assert.True(found, "the gate must PERSIST a row, or the Recommendations tab has nothing to show"); + Assert.True(insufficient); + Assert.Equal(DarlingWorker.PostgresAnalysisNotApplicable, message); + + /* And the specific words that make it honest rather than merely non-empty. */ + Assert.Contains("does not apply to a PostgreSQL target", message, StringComparison.Ordinal); + Assert.Contains("get_pg_blocking", message, StringComparison.Ordinal); + /* And it DISCLAIMS the still-collecting reading rather than avoiding the words: the message + quotes the phrase in order to contrast with it ("This is not \"still collecting\""), so a + DoesNotContain on those words can never pass and asserting it was my error, not the + product's. The property worth pinning is that the disclaimer is present. */ + Assert.Contains("This is not \"still collecting\"", message, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, (cleanup, cleanupCt) => + DeleteAnalysisStateAsync(cleanup, cleanupCt, serverId)); + } + } + + /// + /// The same door against a SQL Server target must NOT take the gate — otherwise the test above would + /// pass on a gate that fires unconditionally, which is the failure mode a presence-assertion is blind to. + /// Asserted by the outcome status alone: a SQL Server target falls through to the real pass, which + /// on a store with no data for this server_id reports insufficient data. Either way it is NOT + /// "analysis not applicable", and that is the discriminator. + /// + [Fact] + public async Task AnalyzeNow_AgainstASqlServerTarget_DoesNotTakeTheEngineGate() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the analyze_now engine-gate test."); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var serverId = ServerIdFor(SqlHost); + + var worker = (DarlingWorker)System.Runtime.CompilerServices.RuntimeHelpers + .GetUninitializedObject(typeof(DarlingWorker)); + SetField(worker, "_serversLock", new object()); + SetField(worker, "_logger", NullLogger.Instance); + SetField(worker, "_postgres", postgres); + + var server = SqlServerLoopState(serverId); + var servers = NewLoopStateList(server); + + var bodySucceeded = false; + try + { + /* The SQL Server path runs the real analysis pass, which needs collaborators the fabricated + worker does not have — so the assertion is that it did NOT short-circuit as the PG arm, which + is observable either as a different status or as a throw from the pass itself. Both prove the + gate is engine-conditional; only "analysis not applicable" would disprove it. */ + string? status = null; + try + { + status = GetOutcomeStatus(await InvokeAnalyzeNowAsync(worker, servers, serverId)); + } + catch (Exception ex) when (ex is not Xunit.Sdk.XunitException) + { + /* Fell through into the pass and hit a missing collaborator — which is itself the proof. */ + Assert.NotNull(ex); + } + + Assert.NotEqual("analysis not applicable", status); + Assert.False(AnalysisStateWritten(server), + "the PostgreSQL once-latch must not be set for a SQL Server target"); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, (cleanup, cleanupCt) => + DeleteAnalysisStateAsync(cleanup, cleanupCt, serverId)); + } + } + + /// + /// The reconcile door, gated (#2230). + /// carries its own engine precondition — "belt to the worker's braces" — and it is the belt that + /// actually stopped the field failure, so it is the one worth pinning. + /// + /// The regression was measured, not theoretical: ungated, this method built a + /// SqlConnection from a PostgreSQL connection string, the ctor threw + /// Keyword not supported: 'host', the caller's catch skipped the latch assignment, and because + /// LongQueryTraceApplied resets to null on every connect it retried EVERY sweep forever — + /// ~1,440 warnings/day/server (#2213 round 2). + /// + /// Both enabled values, because the gate sits ahead of that branch: ungated, the false arm + /// would try to DROP a session over the same impossible connection. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task ReconcileLongQueryCompletions_AgainstAPostgresTarget_ReturnsBeforeBuildingASqlConnection(bool enabled) + { + /* runner is null on purpose: reaching it would mean the gate did not fire. */ + await DarlingXeSessions.ReconcileLongQueryCompletionsAsync( + PostgresRuntime(), runner: null!, enabled, NullLogger.Instance, CancellationToken.None); + } + + /// + /// The proof the test above is not vacuous. Same connection string, ENGINE flipped to SQL Server: the + /// gate no longer applies, the ctor rejects the string, and the exact field exception surfaces. Without + /// this arm, the pin above would pass just as happily against a method that had stopped connecting for + /// some unrelated reason. + /// + [Fact] + public async Task ReconcileLongQueryCompletions_SameStringButSqlServerEngine_ThrowsTheFieldFailure() + { + /* The engine is the ONLY difference from the gated case — same host, same connection string. */ + var ungated = PostgresRuntime(CollectorTargetEngine.SqlServer); + + var ex = await Assert.ThrowsAsync(() => + DarlingXeSessions.ReconcileLongQueryCompletionsAsync( + ungated, runner: null!, enabled: true, NullLogger.Instance, CancellationToken.None)); + + /* The words from the sweep log, so a future reader can match this pin to that incident. */ + Assert.Contains("Keyword not supported", ex.Message, StringComparison.Ordinal); + Assert.Contains("host", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The snapshot_now door, the THIRD dispatch loop and the one that got neither engine gate in #2213's + /// first round (#2230). + /// + /// The regression is phantom success, not a crash. An operator snapshot against a + /// PostgreSQL target dispatched every SQL Server collector. Those collectors do not fail loudly — their + /// own AppliesTo early-returns, yielding zero rows, and RunOneAsync then writes + /// SUCCESS to collection_log. So one click produced a burst of ~40 rows saying collection + /// worked, on an engine where those collectors cannot mean anything. That reads as health, which is + /// strictly worse than an error. + /// + /// Why one collector rather than all of them. The schedule overrides disable everything + /// except wait_stats — SQL-Server-only, and dispatched through the same loop — so the two arms + /// below differ in the ENGINE and nothing else: same collector, same overrides, same store. That keeps + /// the test to a single dispatch decision instead of 49, and makes the SQL Server arm cheap enough to be + /// the non-vacuity proof rather than a second slow test. + /// + [Fact] + public async Task SnapshotNow_AgainstAPostgresTarget_DispatchesNoSqlServerCollector() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the snapshot_now engine-gate test."); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var serverId = ServerIdFor(PgSnapshotHost, "postgres"); + + var bodySucceeded = false; + try + { + var (collectorsRun, _) = await InvokeSnapshotAsync( + postgres, + NewLoopState( + new MonitoredServer { Name = "pg-snapshot-gate", Host = PgSnapshotHost, Engine = "postgres" }, + SnapshotRuntime(PgSnapshotHost, serverId, CollectorTargetEngine.PostgreSql)), + serverId); + + /* 1. THE GATE: the only enabled collector is SQL-Server-only, so a gated loop runs nothing. */ + Assert.Equal(0, collectorsRun); + + /* 2. And it left no trace claiming otherwise. This is the assertion that would have caught the + original defect: ungated, collection_log carries a wait_stats row here, and its status is + SUCCESS — the phantom-success class. Asserting on the ABSENCE of the row rather than on its + status is deliberate, because a gate that dispatched and then failed would also avoid + SUCCESS while still having connected to a PostgreSQL host as SQL Server. */ + Assert.Empty(await ReadLoggedCollectorsAsync(postgres, serverId)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, (cleanup, cleanupCt) => + DeleteCollectionLogAsync(cleanup, cleanupCt, serverId)); + } + } + + /// + /// The proof the test above is not vacuous. Same collector, same overrides, engine flipped to SQL + /// Server: the loop dispatches it, and the run writes itself to collection_log. + /// + /// Without this arm the gate assertion would pass just as well against a snapshot that had stopped + /// dispatching anything at all — a loop broken for some unrelated reason looks identical to a loop + /// gated correctly, and "ran zero collectors" is exactly what a broken snapshot also reports. + /// + /// The connection goes to a loopback port with no listener, so the collector fails immediately + /// rather than waiting out a default timeout. The STATUS is not asserted: whether the attempt lands + /// ERROR or a zero-row SUCCESS depends on how far the collector gets before the connection dies, and the + /// fact under test is that it was dispatched at all. + /// + [Fact] + public async Task SnapshotNow_AgainstASqlServerTarget_DispatchesTheSameCollector() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the snapshot_now engine-gate test."); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var serverId = ServerIdFor(SqlSnapshotHost); + + var bodySucceeded = false; + try + { + var (collectorsRun, success) = await InvokeSnapshotAsync( + postgres, + NewLoopState( + new MonitoredServer { Name = "sql-snapshot-gate", Host = SqlSnapshotHost }, + SnapshotRuntime(SqlSnapshotHost, serverId, CollectorTargetEngine.SqlServer)), + serverId); + + Assert.True(success); + Assert.Equal(1, collectorsRun); + + /* The row the PostgreSQL arm must not have, under the name that identifies it. */ + Assert.Equal(new[] { "wait_stats" }, await ReadLoggedCollectorsAsync(postgres, serverId)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, (cleanup, cleanupCt) => + DeleteCollectionLogAsync(cleanup, cleanupCt, serverId)); + } + } + + /// A PostgreSQL runtime whose connection string is the PostgreSQL shape that SqlConnection + /// cannot parse — the combination that produced the field failure. + private static ServerRuntime PostgresRuntime( + CollectorTargetEngine engine = CollectorTargetEngine.PostgreSql) => new() + { + Config = new MonitoredServer { Name = "pg-reconcile-gate", Host = PgHost, Engine = "postgres" }, + ConnectionString = $"Host={PgHost};Database=postgres;Username=monitor", + Target = new CollectorTargetInfo { Engine = engine }, + StorageName = PgHost, + ServerId = ServerIdFor(PgHost, "postgres"), + }; + + private static void SetField(DarlingWorker worker, string name, object value) => + typeof(DarlingWorker) + .GetField(name, BindingFlags.NonPublic | BindingFlags.Instance)! + .SetValue(worker, value); + + private static async Task InvokeAnalyzeNowAsync( + DarlingWorker worker, object servers, int serverId) + { + var method = typeof(DarlingWorker).GetMethod( + "RunAnalyzeNowAsync", BindingFlags.NonPublic | BindingFlags.Instance)!; + + /* planFetcher / notificationService / config are only touched on the SQL Server path, so the gate + can be driven with nulls — which is itself part of what "short-circuits" means here. */ + var task = (Task)method.Invoke(worker, new object?[] + { + servers, null, null, null, serverId, CancellationToken.None, + })!; + await task; + return task.GetType().GetProperty("Result")!.GetValue(task)!; + } + + /* CommandOutcome is public (DarlingCommandExecutor), so no reflection is needed for the result — + only for ServerLoopState, which is a private nested type. */ + private static bool GetOutcomeSuccess(object outcome) => ((CommandOutcome)outcome).Success; + + private static string? GetOutcomeStatus(object? outcome) => (outcome as CommandOutcome)?.ResultStatus; + + /// + /// DarlingWorker.ServerLoopState is a PRIVATE nested class, so the test cannot name the type and + /// builds it reflectively — the same trade CollectorMemoryKnobTests makes for private gate state. + /// Widening it to internal purely for a test would be a production change to observe behaviour + /// that reflection can already reach. + /// + private static readonly Type LoopStateType = typeof(DarlingWorker) + .GetNestedType("ServerLoopState", BindingFlags.NonPublic)!; + + private static object NewLoopState(MonitoredServer config, ServerRuntime runtime) + { + var state = Activator.CreateInstance(LoopStateType)!; + LoopStateType.GetProperty("Config")!.SetValue(state, config); + LoopStateType.GetProperty("Runtime")!.SetValue(state, runtime); + return state; + } + + private static bool AnalysisStateWritten(object loopState) => + (bool)LoopStateType.GetProperty("PostgresAnalysisStateWritten")!.GetValue(loopState)!; + + /// The parameter is List<ServerLoopState>, so the list is reflective too. + private static object NewLoopStateList(object single) + { + var list = Activator.CreateInstance(typeof(List<>).MakeGenericType(LoopStateType))!; + list.GetType().GetMethod("Add")!.Invoke(list, new[] { single }); + return list; + } + + private static object PostgresLoopState(int serverId) => NewLoopState( + new MonitoredServer { Name = "pg-gate", Host = PgHost, Engine = "postgres" }, + new ServerRuntime + { + Config = new MonitoredServer { Name = "pg-gate", Host = PgHost, Engine = "postgres" }, + ConnectionString = $"Host={PgHost};Database=postgres;Username=monitor", + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql }, + StorageName = PgHost, + ServerId = serverId, + }); + + private static object SqlServerLoopState(int serverId) => NewLoopState( + new MonitoredServer { Name = "sql-gate", Host = SqlHost }, + new ServerRuntime + { + Config = new MonitoredServer { Name = "sql-gate", Host = SqlHost }, + ConnectionString = $"Server={SqlHost};Integrated Security=true", + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.SqlServer }, + StorageName = SqlHost, + ServerId = serverId, + }); + + private static async Task<(bool Found, bool Insufficient, string Message)> ReadAnalysisStateAsync( + NpgsqlDataSource postgres, int serverId) + { + await using var command = postgres.CreateCommand( + "SELECT insufficient_data, message FROM analysis_state WHERE server_id = $1 " + + "ORDER BY analysis_time DESC LIMIT 1"); + command.Parameters.AddWithValue(serverId); + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + if (!await reader.ReadAsync(TestContext.Current.CancellationToken)) + { + return (false, false, string.Empty); + } + + return (true, + !reader.IsDBNull(0) && reader.GetBoolean(0), + reader.IsDBNull(1) ? string.Empty : reader.GetString(1)); + } + + /// + /// Deletes only this test's own synthetic server_id, through LiveStoreCleanup so the teardown runs + /// on its OWN connection rather than the body's (#1902). A finally that tears down on the body's + /// connection throws out of the finally and REPLACES the body's exception with the teardown's — and it is + /// the body's failure that closed the connection in the first place, so the teardown fails because of the + /// thing it then hides. Opening a fresh connection by hand is explicitly not accepted either: it is half + /// the fix and still throws from the finally. + /// + private static async Task DeleteAnalysisStateAsync( + NpgsqlConnection cleanup, CancellationToken cleanupCt, int serverId) + { + await using var command = new NpgsqlCommand( + "DELETE FROM analysis_state WHERE server_id = $1", cleanup); + command.Parameters.AddWithValue(serverId); + await command.ExecuteNonQueryAsync(cleanupCt); + } + + /// + /// A runtime for the snapshot arms. A dispatched collector must fail FAST, because the SQL Server arm's + /// whole purpose is to prove dispatch happened, not to collect anything. + /// + /// The connection host is deliberately NOT the identity host. Identity comes from + /// host — a synthetic .invalid name, so the lookup and the store rows cannot collide with a + /// real server — while the connection goes to 127.0.0.1 port 1, which is the suite's existing + /// unreachable-endpoint idiom (ViewerControlPlaneStage3bTests, the MCP DeadStore + /// constants). Connection-refused from a loopback port with no listener is immediate and depends on + /// nothing outside the runner; resolving a .invalid name instead makes the timing a property of + /// CI's resolver, which is not a thing this test should be measuring. + /// + private static ServerRuntime SnapshotRuntime(string host, int serverId, CollectorTargetEngine engine) => new() + { + Config = new MonitoredServer { Name = host, Host = host }, + ConnectionString = engine == CollectorTargetEngine.PostgreSql + ? "Host=127.0.0.1;Port=1;Database=postgres;Username=monitor;Timeout=1" + /* SQL auth rather than integrated: the failure under test is the connect, and integrated auth on + a Linux runner fails for a platform reason instead, which is a different thing to assert on. */ + : "Server=127.0.0.1,1;User ID=x;Password=x;Connect Timeout=1;Encrypt=false", + Target = new CollectorTargetInfo { Engine = engine }, + StorageName = host, + ServerId = serverId, + }; + + /// + /// Every collector disabled except , as per-server overrides. This is what + /// keeps the two arms to a single dispatch decision — and it goes through the shipped + /// , which the loop consults, rather than reaching past + /// it. + /// + private static IReadOnlyList SingleEnabledCollectorOverrides(int serverId) => + CollectorScheduleDefaults.All.Keys + .Select(name => new ScheduleOverride( + serverId, name, null, null, + Enabled: string.Equals(name, SnapshotCollector, StringComparison.OrdinalIgnoreCase))) + .ToList(); + + /// + /// Drives the real RunSnapshotAsync and returns what its outcome JSON reports. The runner is a + /// REAL rather than a stand-in: a fake would have to reimplement the + /// dispatch it exists to observe, and the collection_log write that carries the assertion happens inside + /// the real path. + /// + private static async Task<(int CollectorsRun, bool Success)> InvokeSnapshotAsync( + NpgsqlDataSource postgres, object loopState, int serverId) + { + var worker = (DarlingWorker)System.Runtime.CompilerServices.RuntimeHelpers + .GetUninitializedObject(typeof(DarlingWorker)); + SetField(worker, "_serversLock", new object()); + SetField(worker, "_logger", NullLogger.Instance); + SetField(worker, "_postgres", postgres); + SetField(worker, "_scheduleOverrides", SingleEnabledCollectorOverrides(serverId)); + + var runner = new DarlingCollectorRunner( + postgres, new CollectorDeltaCalculator(), NullLogger.Instance); + + var method = typeof(DarlingWorker).GetMethod( + "RunSnapshotAsync", BindingFlags.NonPublic | BindingFlags.Instance)!; + var task = (Task)method.Invoke(worker, new object?[] + { + NewLoopStateList(loopState), runner, serverId, TestContext.Current.CancellationToken, + })!; + await task; + + var outcome = (CommandOutcome)task.GetType().GetProperty("Result")!.GetValue(task)!; + + /* collectorsRun is only in the JSON — the outcome record carries the status text, not the count. A + failure shape has no such property, so a missing one is reported as a failed snapshot rather than + silently read as zero. */ + using var json = System.Text.Json.JsonDocument.Parse(outcome.ResultJson!); + var ran = json.RootElement.TryGetProperty("collectorsRun", out var value) ? value.GetInt32() : -1; + return (ran, outcome.Success); + } + + /// The distinct collector names this snapshot logged, ordered so the assertion is stable. + private static async Task> ReadLoggedCollectorsAsync(NpgsqlDataSource postgres, int serverId) + { + var names = new List(); + await using var command = postgres.CreateCommand( + "SELECT DISTINCT collector_name FROM collection_log WHERE server_id = $1 ORDER BY collector_name"); + command.Parameters.AddWithValue(serverId); + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + while (await reader.ReadAsync(TestContext.Current.CancellationToken)) + { + names.Add(reader.GetString(0)); + } + + return names; + } + + private static async Task DeleteCollectionLogAsync( + NpgsqlConnection cleanup, CancellationToken cleanupCt, int serverId) + { + await using var command = new NpgsqlCommand( + "DELETE FROM collection_log WHERE server_id = $1", cleanup); + command.Parameters.AddWithValue(serverId); + await command.ExecuteNonQueryAsync(cleanupCt); + } +} diff --git a/Darling/Darling.Tests/PostgresFaultOutcomeTests.cs b/Darling/Darling.Tests/PostgresFaultOutcomeTests.cs new file mode 100644 index 000000000..8537df862 --- /dev/null +++ b/Darling/Darling.Tests/PostgresFaultOutcomeTests.cs @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Targets; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins how a PostgreSQL fault becomes a collection_log outcome. The point of the mapping is that a +/// PERSISTENT, operator-actionable condition must not log ERROR every cycle forever — and that the ones +/// which genuinely are errors still do. +/// +public class PostgresFaultOutcomeTests +{ + /// + /// PostgresException's public constructor takes the fields the classifier reads. SqlState is the only + /// one that decides anything; the message text just has to survive into the explanation. + /// + private static PostgresException Pg(string sqlState, string message = "boom") + => new(message, "ERROR", "ERROR", sqlState); + + /* A collector that does NOT opt into the lock-timeout yield, so 55P03 stays an error for it. */ + private const string PlainCollector = "pg_wait_stats"; + + [Fact] + public void PermissionDeniedIsRecordedAsPermissionsAndNamesTheRoleThatFixesIt() + { + var (status, explanation) = DarlingWorker.PostgresFaultOutcome(Pg("42501"), PlainCollector); + + Assert.Equal("PERMISSIONS", status); + Assert.Contains("pg_monitor", explanation, StringComparison.Ordinal); + Assert.Contains("42501", explanation, StringComparison.Ordinal); + } + + /// + /// The case this wiring exists for. pg_statement_stats against a database where the extension was + /// never created raises 42P01 on every single cycle; before this it logged ERROR every minute forever. + /// It must degrade quietly AND say plainly that it is not a grant problem, or the PERMISSIONS status + /// sends someone hunting for a GRANT that will not help. + /// + [Theory] + [InlineData("42P01")] + [InlineData("42883")] + public void AMissingObjectDegradesQuietlyAndSaysItIsNotAGrant(string sqlState) + { + var (status, explanation) = DarlingWorker.PostgresFaultOutcome(Pg(sqlState), "pg_statement_stats"); + + Assert.Equal("PERMISSIONS", status); + Assert.Contains("NOT a missing grant", explanation, StringComparison.Ordinal); + Assert.Contains("CREATE EXTENSION", explanation, StringComparison.Ordinal); + } + + /// + /// Aurora does not implement some community sources at all (0A000 for pg_stat_wal) and gates others by + /// parameter group (55006). Neither will change until the platform or the parameter group does, so + /// neither is worth an error every cycle — and neither is a grant. + /// + [Theory] + [InlineData("0A000")] + [InlineData("55000")] + [InlineData("55006")] + public void AnUnsupportedOrDisabledFeatureDegradesQuietly(string sqlState) + { + var (status, explanation) = DarlingWorker.PostgresFaultOutcome(Pg(sqlState), PlainCollector); + + Assert.Equal("PERMISSIONS", status); + Assert.Contains("NOT a missing grant", explanation, StringComparison.Ordinal); + Assert.Contains("parameter group", explanation, StringComparison.Ordinal); + } + + /// + /// A lock timeout is a YIELD only for a collector that deliberately set a short lock timeout. For any + /// other collector it is a genuine error and must reach the general handler. + /// + [Fact] + public void LockTimeoutYieldsOnlyForACollectorThatOptedIn() + { + Assert.Equal("ERROR", DarlingWorker.PostgresFaultOutcome(Pg("55P03"), PlainCollector).Status); + + /* Whichever collectors declare the guard, at least one must yield — otherwise the branch is dead + and the classifier's yieldsOnLockTimeout argument is being ignored. */ + var yielding = Array.Find( + System.Linq.Enumerable.ToArray(CollectorCatalog.All), + d => CollectorCatalog.YieldsOnLockTimeout(d.Name)); + + if (yielding is not null) + { + var (status, explanation) = DarlingWorker.PostgresFaultOutcome(Pg("55P03"), yielding.Name); + Assert.Equal("YIELDED", status); + Assert.Contains("lock contention", explanation, StringComparison.Ordinal); + } + } + + /// + /// A statement timeout is a slow query, not a dead socket. It must fall through to the general handler + /// as an ERROR — and, critically, must NOT be classified ConnectionFatal, because the general handler + /// drops and reprobes the connection for that, turning a tuning problem into a reconnect storm. + /// + [Fact] + public void AStatementTimeoutIsAnErrorButNotAConnectionFailure() + { + Assert.Equal("ERROR", DarlingWorker.PostgresFaultOutcome(Pg("57014"), PlainCollector).Status); + + Assert.Equal( + CollectorTargetFault.CommandTimeout, + PostgresTargetProvider.Instance.Classify(Pg("57014"), yieldsOnLockTimeout: false)); + Assert.NotEqual( + CollectorTargetFault.ConnectionFatal, + PostgresTargetProvider.Instance.Classify(Pg("57014"), yieldsOnLockTimeout: false)); + } + + /// + /// Connection-level failures DO belong to the general handler, and are the cases that force the + /// reconnect. Pinned through the provider, which is what the handler's own filter consults. + /// + [Theory] + [InlineData("08006")] + [InlineData("08003")] + [InlineData("57P01")] + [InlineData("57P02")] + [InlineData("57P03")] + public void ConnectionLevelFailuresReachTheGeneralHandlerAndForceAReprobe(string sqlState) + { + Assert.Equal("ERROR", DarlingWorker.PostgresFaultOutcome(Pg(sqlState), PlainCollector).Status); + + Assert.Equal( + CollectorTargetFault.ConnectionFatal, + PostgresTargetProvider.Instance.Classify(Pg(sqlState), yieldsOnLockTimeout: false)); + } + + /// An unrecognized SQLSTATE stays loud rather than being quietly bucketed as a skip. + [Fact] + public void AnUnknownSqlStateStaysAnError() + { + var (status, explanation) = DarlingWorker.PostgresFaultOutcome(Pg("XX000", "internal error"), PlainCollector); + + Assert.Equal("ERROR", status); + Assert.Contains("internal error", explanation, StringComparison.Ordinal); + } + + /// + /// Every non-ERROR status the mapper can emit must be one the store already understands. Inventing a + /// sixth would break the dashboards, health bands and self-alerts that read this column. + /// + [Theory] + [InlineData("42501")] + [InlineData("42P01")] + [InlineData("0A000")] + [InlineData("55006")] + [InlineData("57014")] + [InlineData("08006")] + [InlineData("XX000")] + public void OnlyEverEmitsAStatusTheStoreAlreadyUnderstands(string sqlState) + { + var status = DarlingWorker.PostgresFaultOutcome(Pg(sqlState), PlainCollector).Status; + + Assert.Contains(status, new[] { "SUCCESS", "PERMISSIONS", "ERROR", "SESSION_MISSING", "YIELDED" }); + } +} diff --git a/Darling/Darling.Tests/PostgresTargetConfigTests.cs b/Darling/Darling.Tests/PostgresTargetConfigTests.cs new file mode 100644 index 000000000..3e25b03f6 --- /dev/null +++ b/Darling/Darling.Tests/PostgresTargetConfigTests.cs @@ -0,0 +1,396 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the config-to-connection path for a PostgreSQL target: how the engine is declared, that +/// omitting it cannot change an existing server's behaviour, and the connection posture the builder +/// produces. +/// +public class PostgresTargetConfigTests +{ + private static MonitoredServer PgServer() => new() + { + Name = "aurora-writer", + Engine = "postgres", + Host = "segments-multi-1.cluster-x.us-east-1.rds.amazonaws.com", + Auth = "sql", + Username = "collector", + }; + + /// + /// The default matters more than the parsing: every darling.json in the field omits "engine", and + /// omitting it must keep those entries on the SQL Server path exactly as before. + /// + [Fact] + public void DefaultsToSqlServerWhenEngineIsAbsent() + { + var server = new MonitoredServer { Host = "SQL2022" }; + + Assert.Equal(CollectorTargetEngine.SqlServer, server.TargetEngine); + Assert.False(server.IsPostgres); + } + + [Theory] + [InlineData("postgres")] + [InlineData("postgresql")] + [InlineData("pg")] + [InlineData("aurora-postgresql")] + [InlineData("aurora")] + [InlineData(" Postgres ")] + [InlineData("POSTGRESQL")] + public void RecognizesThePostgresSpellings(string engine) + { + Assert.Equal(CollectorTargetEngine.PostgreSql, new MonitoredServer { Engine = engine }.TargetEngine); + } + + /// + /// A typo resolves to SQL Server rather than throwing: one bad server entry must not stop the + /// service from starting and monitoring every other server. The mismatch is loud anyway — the SQL + /// Server detection query fails immediately against a Postgres host. + /// + [Theory] + [InlineData("postgrez")] + [InlineData("mysql")] + [InlineData("")] + public void FallsBackToSqlServerOnAnUnrecognizedEngine(string engine) + { + Assert.Equal(CollectorTargetEngine.SqlServer, new MonitoredServer { Engine = engine }.TargetEngine); + } + + [Fact] + public void BuildsAPostgresConnectionStringWithTheIntendedPosture() + { + var raw = MonitoredServerConnection.BuildConnectionString(PgServer(), "secret"); + var built = new NpgsqlConnectionStringBuilder(raw); + + Assert.Equal("segments-multi-1.cluster-x.us-east-1.rds.amazonaws.com", built.Host); + Assert.Equal("postgres", built.Database); // maintenance DB when none is configured + Assert.Equal("collector", built.Username); + Assert.Equal("PerformanceMonitorDarling", built.ApplicationName); + Assert.Equal(15, built.Timeout); // same connect budget as the SQL Server path + Assert.Equal(60, built.CommandTimeout); + Assert.Equal(SslMode.VerifyFull, built.SslMode); // fail-closed by default + } + + [Fact] + public void UsesTheConfiguredDatabaseWhenOneIsGiven() + { + var server = PgServer(); + server.Database = "payment_processing"; + + var built = new NpgsqlConnectionStringBuilder(MonitoredServerConnection.BuildConnectionString(server, "secret")); + + Assert.Equal("payment_processing", built.Database); + } + + /// + /// pg_stat_statements lives in the app database on some of our clusters rather than in postgres, + /// so pointing an entry at a specific database is a supported configuration, not an edge case. + /// + [Fact] + public void HonorsANonDefaultPort() + { + var server = PgServer(); + server.Port = 5433; + + Assert.Equal(5433, new NpgsqlConnectionStringBuilder( + MonitoredServerConnection.BuildConnectionString(server, "secret")).Port); + } + + [Fact] + public void DefaultsToTheStandardPortWhenNoneIsConfigured() + { + Assert.Equal(5432, new NpgsqlConnectionStringBuilder( + MonitoredServerConnection.BuildConnectionString(PgServer(), "secret")).Port); + } + + /// + /// TrustServerCertificate relaxes verification without abandoning TLS — Aurora presents an RDS CA + /// a stock trust store does not know, which is exactly the case this covers. + /// + [Fact] + public void TrustServerCertificateRelaxesVerificationButKeepsTls() + { + var server = PgServer(); + server.TrustServerCertificate = true; + + Assert.Equal(SslMode.Require, new NpgsqlConnectionStringBuilder( + MonitoredServerConnection.BuildConnectionString(server, "secret")).SslMode); + } + + [Fact] + public void OptionalEncryptModeDowngradesToPrefer() + { + var server = PgServer(); + server.EncryptMode = "optional"; + + Assert.Equal(SslMode.Prefer, new NpgsqlConnectionStringBuilder( + MonitoredServerConnection.BuildConnectionString(server, "secret")).SslMode); + } + + /// + /// Integrated auth is rejected loudly rather than producing a connection string that cannot + /// authenticate and failing later, further from the cause. + /// + [Fact] + public void RejectsIntegratedAuthForPostgresTargets() + { + var server = PgServer(); + server.Auth = "integrated"; + + var ex = Assert.Throws( + () => MonitoredServerConnection.BuildConnectionString(server, null)); + Assert.Contains("PostgreSQL", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void RejectsSqlAuthWithNoResolvedPassword() + { + Assert.Throws( + () => MonitoredServerConnection.BuildConnectionString(PgServer(), null)); + } + + /// + /// A Postgres entry still derives its identity through the shared rule, so it lands in the same + /// server registry with a server_id computed the same way as every SQL Server entry. + /// + [Fact] + public void DerivesStorageIdentityThroughTheSharedRule() + { + var server = PgServer(); + server.Database = "segments_horizon"; + server.ReadOnlyIntent = true; + + /* #2218: ":pg" sits between the database and ":RO". A PostgreSQL instance and a SQL Server on one host + used to derive ONE server_id and interleave their histories; the engine token is what separates them, + and its position is fixed so two callers with the same facts cannot produce two names. */ + Assert.Equal( + "segments-multi-1.cluster-x.us-east-1.rds.amazonaws.com:segments_horizon:pg:RO", + server.StorageName); + + /* A port appends after the engine when one is set — the second half of #2218, for two PostgreSQL + instances on one host. */ + server.Port = 6432; + Assert.Equal( + "segments-multi-1.cluster-x.us-east-1.rds.amazonaws.com:segments_horizon:pg:6432:RO", + server.StorageName); + } + + /// + /// The detection query must only touch surfaces a pg_monitor-grade login can read, and must not + /// depend on version() text formatting. + /// + [Fact] + public void PostgresDetectionQueryUsesPortableSurfaces() + { + var sql = DarlingServerConnector.PostgresDetectionQueryText; + + Assert.Contains("server_version_num", sql, StringComparison.Ordinal); + Assert.Contains("pg_is_in_recovery()", sql, StringComparison.Ordinal); + Assert.Contains("aurora_version", sql, StringComparison.Ordinal); + // Aurora detection must not hard-fail on stock PostgreSQL, so it is a pg_proc lookup. + Assert.Contains("pg_proc", sql, StringComparison.Ordinal); + // No T-SQL leaked into the Postgres path. + Assert.DoesNotContain("SERVERPROPERTY", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("@@VERSION", sql, StringComparison.OrdinalIgnoreCase); + } + + private static DarlingConfig ConfigWith(MonitoredServer server) + { + var config = new DarlingConfig(); + config.Postgres.ConnectionString = "Host=localhost;Database=darling;Username=x;Password=y"; + config.Servers.Add(server); + return config; + } + + /// + /// The pre-flight has to reject integrated auth on a PostgreSQL target. The connection builder + /// throws on it too, but that fires at first connect — for a service, hours after deployment and + /// only in a log, where --test-connection would have said so before install. + /// + [Fact] + public void ValidationRejectsIntegratedAuthOnAPostgresTarget() + { + var problems = ConfigWith(new MonitoredServer + { + Name = "aurora-writer", + Engine = "postgres", + Host = "aurora.cluster-x.us-east-1.rds.amazonaws.com", + Auth = "integrated", + }).Validate(); + + Assert.Contains(problems, p => p.Contains("PostgreSQL target requires auth 'sql'", StringComparison.Ordinal)); + } + + /// Integrated auth stays perfectly valid on a SQL Server entry — the new rule is engine-scoped. + [Fact] + public void ValidationStillAllowsIntegratedAuthOnASqlServerTarget() + { + var problems = ConfigWith(new MonitoredServer { Name = "SQL2022", Host = "SQL2022", Auth = "integrated" }) + .Validate(); + + Assert.Empty(problems); + } + + /// A fully specified Postgres entry passes, so the new rules cannot reject a good config. + [Fact] + public void ValidationAcceptsAWellFormedPostgresTarget() + { + var server = PgServer(); + server.Password = "env:PGPASSWORD"; + server.Port = 5432; + + Assert.Empty(ConfigWith(server).Validate()); + } + + /// + /// 0 is the documented "use the driver's default" value and must not be flagged; a real out-of-range + /// port must be. Left unvalidated, a typo'd port surfaces as a connect timeout, which reads like a + /// firewall problem and gets escalated to the wrong team. + /// + [Theory] + [InlineData(0, false)] + [InlineData(5432, false)] + [InlineData(65535, false)] + [InlineData(-1, true)] + [InlineData(65536, true)] + public void ValidationRangeChecksTheOptionalPort(int port, bool expectProblem) + { + var server = PgServer(); + server.Password = "env:PGPASSWORD"; + server.Port = port; + + var problems = ConfigWith(server).Validate(); + + Assert.Equal(expectProblem, problems.Any(p => p.Contains("port must be between", StringComparison.Ordinal))); + } + + /* ─────────────── the store round-trip: darling.json is NOT the live source of truth ─────────────── */ + + /// + /// Every connection-affecting property must have a column in + /// config.config_monitored_servers, because that registry — not darling.json — is what the worker + /// reads once the store has been seeded. + /// This test exists because Engine and Port did not have columns. A PostgreSQL entry + /// was therefore written to the store without its engine, read back as the "sqlserver" property + /// default, and connected to with SqlConnection — on the FIRST start, since the seed is immediately + /// followed by the load that replaces the file's list. Nothing failed to compile and no test covered it; + /// the whole PostgreSQL feature simply did not survive its own registration. A property-driven check is + /// the only kind that catches the NEXT one. + /// + [Fact] + public void EveryRoundTripCriticalServerPropertyHasAStoreColumn() + { + /* Property name -> column name. Deliberately explicit rather than a PascalCase-to-snake_case + convention, because the mapping is the thing under test: a convention would "prove" a column + exists by deriving its name from the property. */ + var mustRoundTrip = new (string Property, string Column)[] + { + ("Name", "name"), + ("Engine", "engine"), + ("Host", "host"), + ("Port", "port"), + ("Database", "database"), + ("Auth", "auth"), + ("Username", "username"), + ("EncryptedPassword", "encrypted_password"), + ("ReadOnlyIntent", "read_only_intent"), + ("TrustServerCertificate", "trust_server_certificate"), + ("EncryptMode", "encrypt_mode"), + ("MultiSubnetFailover", "multi_subnet_failover"), + ("ExcludedDatabases", "excluded_databases"), + ("MonthlyCostUsd", "monthly_cost_usd"), + ("AlertDeliveryModeOverride", "alert_delivery_mode_override"), + /* #2218. The one entry whose property name deliberately does not resemble its column: the + property says STORED because that is the whole point — it is the registry's value, and null + means "no store row yet", which is what makes MonitoredServer.ServerId fall back to the + derivation. It belongs in this list rather than beside Password's exemption because it really + does round-trip, through the table's own PRIMARY KEY. That column existed from V17 and this + read simply did not select it, which is the class of bug this test was written for: a property + and a column that fail to meet, with nothing failing to compile. */ + ("StoredServerId", "server_id"), + }; + + /* Password is the deliberate exception: a plaintext dev password is never persisted, and is + backfilled from the in-memory bootstrap config at read time instead. */ + var properties = typeof(MonitoredServer).GetProperties() + .Where(p => p.CanWrite) + .Select(p => p.Name) + .ToHashSet(StringComparer.Ordinal); + var accountedFor = mustRoundTrip.Select(m => m.Property).Append("Password").ToHashSet(StringComparer.Ordinal); + + Assert.Empty(properties.Except(accountedFor)); + + /* The columns, as the ladder actually leaves them: the CREATE plus every later ADD COLUMN. */ + var ladder = string.Join("\n", PgMigrations.Scripts.Select(s => s.Sql)); + foreach (var (property, column) in mustRoundTrip) + { + Assert.True( + ladder.Contains($" {column} ", StringComparison.Ordinal) + || ladder.Contains($"ADD COLUMN IF NOT EXISTS {column} ", StringComparison.Ordinal), + $"MonitoredServer.{property} has no '{column}' column in config_monitored_servers — it will not " + + "survive the store round-trip, and the store is authoritative once seeded."); + } + } + + /// + /// The two write paths into the registry must both carry the engine. The seed covers a fresh install; the + /// add_servers tool covers every install after the first seed, which is the ONLY path there — a + /// darling.json edit does not add a server to an already-seeded store. + /// + [Fact] + public void TheOnboardingToolPersistsTheEngineAndPort() + { + Assert.Contains("engine", DarlingMcpServerAdminTools.InsertServerSql, StringComparison.Ordinal); + Assert.Contains("port", DarlingMcpServerAdminTools.InsertServerSql, StringComparison.Ordinal); + } + + [Theory] + [InlineData(null, "sqlserver")] + [InlineData("", "sqlserver")] + [InlineData("sqlserver", "sqlserver")] + [InlineData("postgres", "postgres")] + [InlineData("POSTGRESQL", "postgres")] + [InlineData(" aurora ", "postgres")] + public void OnboardingNormalizesTheEngine(string? raw, string expected) + { + var (engine, error) = DarlingMcpServerAdminTools.ResolveEngine(raw); + + Assert.Null(error); + Assert.Equal(expected, engine); + } + + /// + /// A typo must be REFUSED here, not silently resolved to SQL Server the way the file parser does. The + /// parser's leniency protects a whole fleet from one bad line at startup; onboarding is a single + /// deliberate act, and "postgress" quietly becoming a SQL Server target yields a connection failure + /// against port 5432 with nothing pointing at the real mistake. + /// + [Theory] + [InlineData("postgress")] + [InlineData("mysql")] + [InlineData("cockroach")] + public void OnboardingRefusesAnUnrecognizedEngineRatherThanDefaultingIt(string raw) + { + var (_, error) = DarlingMcpServerAdminTools.ResolveEngine(raw); + + Assert.NotNull(error); + Assert.Contains("postgres", error, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/ProvisioningVerdictTests.cs b/Darling/Darling.Tests/ProvisioningVerdictTests.cs new file mode 100644 index 000000000..3a487202c --- /dev/null +++ b/Darling/Darling.Tests/ProvisioningVerdictTests.cs @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2246: the FinOps provisioning verdict, which used to report UNDER_PROVISIONED for every server +/// alive. +/// +/// What it got wrong. The rule tested +/// total_server_memory_mb / target_server_memory_mb > 0.95. Those are the perfmon Total and Target +/// Server Memory counters, and they converge at 1.0 the moment an instance is warmed. Measured on 42 +/// production servers: median 1.0000, min 0.9997, max 1.0002 — so every server tripped it, and +/// OVER_PROVISIONED, whose arm needs the same ratio below 0.5, was unreachable at any workload. +/// +/// Why these tests exist rather than a fleet check. The replacement reads real pressure signals +/// — grant waiters, grant timeouts, forced grants — and across the store's entire retained history those are +/// zero: 0 nonzero of 2,938,711 grant rows, and 0 of 1,000,560 Memory Grants Pending samples. That is +/// the correct answer for a fleet with no memory pressure, and it also means the fleet CANNOT demonstrate +/// that the alarm fires. So the positive control is built here instead, one input at a time. +/// +/// Every threshold asserted below has the fleet distribution behind it, recorded on +/// itself: CPU p95 max 51.0 against a limit of 85, worker ratio max 0.635 +/// against 0.8, grant utilization max 18.8% against 50%. +/// +public sealed class ProvisioningVerdictTests +{ + /// A quiet server with no pressure anywhere: the downsizing candidate the report exists to + /// find, and the verdict the old rule could never reach. + [Fact] + public void AQuietServerWithNoPressure_IsOverProvisioned() + { + Assert.Equal( + ProvisioningVerdict.OverProvisioned, + ProvisioningVerdict.Evaluate( + avgCpuPercent: 6m, maxCpuPercent: 22m, p95CpuPercent: 11m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 0.5m, maxWorkers: 576, currentWorkers: 142)); + } + + /// The fleet's own median server, from the measurement that drove this change: avg 6.5, p95 + /// 11.0, grant utilization 0.5%, worker ratio 0.246. It must come out OVER_PROVISIONED — under the old + /// rule this exact server was reported as starved for memory. + [Fact] + public void TheFleetsMedianServer_IsNotReportedAsStarved() + { + var verdict = ProvisioningVerdict.Evaluate( + avgCpuPercent: 6.5m, maxCpuPercent: 30m, p95CpuPercent: 11.0m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 0.5m, maxWorkers: 576, currentWorkers: 142); + + Assert.NotEqual(ProvisioningVerdict.UnderProvisioned, verdict); + Assert.Equal(ProvisioningVerdict.OverProvisioned, verdict); + } + + /// Busy enough not to shrink, not pressured enough to grow. + [Fact] + public void ABusyButHealthyServer_IsRightSized() + { + Assert.Equal( + ProvisioningVerdict.RightSized, + ProvisioningVerdict.Evaluate( + avgCpuPercent: 35m, maxCpuPercent: 70m, p95CpuPercent: 60m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 12m, maxWorkers: 576, currentWorkers: 200)); + } + + /// THE POSITIVE CONTROL the fleet cannot supply: each pressure input alone must raise + /// UNDER_PROVISIONED, driven one at a time so a single over-broad condition cannot mask a dead one. + /// + [Theory] + [InlineData(1, 0, 0, "a query waited for a workspace-memory grant")] + [InlineData(0, 1, 0, "a grant request timed out")] + [InlineData(0, 0, 1, "a grant was forced through below its request")] + public void AnyMemoryPressureSignal_AloneRaisesUnderProvisioned( + long waiters, long timeouts, long forced, string because) + { + var verdict = ProvisioningVerdict.Evaluate( + avgCpuPercent: 6m, maxCpuPercent: 22m, p95CpuPercent: 11m, + maxGrantWaiters: waiters, grantTimeouts: timeouts, forcedGrants: forced, + grantUtilizationPercent: 0.5m, maxWorkers: 576, currentWorkers: 142); + + /* The CPU numbers here are the quiet ones from the OVER_PROVISIONED case above, so this also pins + the ORDERING: pressure outranks idleness. Recommending a smaller instance for a server whose + queries are queueing for memory would be the worst possible answer. */ + Assert.Equal(ProvisioningVerdict.UnderProvisioned, verdict); + Assert.NotEqual(ProvisioningVerdict.OverProvisioned, verdict); + Assert.False(string.IsNullOrEmpty(because)); + } + + /// Sustained CPU still means under-provisioned. Threshold unchanged from the rule this + /// replaces; the fleet's highest p95 is 51.0, so it stays reachable rather than academic. + [Fact] + public void SustainedHighCpu_IsUnderProvisioned() + { + Assert.Equal( + ProvisioningVerdict.UnderProvisioned, + ProvisioningVerdict.Evaluate( + avgCpuPercent: 70m, maxCpuPercent: 99m, p95CpuPercent: 92m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 5m, maxWorkers: 576, currentWorkers: 200)); + } + + /// Worker-thread saturation, the term the Full Dashboard's view has always carried and the app + /// copies dropped — so a worker-starved server was invisible to both of them. + [Fact] + public void WorkerThreadSaturation_IsUnderProvisioned() + { + Assert.Equal( + ProvisioningVerdict.UnderProvisioned, + ProvisioningVerdict.Evaluate( + avgCpuPercent: 6m, maxCpuPercent: 22m, p95CpuPercent: 11m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 0.5m, maxWorkers: 100, currentWorkers: 81)); + } + + /// Unknown is not saturation. A sample that never reported a worker ceiling must not imply + /// exhaustion — the same rule the collector gates follow for an unclassified target, and without it a + /// missing column would silently flag every server. + [Fact] + public void AnUnknownWorkerCeiling_IsNotSaturation() + { + Assert.Equal( + ProvisioningVerdict.OverProvisioned, + ProvisioningVerdict.Evaluate( + avgCpuPercent: 6m, maxCpuPercent: 22m, p95CpuPercent: 11m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 0.5m, maxWorkers: 0, currentWorkers: 9999)); + } + + /// Quiet on CPU but working its semaphore hard: not a downsizing candidate. This is the term + /// that stops "idle" from being decided on CPU alone, and the fleet's peak utilization is 18.8%, so it + /// excludes nothing real today. + [Fact] + public void IdleCpuButHighGrantUtilization_IsNotOverProvisioned() + { + var verdict = ProvisioningVerdict.Evaluate( + avgCpuPercent: 6m, maxCpuPercent: 22m, p95CpuPercent: 11m, + maxGrantWaiters: 0, grantTimeouts: 0, forcedGrants: 0, + grantUtilizationPercent: 65m, maxWorkers: 576, currentWorkers: 142); + + Assert.Equal(ProvisioningVerdict.RightSized, verdict); + } + + /// + /// The REASON must name the condition that actually fired. The UI used to derive this itself as + /// "p95 > 85 ? CPU : blame the memory ratio", so once the verdict gained grant-pressure and + /// worker-thread reasons, every one of those would have been explained as a memory ratio that no longer + /// decides anything — citing a threshold the code does not check. These pin that each cause explains + /// itself. + /// + [Fact] + public void TheReasonNamesTheConditionThatFired() + { + var cpu = ProvisioningVerdict.UnderProvisionedReason(92m, 0, 0, 0, 576, 200); + Assert.Contains("CPU p95 is 92.0%", cpu, System.StringComparison.Ordinal); + + /* Grant pressure with QUIET cpu: the old text would have blamed the memory ratio here. */ + var grants = ProvisioningVerdict.UnderProvisionedReason(11m, 3, 1, 2, 576, 142); + Assert.Contains("workspace memory", grants, System.StringComparison.Ordinal); + Assert.Contains("3 grant waiter(s)", grants, System.StringComparison.Ordinal); + Assert.Contains("1 grant timeout(s)", grants, System.StringComparison.Ordinal); + Assert.Contains("2 forced grant(s)", grants, System.StringComparison.Ordinal); + /* And it must NOT cite the retired threshold. */ + Assert.DoesNotContain("0.95", grants, System.StringComparison.Ordinal); + Assert.DoesNotContain("memory ratio", grants, System.StringComparison.Ordinal); + + var workers = ProvisioningVerdict.UnderProvisionedReason(11m, 0, 0, 0, 100, 81); + Assert.Contains("Worker threads", workers, System.StringComparison.Ordinal); + Assert.Contains("81 of 100", workers, System.StringComparison.Ordinal); + + /* Asked about inputs that are not under-provisioned, it says so rather than inventing a cause. */ + var none = ProvisioningVerdict.UnderProvisionedReason(11m, 0, 0, 0, 576, 142); + Assert.Contains("No under-provisioning condition", none, System.StringComparison.Ordinal); + } + + /// The boundaries themselves, since every one of them is a published constant that something + /// downstream will eventually be tuned against. Strict comparisons, so a value sitting exactly ON a + /// limit does not trip it. + [Fact] + public void TheThresholdsAreStrict_AtTheirExactValues() + { + /* p95 exactly 85 is not "> 85". */ + Assert.NotEqual( + ProvisioningVerdict.UnderProvisioned, + ProvisioningVerdict.Evaluate(50m, 90m, ProvisioningVerdict.HighCpuP95Percent, + 0, 0, 0, 5m, 576, 200)); + + /* avg exactly 15 is not "< 15", so it cannot be over-provisioned. */ + Assert.Equal( + ProvisioningVerdict.RightSized, + ProvisioningVerdict.Evaluate(ProvisioningVerdict.IdleAvgCpuPercent, 30m, 20m, + 0, 0, 0, 5m, 576, 142)); + + /* worker ratio exactly 0.8 is not "> 0.8". */ + Assert.Equal( + ProvisioningVerdict.OverProvisioned, + ProvisioningVerdict.Evaluate(6m, 22m, 11m, 0, 0, 0, 0.5m, maxWorkers: 100, currentWorkers: 80)); + } +} diff --git a/Darling/Darling.Tests/PvsStatsStoreTests.cs b/Darling/Darling.Tests/PvsStatsStoreTests.cs index 5e1a7eb40..6a495a891 100644 --- a/Darling/Darling.Tests/PvsStatsStoreTests.cs +++ b/Darling/Darling.Tests/PvsStatsStoreTests.cs @@ -45,8 +45,10 @@ public void V47_MigrationIdentity_AndStorageVersionTracksTheNewestRung() (#2060, the persisted finding drill-down), then V53 (#2068, the store self-metrics table) followed this migration — the newest-rung pins track the newest, the V47 identity pins below are unchanged. */ - Assert.Equal(54, PgMigrations.Scripts[^1].Version); - Assert.Equal(54, StorageVersion.SchemaVersion); + /* The invariant the test name states, with no literal to go stale: the build's schema version IS + the newest registered rung. Three in-flight branches bumping versions made the literal form a + recurring multi-test failure (#2210 round, again here at V62). */ + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); /* collect.-qualified like V44 and V34, and idempotent so a re-run is a no-op. */ Assert.Contains("CREATE TABLE IF NOT EXISTS collect.pvs_stats (", v47.Sql, StringComparison.Ordinal); @@ -86,8 +88,9 @@ public void ViewerSchemaGate_KnowsV47_SoAFullyMigratedStoreIsNotRefused() compares the result against RequiredStoreSchemaVersion. A probe that cannot SEE the newest migration reports every healthy store as skewed and refuses to open it — permanently. (53 since #2068's store self-metrics table; the full-sentinel pin lives in - ViewerDataServiceTests.) */ - Assert.Equal(54, ViewerDataService.RequiredStoreSchemaVersion); + ViewerDataServiceTests.) Invariant form, no literal to go stale: the gate always + requires exactly the build's schema version. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); Assert.Contains("table_name = 'pvs_stats'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); /* The V47 arm: pvs_stats present (and nothing newer) maps to exactly 47. */ diff --git a/Darling/Darling.Tests/QueryStatExtremesTests.cs b/Darling/Darling.Tests/QueryStatExtremesTests.cs new file mode 100644 index 000000000..1f09e1e87 --- /dev/null +++ b/Darling/Darling.Tests/QueryStatExtremesTests.cs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2235's minor finding: min/max CPU and elapsed on the top-queries/top-procedures reads are +/// lifetime extremes for the plan's cache residency, and a lifetime max can EXCEED the windowed +/// total — which reads as impossible unless labeled. These pin the conditional note: it fires only +/// on the provable case and names exactly the column(s) that prove it. This SAME table is pinned +/// identically in Lite.Tests so the two SKUs cannot drift. +/// +public sealed class QueryStatExtremesTests +{ + /// The ordinary row: extremes inside the window's totals say nothing. + [Fact] + public void ExtremesWithinTotalsCarryNoNote() + { + Assert.Null(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 1000, maxCpu: 400, totalElapsed: 2000, maxElapsed: 900)); + } + + /// + /// Equality is NOT the provable case — a single-execution window has max == total by + /// construction, and flagging it would put the note on every one-shot query. + /// + [Fact] + public void ExactEqualityCarriesNoNote() + { + Assert.Null(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 400, maxCpu: 400, totalElapsed: 900, maxElapsed: 900)); + } + + /// + /// The field case that filed this: dbo.ClosingReportV6, window total 158,906 ms against a + /// lifetime max of 322,066 ms. The note fires and names the CPU column. + /// + [Fact] + public void CpuExtremeBeyondTheWindowTotalIsFlagged() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 158906, maxCpu: 322066, totalElapsed: 500000, maxElapsed: 400000); + + Assert.NotNull(note); + Assert.Contains("max_cpu_ms exceeds", note, System.StringComparison.Ordinal); + Assert.DoesNotContain("max_elapsed_ms", note, System.StringComparison.Ordinal); + } + + /// Elapsed alone proves it too, and the note names that column instead. + [Fact] + public void ElapsedExtremeBeyondTheWindowTotalIsFlagged() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 500000, maxCpu: 400000, totalElapsed: 158906, maxElapsed: 322066); + + Assert.NotNull(note); + Assert.Contains("max_elapsed_ms exceeds", note, System.StringComparison.Ordinal); + Assert.DoesNotContain("max_cpu_ms", note, System.StringComparison.Ordinal); + } + + /// Both provable, both named — a reader should not have to re-derive which half. + [Fact] + public void BothExtremesBeyondTotalsNameBoth() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 100, maxCpu: 200, totalElapsed: 100, maxElapsed: 200); + + Assert.NotNull(note); + Assert.Contains("max_cpu_ms and max_elapsed_ms exceed", note, System.StringComparison.Ordinal); + } + + /// + /// A zero-total window with a nonzero lifetime max is the purest form of the problem — the plan + /// did nothing this window and the extreme is entirely inherited. It must flag. + /// + [Fact] + public void InheritedExtremeOnAnIdleWindowIsFlagged() + { + Assert.NotNull(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 0, maxCpu: 5, totalElapsed: 0, maxElapsed: 5)); + } +} diff --git a/Darling/Darling.Tests/QueryStoreBackfillTests.cs b/Darling/Darling.Tests/QueryStoreBackfillTests.cs index e3bfea500..47b490e90 100644 --- a/Darling/Darling.Tests/QueryStoreBackfillTests.cs +++ b/Darling/Darling.Tests/QueryStoreBackfillTests.cs @@ -78,6 +78,81 @@ Derived so a retention change moves this automatically — the #1937 rule. */ "the backfill horizon must sit strictly inside raw retention, or a slice could land rows the next purge immediately drops"); } + [Fact] + public void BoundSliceFloor_CapsWideRanges_AndPassesNarrowOnesThrough() + { + /* #2102: a slice queries at most the top MaxSliceSpan of its remaining range — the byte + budget bounds what ships, not what the query aggregates and sorts, so an unchunked wide + window on a big database re-times-out every tick and the range never drains. The caller + reads the verdict from the result: floor moved = chunk (an empty slice shrinks the + ceiling and keeps walking); floor unmoved = the whole remainder was asked (an empty + slice is terminal, the pre-chunking semantics). */ + var ceiling = new DateTime(2026, 8, 7, 12, 0, 0, DateTimeKind.Utc); + + var wideFloor = ceiling.AddHours(-23); + Assert.Equal(ceiling - QueryStoreBackfillState.MaxSliceSpan, QueryStoreBackfillState.BoundSliceFloor(wideFloor, ceiling)); + + var narrowFloor = ceiling.AddMinutes(-25); + Assert.Equal(narrowFloor, QueryStoreBackfillState.BoundSliceFloor(narrowFloor, ceiling)); + + /* Exactly MaxSliceSpan wide is narrow enough — one slice takes it whole, so its empty + verdict stays terminal rather than saving a zero-width hole. */ + var exactFloor = ceiling - QueryStoreBackfillState.MaxSliceSpan; + Assert.Equal(exactFloor, QueryStoreBackfillState.BoundSliceFloor(exactFloor, ceiling)); + } + + [Fact] + public void AdaptiveSpan_HalvesPerFailure_FloorsAtFifteenMinutes_AndResetsAtZero() + { + /* #2111 promoted from reserve on field evidence: a member whose 1h window intermittently + exceeds the command timeout stayed stuck for hours (Redstone, 3+ hours flat overnight) — + halving toward a floor gives it a window that fits, and the skipped range rides the same + hole records the clamp writes. Zero failures = full width, success resets the counter at + every call site, and the exponent cap keeps the shift math from wrapping. */ + var full = QueryStoreBackfillState.MaxSliceSpan; + + Assert.Equal(full, QueryStoreBackfillState.AdaptiveSpan(full, 0)); + Assert.Equal(TimeSpan.FromMinutes(30), QueryStoreBackfillState.AdaptiveSpan(full, 1)); + Assert.Equal(TimeSpan.FromMinutes(15), QueryStoreBackfillState.AdaptiveSpan(full, 2)); + Assert.Equal(QueryStoreBackfillState.MinAdaptiveSpan, QueryStoreBackfillState.AdaptiveSpan(full, 3)); + Assert.Equal(QueryStoreBackfillState.MinAdaptiveSpan, QueryStoreBackfillState.AdaptiveSpan(full, 100)); + + Assert.Equal(TimeSpan.FromMinutes(15), QueryStoreBackfillState.MinAdaptiveSpan); + } + + [Fact] + public void BoundSliceFloor_AdaptiveForm_CapsToThePassedSpan() + { + var ceiling = new DateTime(2026, 8, 8, 12, 0, 0, DateTimeKind.Utc); + var wideFloor = ceiling.AddHours(-23); + + Assert.Equal( + ceiling - TimeSpan.FromMinutes(15), + QueryStoreBackfillState.BoundSliceFloor(wideFloor, ceiling, TimeSpan.FromMinutes(15))); + + /* The parameterless form stays the full-span behavior. */ + Assert.Equal( + ceiling - QueryStoreBackfillState.MaxSliceSpan, + QueryStoreBackfillState.BoundSliceFloor(wideFloor, ceiling)); + } + + [Fact] + public void ShouldYieldToLive_YieldsInsideTheWindow_RunsOutsideIt_AndNeverOnNull() + { + /* #2111: a live query_store failure inside the window means the replica is contended NOW — + the slice yields. At or beyond the window (or never failed), backfill runs. The window is + two poll cycles: current-or-previous-cycle failures count, older ones are history. */ + var now = new DateTime(2026, 8, 7, 17, 0, 0, DateTimeKind.Utc); + + Assert.False(QueryStoreBackfillState.ShouldYieldToLive(null, now)); + Assert.True(QueryStoreBackfillState.ShouldYieldToLive(now.AddMinutes(-1), now)); + Assert.True(QueryStoreBackfillState.ShouldYieldToLive(now - QueryStoreBackfillState.YieldToLiveWindow + TimeSpan.FromSeconds(1), now)); + Assert.False(QueryStoreBackfillState.ShouldYieldToLive(now - QueryStoreBackfillState.YieldToLiveWindow, now)); + Assert.False(QueryStoreBackfillState.ShouldYieldToLive(now.AddHours(-2), now)); + + Assert.Equal(TimeSpan.FromMinutes(10), QueryStoreBackfillState.YieldToLiveWindow); + } + [Fact] public void StateIdentity_IsTheWorkersOwn_NotTheDefinitions() { diff --git a/Darling/Darling.Tests/QueryStoreCorrectedRollupLiveTests.cs b/Darling/Darling.Tests/QueryStoreCorrectedRollupLiveTests.cs index 7d8ed0400..3cf475f5a 100644 --- a/Darling/Darling.Tests/QueryStoreCorrectedRollupLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreCorrectedRollupLiveTests.cs @@ -51,6 +51,63 @@ public sealed class QueryStoreCorrectedRollupLiveTests /// closed form so the expectation is derived, not a magic number transcribed from a test run. private const long InflatedSum = ReCollections * (ReCollections + 1L) / 2L; + /// + /// L1's retention horizon in days (#2223), from the product's own TIMESPAN rather than re-parsing the + /// interval string. + /// + /// Review catch: the first cut split IntervalRetentionInterval on whitespace and parsed the + /// leading number, reimplementing a conversion that already exists typed — and one that would quietly + /// produce a wrong seed depth if the interval were ever written "1 week". It matters more here than + /// it looks: this feeds a static initializer, so a bad parse fails as a + /// TypeInitializationException across the whole class rather than as anything readable. + /// TimescaleContinuousAggregateTests pins the span equal to the string, so this cannot drift from + /// the SQL the policy is created with. + /// + private static readonly int L1RetentionDays = TimescaleSupport.IntervalRetentionSpan.Days; + + /// + /// How deep every retention-arming test in this class seeds its history — DERIVED from L1's horizon, with + /// two days of margin, so nothing a test seeds is ever eligible for deletion by a policy that test itself + /// arms. + /// + /// #2223, and it was a class-wide contradiction rather than one test's bug. All four tests here + /// that call EnsureRetentionPoliciesAsync used to seed a hardcoded 9 days while step 1 of each arms + /// L1's retention at drop_after => INTERVAL '7 days'. Each therefore armed a policy guaranteed to + /// want to delete two days of its own fixture, and then asserted against floors those buckets defined. + /// + /// How it failed. In the re-hold test, l1Floor is read once and the backfill it is + /// compared against runs ~20 lines later. If the armed retention job fires inside that window it drops L1's + /// two oldest buckets, so the rebuilt consumer can only backfill to now - 7 days while the assertion + /// still holds the pre-deletion floor. The reported failure was consumer 08-05 against L1 + /// 08-03 on a fixture seeded from 08-03 — the consumer floor was EXACTLY + /// now - IntervalRetentionInterval, which is what identified the mechanism. Intermittent because it + /// depends on the job firing in that window: solo runs passed every time, full-suite runs failed about half, + /// and more wall clock means more chances. + /// + /// Derived rather than re-hardcoded to a smaller number so that changing the product horizon moves + /// every fixture with it, instead of silently reintroducing this the next time it is tuned. The relationship + /// is pinned by . + /// + private static readonly int SeedDepthDays = L1RetentionDays - 2; + + /// + /// The #2223 invariant itself: a test may not seed history that a policy it arms would delete. Pinned as a + /// test rather than trusted as arithmetic, because the product horizon is tunable and the whole failure mode + /// was a fixture silently drifting outside it. + /// + [Fact] + public void SeededHistoryStaysInsideTheHorizonItArms() + { + Assert.True(SeedDepthDays < L1RetentionDays, + $"seeded history ({SeedDepthDays}d) must stay inside L1's retention horizon " + + $"({TimescaleSupport.IntervalRetentionInterval}) or the retention job these tests arm can delete " + + $"the buckets they assert on — that is #2223."); + + /* And deep enough to still be the scenario: the coverage gap has to span more than one daily bucket. */ + Assert.True(SeedDepthDays >= 3, + $"seeded history ({SeedDepthDays}d) must span at least three days for a multi-bucket coverage gap."); + } + [Fact] public async Task CorrectedRollups_DedupTheReCollectedInterval_WhileTheOldPairKeepsInflating() { @@ -174,7 +231,7 @@ public async Task RawPurgeArming_HeldWhileTheCorrectedIntervalLayerIsShort_ThenR await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); - var oldest = now.Date.AddDays(-9); + var oldest = now.Date.AddDays(-SeedDepthDays); /* Pre-existing raw history, one interval per hour, going back well past any refresh policy's 3-day window — the shape every real store upgrading into this build is in. */ @@ -391,7 +448,7 @@ public async Task IntervalLayerPurgeArming_HeldWhileTheDayGrainLayerIsShort_Then await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); - var oldest = now.Date.AddDays(-9); + var oldest = now.Date.AddDays(-SeedDepthDays); for (var offset = 0; oldest.AddHours(offset) < now; offset += 3) { @@ -495,7 +552,7 @@ public async Task ArmedIntervalPurge_IsReHeldWhenItsCoverageListGainsAnEmptyCons await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); - var oldest = now.Date.AddDays(-9); + var oldest = now.Date.AddDays(-SeedDepthDays); var span = (From: oldest.AddDays(-1), To: now.AddDays(1)); for (var offset = 0; oldest.AddHours(offset) < now; offset += 3) @@ -603,7 +660,7 @@ public async Task RetentionSweep_TreatsAFailedProbeAsUnknown_NeverAsACoverageReg await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); - var oldest = now.Date.AddDays(-9); + var oldest = now.Date.AddDays(-SeedDepthDays); var span = (From: oldest.AddDays(-1), To: now.AddDays(1)); for (var offset = 0; oldest.AddHours(offset) < now; offset += 3) diff --git a/Darling/Darling.Tests/QueryStoreHealthStoreTests.cs b/Darling/Darling.Tests/QueryStoreHealthStoreTests.cs new file mode 100644 index 000000000..d6a50b0d4 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreHealthStoreTests.cs @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// The V76 per-database Query Store health collector (#2319) — the instrument #2312's investigation was +/// missing: database_config's single is_query_store_on bit cannot say whether Query Store +/// actually works (desired READ_WRITE with actual READ_ONLY after the cap hit is the classic silent +/// failure) or how close to its cap it sits. These facts pin the rung's place on the ladder, the viewer +/// probe, the enumeration SQL's load-bearing filters, the per-item query's identity-quoting and honesty +/// contract, and the schedule decision. +/// +public sealed class QueryStoreHealthStoreTests +{ + /* ---------------- the rung ---------------- */ + + [Fact] + public void TheRungIsTheTopOfADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + Assert.Equal(76, versions.Max()); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + + /* Dense above the one sanctioned historical hole at V45. */ + var above = versions.Where(v => v > 45).OrderBy(v => v).ToList(); + Assert.Equal(Enumerable.Range(above[0], above.Count), above); + + Assert.Equal("query-store-health", PgMigrations.Scripts.Single(s => s.Version == 76).Name); + } + + /* ---------------- the viewer probe ---------------- */ + + [Fact] + public void TheProbeMapsAFullyMigratedStoreTo76() + { + Assert.Equal(76, StorageVersion.SchemaVersion); + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + /* 51 positional sentinels then the V76 one by name — the map takes 52 parameters. Present => 76, + newest-first; absent => the previous arm still answers 75 rather than falling through. */ + var all = Enumerable.Repeat(true, 51).Cast().ToArray(); + + Assert.Equal(76, InvokeMap(all, hasQueryStoreHealth: true)); + Assert.Equal(75, InvokeMap(all, hasQueryStoreHealth: false)); + } + + [Fact] + public void TheProbeAsksForTheTable_AndTheThreePlacesAgree() + { + Assert.Contains("table_name = 'query_store_health'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + + var mapParameters = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .GetParameters().Length; + + var viewerSource = ReadViewerSource(); + + /* The reader must hand over exactly one argument per map parameter: ordinals are 0-based, so the + highest is Count - 1, and the next one up must NOT appear. */ + Assert.Contains($"reader.GetBoolean({mapParameters - 1})", viewerSource, StringComparison.Ordinal); + Assert.DoesNotContain($"reader.GetBoolean({mapParameters})", viewerSource, StringComparison.Ordinal); + } + + /* ---------------- the definition ---------------- */ + + /// + /// The enumeration list's load-bearing filters, each of which cost real rounds elsewhere: + /// HAS_DBACCESS self-skip (#1823 — a least-privilege login without per-db access raised 916 per db + /// per cycle), the AG filter (a readable-secondary's databases answer for the primary's identity), + /// ONLINE only (a RESTORING database's catalog views are unreachable), and the house RECOMPILE. + /// + [Fact] + public void TheEnumerationCarriesTheLoadBearingFilters() + { + var context = TestContext(isAzure: false); + var query = QueryStoreHealthCollector.Instance.BuildEnumerationQuery(context)!; + + Assert.Contains("HAS_DBACCESS(d.name) = 1", query.Text, StringComparison.Ordinal); + Assert.Contains("drs.is_primary_replica = 1", query.Text, StringComparison.Ordinal); + Assert.Contains("d.state_desc = N'ONLINE'", query.Text, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE)", query.Text, StringComparison.Ordinal); + Assert.DoesNotContain("/*EXCLUSION_FILTER*/", query.Text, StringComparison.Ordinal); + + /* Azure lists all online databases — from master, HAS_DBACCESS returns 0 for every user database + and there is no AG catalog, so the on-prem filters would enumerate NOTHING there. */ + var azure = QueryStoreHealthCollector.Instance.BuildEnumerationQuery(TestContext(isAzure: true))!; + Assert.DoesNotContain("HAS_DBACCESS", azure.Text, StringComparison.Ordinal); + Assert.DoesNotContain("dm_hadr_database_replica_states", azure.Text, StringComparison.Ordinal); + } + + /// A database named with a closing bracket must not escape its identifier — the same + /// quote-doubling every sibling per-database collector carries. + [Fact] + public void ThePerItemQueryDoublesClosingBrackets() + { + var query = QueryStoreHealthCollector.Instance.BuildPerItemQuery("we]ird", TestContext(isAzure: false)); + + Assert.Contains("EXECUTE [we]]ird].sys.sp_executesql", query.Text, StringComparison.Ordinal); + Assert.Contains("sys.database_query_store_options", query.Text, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE)", query.Text, StringComparison.Ordinal); + } + + /// + /// The honesty contract: the database list is deliberately NOT filtered to is_query_store_on — the + /// options view answers one row even for a QS-off database, so OFF is recorded as OFF and an absent + /// row can only mean "not collected". Filtering the list would make those two states identical. + /// + [Fact] + public void TheEnumerationDoesNotFilterToQueryStoreOn() + { + var query = QueryStoreHealthCollector.Instance.BuildEnumerationQuery(TestContext(isAzure: false))!; + + Assert.DoesNotContain("is_query_store_on", query.Text, StringComparison.Ordinal); + } + + /// + /// Hourly, NOT the config family's on-load cadence: actual_state, readonly_reason and the storage + /// numbers change BY THEMSELVES, and the cap-hit READ_ONLY transition is the point of collecting + /// this — an on-load snapshot would miss it until the next reconnect. + /// + [Fact] + public void TheScheduleIsHourlyWithConfigFamilyRetention() + { + var schedule = CollectorScheduleDefaults.All["query_store_health"]; + + Assert.Equal(60, schedule.FrequencyMinutes); + Assert.Equal(30, schedule.RetentionDays); + } + + /// + /// The collector gates on 2016+ (review catch: sys.database_query_store_options does not exist + /// before v13, so an ungated pre-2016 target would error once per database per hour) — the same + /// condition QueryStoreCollector carries, so Lite and Darling skip identically. + /// + [Theory] + [InlineData(11, false)] /* 2012 — no Query Store catalog */ + [InlineData(12, false)] /* 2014 — no Query Store catalog */ + [InlineData(13, true)] /* 2016 — Query Store ships */ + [InlineData(16, true)] + [InlineData(0, true)] /* version unknown = assume newest */ + public void TheCollectorGatesOnQueryStoresExistence(int majorVersion, bool applies) + => Assert.Equal(applies, QueryStoreHealthCollector.Instance.AppliesTo( + new CollectorTargetInfo { SqlMajorVersion = majorVersion })); + + [Fact] + public void AzureAlwaysApplies() + { + Assert.True(QueryStoreHealthCollector.Instance.AppliesTo(new CollectorTargetInfo { SqlMajorVersion = 11, IsAzureSqlDb = true })); + Assert.True(QueryStoreHealthCollector.Instance.AppliesTo(new CollectorTargetInfo { SqlMajorVersion = 11, IsAzureManagedInstance = true })); + } + + /// WITHIN the view, every selected column exists from 2016 on — no per-column gates; + /// pinned so a gated column cannot be added without revisiting this claim. + [Fact] + public void ThePayloadIsUngatedAndOrdered() + { + var columns = QueryStoreHealthCollector.Instance.PayloadColumns.Select(c => c.Name).ToArray(); + + Assert.Equal(new[] + { + "database_name", "actual_state", "desired_state", "readonly_reason", + "current_storage_size_mb", "max_storage_size_mb", "size_based_cleanup_mode", + "stale_query_threshold_days", "max_plans_per_query", "interval_length_minutes", + }, columns); + } + + /* ---------------- helpers ---------------- */ + + private static CollectorContext TestContext(bool isAzure) => new() + { + ServerId = -640001, + ServerName = "query-store-health-pins", + CollectionTime = DateTime.UtcNow, + Deltas = null!, + Target = new CollectorTargetInfo + { + SqlMajorVersion = 16, + IsAzureSqlDb = isAzure, + }, + }; + + private static int InvokeMap(object[] leading, bool hasQueryStoreHealth) + { + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + + var args = leading.Concat(new object[] { hasQueryStoreHealth }).ToArray(); + Assert.Equal(method.GetParameters().Length, args.Length); + + return (int)method.Invoke(null, args)!; + } + + private static string ReadViewerSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/QueryStoreOpenIntervalStateTests.cs b/Darling/Darling.Tests/QueryStoreOpenIntervalStateTests.cs new file mode 100644 index 000000000..fb4588db3 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreOpenIntervalStateTests.cs @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins for (#2312) — the per-database stamp that decides +/// which cycles re-read the OPEN Query Store interval. The decision table matters because every wrong +/// answer is silent in a different direction: a false "skip" starves the current hour's view, a false +/// "include" quietly keeps paying the 40–110 s open-interval bill this state exists to cut. +/// +public sealed class QueryStoreOpenIntervalStateTests +{ + private static readonly DateTime Now = new(2026, 8, 17, 12, 0, 0, DateTimeKind.Utc); + + private static Dictionary StateWith(string databaseName, string raw) => + new(StringComparer.Ordinal) { [QueryStoreOpenIntervalState.KeyFor(databaseName)] = raw }; + + private static string StampedAt(DateTime utc) => QueryStoreOpenIntervalState.Format(utc); + + /// + /// Include is the CONSERVATIVE default: a first run, a restarted host, a broken store and a + /// clock-skewed stamp all behave exactly like today's collector rather than silently going stale. + /// + [Fact] + public void AbsentMalformedAndFutureStampsAllInclude() + { + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(null, "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + new Dictionary(StringComparer.Ordinal), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", ""), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", "not-a-number"), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", "-5"), "SO", Now)); + /* Numeric but beyond FromUnixTimeSeconds's year-9999 ceiling (review catch): parses as a long, + so it must fall to the ArgumentOutOfRangeException guard, not throw through it. */ + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", "999999999999999"), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", long.MaxValue.ToString()), "SO", Now)); + /* Future stamp = the clock moved backwards; honoring it would pin the snapshot stale for as + long as the skew lasts. */ + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + StateWith("SO", StampedAt(Now.AddMinutes(10))), "SO", Now)); + } + + /// The refresh boundary, both sides, inclusive at exactly RefreshEvery. + [Fact] + public void FreshStampSkips_StaleStampIncludes() + { + Assert.False(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + StateWith("SO", StampedAt(Now.AddMinutes(-5))), "SO", Now)); + Assert.False(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + StateWith("SO", StampedAt(Now - QueryStoreOpenIntervalState.RefreshEvery + TimeSpan.FromSeconds(1))), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + StateWith("SO", StampedAt(Now - QueryStoreOpenIntervalState.RefreshEvery)), "SO", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + StateWith("SO", StampedAt(Now.AddHours(-2))), "SO", Now)); + } + + /// Databases are independent: one database's fresh stamp must not skip another's refresh. + [Fact] + public void StampsArePerDatabase() + { + var state = StateWith("A", StampedAt(Now.AddMinutes(-1))); + + Assert.False(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(state, "A", Now)); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(state, "B", Now)); + } + + /// + /// The owner seam, pinned from both ends like the plan and text watermarks: the definition declares + /// no state keys, and this state's owner name differs from the collector's — a row written under + /// "query_store" would never be read back, so the skip would silently never apply and collection + /// would quietly keep paying full price. + /// + [Fact] + public void OwnerIsItsOwnStateCollectorName() + { + Assert.Empty(QueryStoreCollector.Instance.StateKeys); + Assert.NotEqual(QueryStoreOpenIntervalState.StateCollectorName, QueryStoreCollector.Instance.Name); + Assert.Equal("query_store_open_interval", QueryStoreOpenIntervalState.StateCollectorName); + Assert.Equal("qsowm:", QueryStoreOpenIntervalState.WatermarkKeyPrefix); + Assert.Equal("qsowm:SO", QueryStoreOpenIntervalState.KeyFor("SO")); + } + + /// Format/parse round-trip at second granularity, and the 15-minute horizon stays a recorded decision. + [Fact] + public void FormatRoundTripsAndTheHorizonIsPinned() + { + var stamp = QueryStoreOpenIntervalState.Format(Now); + Assert.False(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", stamp), "SO", Now.AddMinutes(14))); + Assert.True(QueryStoreOpenIntervalState.ShouldIncludeOpenInterval(StateWith("SO", stamp), "SO", Now.AddMinutes(15))); + Assert.Equal(TimeSpan.FromMinutes(15), QueryStoreOpenIntervalState.RefreshEvery); + } +} diff --git a/Darling/Darling.Tests/QueryStorePlanSizeLearnTests.cs b/Darling/Darling.Tests/QueryStorePlanSizeLearnTests.cs new file mode 100644 index 000000000..450ee1426 --- /dev/null +++ b/Darling/Darling.Tests/QueryStorePlanSizeLearnTests.cs @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// Decision-table pins for (#2312 Finding 1) — the fold +/// that finally wires the adaptive candidate sizing its own doc comment promised. The stakes, from +/// the field: sizing every pass from the 160KB seed put K at ~118 on every database, and on the +/// churn-heavy ones the fetch decompressed that window every cycle whether the database's real +/// average was 15KB or 162KB. +/// +public sealed class QueryStorePlanSizeLearnTests +{ + /// + /// An empty pass proves the walk CAUGHT UP (nothing qualified past the watermark) but teaches + /// nothing about plan size — catch-up clears, the carried average stands. + /// + [Fact] + public void AnEmptyPassClearsCatchUpAndKeepsTheAverage() + { + var previous = new QueryStorePlanXmlState.PlanSizeEstimate(40_000, CatchUpInProgress: true); + + var next = QueryStorePlanXmlState.Learn(previous, bytesShipped: 0, plansShipped: 0, plansMeasured: 0, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.Equal(40_000, next.AvgBytes); + Assert.False(next.CatchUpInProgress); + } + + /// A pass that consumed its whole candidate window proves a backlog remains. + [Fact] + public void AFullWindowPassSetsCatchUp() + { + var next = QueryStorePlanXmlState.Learn( + default, bytesShipped: 1_180_000, plansShipped: 118, plansMeasured: 118, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.True(next.CatchUpInProgress); + Assert.Equal(10_000, next.AvgBytes); + } + + /// + /// A pass cut by the byte budget proves the same thing from the other bound — the shipped set + /// can overshoot the budget by up to one plan (the predicate admits the plan that crosses the + /// line), so >= is the correct comparison. + /// + [Fact] + public void ABudgetCutPassSetsCatchUp() + { + var next = QueryStorePlanXmlState.Learn( + default, bytesShipped: 13_000_000, plansShipped: 80, plansMeasured: 80, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.True(next.CatchUpInProgress); + Assert.Equal(162_500, next.AvgBytes); + } + + /// An ordinary partial pass learns its average and clears catch-up. + [Fact] + public void AnOrdinaryPassLearnsAndClearsCatchUp() + { + var previous = new QueryStorePlanXmlState.PlanSizeEstimate(160_000, CatchUpInProgress: true); + + var next = QueryStorePlanXmlState.Learn( + previous, bytesShipped: 450_000, plansShipped: 30, plansMeasured: 30, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.Equal(15_000, next.AvgBytes); + Assert.False(next.CatchUpInProgress); + } + + /// + /// Zero bytes with a nonzero count (every plan NULL — possible, plan XML can be unavailable) + /// must not zero the carried average: ObservedAvgPlanBytes yields null there and the previous + /// estimate stands, exactly like the quiet pass. + /// + [Fact] + public void AnAllNullPassKeepsThePreviousAverage() + { + var previous = new QueryStorePlanXmlState.PlanSizeEstimate(52_000, CatchUpInProgress: false); + + var next = QueryStorePlanXmlState.Learn( + previous, bytesShipped: 0, plansShipped: 5, plansMeasured: 0, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.Equal(52_000, next.AvgBytes); + } + + /// + /// The review catch, pinned: NULL-XML plans ship as rows (the watermark must pass unpersistable + /// plans) so they count for the window comparison — but averaging real bytes over a NULL-inflated + /// divisor would understate plan size and INFLATE the next window, the unsafe direction. A pass + /// of 10 rows where only 5 carried XML averages over 5. + /// + [Fact] + public void AMixedPassAveragesOverOnlyTheMeasuredPlans() + { + var next = QueryStorePlanXmlState.Learn( + default, bytesShipped: 500_000, plansShipped: 10, plansMeasured: 5, candidateWindow: 118, budgetBytes: 12_582_912); + + Assert.Equal(100_000, next.AvgBytes); + } + + /// + /// The never-learned default round-trips as "pass null to CandidatePlanCount": zero AvgBytes is + /// the sentinel the call site tests, so the seed applies exactly until the first real sample. + /// + [Fact] + public void TheDefaultEstimateMeansNeverLearned() + { + QueryStorePlanXmlState.PlanSizeEstimate estimate = default; + + Assert.Equal(0, estimate.AvgBytes); + Assert.False(estimate.CatchUpInProgress); + } +} diff --git a/Darling/Darling.Tests/QueryStorePlanWatermarkTests.cs b/Darling/Darling.Tests/QueryStorePlanWatermarkTests.cs new file mode 100644 index 000000000..13c020c2e --- /dev/null +++ b/Darling/Darling.Tests/QueryStorePlanWatermarkTests.cs @@ -0,0 +1,640 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2164: the plan-XML watermark. 97% of the plan XML shipped in a three-hour fleet window was for plans the +/// store already held, and since drain is 94-97% of a pass and costs per-row LOB bytes, not fetching is worth +/// far more than fetching less. +/// +/// Driven entirely through the collector's PUBLIC surface — BuildPerItemQuery, +/// BuildBackfillPerItemQuery, ReadItemAsync — rather than reaching for the internal helpers, so +/// no production visibility is widened for the tests' benefit. It also makes the state format an explicit +/// pin: the stored string is written out literally here instead of being produced by the same formatter under +/// test, which would have agreed with itself no matter what it emitted. +/// +public class QueryStorePlanWatermarkTests +{ + private const string Db = "probedb"; + private static readonly DateTime Now = new(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc); + + /* ---------- #2210: the SQL-side narrowing is gone; QueryStorePlanXmlState.Resolve is what remains ---------- + The ROW_NUMBER-gated CASE and its in-stream `AND qsp.plan_id > ` predicate are DELETED, not reworked — + BuildPlanFetchQuery is the only thing that reads plan XML now, and its `watermark` parameter is resolved + by the host calling Resolve directly rather than derived inline in this query. These tests used to drive + that predicate through the live SQL; they now drive Resolve directly, which is exactly what the host + still calls to get BuildPlanFetchQuery's watermark argument, so the state-format/malformed/expired/ + future-stamp/per-database coverage stays live even though the SQL it used to narrow does not exist. */ + + [Fact] + public void Resolve_Fresh_ReturnsTheStoredPlanId() + { + var resolved = QueryStorePlanXmlState.Resolve(Stored(900_000, Now), Db, Now); + + Assert.Equal(900_000, resolved); + } + + [Fact] + public void Resolve_Absent_ReturnsZero() + { + /* Absent is what a first run, a restarted host and a broken store all look like, and all three must + resolve to "fetch everything" rather than skip. */ + var resolved = QueryStorePlanXmlState.Resolve(new Dictionary(), Db, Now); + + Assert.Equal(0, resolved); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("900000")] /* no stamp */ + [InlineData("900000:")] /* empty stamp */ + [InlineData("notanumber:1786449600")] + [InlineData("900000:notanumber")] + [InlineData("900000:1786449600:extra")] + [InlineData("0:1786449600")] /* plan_id 0 is not a plan */ + [InlineData("-5:1786449600")] + public void Resolve_Malformed_ReturnsZero(string raw) + { + /* Anything unparseable degrades to a full fetch. Trusting a partially parsed value would suppress + plan XML based on a number nobody wrote. */ + var state = new Dictionary { [QueryStorePlanXmlState.WatermarkKeyPrefix + Db] = raw }; + + Assert.Equal(0, QueryStorePlanXmlState.Resolve(state, Db, Now)); + } + + [Fact] + public void Resolve_Expired_ReturnsZero_ButStillFreshReturnsTheStoredPlanId() + { + /* Bounded staleness is why the stamp is stored beside the id. Query Store can rewrite a plan's XML in + place (memory grant feedback and friends) without issuing a new plan_id, and a permanent watermark + would never look again. It also bounds the documented dormant-plan gap. */ + var stampedLongAgo = Now - QueryStorePlanXmlState.RefreshAfter - TimeSpan.FromMinutes(1); + + var expired = QueryStorePlanXmlState.Resolve(Stored(900_000, stampedLongAgo), Db, Now); + var stillFresh = QueryStorePlanXmlState.Resolve(Stored(900_000, Now - TimeSpan.FromMinutes(1)), Db, Now); + + Assert.Equal(0, expired); + Assert.Equal(900_000, stillFresh); + } + + [Fact] + public void Resolve_StampedInTheFuture_ReturnsZero() + { + /* A clock that moved backwards would otherwise pin the watermark for as long as the skew lasts. */ + var resolved = QueryStorePlanXmlState.Resolve(Stored(900_000, Now.AddDays(3)), Db, Now); + + Assert.Equal(0, resolved); + } + + [Fact] + public void Resolve_IsKeyedPerDatabase() + { + /* plan_id is monotonic WITHIN a database and means nothing across them, so one database's watermark + must never be read for another. */ + var state = new Dictionary + { + [QueryStorePlanXmlState.WatermarkKeyPrefix + "alpha"] = "900000:" + Unix(Now), + }; + + Assert.Equal(900_000, QueryStorePlanXmlState.Resolve(state, "alpha", Now)); + Assert.Equal(0, QueryStorePlanXmlState.Resolve(state, "beta", Now)); + } + + [Fact] + public void TheDefinitionDeclaresNoStateKeys_TheHostOwnsThisState() + { + /* The watermark keys are one per DATABASE and only known at runtime, so the definition could not + declare them even if it wanted to. More importantly it MUST NOT: a state-declaring definition is a + two-host contract (CollectorStateContractTests pins default_trace_events as the only one), while + this is host bookkeeping. The QueryStoreBackfillState seam — a separate state owner name — is what + lets the host persist per-database state without the definition claiming any. + + The failure mode if this ever flips is silent, which is why it is pinned from both ends: a row + written under the DEFINITION's name is never read back, so the watermark would resolve absent + forever and collection would quietly keep paying full price. */ + Assert.Empty(QueryStoreCollector.Instance.StateKeys); + Assert.NotEqual(QueryStorePlanXmlState.StateCollectorName, QueryStoreCollector.Instance.Name); + Assert.Equal("query_store_plan_xml", QueryStorePlanXmlState.StateCollectorName); + Assert.Equal("planwm:", QueryStorePlanXmlState.WatermarkKeyPrefix); + } + + /* ---------- #2210: the runtime-stats query itself, now flag-independent ---------- */ + + [Fact] + public void LiveQuery_NeverCarriesThePlanIdPredicateOrTheRowNumberGate_RegardlessOfWatermarkState() + { + /* The predicate and the CASE it used to narrow are gone from the query entirely, not just from the + conservative path — there is no live watermark state under which either can reappear. */ + var withState = LiveSql(Context(capturePlanXml: true, state: Stored(900_000, Now))); + var withoutState = LiveSql(Context(capturePlanXml: true, state: new Dictionary())); + + Assert.DoesNotContain("qsp.plan_id > ", withState, StringComparison.Ordinal); + Assert.DoesNotContain("qsp.plan_id > ", withoutState, StringComparison.Ordinal); + Assert.DoesNotContain("ROW_NUMBER()", withState, StringComparison.Ordinal); + Assert.DoesNotContain("query_plan_text = CASE", withState, StringComparison.Ordinal); + } + + [Fact] + public void LiveQuery_EmitsThePlaceholder_RegardlessOfCapturePlanXml() + { + /* #2210: both branches of the old ternary now emit the same nvarchar(1) NULL placeholder — the + runtime query carries no plan XML in either mode, so CapturePlanXml no longer changes this query's + text at all. Darling (on) and Lite (off) get byte-identical SQL here; the only thing CapturePlanXml + still gates is the separate BuildPlanFetchQuery fetch. */ + var on = LiveSql(Context(capturePlanXml: true, state: new Dictionary())); + var off = LiveSql(Context(capturePlanXml: false, state: new Dictionary())); + + Assert.Contains("query_plan_text = CONVERT(nvarchar(1), NULL),", on, StringComparison.Ordinal); + Assert.Contains("query_plan_text = CONVERT(nvarchar(1), NULL),", off, StringComparison.Ordinal); + Assert.True(string.Equals(on, off, StringComparison.Ordinal), "CapturePlanXml must no longer change this query's text"); + } + + /* ---------- write-back, driven through the real read loop ---------- */ + + [Fact] + public async Task WriteBack_NormalPass_AdvancesToTheHighestStoredPlanId() + { + var context = Context(capturePlanXml: true, state: new Dictionary()); + await Read(context, Plan(10, xml: true), Plan(20, xml: true), Plan(30, xml: true)); + + Assert.Equal("30:" + Unix(Now), Written(context)); + } + + [Fact] + public async Task WriteBack_CountsOnlyPlansWhoseXmlActuallyShipped() + { + /* The ROW_NUMBER gate NULLs the XML on all but one interval per plan, so "seen" and "stored" differ + on every real pass. Advancing on a plan whose XML was NULL would suppress that plan's XML from then + on without ever having sent it. */ + var context = Context(capturePlanXml: true, state: new Dictionary()); + await Read(context, Plan(10, xml: true), Plan(40, xml: false)); + + Assert.Equal("10:" + Unix(Now), Written(context)); + } + + /* WriteBack_BudgetCutPass_DoesNotAdvanceAtAll (#2164) deleted with the shape it pinned: its own comment + said it recorded known-broken behaviour under the last_execution_time-ordered fetch — the watermark + could never advance on a budget cut because a cut left an arbitrary SUBSET of plan_ids. #2210's + plan_id-ordered fetch removes that premise; the replacement is + AdvanceWatermark_OnABudgetCutPass_StillAdvances below, which pins the new behaviour directly against + QueryStorePlanXmlState.AdvanceWatermark. */ + + [Fact] + public async Task WriteBack_QuietWindow_NeverMovesTheWatermarkBackward() + { + /* This is the case that killed the first design. A window whose newest-EXECUTING plan is older than + the newest-COMPILED one is an ordinary quiet window, and on a steady workload it is most windows. + The first cut read it as a Query Store reset and dropped the watermark, which would have refetched + the whole catalog on nearly every pass — the exact cost being removed. */ + var context = Context(capturePlanXml: true, state: Stored(900_000, Now)); + await Read(context, Plan(800_000, xml: true), Plan(850_000, xml: true)); + + Assert.Null(Written(context)); + } + + [Fact] + public async Task WriteBack_Advance_CarriesTheOriginalStampForward_SoTheHorizonStillFires() + { + /* The stamp dates the last FULL fetch. If an advance re-stamped it to now, then any database that + keeps compiling new plans would push its refresh horizon out forever — and those are the busy + databases where a stale plan matters most. The bounded refresh would silently never happen. */ + var fetchedAt = Now - TimeSpan.FromHours(20); + var context = Context(capturePlanXml: true, state: Stored(900_000, fetchedAt)); + + await Read(context, Plan(950_000, xml: true)); + + Assert.Equal("950000:" + Unix(fetchedAt), Written(context)); + } + + [Fact] + public async Task WriteBack_AfterExpiry_StampsTheFullFetchAtNow() + { + /* The other half of the same rule: an expired watermark means THIS pass refetched everything, so it + is the one case that legitimately re-dates the horizon. Without this the stamp would never move and + every pass after the first expiry would be a full fetch. */ + var longAgo = Now - QueryStorePlanXmlState.RefreshAfter - TimeSpan.FromHours(1); + var context = Context(capturePlanXml: true, state: Stored(900_000, longAgo)); + + await Read(context, Plan(950_000, xml: true)); + + Assert.Equal("950000:" + Unix(Now), Written(context)); + } + + [Fact] + public async Task WriteBack_PlanCaptureOff_WritesNothing() + { + /* Lite reads the same rows with no XML. It must not leave a watermark behind that a plan-capturing + host would later honor, having never shipped a single plan. */ + var context = Context(capturePlanXml: false, state: new Dictionary()); + await Read(context, Plan(10, xml: true), Plan(30, xml: true)); + + Assert.Null(Written(context)); + } + + /* ---------- helpers ---------- */ + + private static Dictionary Stored(long planId, DateTime stampedAt) => + new() + { + [QueryStorePlanXmlState.WatermarkKeyPrefix + Db] = + planId.ToString(CultureInfo.InvariantCulture) + ":" + Unix(stampedAt), + }; + + private static string Unix(DateTime utc) => + new DateTimeOffset(DateTime.SpecifyKind(utc, DateTimeKind.Utc)).ToUnixTimeSeconds() + .ToString(CultureInfo.InvariantCulture); + + private static string? Written(CollectorContext context) => + context.PendingState.TryGetValue(QueryStorePlanXmlState.WatermarkKeyPrefix + Db, out var value) + ? value + : null; + + private static string LiveSql(CollectorContext context) => + QueryStoreCollector.Instance.BuildPerItemQuery(Db, context).Text; + + private static CollectorContext Context( + bool capturePlanXml, + IReadOnlyDictionary state, + int? budgetOverride = null) + { + var context = new CollectorContext + { + ServerId = 1, + ServerName = "probe", + CollectionTime = Now, + Deltas = new CollectorDeltaCalculator(), + CapturePlanXml = capturePlanXml, + State = state, + TextByteBudgetOverride = budgetOverride, + }; + context.CurrentDatabaseName = Db; + return context; + } + + private static (long PlanId, bool Xml) Plan(long planId, bool xml) => (planId, xml); + + private static async Task Read(CollectorContext context, params (long PlanId, bool Xml)[] plans) + { + using var reader = MakeReader(plans); + var rows = new List(); + await QueryStoreCollector.Instance.ReadItemAsync(Db, reader, rows, context, CancellationToken.None); + } + + /// + /// A real DbDataReader over the collector's OWN payload shape, generated from + /// PayloadColumns minus database_name (which the on-prem path takes from the enumerated + /// item, not the reader). Generated rather than hand-listed so a column added to the collector cannot + /// silently shift the ordinals the read loop depends on. + /// + private static DataTableReader MakeReader((long PlanId, bool Xml)[] plans) + { + var table = new DataTable("payload"); + var columns = QueryStoreCollector.Instance.PayloadColumns.Skip(1).ToList(); + + foreach (var column in columns) + { + table.Columns.Add(column.Name, ClrType(column.Name, column.Type)); + } + + for (var i = 0; i < plans.Length; i++) + { + var row = table.NewRow(); + + foreach (var column in columns) + { + row[column.Name] = column.Type switch + { + CollectorColumnType.BigInt => 0L, + CollectorColumnType.Integer => 160, + CollectorColumnType.Boolean => false, + /* Distinct per row, so a budget cut's boundary tie group ends on the very next row. */ + CollectorColumnType.Timestamp when ClrType(column.Name, column.Type) == typeof(DateTime) + => Now.AddMinutes(i), + CollectorColumnType.Timestamp => new DateTimeOffset(Now.AddMinutes(i), TimeSpan.Zero), + _ => "x", + }; + } + + row["query_id"] = plans[i].PlanId * 10; + row["plan_id"] = plans[i].PlanId; + row["execution_count"] = 1L; + /* Must not contain the self-query marker, or the read loop skips the row entirely. */ + row["query_text"] = "SELECT 1 FROM dbo.Whatever"; + row["query_plan_text"] = plans[i].Xml ? new string('p', 4096) : (object)DBNull.Value; + + table.Rows.Add(row); + } + + var dataSet = new DataSet(); + dataSet.Tables.Add(table); + return dataSet.CreateDataReader(); + } + + /// + /// The provider types the read loop actually expects, which are NOT uniform across the timestamp + /// columns: first_execution_time / last_execution_time come out of Query Store as + /// datetimeoffset and are read as DateTimeOffset, while interval_start_time_utc is + /// computed datetime2 and read with GetDateTime. A harness that types all three the same + /// way throws InvalidCastException inside the loop — which is how this was found. + /// + private static Type ClrType(string name, CollectorColumnType type) => type switch + { + CollectorColumnType.BigInt => typeof(long), + CollectorColumnType.Integer => typeof(int), + CollectorColumnType.Boolean => typeof(bool), + CollectorColumnType.Timestamp => + name.Equals("interval_start_time_utc", StringComparison.Ordinal) ? typeof(DateTime) : typeof(DateTimeOffset), + _ => typeof(string), + }; + + /* ---- #2210: the plan_id-ordered fetch policy. Pure functions, pinned like QueryStoreBackfillState + .AdaptiveSpan, because the candidate window and the watermark advance are the two places this + optimization can silently do nothing (attempt one) or silently lose plans (the ordering precondition). */ + + /// + /// The candidate window sits just past what the budget can actually ship, at every plan size the fleet + /// ACTUALLY exhibits — per-quartile averages of 162 / 80 / 39 / 15 KB measured across 2,166 budget-cut + /// passes. The point of the pin is that none of these clamp: if a real fleet plan size hit a bound, the + /// bound would be doing the sizing instead of the measurement. + /// + [Theory] + [InlineData(162, 114)] + [InlineData(80, 231)] + [InlineData(39, 473)] + [InlineData(15, 1229)] + public void CandidatePlanCount_SitsJustPastTheBudget_AtEveryMeasuredFleetPlanSize(int avgKb, int expected) + { + var k = QueryStorePlanXmlState.CandidatePlanCount(avgKb * 1024L, 12L * 1024 * 1024, out var clamped); + + Assert.Equal(expected, k); + Assert.False(clamped, "a plan size the fleet actually shows must not hit a bound"); + + /* Just past, not far past: the window is the coarse bound and the running byte total is the exact one, + and every plan IN the window is decompressed to compute that total. */ + var actuallyFit = (12L * 1024 * 1024) / (avgKb * 1024L); + Assert.InRange(k / (double)actuallyFit, 1.4, 1.6); + } + + /// + /// First contact assumes LARGE plans on purpose. The estimate is a divisor, so over-stating plan size + /// yields a small window — and small is the safe direction: it only slows the watermark down, where too + /// large decompresses a catalog to discover what fits, which is the trap the window exists to prevent. + /// + [Fact] + public void CandidatePlanCount_WithNoPreviousPass_IsConservativelySmall() + { + var seed = QueryStorePlanXmlState.CandidatePlanCount(null, 12L * 1024 * 1024, out var clamped); + var atLargestMeasured = QueryStorePlanXmlState.CandidatePlanCount(162 * 1024L, 12L * 1024 * 1024, out _); + + Assert.False(clamped); + Assert.InRange(seed, atLargestMeasured - 10, atLargestMeasured + 10); + } + + /// Bounds hold, and every clamp REPORTS itself — a window silently pinned at its ceiling reads + /// exactly like one that fit, which is how a cap becomes invisible. + [Theory] + [InlineData(1, 12L * 1024 * 1024, QueryStorePlanXmlState.MaxCandidatePlans)] + [InlineData(64 * 1024, 12L * 1024 * 1024, QueryStorePlanXmlState.MinCandidatePlans)] + public void CandidatePlanCount_ClampsAndSaysSo(long avgKb, long budget, int expected) + { + var k = QueryStorePlanXmlState.CandidatePlanCount(avgKb * 1024L, budget, out var clamped); + + Assert.Equal(expected, k); + Assert.True(clamped, "a clamped window must be reportable so the caller can log it"); + } + + /// + /// `clamped` means a bound CHANGED the answer, not that the answer equals one. A window whose measured size + /// lands naturally on a bound was sized by the measurement and needs no log line; reporting it as clamped is + /// a false positive, and a caller that logs on it trains its reader to ignore the message. + /// + [Fact] + public void CandidatePlanCount_LandingNaturallyOnABound_IsNotReportedAsClamped() + { + /* Budget chosen so budget/avg*margin is exactly MinCandidatePlans: 32 / 1.5 = 21.33 plans of 1 byte. */ + var exactlyTheFloor = (long)(QueryStorePlanXmlState.MinCandidatePlans / QueryStorePlanXmlState.CandidatePlanMargin); + var k = QueryStorePlanXmlState.CandidatePlanCount(1, exactlyTheFloor, out var clamped); + + Assert.Equal(QueryStorePlanXmlState.MinCandidatePlans, k); + Assert.False(clamped, "the measurement produced this value; no bound changed it"); + } + + /// + /// While catch-up is in progress the observed average is floored at the seed, because the sample is biased + /// then and measurably so: on one production catalog the plans the fetch shipped averaged 15 KB while the + /// newest 300 in the same catalog averaged 46 KB. Trusting the low figure inflates K threefold and + /// decompresses that much more than the budget can ship. After convergence the observed value is trusted. + /// + [Fact] + public void CandidatePlanCount_DuringCatchUp_FloorsTheEstimateAtTheSeed() + { + const long budget = 12L * 1024 * 1024; + var biased = 15 * 1024L; + + var duringCatchUp = QueryStorePlanXmlState.CandidatePlanCount(biased, budget, catchUpInProgress: true, out _); + var converged = QueryStorePlanXmlState.CandidatePlanCount(biased, budget, catchUpInProgress: false, out _); + var seeded = QueryStorePlanXmlState.CandidatePlanCount(null, budget, out _); + + Assert.Equal(seeded, duringCatchUp); + Assert.True(converged > duringCatchUp, "the un-floored estimate must still be trusted once converged"); + + /* A large observed average is NOT raised by the floor — over-estimating plan size is the safe direction + and the floor only ever makes the window smaller. */ + var large = 200 * 1024L; + Assert.Equal( + QueryStorePlanXmlState.CandidatePlanCount(large, budget, catchUpInProgress: false, out _), + QueryStorePlanXmlState.CandidatePlanCount(large, budget, catchUpInProgress: true, out _)); + } + + /// + /// #2210, ruling item 4: a FULL cursor sweep over a healthy catalog leaves the watermark byte-identical. + /// Only the reset arm may ever zero it. + /// + /// Simulated through the real functions rather than mocked, because the property is about them: a + /// healthy catalog means every slice the cursor walks finds its stored hash matching the live one, so + /// nothing is re-fetched and the pass lands no plan ids. The sweep is then ceil(range / slice) calls + /// to with nothing landed, and the persisted string + /// has to come out the same at the end — including its stamp, since re-stamping on a no-op sweep would push + /// the refresh period out forever on any database the cursor keeps visiting. + /// + [Fact] + public void AFullCursorSweepOverAHealthyCatalog_LeavesTheWatermarkByteIdentical() + { + const long watermark = 77_176; + var stamp = Now; + var before = QueryStorePlanXmlState.Format(watermark, stamp); + + var slice = QueryStorePlanMap.CursorSliceWidth(watermark, QueryStorePlanXmlState.RefreshAfter, TimeSpan.FromMinutes(5)); + Assert.True(slice > 0); + + var standing = watermark; + var passes = 0; + for (var floor = 0L; floor < watermark; floor += slice) + { + /* Healthy: hashes match across the slice, so nothing is re-fetched and nothing lands. */ + var advance = QueryStorePlanXmlState.AdvanceWatermark(standing, Array.Empty()); + + Assert.True(advance.ArrivedInPlanIdOrder); + Assert.Equal(standing, advance.Watermark); + standing = advance.Watermark; + passes++; + } + + Assert.Equal(watermark, standing); + Assert.Equal(before, QueryStorePlanXmlState.Format(standing, stamp)); + + /* And the sweep genuinely covered the range in one refresh period rather than needing a second. */ + Assert.True((long)passes * slice >= watermark); + Assert.True(passes <= QueryStorePlanXmlState.RefreshAfter.Ticks / TimeSpan.FromMinutes(5).Ticks); + } + + /// A misconfigured budget floors the window rather than producing zero or a negative one. + [Fact] + public void CandidatePlanCount_WithNonPositiveBudget_FloorsAndReportsClamped() + { + var k = QueryStorePlanXmlState.CandidatePlanCount(160 * 1024L, 0, out var clamped); + + Assert.Equal(QueryStorePlanXmlState.MinCandidatePlans, k); + Assert.True(clamped); + } + + /// + /// The estimator reproduces the measured fleet numbers from the same two inputs a pass already has, which + /// is the whole reason no probe is needed: 12.1 MB over 78 plans is the q1 average, 12.3 MB over 828 is q4. + /// + [Theory] + [InlineData(12.1, 78, 158)] + [InlineData(12.3, 828, 15)] + public void ObservedAvgPlanBytes_ReproducesTheMeasuredQuartiles(double shippedMb, int plans, int expectedKb) + { + var avg = QueryStorePlanXmlState.ObservedAvgPlanBytes((long)(shippedMb * 1024 * 1024), plans); + + Assert.NotNull(avg); + Assert.Equal(expectedKb, avg!.Value / 1024); + } + + /// A pass that shipped no plans teaches nothing about plan size and must leave the previous + /// estimate standing rather than replace it with a fallback. + [Fact] + public void ObservedAvgPlanBytes_OnAQuietPass_IsNull() + { + Assert.Null(QueryStorePlanXmlState.ObservedAvgPlanBytes(0, 0)); + Assert.Null(QueryStorePlanXmlState.ObservedAvgPlanBytes(5_000, 0)); + } + + /// + /// THE POINT OF THE WHOLE REDESIGN: a budget-cut pass still advances. Under plan_id-ordered shipping a cut + /// truncates a SUFFIX, so the highest landed id is safe. The previous design shipped in + /// last_execution_time order, where a cut left an arbitrary subset, no value was safe, and the guard that + /// followed meant the watermark could not advance on 97.8% of passes. + /// + [Fact] + public void AdvanceWatermark_OnABudgetCutPass_StillAdvances() + { + var cut = QueryStorePlanXmlState.AdvanceWatermark(100, new long[] { 101, 102 }); + + Assert.Equal(102, cut.Watermark); + Assert.True(cut.ArrivedInPlanIdOrder); + } + + /// Never backward, and a quiet pass earns nothing: lowering the watermark refetches the catalog, + /// and "no new plans this window" is an ordinary pass, not a reset. + [Theory] + [InlineData(new long[0], 100L)] + [InlineData(new[] { 98L, 99L }, 100L)] + [InlineData(new[] { 101L, 102L, 103L }, 103L)] + [InlineData(new[] { 101L, 101L, 102L }, 102L)] + public void AdvanceWatermark_NeverMovesBackward(long[] landed, long expected) + { + Assert.Equal(expected, QueryStorePlanXmlState.AdvanceWatermark(100, landed).Watermark); + } + + /// + /// A descent ABANDONS the advance rather than honouring the leading ascending run. Honouring it looks + /// safer and is not: given {105, 101} it would advance to 105, and with ordering broken there is no basis + /// for inferring that every SELECTED plan below 105 landed — so a plan whose XML never arrived would be + /// suppressed until the refresh horizon. One lost pass of progress is the cheap side of that trade. + /// + [Theory] + [InlineData(new[] { 101L, 102L, 99L, 105L })] + [InlineData(new[] { 105L, 101L })] + public void AdvanceWatermark_WhenOrderingIsViolated_RefusesToAdvance(long[] landed) + { + var refused = QueryStorePlanXmlState.AdvanceWatermark(100, landed); + + Assert.Equal(100, refused.Watermark); + Assert.False(refused.ArrivedInPlanIdOrder, + "the caller needs this to LOG the violation instead of just watching the watermark stop"); + } + + /// + /// #2210: the plan fetch admits a plan when the total BEFORE it was under budget, never on the cumulative + /// total alone. The naive running_bytes <= budget is a per-database STALL — a plan bigger than the + /// whole budget exceeds it on its own row, so it is excluded, every later row is excluded too, the pass ships + /// nothing, the watermark holds, and the next pass re-selects the same plan forever. One 13 MB plan against + /// the 12 MB default does it. + /// + /// A SHAPE pin, and worth being clear about its limit: it asserts the predicate the SQL carries, not + /// what SQL Server does with it. The two behavioural cases the reviewer asked for — an oversized plan ships + /// alone and advances the watermark, and an oversized plan mid-window cuts AFTER it rather than dropping it — + /// need a real Query Store to execute and belong to the measurement session before this leaves draft. What + /// this catches is the regression that reintroduces the naive form, which is the cheap half and the half a + /// future editor is most likely to trip. + /// + [Fact] + public void PlanFetch_AdmitsAPlanOnTheTotalBeforeIt_SoOneOversizedPlanCannotStall() + { + var sql = QueryStoreCollector.Instance.BuildPlanFetchQuery( + Db, Context(capturePlanXml: true, state: new Dictionary()), + watermark: 900_000, candidatePlans: 114, budgetBytes: 12L * 1024 * 1024).Text; + + Assert.Contains("b.running_bytes - b.plan_bytes < 12582912", sql, StringComparison.Ordinal); + Assert.DoesNotContain("b.running_bytes <= ", sql, StringComparison.Ordinal); + + /* The coarse bound sorts and filters on plan_id alone — no XML touched — which is what caps the + decompression the exact bound would otherwise pay across a whole catalog. */ + /* The CONVERT sits INSIDE the window and the running total measures the converted text, so a shipped + plan is decompressed once. Measured on a 73,163-plan production catalog: this shape 133ms against + 274ms for measuring DATALENGTH(qsp.query_plan) in the window and joining back for the text, same 114 + rows and 1.7MB out of both. Plan-id-only with no XML was 114ms. */ + Assert.Contains("query_plan_text = CONVERT(nvarchar(max), qsp.query_plan)", sql, StringComparison.Ordinal); + Assert.Contains("SELECT TOP (114)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("JOIN sys.query_store_plan", sql, StringComparison.Ordinal); + + /* A NULL query_plan counts as zero bytes and still ships. Letting NULL propagate would make the budget + predicate NULL, filter the row out, and a window of all-NULL plans would then hold the watermark and + re-select forever — the oversized-plan stall by another route. */ + Assert.Contains("COALESCE(DATALENGTH(c.query_plan_text), 0)", sql, StringComparison.Ordinal); + Assert.Contains("WHERE qsp.plan_id > 900000", sql, StringComparison.Ordinal); + Assert.Contains("ROWS UNBOUNDED PRECEDING", sql, StringComparison.Ordinal); + } + + /// The ordering verdict rides along with the advance on the cases that are fine. + [Theory] + [InlineData(new[] { 101L, 102L, 103L })] + [InlineData(new[] { 101L, 101L, 102L })] + [InlineData(new[] { 7L })] + [InlineData(new long[0])] + public void AdvanceWatermark_AcceptsNonDescendingArrival(long[] landed) + { + Assert.True(QueryStorePlanXmlState.AdvanceWatermark(100, landed).ArrivedInPlanIdOrder); + } +} diff --git a/Darling/Darling.Tests/QueryStoreReplicaSplitAnalysisLiveTests.cs b/Darling/Darling.Tests/QueryStoreReplicaSplitAnalysisLiveTests.cs index 317af5194..1c72f8cc0 100644 --- a/Darling/Darling.Tests/QueryStoreReplicaSplitAnalysisLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreReplicaSplitAnalysisLiveTests.cs @@ -180,6 +180,111 @@ and everything below SQL Server 2022. This is the arm that would have caught an Assert.Equal("", row.GetProperty("replica_role").GetString()); } + /* ── #2138 CPU-primary scoring, against the REAL Postgres SQL. Duration-only: CPU flat, + duration 5x worse. Duration alone is confounded by blocking, IO waits and machine + contention that no plan choice caused — under the old GREATEST this fired at 5.0; now + the CPU path (1x < 2) and the corroboration gate (1x < 1.25) both decline it. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedSplitSignalsAsync(connection, periodStart, periodEnd, + goodCpuUs: 100_000, goodDurUs: 120_000, badCpuUs: 100_000, badDurUs: 600_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + Assert.Null(await CollectPlanRegressionFactAsync(postgres, context)); + /* The drill-down runs the same scoring — a row here that the fact never counted would be + incoherent in the report. */ + Assert.Empty(await CollectRegressedQueriesDrillDownAsync(postgres, context)); + } + + /* ── #2138: extreme duration (6x) with mild CPU corroboration (1.5x) DOES fire, at HALF the + duration ratio — 3.0, not 6.0 — so it competes honestly with CPU-detected rows. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedSplitSignalsAsync(connection, periodStart, periodEnd, + goodCpuUs: 100_000, goodDurUs: 100_000, badCpuUs: 150_000, badDurUs: 600_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var fact = await CollectPlanRegressionFactAsync(postgres, context); + Assert.NotNull(fact); + Assert.Equal(3.0, fact!.Metadata["worst_regression_factor"], precision: 1); + /* A duration-fired row reports the duration dimension. */ + Assert.Equal(2.0, fact.Metadata["regressed_dimension"]); + } + + /* ── #2138 review catch: CPU has PRECEDENCE, so cpu 2.5x with duration 10x (a genuine CPU + regression that also picked up blocking) fires the CPU branch at 2.5 — and must be + LABELED cpu. Comparing raw ratio magnitudes, correct under the old GREATEST, would + call this duration-caused. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedSplitSignalsAsync(connection, periodStart, periodEnd, + goodCpuUs: 100_000, goodDurUs: 100_000, badCpuUs: 250_000, badDurUs: 1_000_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var fact = await CollectPlanRegressionFactAsync(postgres, context); + Assert.NotNull(fact); + Assert.Equal(2.5, fact!.Metadata["worst_regression_factor"], precision: 1); + Assert.Equal(1.0, fact.Metadata["regressed_dimension"]); + } + + /* ── #2138: below the spend floor. A 12x CPU ratio on a query burning 1.2 CPU-seconds across + the whole window (100 execs x 12ms) is sampling jitter, not a finding. Same 12x ratio as + the arms above — the only difference is absolute spend, so this pins the 10-CPU-second + noise floor and nothing else. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedSplitSignalsAsync(connection, periodStart, periodEnd, + goodCpuUs: 1_000, goodDurUs: 21_000, badCpuUs: 12_000, badDurUs: 32_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + Assert.Null(await CollectPlanRegressionFactAsync(postgres, context)); + Assert.Empty(await CollectRegressedQueriesDrillDownAsync(postgres, context)); + } + + /* ── #2138 gap 3: the regressed query's hash ALSO shows the parameter-sensitivity signature + in the plan cache (ratio 20x, past every detector floor) — the drill-down row must + carry the flag, because the force-plan caution and the future bot's never-auto-force + gate both read it. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedReplicaAsync(connection, periodStart, periodEnd, role: null, BadCpuUsPrimary, offsetSeconds: 0, ct); + await SeedPlanCacheRowAsync(connection, periodStart, periodEnd, minWorkerUs: 15_000, maxWorkerUs: 300_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var row = Assert.Single(await CollectRegressedQueriesDrillDownAsync(postgres, context)); + Assert.True(row.GetProperty("parameter_sensitivity_cofired").GetBoolean()); + } + + /* ── A 2x worker-time spread is ordinary variance, not the >= 10x signature: the flag stays + false. Pins that the flag uses the detector's own threshold, not mere cache presence. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedReplicaAsync(connection, periodStart, periodEnd, role: null, BadCpuUsPrimary, offsetSeconds: 0, ct); + await SeedPlanCacheRowAsync(connection, periodStart, periodEnd, minWorkerUs: 150_000, maxWorkerUs: 300_000, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var row = Assert.Single(await CollectRegressedQueriesDrillDownAsync(postgres, context)); + Assert.False(row.GetProperty("parameter_sensitivity_cofired").GetBoolean()); + } + bodySucceeded = true; } finally @@ -233,15 +338,32 @@ private static async Task SeedReplicaAsync( var collectionTime = periodEnd.AddMinutes(-10 + collection).AddSeconds(offsetSeconds); await SeedRowAsync(connection, collectionTime, planId: 1, GoodPlanHash, intervalId: 1, - goodFirstExec, goodLastExec, GoodCpuUs, role, ct); + goodFirstExec, goodLastExec, GoodCpuUs, role, avgDurUs: null, ct); await SeedRowAsync(connection, collectionTime, planId: 2, BadPlanHash, intervalId: 2, - badFirstExec, badLastExec, badCpuUs, role, ct); + badFirstExec, badLastExec, badCpuUs, role, avgDurUs: null, ct); } } + /// + /// One replica-less query, two plans, CPU and duration controlled INDEPENDENTLY — the seed shape for + /// the #2138 CPU-primary scoring arms, where which signal moved is the entire test. + /// + private static async Task SeedSplitSignalsAsync( + NpgsqlConnection connection, DateTime periodStart, DateTime periodEnd, + long goodCpuUs, long goodDurUs, long badCpuUs, long badDurUs, CancellationToken ct) + { + var collectionTime = periodEnd.AddMinutes(-10); + + await SeedRowAsync(connection, collectionTime, planId: 1, GoodPlanHash, intervalId: 1, + periodStart.AddDays(-6), periodStart.AddDays(-5), goodCpuUs, role: null, goodDurUs, ct); + await SeedRowAsync(connection, collectionTime, planId: 2, BadPlanHash, intervalId: 2, + periodStart.AddDays(-1), periodEnd, badCpuUs, role: null, badDurUs, ct); + } + private static async Task SeedRowAsync( NpgsqlConnection connection, DateTime collectionTime, long planId, string planHash, long intervalId, - DateTime firstExecutionTime, DateTime lastExecutionTime, long avgCpuUs, string? role, CancellationToken ct) + DateTime firstExecutionTime, DateTime lastExecutionTime, long avgCpuUs, string? role, + long? avgDurUs, CancellationToken ct) { const string sql = @" INSERT INTO query_store_stats @@ -271,16 +393,65 @@ INSERT INTO query_store_stats after the dedup collapses the repeat collections down to one row. */ command.Parameters.AddWithValue(100L); command.Parameters.AddWithValue(avgCpuUs); - /* Duration tracks CPU so GREATEST(cpu ratio, duration ratio) is unambiguous. */ - command.Parameters.AddWithValue(avgCpuUs + 20_000); + /* Duration defaults to CPU + 20ms, so the classic seeds regress on BOTH signals and fire the + CPU-primary path (#2138); the split-signal arms pass avgDurUs to move one without the other. */ + command.Parameters.AddWithValue(avgDurUs ?? avgCpuUs + 20_000); await command.ExecuteNonQueryAsync(ct); } - private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + /// + /// One plan-cache row for THE regressed query's hash ('0xREGRESSQH'), with the worker-time spread + /// as the only dial — the #2138 gap-3 PSP-signature seed. Grants flat, no spills. + /// + private static async Task SeedPlanCacheRowAsync( + NpgsqlConnection connection, DateTime periodStart, DateTime periodEnd, + long minWorkerUs, long maxWorkerUs, CancellationToken ct) { - await using var command = new NpgsqlCommand( - "DELETE FROM query_store_stats WHERE server_id = $1", connection); + const string sql = @" +INSERT INTO query_stats + (collection_id, collection_time, server_id, server_name, database_name, + query_hash, query_plan_hash, creation_time, execution_count, + min_worker_time, max_worker_time, min_grant_kb, max_grant_kb, + min_spills, max_spills, query_text, delta_execution_count) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"; + + await using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(periodEnd.AddMinutes(-5)); command.Parameters.AddWithValue(TestServerId); + command.Parameters.AddWithValue(TestServerName); + command.Parameters.AddWithValue(Db); + command.Parameters.AddWithValue("0xREGRESSQH"); + command.Parameters.AddWithValue(BadPlanHash); + command.Parameters.AddWithValue(periodStart.AddDays(-3)); + command.Parameters.AddWithValue(100L); + command.Parameters.AddWithValue(minWorkerUs); + command.Parameters.AddWithValue(maxWorkerUs); + command.Parameters.AddWithValue(1024L); + command.Parameters.AddWithValue(1024L); + command.Parameters.AddWithValue(0L); + command.Parameters.AddWithValue(0L); + command.Parameters.AddWithValue("SELECT * FROM dbo.Orders WHERE CustomerId = @id"); + command.Parameters.AddWithValue(50L); await command.ExecuteNonQueryAsync(ct); } + + private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + /* Two commands, not one multi-statement string: Npgsql does not allow parameters in + multi-statement commands. query_stats carries the #2138 gap-3 plan-cache seeds. */ + await using (var command = new NpgsqlCommand( + "DELETE FROM query_store_stats WHERE server_id = $1", connection)) + { + command.Parameters.AddWithValue(TestServerId); + await command.ExecuteNonQueryAsync(ct); + } + + await using (var command = new NpgsqlCommand( + "DELETE FROM query_stats WHERE server_id = $1", connection)) + { + command.Parameters.AddWithValue(TestServerId); + await command.ExecuteNonQueryAsync(ct); + } + } } diff --git a/Darling/Darling.Tests/QueryStoreSliceRepairLiveTests.cs b/Darling/Darling.Tests/QueryStoreSliceRepairLiveTests.cs index bd21e6c67..5182ba7f6 100644 --- a/Darling/Darling.Tests/QueryStoreSliceRepairLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreSliceRepairLiveTests.cs @@ -34,6 +34,16 @@ namespace Darling.Tests; /// public sealed class QueryStoreSliceRepairLiveTests { + [Fact] + public void SliceStatementTimeout_IsGenerousButBounded() + { + /* #2105 field failure: Npgsql's default 30s killed the stage aggregation on a store fresh + off a large catch-up, surfacing as "Exception while reading from stream" with no mention + of a timeout. Bounded on purpose - the slice transaction holds chunk locks the live + service's compression jobs also want, so infinite (the VACUUM precedent) is wrong here. */ + Assert.Equal(900, QueryStoreSliceRepair.SliceStatementTimeoutSeconds); + } + private const int TestServerId = -919120; /// @@ -238,6 +248,7 @@ verb like this could have — it exists so an operator can look before committin var dryText = dryOut.ToString(); Assert.Contains("Split intervals found : 1", dryText, StringComparison.Ordinal); Assert.Contains("DRY RUN — nothing was changed.", dryText, StringComparison.Ordinal); + Assert.DoesNotContain("[OK]", dryText, StringComparison.Ordinal); await using (var check = new NpgsqlConnection(scratch.ConnectionString)) { @@ -252,6 +263,12 @@ verb like this could have — it exists so an operator can look before committin var runText = runOut.ToString(); Assert.Contains("Collapsed. Rows removed: 1", runText, StringComparison.Ordinal); Assert.Contains("DONE", runText, StringComparison.Ordinal); + /* Per-slice progress: the real run announces each slice with its removal count and span + percent — a big backlog is no longer a silent console between the banner and DONE. The + removed figure inside the [OK] line is the same deleted-minus-reinserted derivation the + summary total uses, so the two cannot disagree. */ + Assert.Contains("[OK]", runText, StringComparison.Ordinal); + Assert.Contains("1 removed (100% of span, 1 total)", runText, StringComparison.Ordinal); await using (var check = new NpgsqlConnection(scratch.ConnectionString)) { @@ -275,6 +292,68 @@ verb like this could have — it exists so an operator can look before committin } } + /// + /// The SECOND #2105 field failure, pinned the way DarlingRetentionTests pins the first of this class + /// (#1564): the collapse's DELETE touches COMPRESSED chunks — a store old enough to need this repair has + /// had its compression policy running the whole time — and TimescaleDB rails DML decompression at 100k + /// tuples per transaction by default, so the field run died at 53400: tuple decompression limit + /// exceeded four minutes in. The fix is the SET LOCAL ... = 0 lift at the top of the slice + /// transaction, and this test is its tripwire: the session arms the rail at ONE tuple before calling the + /// collapse, so the transaction-local lift is the only thing standing between the DELETE and the exact + /// field error. Drop the lift and this fails with the operator's 53400 instead of a silent coverage hole. + /// Compression is applied SYNCHRONOUSLY (the retention test's pattern — no background-job race), and the + /// compressed shape is PROVEN before the repair runs, not assumed. + /// + [Fact] + public async Task CollapsingRowsInsideACompressedChunk_SurvivesTheDecompressionRail_TheSecondFieldFailure() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #2105 compressed-chunk collapse test (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + + var hour = new DateTime(2026, 6, 12, 8, 0, 0, DateTimeKind.Unspecified); + await SeedSliceAsync(connection, hour.AddMinutes(5), intervalId: 8201, queryId: 91, planId: 111, + intervalStart: hour, executionCount: FlushedCount, avgDurationUs: FlushedAvgUs, ct: ct); + await SeedSliceAsync(connection, hour.AddMinutes(5), intervalId: 8201, queryId: 91, planId: 111, + intervalStart: hour, executionCount: MemoryCount, avgDurationUs: MemoryAvgUs, ct: ct); + + /* Compression enablement lives in ApplyCompressionPolicyAsync (a separate service-start step this + test deliberately skips — a background policy racing the assertions is pure interference), so + enable it directly, exactly like the retention test's compressed-chunk pin; then compress the + seeded chunk synchronously and PROVE the compressed shape is what the collapse runs against. */ + await ExecAsync(connection, + "ALTER TABLE collect.query_store_stats SET (timescaledb.compress, timescaledb.compress_segmentby = 'server_id')", ct); + await ExecAsync(connection, + "SELECT compress_chunk(c, if_not_compressed => true) FROM show_chunks('collect.query_store_stats') c", ct); + Assert.True(await ScalarAsync(connection, @" +SELECT count(*) +FROM timescaledb_information.chunks +WHERE hypertable_name = 'query_store_stats' + AND is_compressed", ct) >= 1, + "expected the seeded query_store_stats chunk to be compressed — the fixture is not exercising the compressed-chunk shape"); + + /* Arm the rail at ONE tuple for this session. The DELETE must decompress the seeded rows' batch + (two tuples at minimum), so only the transaction-local SET LOCAL lift lets the collapse commit. */ + await ExecAsync(connection, "SET timescaledb.max_tuples_decompressed_per_dml_transaction = 1", ct); + + var removed = await QueryStoreSliceRepair.CollapseSliceAsync(connection, hour, hour.AddHours(1), ct); + Assert.Equal(1, removed); + + Assert.Equal(1, await RawRowCountAsync(connection, queryId: 91, ct)); + Assert.Equal(TrueCount, await RawExecutionCountAsync(connection, queryId: 91, ct)); + } + /* ─────────────────────────── helpers ─────────────────────────── */ private static async Task SeedSliceAsync( diff --git a/Darling/Darling.Tests/QueryStoreStatePruneLivePostgresTests.cs b/Darling/Darling.Tests/QueryStoreStatePruneLivePostgresTests.cs new file mode 100644 index 000000000..8a8793fb4 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreStatePruneLivePostgresTests.cs @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The #2188 prune against a REAL store, driving +/// — the production method, +/// not a re-implementation. Split from the pure per the shape +/// LivePostgresCollectionHygieneTests asks for, so the source and policy pins do not serialize behind +/// the shared store. +/// +/// The statement's whole risk lives in how PostgreSQL evaluates two guards and an anti-join together — +/// a NULL-valued aggregate over an empty snapshot, a timestamp comparison, and a correlated NOT EXISTS — and +/// no source pin can speak to any of it. +/// +[Collection("live-postgres")] +public sealed class QueryStoreStatePruneLivePostgresTests +{ + /// Distinctive fake ids — a real server_id is a storage-name hash, never these. + private const int LiveServerId = -218800; + private const int NeighborServerId = -218801; + private const string ServerName = "PLANWM-PRUNE-SRV"; + + /// The snapshot's collection_time in every case below; state rows are dated relative to it. + private static readonly DateTime Newest = new(2026, 8, 11, 9, 0, 0, DateTimeKind.Unspecified); + + /// Old enough to be judged by — the ordinary case for a real state row. + private static readonly DateTime BeforeNewest = Newest.AddHours(-1); + + private static string Planwm(string database) => QueryStorePlanXmlState.WatermarkKeyPrefix + database; + private static string Done(string database) => QueryStoreBackfillState.DoneKeyPrefix + database; + private static string Hole(string database) => QueryStoreBackfillState.HoleKeyPrefix + database; + + private static string EncodedHole() => QueryStoreBackfillState.EncodeHole( + new DateTime(2026, 8, 10, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2026, 8, 10, 6, 0, 0, DateTimeKind.Utc)); + + /// + /// One live pass over every case that separates a correct prune from a destructive one. + /// + [Fact] + public async Task Prune_RetiresOnlyDroppedDatabases_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live query_store state prune test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var logger = new CapturingTestLogger(); + var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator(), logger); + + var bodySucceeded = false; + try + { + /* The snapshot: what sys.databases still holds. "Parked" is the case the whole design turns on — + a database that EXISTS but that query_store's enumeration would never return (it screens + state_desc = ONLINE), so a prune keyed on the enumeration deletes it and a prune keyed on + sys.databases keeps it. + + "App" is present and "AppArchive" is not, which is the name-shape trap: writing the anti-join + as starts_with(state_key, prefix || ds.database_name) instead of an equality is a very + plausible variant, and it would spare planwm:AppArchive forever because "planwm:App" is a + prefix of it. For an issue whose subject is database name churn, that case has to be here. */ + await SnapshotAsync(connection, ct, Newest, "Live", "Parked", "App"); + + /* An OLDER snapshot still naming the dropped databases. If the prune read any snapshot but the + newest, nothing would ever be retired. */ + await SnapshotAsync(connection, ct, Newest.AddMinutes(-15), "Live", "Parked", "App", "Dropped", "AppArchive"); + + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Live"), "900000:1786449600"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Parked"), "800000:1786449600"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Dropped"), "700000:1786449600"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("App"), "500000:1786449600"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("AppArchive"), "400000:1786449600"); + + /* The backfill worker's per-database keys, which orphan identically. Both prefixes get a + survivor as well as a casualty: with only a delete case, a statement that deleted + unconditionally would still pass. */ + await StateAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Done("Live"), "2026-08-11T09:00:00.0000000Z"); + await StateAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"), "2026-08-11T09:00:00.0000000Z"); + await StateAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Hole("Live"), EncodedHole()); + await StateAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Hole("Dropped"), EncodedHole()); + + /* A key under the SAME owner that is not database-keyed. The prefix filter is what protects it; + a prune written as "every key of this collector" would take it. */ + await StateAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping", "keep me"); + + /* Another collector's state entirely, and a NEIGHBOUR SERVER whose database really was dropped + here — server scoping is the difference between pruning one server and pruning the fleet. */ + await StateAsync(connection, ct, LiveServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey, @"S:\MSSQL\Log\log_766.trc"); + await StateAsync(connection, ct, NeighborServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("Dropped"), "600000:1786449600"); + + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + + /* Retired: gone from the newest snapshot, on every prefix it could have left behind. */ + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Dropped"))); + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"))); + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Hole("Dropped"))); + + /* Retired even though a LIVE database's name is a prefix of it. */ + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("AppArchive"))); + + /* Kept: still collected. */ + Assert.Equal("900000:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Live"))); + Assert.Equal("500000:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("App"))); + Assert.Equal("2026-08-11T09:00:00.0000000Z", + await ValueAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Done("Live"))); + Assert.Equal(EncodedHole(), + await ValueAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, Hole("Live"))); + + /* Kept, and this is the assertion the change exists for: present in sys.databases, absent from + every enumeration query_store runs. Pruning it costs a full plan-XML refetch of a database that + never went anywhere, on precisely the servers that keep databases parked. */ + Assert.Equal("800000:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Parked"))); + + /* Kept: not database-keyed, not this collector, not this server. */ + Assert.Equal("keep me", + await ValueAsync(connection, ct, LiveServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping")); + Assert.Equal(@"S:\MSSQL\Log\log_766.trc", + await ValueAsync(connection, ct, LiveServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey)); + Assert.Equal("600000:1786449600", + await ValueAsync(connection, ct, NeighborServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Dropped"))); + + /* The DIAGNOSTIC, which is the only thing that could ever make a wrong delete visible — the other + symptom is a silent refetch. It comes from the statement's RETURNING clause, so if that ever + stopped yielding rows the deletes would still happen and the log would simply go quiet: no + assertion on the store's contents can see that, which is why it is asserted on the log. */ + Assert.Contains("Dropped", logger.Joined, StringComparison.Ordinal); + Assert.Contains("AppArchive", logger.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("Parked", logger.Joined, StringComparison.Ordinal); + + /* Idempotent — it runs on every query_store cycle of every server, so a second pass over a clean + store must touch nothing. Seven survivors: planwm for Live, Parked and App; done and hole for + Live; the non-database-keyed bookkeeping row; and the other collector's key. */ + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + Assert.Equal(7, await CountAsync(connection, ct, LiveServerId)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The freshness guard, which is the difference between correct and merely usually-correct. + /// + /// A snapshot that EXISTS is not a snapshot that is CURRENT. If database_states stops collecting + /// for a server, its newest snapshot freezes, and every database created after that instant is missing + /// from it while being perfectly alive. Pruning on presence alone would delete such a database's + /// watermark on every cycle forever — paying a full plan-XML refetch each time, which is the exact cost + /// #2164 exists to remove, while logging that a live database is gone. A snapshot cannot judge a row + /// written after it was taken. + /// + [Fact] + public async Task Prune_LeavesStateWrittenAfterTheSnapshot_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live prune freshness test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator()); + + var bodySucceeded = false; + try + { + /* A frozen snapshot: database_states stopped collecting at Newest and names only OldDb. */ + await SnapshotAsync(connection, ct, Newest, "OldDb"); + + /* Created after the snapshot froze — absent from it, and entirely alive. */ + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("BornAfterTheSnapshot"), "10:1786449600", updatedAt: Newest.AddMinutes(30)); + + /* Dropped before the snapshot froze: absent from it, and its last state write PRECEDES it, which + is what still makes it prunable. Without this the test would pass for a prune that had simply + stopped working. */ + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("DroppedLongAgo"), "20:1786449600", updatedAt: BeforeNewest); + + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + + Assert.Equal("10:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("BornAfterTheSnapshot"))); + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("DroppedLongAgo"))); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The empty-snapshot guard, isolated. A server with NO database_states snapshot must lose nothing — + /// this is the case that turns a hygiene sweep into a fleet-wide data event, and ordinary configurations + /// reach it: Azure SQL DB never collects database_states at all + /// (DatabaseStateCollector.AppliesTo), and a server whose rows have aged out of the raw retention + /// tier looks identical from here. + /// + [Fact] + public async Task Prune_WithNoDatabaseSnapshot_RetiresNothing_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live prune guard test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator()); + + var bodySucceeded = false; + try + { + /* No snapshot for THIS server. A neighbour's snapshot exists and names none of these databases, + so a prune that forgot to scope the snapshot read by server would wipe every row here. */ + await SnapshotAsync(connection, ct, Newest, NeighborServerId, "SomeOtherServersDatabase"); + + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Alpha"), "1:1786449600"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Beta"), "2:1786449600"); + + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + + Assert.Equal("1:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Alpha"))); + Assert.Equal("2:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Beta"))); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The race the issue asks about, driven in the order that would lose data if the write-back were not an + /// upsert: a cycle loads state, the prune deletes that key underneath it, and the cycle then persists + /// what it observed. The row must come back. + /// + /// The prune cannot actually target a live database — its predicate is absence from the newest + /// sys.databases snapshot — so this drives the adversarial case DELIBERATELY, by pruning while the name + /// is missing from the snapshot. That is what makes the consequence a measured fact instead of an + /// argument: even a prune that fires on a database it should not have costs one refetch, never a + /// row. + /// + [Fact] + public async Task Prune_RacingAnInFlightCycle_CannotLoseTheWatermark_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live prune race test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteLiveRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator()); + + var bodySucceeded = false; + try + { + /* A snapshot that does NOT name Racer, and a state row old enough to be judged by it — the + adversarial setup, since neither is true of a real live database. */ + await SnapshotAsync(connection, ct, Newest, "Live"); + await StateAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, + Planwm("Racer"), "900000:1786449600", updatedAt: BeforeNewest); + + /* Cycle start: the collection pass reads its state. */ + var loaded = await runner.GetCollectorStateAsync(LiveServerId, QueryStorePlanXmlState.StateCollectorName, ct); + Assert.Equal("900000:1786449600", Assert.Contains(Planwm("Racer"), loaded)); + + /* Mid-flight: the prune fires and takes the row this cycle is still working from. */ + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + Assert.Null(await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Racer"))); + + /* Cycle end: the write-back is an INSERT ... ON CONFLICT, so it restores rather than failing on + a row that is no longer there. The database keeps collecting; the delete cost nothing. */ + await runner.SaveCollectorStateAsync( + LiveServerId, QueryStorePlanXmlState.StateCollectorName, + new Dictionary(StringComparer.Ordinal) { [Planwm("Racer")] = "950000:1786449600" }, + ct); + + Assert.Equal("950000:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Racer"))); + + /* And it stays: the restored row is stamped NOW, which is after the snapshot, so the freshness + guard keeps the next cycle's prune off it too. Without that the two would fight forever. */ + await runner.PruneOrphanedQueryStoreDatabaseStateAsync(LiveServerId, ct); + Assert.Equal("950000:1786449600", + await ValueAsync(connection, ct, LiveServerId, QueryStorePlanXmlState.StateCollectorName, Planwm("Racer"))); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteLiveRowsAsync(cleanup, cleanupCt)); + } + } + + /* ---------------- helpers ---------------- */ + + private static Task SnapshotAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, params string[] databases) + => SnapshotAsync(connection, ct, at, LiveServerId, databases); + + private static async Task SnapshotAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, int serverId, params string[] databases) + { + foreach (var database in databases) + { + /* state_desc is deliberately never read by the prune — existence is the only question it asks — + and "Parked" carrying OFFLINE is what makes that testable: the assertion that it survives + fails the moment anyone adds a state filter to the anti-join. */ + using var command = new NpgsqlCommand(@" +INSERT INTO collect.database_states (collection_id, collection_time, server_id, server_name, database_name, database_id, state_desc, is_in_standby) +VALUES (0, $1, $2, $3, $4, 5, $5, false)", connection); + command.Parameters.AddWithValue(at); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(database); + command.Parameters.AddWithValue(string.Equals(database, "Parked", StringComparison.Ordinal) ? "OFFLINE" : "ONLINE"); + await command.ExecuteNonQueryAsync(ct); + } + } + + private static async Task StateAsync( + NpgsqlConnection connection, CancellationToken ct, int serverId, string owner, string key, string value, + DateTime? updatedAt = null) + { + using var command = new NpgsqlCommand(@" +INSERT INTO collect.collector_state (server_id, collector_name, state_key, state_value, updated_at) +VALUES ($1, $2, $3, $4, $5)", connection); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(owner); + command.Parameters.AddWithValue(key); + command.Parameters.AddWithValue(value); + command.Parameters.AddWithValue(updatedAt ?? BeforeNewest); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task ValueAsync( + NpgsqlConnection connection, CancellationToken ct, int serverId, string owner, string key) + { + using var command = new NpgsqlCommand( + "SELECT state_value FROM collect.collector_state WHERE server_id = $1 AND collector_name = $2 AND state_key = $3", + connection); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(owner); + command.Parameters.AddWithValue(key); + var value = await command.ExecuteScalarAsync(ct); + return value is DBNull or null ? null : (string)value; + } + + private static async Task CountAsync(NpgsqlConnection connection, CancellationToken ct, int serverId) + { + using var command = new NpgsqlCommand( + "SELECT COUNT(*) FROM collect.collector_state WHERE server_id = $1", connection); + command.Parameters.AddWithValue(serverId); + return (long)(await command.ExecuteScalarAsync(ct))!; + } + + private static async Task DeleteLiveRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + var live = LiveServerId.ToString(CultureInfo.InvariantCulture); + var neighbor = NeighborServerId.ToString(CultureInfo.InvariantCulture); + using var cleanup = new NpgsqlCommand( + $"DELETE FROM collect.collector_state WHERE server_id IN ({live}, {neighbor});" + + $"DELETE FROM collect.database_states WHERE server_id IN ({live}, {neighbor});", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/QueryStoreStatePruneTests.cs b/Darling/Darling.Tests/QueryStoreStatePruneTests.cs new file mode 100644 index 000000000..2c0c3e0ae --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreStatePruneTests.cs @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2188: retiring the per-database collector_state rows query_store leaves behind for databases the +/// server no longer has. The #2164 plan-XML watermark writes one planwm: row per database and the +/// #2022 backfill worker writes done: and hole: rows the same way, and until this nothing +/// deleted any of them for a dropped or renamed database. +/// +/// What actually needs pinning is the input, not the delete. A delete keyed on the wrong list is +/// the failure mode: query_store's own enumeration is filtered by ONLINE state, AG primary-ness, the +/// excluded-database list, a vendor-name screen, HAS_DBACCESS, and a per-database probe that can fail, +/// so a database absent from one cycle's items is far more often offline or unprobeable than dropped. Pruning +/// on that absence would delete LIVE watermarks on exactly the servers that have such databases, and because +/// the consequence is a silent refetch rather than an error, nothing downstream would ever report it. +/// is where that is proven against a real store; this +/// class holds the policy and the cross-host wiring, which no store can see. +/// +/// Both SKUs. Lite writes no planwm: (it never sets +/// CollectorContext.CapturePlanXml) but it DOES write done: and hole: through its own +/// backfill worker, and it only ever deletes a hole it services or expires — so the orphan class is real on +/// both sides and the prune is ported, not declared Darling-only. +/// pins that in both directions, and the key set +/// itself lives in the shared so a prefix cannot end up pruned on +/// one SKU and orphaning on the other. +/// +public sealed class QueryStoreStatePruneTests +{ + private static string Planwm(string database) => QueryStorePlanXmlState.WatermarkKeyPrefix + database; + + /* ---------------- the design's premise, pinned without a store ---------------- */ + + [Fact] + public void ThePruneChecksSysDatabases_NotTheCollectorsFilteredEnumeration() + { + /* The whole correctness of this change is which relation answers "does this database still exist". + database_states is an unfiltered SELECT ... FROM sys.databases; query_store's enumeration is not. + Pinned on the statement text because the difference is invisible to any test that only seeds + databases which are both present AND collectable — which is every naive fixture. */ + Assert.Contains("FROM database_states", DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, StringComparison.Ordinal); + + /* The snapshot guard. MAX() over zero rows yields one row holding NULL, so without this an anti-join + against a server that has never collected database_states matches EVERY key and deletes the lot. */ + Assert.Contains("snapshot.newest IS NOT NULL", DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, StringComparison.Ordinal); + + /* The freshness guard, which is the stronger of the two: a snapshot cannot judge a state row written + AFTER it was taken. Without it, a server whose database_states collection has stopped prunes every + database created since — live ones — on every cycle forever. */ + Assert.Contains("s.updated_at < snapshot.newest", DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, StringComparison.Ordinal); + + /* Newest snapshot only: an older one names databases that have since been dropped, which would make + the prune permanently unable to retire anything. */ + Assert.Contains("MAX(collection_time)", DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, StringComparison.Ordinal); + } + + [Fact] + public void LitesTwinCarriesTheSameTwoGuards() + { + /* The DuckDB statement is a separate string in a separate project, so nothing but a source pin keeps + it honest. Both guards are what stop a hygiene sweep becoming a data event, and Lite is the SKU + where a mistake lands on somebody's laptop with no DBA watching a fleet dashboard. + + DuckDB gets the freshness guard for free as the empty-snapshot guard too — `<` against a NULL MAX + is NULL — so it carries one predicate where Darling spells out two; what must not drift is that + the comparison against the snapshot's own timestamp is THERE. */ + var root = FindRepoRoot(); + Assert.True(root is not null, "repo root not found -- the source pin cannot run"); + + var liteBackfill = File.ReadAllText(Path.Combine( + root!, "Lite", "Services", "RemoteCollectorService.QueryStoreBackfill.cs")); + + Assert.Contains("FROM database_states", liteBackfill, StringComparison.Ordinal); + Assert.Contains( + "updated_at < (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1)", + liteBackfill, StringComparison.Ordinal); + + /* And the anti-join is NOT EXISTS on both sides, not NOT IN. They are not equivalent: a single NULL + anywhere in a NOT IN list makes the whole predicate NULL, so the prune would silently stop + retiring anything. Fail-safe, and therefore exactly the kind of divergence that would sit + undetected for a release — the two statements answer the same question the same way or the SKUs + have quietly forked. */ + Assert.Contains("AND NOT EXISTS", liteBackfill, StringComparison.Ordinal); + Assert.DoesNotContain("NOT IN", liteBackfill, StringComparison.Ordinal); + Assert.Contains("AND NOT EXISTS", DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, StringComparison.Ordinal); + } + + [Fact] + public void EveryPerDatabaseKeyPrefixQueryStoreOwnsIsAccountedFor() + { + /* The drift guard that matters more than the prune itself: a fourth key prefix added to either state + class is a new orphan class if it is per-database, and nothing about adding one would fail a test. + Derived from the state classes' own consts rather than a hand-written list, so the two cannot + disagree. + + It demands a DECISION rather than an addition. A prefix must appear in exactly one of the two + shared lists, because the wrong answer here is not "forgot to prune" — it is adding a + SERVER-scoped key to PrunableKeys, whose rows can never equal prefix || databaseName and so would + be deleted on every single cycle. The message has to say that, or the obvious fix is the bug. */ + /* Every query_store state class in the collectors assembly, discovered rather than listed: a + hand-written pair would have made a THIRD state class invisible to this guard, which is the same + silent-omission shape the guard exists to catch. */ + var stateClasses = typeof(QueryStorePlanXmlState).Assembly.GetTypes() + .Where(type => type.IsClass && type.IsAbstract && type.IsSealed /* static */ + && type.Name.StartsWith("QueryStore", StringComparison.Ordinal) + && type.Name.EndsWith("State", StringComparison.Ordinal)) + .ToArray(); + + Assert.Contains(typeof(QueryStorePlanXmlState), stateClasses); + Assert.Contains(typeof(QueryStoreBackfillState), stateClasses); + /* #2150 added a third, and the discovery above found it without being told — which is the property + this guard exists for. Named here anyway so a rename that quietly drops it out of the pattern + fails rather than silently shrinking the set under test. */ + Assert.Contains(typeof(QueryStoreTextState), stateClasses); + /* #2312 added a fourth, same treatment. */ + Assert.Contains(typeof(QueryStoreOpenIntervalState), stateClasses); + + var declared = stateClasses + .SelectMany(type => type.GetFields(BindingFlags.Public | BindingFlags.Static)) + .Where(field => field.IsLiteral && field.FieldType == typeof(string) + && field.Name.EndsWith("KeyPrefix", StringComparison.Ordinal)) + .Select(field => (string)field.GetRawConstantValue()!) + .ToArray(); + + Assert.NotEmpty(declared); + + var pruned = QueryStorePerDatabaseState.PrunableKeys.Select(pair => pair.Prefix).ToArray(); + + foreach (var prefix in declared) + { + Assert.True( + pruned.Contains(prefix, StringComparer.Ordinal) + || QueryStorePerDatabaseState.NotKeyedByDatabase.Contains(prefix, StringComparer.Ordinal), + $"The query_store state key prefix '{prefix}' is in neither shared list, so #2188 has no " + + "verdict on it. Decide which it is:\n" + + " - keyed by DATABASE NAME (the key is prefix + databaseName): add it to " + + "QueryStorePerDatabaseState.PrunableKeys with its owning collector_name, and both hosts " + + "prune it when the database is dropped.\n" + + " - keyed by anything ELSE (server-scoped, or a compound key): add it to " + + "QueryStorePerDatabaseState.NotKeyedByDatabase. Do NOT put it in PrunableKeys to silence " + + "this — both prunes test a key by rebuilding it as prefix + databaseName, so a key that " + + "is not shaped that way matches no live database and gets DELETED every cycle."); + } + + /* Owner and prefix must travel together: a prefix pruned under the wrong collector_name silently + deletes nothing, which looks exactly like "there was nothing to prune". */ + Assert.Contains( + (QueryStorePlanXmlState.StateCollectorName, QueryStorePlanXmlState.WatermarkKeyPrefix), + QueryStorePerDatabaseState.PrunableKeys); + Assert.Contains( + (QueryStoreBackfillState.StateCollectorName, QueryStoreBackfillState.HoleKeyPrefix), + QueryStorePerDatabaseState.PrunableKeys); + /* #2150: paired with its OWN collector name, not the plan fetch's. The two watermarks are stored + separately on purpose (they walk different catalogs at different rates), so borrowing the plan + fetch's owner here would prune nothing and look exactly like having nothing to prune. */ + Assert.Contains( + (QueryStoreTextState.StateCollectorName, QueryStoreTextState.WatermarkKeyPrefix), + QueryStorePerDatabaseState.PrunableKeys); + /* #2312: the open-interval stamp, per database like the three above, under its own owner. */ + Assert.Contains( + (QueryStoreOpenIntervalState.StateCollectorName, QueryStoreOpenIntervalState.WatermarkKeyPrefix), + QueryStorePerDatabaseState.PrunableKeys); + } + + [Fact] + public void BothHostsPruneOnTheQueryStoreCycle() + { + /* Wiring invisible to everything else here: delete either call and every assertion in this file + still passes, because they drive the prunes directly. The rows would simply never be pruned in + production. Source-pinned in BOTH hosts together, for the reason CollectorStateContractTests + gives — a fix applied to one host and not the other is the drift this product keeps paying for. + + The GATE is pinned, not just the call: an ungated prune would run for all 38 collectors, and + since it deletes by collector_name that would be 37 harmless no-ops hiding one real behaviour + change nobody chose. */ + var root = FindRepoRoot(); + Assert.True(root is not null, "repo root not found -- the source pin cannot run"); + + var hosts = new[] + { + Path.Combine(root!, "Darling", "PerformanceMonitor.Darling.Service", "DarlingCollectorRunner.cs"), + Path.Combine(root!, "Lite", "Services", "RemoteCollectorService.DefinitionRunner.cs"), + }; + + foreach (var host in hosts) + { + var source = File.ReadAllText(host); + var name = Path.GetFileName(host); + + var call = source.IndexOf("await PruneOrphanedQueryStoreDatabaseStateAsync(", StringComparison.Ordinal); + Assert.True(call >= 0, $"{name} must prune orphaned per-database query_store state"); + + /* The 400 characters immediately BEFORE the call — the `if` that guards it. Checking the whole + file would be vacuous: DarlingCollectorRunner already tests definition.Name against + query_store in five other places for unrelated reasons, so a file-wide Contains would pass + for an ungated prune. */ + var guard = source.Substring(Math.Max(0, call - 400), Math.Min(400, call)); + + Assert.Contains( + "string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)", + guard, StringComparison.Ordinal); + + /* And on the collector that supplies the snapshot: without this the prune runs three guaranteed + no-op deletes per cycle forever on Azure SQL DB, and #2191's boundary is emergent rather than + declared. */ + Assert.Contains("DatabaseStateCollector.Instance.AppliesTo(", guard, StringComparison.Ordinal); + } + } + + [Fact] + public void LiteWritesTheBackfillKeysButNeverTheWatermark() + { + /* The parity FACT, which the first cut of this change got wrong: Lite writes no planwm: (it never + sets CapturePlanXml) but it DOES write done: and hole: through its own backfill worker, and it + only ever deletes a hole it services or expires. So the orphan class is real on both SKUs and the + prune had to be ported, not declared Darling-only. + + Pinned at source in both directions so neither half can rot: if Lite ever starts capturing plans + it inherits a planwm: prune that is already there (the shared PrunableKeys carries the watermark + on both hosts precisely so that day needs no code change), and if Lite ever stops writing the + backfill keys this test says so rather than leaving a prune nobody needs. */ + var root = FindRepoRoot(); + Assert.True(root is not null, "repo root not found -- the source pin cannot run"); + + var liteRunner = File.ReadAllText(Path.Combine( + root!, "Lite", "Services", "RemoteCollectorService.DefinitionRunner.cs")); + var liteBackfill = File.ReadAllText(Path.Combine( + root!, "Lite", "Services", "RemoteCollectorService.QueryStoreBackfill.cs")); + + Assert.False( + liteRunner.Contains("CapturePlanXml", StringComparison.Ordinal), + "Lite's definition runner now sets CapturePlanXml, so Lite writes planwm: rows too. The shared " + + "PrunableKeys already covers that prefix on both hosts, so the prune needs no change — but " + + "QueryStorePlanWatermarkTests.WriteBack_PlanCaptureOff_WritesNothing and this file's prose " + + "both describe Lite as never writing them, and that is now wrong."); + + foreach (var prefix in new[] { "DoneKeyPrefix", "HoleKeyPrefix" }) + { + /* The per-database KEY SHAPE, not merely a mention of the prefix: `prefix + databaseName` is + exactly what makes these rows orphan when the database goes away, and it is what both prunes + reconstruct to test a key against the live database list. A worker that started keying these + some other way would leave the prune matching nothing while every "is the prefix used?" check + still passed. */ + Assert.True( + liteBackfill.Contains( + $"QueryStoreBackfillState.{prefix} + databaseName", StringComparison.Ordinal), + $"Lite's backfill worker no longer keys QueryStoreBackfillState.{prefix} by database name. " + + "The #2188 prune rebuilds keys as prefix + databaseName to test them against the newest " + + "sys.databases snapshot, so it now matches nothing for this prefix — revisit both."); + } + } + + /// + /// The recreate-with-the-same-name case, which is the only shape here that could cost data rather than a + /// refetch: a dropped and recreated database restarts Query Store's plan_id numbering at 1, so every plan + /// in the NEW database sorts below the OLD database's watermark and has its XML suppressed. + /// + /// #2183 ships no reset detection — it was written, found unsound, and removed, because the + /// tempting test ("the highest plan_id seen this pass is below the standing watermark") is TRUE in any + /// ordinary window where nothing new compiled. What actually bounds this is + /// : the stamp dates the last FULL fetch, so a stale + /// watermark stops applying within a day no matter what. This test states that mechanism explicitly, so + /// the claim is a checked fact rather than a PR-description assertion. + /// + /// The prune strictly improves on that bound without replacing it — it removes the row outright + /// when the drop is observed between cycles — but it cannot be the guarantee, because a drop and recreate + /// entirely within one cycle is never observed as an absence at all. + /// + [Fact] + public void RecreatedDatabase_IsBoundedByTheRefreshHorizon_NotByResetDetection() + { + var now = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc); + var state = new Dictionary(StringComparer.Ordinal) + { + [Planwm("Recreated")] = QueryStorePlanXmlState.Format(900_000, now - QueryStorePlanXmlState.RefreshAfter), + }; + + /* At the horizon the watermark stops applying, so the recreated database's plan_ids (which start at 1 + and would all fail a > 900000 predicate) are fetched again. */ + Assert.Equal(0, QueryStorePlanXmlState.Resolve(state, "Recreated", now)); + + /* And one second inside it, the stale watermark DOES still apply — which is the exposure this bounds, + and the reason the prune is worth having even though it is not the guarantee. */ + Assert.Equal(900_000, QueryStorePlanXmlState.Resolve(state, "Recreated", now - TimeSpan.FromSeconds(1))); + + /* A pruned row is simply absent, and absent is the conservative full-fetch path — so a recreate that + happens after an observed drop inherits nothing at all. */ + Assert.Equal(0, QueryStorePlanXmlState.Resolve(new Dictionary(StringComparer.Ordinal), "Recreated", now)); + } + + /// + /// Walks up from the test output directory to the repo root — the same walk-up idiom + /// CollectorStateContractTests uses. + /// + private static string? FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + for (var i = 0; i < 10 && directory is not null; i++) + { + if (File.Exists(Path.Combine(directory.FullName, "PerformanceMonitor.sln"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/Darling/Darling.Tests/QueryStoreTextStoreTests.cs b/Darling/Darling.Tests/QueryStoreTextStoreTests.cs new file mode 100644 index 000000000..1eea31cbc --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreTextStoreTests.cs @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2150 V74: where the query-text fetch lands statement text, and the ladder invariants around it. +/// +/// The payload used to select query_sql_text (nvarchar(max)) inside a +/// TOP ... WITH TIES ... ORDER BY last_execution_time, and a Top-N Sort carries every output column +/// through the sort while reading ALL of its input before emitting row one — so choosing the rows to ship +/// materialized text for the entire qualifying set. With #2210's plan XML already gone and that column as +/// the only difference: time-to-first-row 4.67s against 0.45s at 1,505 rows, 5.02s against 0.57s at 4,037. +/// Neither the row cap nor the client byte budget bounds it (TOP (500) measured the same as TOP (50000); +/// wall time flat from a 4 MB to a 256 MB budget). +/// +/// The fetch is live as of the reader conversion. FetchQueryTextSeparately is on for the +/// sweep, so the payload's inline query_sql_text is null for newly collected rows and all six reader +/// surfaces resolve text from this table, falling back to the inline column for history collected before the +/// cutover. The flip and that conversion had to land together: an unconverted reader would have shown blank +/// text for new rows while looking perfectly healthy. +/// +public sealed class QueryStoreTextStoreTests +{ + /// + /// The rung and the helper's own DDL must agree, or a fresh store and an upgraded one get different + /// tables — the same discipline the ladder diff enforces for collector tables, applied by hand because a + /// non-collector table is outside that generator. + /// + [Fact] + public void TheRungMatchesTheHelpersCreateTableSql() + { + var rung = PgMigrations.Scripts.Single(s => s.Version == 74); + + Assert.Equal("query-store-text", rung.Name); + Assert.Equal(Normalize(QueryStoreTextStore.CreateTableSql), Normalize(rung.Sql)); + } + + /// + /// The ladder's invariants: this rung is the top, the ladder is ordered and dense above the one + /// sanctioned historical hole, and the build's schema version tracks it. A gap is skipped SILENTLY on + /// every upgraded store, so the objects would never exist and no later upgrade would repair it. + /// + [Fact] + public void TheRungIsTheTopOfADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + /* #2316 added V75, so this rung is no longer the top — the "I am the top" claim moves to the + newest rung's own test (PlanContentRetentionTests) and this one keeps the invariants that stay + true forever: the rung is PRESENT, the ladder is ordered and dense, and the build's schema + version tracks the maximum. */ + Assert.Contains(74, versions); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + + var above = versions.Where(v => v > 45).OrderBy(v => v).ToList(); + Assert.Equal(Enumerable.Range(above[0], above.Count), above); + } + + /// + /// The Viewer's connect-time gate has to be able to SEE a fully-migrated store, or every store at the + /// newest version reads as below the required one and the Viewer refuses to open. Three things move in + /// lockstep for that — a probe column, a reader argument and a map parameter — which is why this asserts + /// the mapping rather than trusting the count. + /// + [Fact] + public void TheProbeMapsAFullyMigratedStoreTo74() + { + /* #2316: no longer the top (that claim lives in PlanContentRetentionTests) — this fact keeps + pinning that a store at exactly 74 maps to 74 and one at 73 maps to 73, forever. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + /* 49 positional sentinels then the V74 one by name — the map takes 50 parameters. Present => 74, + newest-first. */ + var all = Enumerable.Repeat(true, 49).Cast().ToArray(); + + Assert.Equal(74, InvokeMap(all, hasQueryStoreText: true)); + + /* And absent => the previous arm still answers 73 rather than falling through to the floor. */ + Assert.Equal(73, InvokeMap(all, hasQueryStoreText: false)); + } + + /// + /// The probe SQL must ASK for the table, and the three places that move in lockstep must agree: a probe + /// column, a reader argument, and a map parameter. A probe that cannot SEE the newest object maps every + /// fully-migrated store below the required version and the Viewer refuses to open. + /// + /// Asserted on the reader's highest ordinal rather than by counting EXISTS (, because one + /// probe column is a COMPOUND EXISTS(...) OR EXISTS(...) — so the occurrence count is one higher + /// than the column count and a naive comparison fails for a reason that has nothing to do with this + /// rung. + /// + [Fact] + public void TheProbeAsksForTheTable_AndTheThreePlacesAgree() + { + Assert.Contains("table_name = 'query_store_text'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + + var mapParameters = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .GetParameters().Length; + + var viewerSource = ReadViewerSource(); + + /* The reader must hand over exactly one argument per map parameter: ordinals are 0-based, so the + highest is Count - 1, and the next one up must NOT appear. */ + Assert.Contains($"reader.GetBoolean({mapParameters - 1})", viewerSource, StringComparison.Ordinal); + Assert.DoesNotContain($"reader.GetBoolean({mapParameters})", viewerSource, StringComparison.Ordinal); + } + + /// + /// The table's shape: keyed on query_id — which is already a stored fact column, so this rung + /// adds a table and touches nothing existing — and indexed on the column it is pruned by. + /// + [Fact] + public void TheTableIsKeyedOnQueryIdAndIndexedForThePrune() + { + Assert.Contains("PRIMARY KEY (server_id, database_name, query_id)", QueryStoreTextStore.CreateTableSql, StringComparison.Ordinal); + Assert.Contains("idx_query_store_text_last_seen", QueryStoreTextStore.CreateTableSql, StringComparison.Ordinal); + Assert.Contains(QueryStoreTextStore.LastSeenColumn, QueryStoreTextStore.CreateTableSql, StringComparison.Ordinal); + } + + /// + /// The upsert overwrites the TEXT, not just the stamp, and that is load-bearing rather than defensive: + /// query_id is unique within a database only until Query Store is RESET, which renumbers from the + /// start — so id 5 afterwards is a different statement than id 5 before. The refresh horizon brings us + /// back to re-read it, and this is where the corrected text has to land. Touching only last_seen + /// would leave the old statement's text attached to the new id forever, which reads as a plausible wrong + /// answer rather than as missing data. + /// + [Fact] + public void TheUpsertOverwritesTheTextAndOrdersByTheConflictKey() + { + Assert.Contains("query_sql_text = EXCLUDED.query_sql_text", QueryStoreTextStore.UpsertSql, StringComparison.Ordinal); + Assert.Contains("last_seen = EXCLUDED.last_seen", QueryStoreTextStore.UpsertSql, StringComparison.Ordinal); + + /* #1801: concurrent batches touching overlapping keys in different orders deadlock. */ + Assert.Contains("ORDER BY server_id, database_name, query_id", QueryStoreTextStore.UpsertSql, StringComparison.Ordinal); + + /* Monotonic stamp, so an out-of-order write cannot age a row backwards into the prune's reach. */ + Assert.Contains("WHERE EXCLUDED.last_seen >= query_store_text.last_seen", QueryStoreTextStore.UpsertSql, StringComparison.Ordinal); + } + + /// + /// The prune is bounded to roughly one chunk-width of the OLDEST rows per call, so a single sweep cannot + /// take an unbounded row lock — the same shape as the plan map's. + /// + [Fact] + public void ThePruneIsBoundedToOneChunkWidth() + { + var sql = QueryStoreTextStore.PruneSql(7); + + Assert.StartsWith("DELETE FROM collect.query_store_text WHERE last_seen < $1", sql, StringComparison.Ordinal); + Assert.Contains("INTERVAL '7 days'", sql, StringComparison.Ordinal); + Assert.Contains("SELECT min(last_seen)", sql, StringComparison.Ordinal); + } + + /// + /// The retention margin is ADDED to the fact horizon, never subtracted. Text has to OUTLIVE the rows + /// that reference it: retired early, a fact's statement reads as absent, which nothing distinguishes + /// from a statement that never had text. Being late costs a few unread rows. + /// + [Fact] + public void TheRetentionMarginKeepsTextAliveLongerThanTheFacts() + => Assert.True(QueryStoreTextStore.PruneMarginDays > 0, + "a zero or negative margin would retire text at or before the facts that reference it"); + + /// + /// The flip is ON for the scheduled sweep and OFF for the on-demand read, and that difference is not + /// cosmetic: the sweep STORES the text it stops shipping inline, while FetchRowsAsync hands rows + /// straight back to its caller and writes nothing — no store insert, no text fetch, no watermark — so + /// nulling the inline column there would lose the statement outright rather than relocate it. + /// Asserted by splitting the source on the method rather than by counting occurrences, because the + /// failure mode worth catching is a flag flipped in the WRONG context, which any count would pass. + /// + [Fact] + public void TheFetchIsOnForTheSweepAndOffForTheOnDemandRead() + { + var source = ReadRunnerSource(); + + var split = source.IndexOf("FetchRowsAsync", StringComparison.Ordinal); + Assert.True(split > 0, + "FetchRowsAsync was renamed or moved — this guard can no longer tell the sweep context from the on-demand one"); + + var sweep = source[..split]; + var onDemand = source[split..]; + + Assert.Contains("FetchQueryTextSeparately = true,", sweep, StringComparison.Ordinal); + Assert.DoesNotContain("FetchQueryTextSeparately = false,", sweep, StringComparison.Ordinal); + Assert.Contains("FetchQueryTextSeparately = false,", onDemand, StringComparison.Ordinal); + Assert.DoesNotContain("FetchQueryTextSeparately = true,", onDemand, StringComparison.Ordinal); + + /* The pass the flip depends on: with the flag on and this call gone, the payload would ship no text + and nothing would land any, which is the one combination that loses data silently. */ + Assert.Contains("FetchAndStoreQueryTextAsync(textFetchConnection", source, StringComparison.Ordinal); + } + + /// + /// The text watermark is saved under its OWN state owner. The load merges both owners into one + /// dictionary, so writing it under the plan fetch's owner would still READ back — and then never be + /// pruned, because the shared prune set pairs textwm: with query_store_text and a prefix + /// pruned under the wrong owner deletes nothing. + /// + [Fact] + public void TheTextWatermarkIsSavedUnderItsOwnOwner() + { + var source = ReadRunnerSource(); + + Assert.Contains("QueryStoreTextState.StateCollectorName, textKeys", source, StringComparison.Ordinal); + Assert.Contains("QueryStoreTextState.WatermarkKeyPrefix", source, StringComparison.Ordinal); + Assert.Contains( + (QueryStoreTextState.StateCollectorName, QueryStoreTextState.WatermarkKeyPrefix), + QueryStorePerDatabaseState.PrunableKeys); + } + + /// + /// Calls the map with 49 positional sentinels plus the V74 one, by reflection — the signature has 50 + /// parameters and hand-writing that many literals is how a miscount turns into a confusing compile + /// error instead of a clear failure. + /// + private static int InvokeMap(object[] leading, bool hasQueryStoreText) + { + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + + /* #2316 and #2319 appended parameters after this rung's — pass them FALSE so these facts keep + exercising the V74/V73 arms rather than the newer ones. */ + var args = leading.Concat(new object[] { hasQueryStoreText, false, false }).ToArray(); + Assert.Equal(method.GetParameters().Length, args.Length); + + return (int)method.Invoke(null, args)!; + } + + private static string Normalize(string sql) => + string.Join(" ", sql.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + + private static string ReadViewerSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } + + private static string ReadRunnerSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingCollectorRunner.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/QueryStoreTextWatermarkTests.cs b/Darling/Darling.Tests/QueryStoreTextWatermarkTests.cs new file mode 100644 index 000000000..536f5bc95 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreTextWatermarkTests.cs @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2150: the per-database watermark for the query-text fetch — the sibling of +/// , pinning the same conservative-zero rules on a second +/// catalog. +/// +/// Why the text fetch exists. The runtime payload selected query_sql_text +/// (nvarchar(max)) inside a TOP ... WITH TIES ... ORDER BY last_execution_time. A Top-N Sort +/// carries every output column through the sort and reads ALL of its input before emitting a row, so +/// choosing the rows to ship materialized text for the entire qualifying set. With #2210's plan XML +/// already gone and that column as the only difference, time-to-first-row measured 4.67s against 0.45s at +/// 1,505 rows and 5.02s against 0.57s at 4,037. Neither the row cap nor the client byte budget bounds it: +/// TOP (500) measured the same as TOP (50000), and wall time was flat from a 4 MB to a +/// 256 MB budget, because the server is finished before the client sees a byte. +/// +/// Every zero below is the same deliberate choice: an absent, malformed, expired or future-stamped +/// watermark means "fetch everything", because a first run, a restarted host and a broken store are +/// indistinguishable from here and all three must refetch rather than skip. +/// +public sealed class QueryStoreTextWatermarkTests +{ + private const string Db = "SO"; + + private static DateTime Now => new(2026, 8, 16, 12, 0, 0, DateTimeKind.Utc); + + private static Dictionary StateWith(long queryId, DateTime stampedAt) => + new() { [QueryStoreTextState.KeyFor(Db)] = QueryStoreTextState.Format(queryId, stampedAt) }; + + [Fact] + public void AFreshWatermarkRoundTrips() + { + Assert.Equal(900, QueryStoreTextState.Resolve(StateWith(900, Now), Db, Now)); + Assert.Equal(Now, QueryStoreTextState.ResolveStamp(StateWith(900, Now), Db)); + } + + /// + /// Past the refresh horizon the watermark expires to 0 — a full re-walk. + /// + /// Not decoration: query_id is monotonic in FIRST-SEEN order, not in "we have stored it", + /// so a Query Store reset renumbers ids from the start and every text would arrive below a standing + /// watermark. Without a bounded horizon that suppresses text forever. + /// + [Fact] + public void PastTheRefreshHorizonItRefetchesEverything() + { + var state = StateWith(900, Now); + + Assert.Equal(900, QueryStoreTextState.Resolve(state, Db, Now + QueryStoreTextState.RefreshAfter - TimeSpan.FromMinutes(1))); + Assert.Equal(0, QueryStoreTextState.Resolve(state, Db, Now + QueryStoreTextState.RefreshAfter)); + } + + /// + /// A future stamp is refused, or a backwards clock would pin the watermark for as long as the skew + /// lasts. + /// + [Fact] + public void AFutureStampIsRefused() + => Assert.Equal(0, QueryStoreTextState.Resolve(StateWith(900, Now), Db, Now.AddHours(-1))); + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("garbage")] + [InlineData("900")] + [InlineData(":123")] + [InlineData("900:")] + [InlineData("-5:123")] + [InlineData("900:notanumber")] + public void AMalformedWatermarkRefetchesEverything(string raw) + { + var state = new Dictionary { [QueryStoreTextState.KeyFor(Db)] = raw }; + + Assert.Equal(0, QueryStoreTextState.Resolve(state, Db, Now)); + Assert.Null(QueryStoreTextState.ResolveStamp(state, Db)); + } + + [Fact] + public void AnAbsentDatabaseRefetchesEverything() + { + Assert.Equal(0, QueryStoreTextState.Resolve(StateWith(900, Now), "somewhere-else", Now)); + Assert.Equal(0, QueryStoreTextState.Resolve(new Dictionary(), Db, Now)); + Assert.Equal(0, QueryStoreTextState.Resolve(null!, Db, Now)); + } + + [Fact] + public void TheWatermarkAdvancesToTheHighestLandedId() + { + var advance = QueryStoreTextState.AdvanceWatermark(100, new long[] { 101, 102, 103 }); + + Assert.Equal(103, advance.Watermark); + Assert.True(advance.ArrivedInQueryIdOrder); + } + + /// + /// Out-of-order arrival HOLDS the watermark, because the ordering is the whole safety argument: a + /// budget cut is only a suffix if the ids arrived sorted, and advancing past a gap would strand + /// unstored text behind a strict comparison permanently. + /// + [Fact] + public void OutOfOrderArrivalHoldsTheWatermark() + { + var advance = QueryStoreTextState.AdvanceWatermark(100, new long[] { 101, 99, 102 }); + + Assert.Equal(100, advance.Watermark); + Assert.False(advance.ArrivedInQueryIdOrder); + } + + /// + /// A quiet pass is a quiet pass, not a reset. Lowering the watermark because nothing new arrived would + /// refetch the catalog on every idle cycle. + /// + [Fact] + public void AQuietPassNeverLowersTheWatermark() + { + Assert.Equal(100, QueryStoreTextState.AdvanceWatermark(100, Array.Empty()).Watermark); + Assert.Equal(100, QueryStoreTextState.AdvanceWatermark(100, null!).Watermark); + Assert.Equal(100, QueryStoreTextState.AdvanceWatermark(100, new long[] { 5, 6 }).Watermark); + Assert.True(QueryStoreTextState.AdvanceWatermark(100, Array.Empty()).ArrivedInQueryIdOrder); + } + + /// + /// The stamp survives an advance, which is what makes the refresh horizon reachable at all: re-stamping + /// on every advance would push it out forever on any database that keeps seeing new statements — which + /// is exactly where a Query Store reset would hurt most. + /// + [Fact] + public void AnAdvanceCanCarryTheOriginalStampForward() + { + var originalStamp = Now.AddHours(-6); + var carried = QueryStoreTextState.Format(950, originalStamp); + var state = new Dictionary { [QueryStoreTextState.KeyFor(Db)] = carried }; + + Assert.Equal(950, QueryStoreTextState.Resolve(state, Db, Now)); + Assert.Equal(originalStamp, QueryStoreTextState.ResolveStamp(state, Db)); + /* Six hours in, the horizon is still six hours closer than a re-stamp would have left it. */ + Assert.Equal(0, QueryStoreTextState.Resolve(state, Db, originalStamp + QueryStoreTextState.RefreshAfter)); + } + + /// + /// Text and plan watermarks live under DIFFERENT collector names. They walk different catalogs at + /// different rates, and sharing state would let one side's reset drop the other's watermark for no + /// reason. + /// + [Fact] + public void TheTextWatermarkIsStoredSeparatelyFromThePlanWatermark() + { + Assert.NotEqual(QueryStorePlanXmlState.StateCollectorName, QueryStoreTextState.StateCollectorName); + Assert.NotEqual(QueryStorePlanXmlState.WatermarkKeyPrefix, QueryStoreTextState.WatermarkKeyPrefix); + Assert.NotEqual(QueryStorePlanXmlState.KeyFor(Db), QueryStoreTextState.KeyFor(Db)); + } +} diff --git a/Darling/Darling.Tests/RegistrationCollisionTests.cs b/Darling/Darling.Tests/RegistrationCollisionTests.cs new file mode 100644 index 000000000..7366d07bb --- /dev/null +++ b/Darling/Darling.Tests/RegistrationCollisionTests.cs @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2280: add_servers refuses a registration whose connection lands in a database another registration +/// already covers — and, in the same method, keys its duplicate gate on the identity the store actually derives. +/// +/// The defect being prevented. Identity is registration-derived, so N registrations that silently +/// resolve to one database get N identities and N full copies of every collected row. #2220 reported it as +/// byte-identical deadlock graphs under six server_ids, one real incident alerting six times. #2277 added +/// the connect-time tripwire that reports it; this refuses it at the point of creation, which is the only place +/// it can be prevented rather than described. +/// +/// Why the probe is what makes it possible. No comparison of configuration can decide this — the two +/// registrations genuinely differ, which is why #2158 and #2218 (identity assigned, then widened) could not touch +/// it. Only the server can say which database a connection reached, and add_servers already probes +/// in-process, so the answer is in hand at exactly the moment the decision has to be made. +/// +public sealed class RegistrationCollisionTests +{ + private static DarlingMcpServerAdminTools.ParsedServerEntry Entry( + string host, string? database, bool readOnlyIntent = false, string engine = "sqlserver", int port = 0) + { + var config = new MonitoredServer + { + Name = host, + Host = host, + Database = database, + ReadOnlyIntent = readOnlyIntent, + Engine = engine, + Port = port, + Auth = "integrated", + }; + + var key = ServerIdHelper.BuildStorageName(host, database, readOnlyIntent, engine, port); + return new DarlingMcpServerAdminTools.ParsedServerEntry(0, host, key, config, null); + } + + private static HashSet Claimed(params string[] keys) => + new(keys, StringComparer.OrdinalIgnoreCase); + + /// + /// THE CASE: the entry names one database, its connection lands in another, and that other one is already + /// monitored. Adding it would give one real database two identities. + /// + [Fact] + public void ARegistrationLandingOnAnAlreadyMonitoredDatabaseIsRefused() + { + var entry = Entry("azure.example.net", "Sibling-A"); + var claimed = Claimed(ServerIdHelper.BuildStorageName("azure.example.net", "Source-DB", false, "sqlserver", 0)); + + var reason = DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "Source-DB", claimed); + + Assert.NotNull(reason); + Assert.Contains("Sibling-A", reason, StringComparison.Ordinal); + Assert.Contains("Source-DB", reason, StringComparison.Ordinal); + /* It has to say what the harm is, or it reads as pedantry about naming. */ + Assert.Contains("two identities", reason, StringComparison.Ordinal); + Assert.Contains("Initial Catalog", reason, StringComparison.Ordinal); + } + + /// + /// An entry that reaches the database it NAMES is the ordinary case and passes — the declared gate has + /// already ruled on it, so firing here would only re-detect that gate's decision under a worse name. + /// + [Theory] + [InlineData("SalesDB", "SalesDB")] + [InlineData("SalesDB", "salesdb")] + [InlineData("SalesDB", " SalesDB ")] + public void AnEntryThatReachesWhatItNamesIsAllowed(string declared, string connected) + { + var entry = Entry("host1", declared); + var claimed = Claimed(ServerIdHelper.BuildStorageName("host1", declared, false, "sqlserver", 0)); + + Assert.Null(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, connected, claimed)); + } + + /// + /// Landing somewhere else is fine as long as nobody else covers it — this refuses a COLLISION, not a + /// misconfiguration. #2277's tripwire is what reports the latter, at every connect. + /// + [Fact] + public void LandingElsewhereIsAllowedWhenNobodyElseCoversIt() + { + var entry = Entry("host1", "Declared-DB"); + var claimed = Claimed(ServerIdHelper.BuildStorageName("host1", "Declared-DB", false, "sqlserver", 0)); + + Assert.Null(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "Somewhere-Else", claimed)); + } + + /// + /// An absent probe answer is UNKNOWN, not colliding. Refusing on a missing value would block registrations + /// for a reason nobody could act on — and a stub probe returns exactly this. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void AnUnknownConnectedDatabaseNeverRefuses(string? connected) + { + var entry = Entry("host1", "Declared-DB"); + var claimed = Claimed(ServerIdHelper.BuildStorageName("host1", "Other-DB", false, "sqlserver", 0)); + + Assert.Null(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, connected, claimed)); + } + + /// + /// THE VALID PAIR THAT MUST STILL BE ALLOWED: a read-only-intent registration alongside a read-write one for + /// the same database. read_only_intent is part of the identity, so comparing on (host, database) alone + /// would refuse a legitimate configuration — which is why the check keys on the FULL identity. + /// + [Fact] + public void AReadOnlyIntentRegistrationDoesNotCollideWithItsReadWriteTwin() + { + /* Read-only entry that lands in Source-DB; the read-WRITE registration of Source-DB is already claimed. */ + var entry = Entry("ag-listener", "Sibling-A", readOnlyIntent: true); + var claimed = Claimed(ServerIdHelper.BuildStorageName("ag-listener", "Source-DB", false, "sqlserver", 0)); + + Assert.Null(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "Source-DB", claimed)); + + /* But it DOES collide with another read-only registration of the same database. */ + var roClaimed = Claimed(ServerIdHelper.BuildStorageName("ag-listener", "Source-DB", true, "sqlserver", 0)); + Assert.NotNull(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "Source-DB", roClaimed)); + } + + /// + /// Engine is part of the identity too (#2218), so a PostgreSQL entry landing in a database name that a SQL + /// Server registration covers is not a collision — they are different instances on one host. + /// + [Fact] + public void APostgresEntryDoesNotCollideWithASqlServerRegistration() + { + var entry = Entry("box01", "declared", engine: "postgres"); + var sqlServerClaim = Claimed(ServerIdHelper.BuildStorageName("box01", "actual", false, "sqlserver", 0)); + + Assert.Null(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "actual", sqlServerClaim)); + + /* Another PostgreSQL registration of that database IS a collision. */ + var pgClaim = Claimed(ServerIdHelper.BuildStorageName("box01", "actual", false, "postgres", 0)); + Assert.NotNull(DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "actual", pgClaim)); + } + + /// + /// A server-scoped entry (no database named) that lands somewhere already covered is still refused — the + /// message says "no database" rather than pretending it named one, so the operator can tell which of their + /// registrations is the vague one. + /// + [Fact] + public void AServerScopedEntryThatLandsOnACoveredDatabaseIsRefusedAndSaysSo() + { + var entry = Entry("host1", null); + var claimed = Claimed(ServerIdHelper.BuildStorageName("host1", "master", false, "sqlserver", 0)); + + var reason = DarlingMcpServerAdminTools.ActualIdentityCollision(entry, "master", claimed); + + Assert.NotNull(reason); + Assert.Contains("no database", reason, StringComparison.Ordinal); + } + + /// + /// #2218's regression in this method, pinned: the duplicate gate reads engine and port, so it + /// keys on the identity the store actually derives. + /// + /// Without them the gate keys on a NARROWER identity than the product does, and a PostgreSQL instance + /// on a host that already has a SQL Server registration reads as a duplicate and is refused — a valid pair + /// rejected because the gate could not see what distinguishes them. It compiles and every existing test + /// passes, which is why it is worth an explicit pin. + /// + [Fact] + public void TheDuplicateGateReadsTheFullIdentity() + { + Assert.Contains("engine", DarlingMcpServerAdminTools.ExistingServersSql, StringComparison.Ordinal); + Assert.Contains("port", DarlingMcpServerAdminTools.ExistingServersSql, StringComparison.Ordinal); + Assert.Contains("read_only_intent", DarlingMcpServerAdminTools.ExistingServersSql, StringComparison.Ordinal); + + /* And the two identities it would compare genuinely differ, so the columns are load-bearing rather than + decorative. */ + Assert.NotEqual( + ServerIdHelper.BuildStorageName("box01", null, false, "sqlserver", 0), + ServerIdHelper.BuildStorageName("box01", null, false, "postgres", 0)); + } +} diff --git a/Darling/Darling.Tests/ServerIdentityFromStoreTests.cs b/Darling/Darling.Tests/ServerIdentityFromStoreTests.cs new file mode 100644 index 000000000..0325ad709 --- /dev/null +++ b/Darling/Darling.Tests/ServerIdentityFromStoreTests.cs @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2218 step one: a monitored server's server_id comes from the STORE, and is derived in exactly one +/// place when there is nothing stored yet. +/// +/// What was wrong. config.config_monitored_servers.server_id is that table's PRIMARY KEY +/// and is authoritative once seeded — but the read selected fifteen columns and not that one, so the registry's +/// own key was discarded and twelve downstream sites re-derived it from host, database and +/// read_only_intent: every operator-command lookup, the reconcile, the self-alert stamps, the schedule +/// resolution. Identity-derived-from-editable-config cannot be replaced while that is true, because a stored +/// surrogate is only worth anything if nothing re-derives it behind the store's back. +/// +/// Behaviour is unchanged today and that is the point. The seed and the Viewer both write exactly +/// the hash this used to recompute, so stored and derived are equal on every existing store and no data moves. +/// What changed is that the derivation is now the FALLBACK, in one property, so the later change is that +/// property rather than twelve call sites — see #2158 and #2228 for what it is a prerequisite for. +/// +/// The live arm therefore does the one thing production cannot yet produce: a row whose stored +/// server_id DISAGREES with the hash of its own host. Without that, every assertion here would pass +/// just as well against the old code, since the two values coincide. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. The one live test reaches DARLING_TEST_PG + only to CREATE and DROP its own database through ScratchPostgres, then works entirely inside it — it never + touches the shared database's tables, so it cannot race the live collection and serializing every pure test + here alongside it would be pure slowdown. Same shape and same reason as DarlingAlertTuningKnobsTests and + DarlingDeliveryModeTests, which also pair pure pins with one scratch-store seed/read round-trip. Kept in one + class rather than split, for that symmetry and because the split would leave a single-test file whose + subject is the same seam the pure pins above cover. This comment is here so the next sweep does not "fix" + it. */ +public sealed class ServerIdentityFromStoreTests +{ + private static int Derived(string host, string? database = null, bool readOnlyIntent = false) => + ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(host, database, readOnlyIntent)); + + /// A darling.json entry before the first seed has no store row, so identity is derived — + /// which is what makes the seed able to mint it, and what keeps --test-connection working against a + /// file on a host that has never started the service. + [Theory] + [InlineData("sql01", null, false)] + [InlineData("myazure.database.windows.net", "AdventureWorks", false)] + [InlineData("ag-listener", null, true)] + public void WithNothingStored_IdentityIsTheDerivation(string host, string? database, bool readOnlyIntent) + { + var server = new MonitoredServer { Host = host, Database = database, ReadOnlyIntent = readOnlyIntent }; + + Assert.Null(server.StoredServerId); + Assert.Equal(Derived(host, database, readOnlyIntent), server.ServerId); + } + + /// + /// THE SEAM. A stored id wins over the derivation, and keeps winning when the fields the derivation reads + /// change underneath it. + /// + /// This is the property the whole change exists for: an operator repointing a server at a new host + /// — the #2158 case, and what the use1 fleet did on 2026-08-09 — must not move the identity its collected + /// history and its per-server config are keyed on. Note the assertion is not merely "stored equals + /// ServerId": it is that ServerId is stable across an edit that changes the hash. + /// + [Fact] + public void AStoredIdWins_AndSurvivesAnEditThatChangesTheHash() + { + var server = new MonitoredServer { Host = "old-host", StoredServerId = 424242 }; + + Assert.Equal(424242, server.ServerId); + Assert.NotEqual(Derived("old-host"), server.ServerId); + + server.Host = "new-host"; + server.ReadOnlyIntent = true; + server.Database = "someDb"; + + /* The derivation moved. The identity did not. */ + Assert.NotEqual(Derived("old-host"), Derived("new-host", "someDb", true)); + Assert.Equal(424242, server.ServerId); + } + + /// + /// The store is the only authority for a stored id — darling.json cannot set one. + /// + /// Deliberate rather than incidental: the registry is authoritative for identity once seeded, so a + /// file that could pin server_id would be a second authority able to disagree with it, and disagree + /// SILENTLY, because nothing downstream re-checks. Both spellings are tried because a reader guessing at + /// the property name is exactly who would try this. + /// + [Theory] + [InlineData("""{"host":"sql01","storedServerId":999}""")] + [InlineData("""{"host":"sql01","serverId":999}""")] + [InlineData("""{"host":"sql01","server_id":999}""")] + public void ADarlingJsonEntryCannotPinAnIdentity(string json) + { + var server = JsonSerializer.Deserialize(json, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + + Assert.Null(server.StoredServerId); + Assert.Equal(Derived("sql01"), server.ServerId); + } + + /// + /// Nothing in the service re-derives identity any more. Three files, because between them they held every + /// one of the twelve converted sites — the worker's lookups and stamps, the connect-time stamp the + /// collectors inherit, and the MCP host's plan-fetch map. + /// + /// A source pin rather than a behavioural one because the failure it guards is a NEW call site being + /// added later, which no test of today's behaviour can see: a fresh + /// GetDeterministicHashCode(config.StorageName) would agree with the stored id on every store that + /// exists right now, and only start lying once ids stop being derivable — i.e. long after the commit that + /// introduced it. + /// + [Theory] + [InlineData("Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs")] + [InlineData("Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs")] + [InlineData("Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs")] + public void TheServiceNoLongerDerivesIdentity(string relativePath) + { + var root = FindRepoRoot(); + Assert.True(root is not null, "repo root not found -- the source pin cannot run"); + + var offenders = File.ReadAllLines(Path.Combine(root!, relativePath)) + .Select((line, index) => (Line: line, Number: index + 1)) + /* Doc comments name the helper on purpose (the cadence-jitter doc explains what its input is). */ + .Where(l => !l.Line.TrimStart().StartsWith("///", StringComparison.Ordinal)) + .Where(l => l.Line.Contains("GetDeterministicHashCode(", StringComparison.Ordinal)) + .Select(l => $"{relativePath}:{l.Number}") + .ToList(); + + Assert.Empty(offenders); + } + + /// + /// The remaining derivations are the ALLOCATION sites, and they are exactly these. Pinned as a closed set + /// so a new one has to be argued for in review rather than appearing: an allocation site is where identity + /// is minted and made permanent, so an unnoticed extra one mints identities nothing else agrees with. + /// + [Fact] + public void IdentityIsMintedInExactlyThreePlaces() + { + var root = FindRepoRoot(); + Assert.True(root is not null, "repo root not found -- the source pin cannot run"); + + var expected = new[] + { + /* The single fallback: MonitoredServer.ServerId. */ + "Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs", + /* add_servers, which hashes a storage key string rather than a MonitoredServer. */ + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs", + /* The Viewer, which writes registry rows without ever building a MonitoredServer. */ + "Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.MonitoredServers.cs", + }; + + var found = new[] + { + "Darling/PerformanceMonitor.Darling.Service", + "Darling/PerformanceMonitor.Darling.Viewer", + "Darling/PerformanceMonitor.Darling.Storage", + } + .SelectMany(dir => Directory.EnumerateFiles(Path.Combine(root!, dir), "*.cs", SearchOption.AllDirectories)) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.Ordinal) + && !path.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", StringComparison.Ordinal)) + .Where(path => File.ReadAllLines(path) + .Any(line => !line.TrimStart().StartsWith("///", StringComparison.Ordinal) + && !line.TrimStart().StartsWith("*", StringComparison.Ordinal) + && line.Contains("GetDeterministicHashCode(", StringComparison.Ordinal))) + .Select(path => Path.GetRelativePath(root!, path).Replace(Path.DirectorySeparatorChar, '/')) + .OrderBy(path => path, StringComparer.Ordinal) + .ToList(); + + Assert.Equal(expected.OrderBy(p => p, StringComparer.Ordinal), found); + } + + /* ---------------- live (DARLING_TEST_PG): the store's id is what the service uses ---------------- */ + + /// + /// The round-trip, against a row whose stored server_id is deliberately NOT the hash of its host — + /// which is the only way to tell the new read from the old one, since production rows have the two equal. + /// + /// It also asserts every other column the read projects, because adding a column to a positional + /// reader is exactly how a silent mis-map happens: shift one ordinal and auth arrives in + /// username, which no compiler and no id assertion would catch. Distinctive values per column, so a + /// mis-map cannot coincide with a plausible default. + /// + /// Its own scratch database () because SeedIfEmptyAsync no-ops on + /// a store any earlier test already seeded. + /// + [Fact] + public async Task TheLoadedServerCarriesTheStoresOwnId_NotAFreshHash() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the server_id round-trip (the test mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using (var connection = new NpgsqlConnection(scratch.ConnectionString)) + { + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + } + + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + var provider = new StoreConfigProvider(dataSource); + + var seeded = new MonitoredServer + { + Name = "identity-roundtrip", + Host = "identity-host", + Database = "identityDb", + Auth = "sql", + Username = "identity-user", + EncryptedPassword = "not-a-real-blob", + EncryptMode = "Strict", + TrustServerCertificate = true, + ReadOnlyIntent = true, + MultiSubnetFailover = true, + ExcludedDatabases = { "excluded-one", "excluded-two" }, + MonthlyCostUsd = 1234.56m, + AlertDeliveryModeOverride = AlertNotificationMode.PerEvent, + Engine = "postgres", + Port = 6432, + }; + + var config = new DarlingConfig(); + config.Servers.Add(seeded); + await provider.SeedIfEmptyAsync(config, ct); + + /* The seed writes the derivation, which is what makes every existing store migration-free. Assert it + rather than assume it -- if this ever stops holding, the "no data moves" claim stops holding too. */ + /* #2218: derived from the SERVER'S OWN StorageName rather than from a re-statement of the rule here. + This test previously hashed (host, database, readOnlyIntent) by hand, which silently stopped matching + the moment the derivation grew the engine and port the seeded server actually carries — and the + failure reads as "the seed wrote the wrong id" rather than "the test's copy of the rule is stale". */ + var seededId = ServerIdHelper.GetDeterministicHashCode(seeded.StorageName); + Assert.Equal(seededId, await ReadServerIdByNameAsync(dataSource, "identity-roundtrip", ct)); + + /* Now the thing production cannot produce yet: re-key the row so the stored id disagrees with the hash + of its own host. Legal precisely because nothing references config_monitored_servers -- there is not + one foreign key to it, which is separately why an edit orphans a server's config today (#2158). */ + const int Surrogate = 20260814; + Assert.NotEqual(Surrogate, seededId); + await using (var rekey = dataSource.CreateCommand( + "UPDATE config_monitored_servers SET server_id = $1 WHERE name = $2")) + { + rekey.Parameters.AddWithValue(Surrogate); + rekey.Parameters.AddWithValue("identity-roundtrip"); + Assert.Equal(1, await rekey.ExecuteNonQueryAsync(ct)); + } + + var view = await provider.LoadViewAsync(new DarlingConfig(), ct); + Assert.NotNull(view); + var loaded = Assert.Single(view!.EnabledServers, s => s.Name == "identity-roundtrip"); + + /* THE ASSERTION: the surrogate, not the hash. Old code returns the hash here. */ + Assert.Equal(Surrogate, loaded.StoredServerId); + Assert.Equal(Surrogate, loaded.ServerId); + Assert.NotEqual(Derived(loaded.Host, loaded.Database, loaded.ReadOnlyIntent), loaded.ServerId); + + /* Every other projected column, in the reader's own order, because that is what a shifted ordinal + breaks. Each value is distinctive: a mis-map surfaces as a wrong value, not a plausible default. */ + Assert.Equal("identity-roundtrip", loaded.Name); + Assert.Equal("identity-host", loaded.Host); + Assert.Equal("identityDb", loaded.Database); + Assert.Equal("sql", loaded.Auth); + Assert.Equal("identity-user", loaded.Username); + Assert.Equal("not-a-real-blob", loaded.EncryptedPassword); + Assert.Equal("Strict", loaded.EncryptMode); + Assert.True(loaded.TrustServerCertificate); + Assert.True(loaded.ReadOnlyIntent); + Assert.True(loaded.MultiSubnetFailover); + Assert.Equal(new[] { "excluded-one", "excluded-two" }, loaded.ExcludedDatabases); + Assert.Equal(1234.56m, loaded.MonthlyCostUsd); + Assert.Equal(AlertNotificationMode.PerEvent, loaded.AlertDeliveryModeOverride); + Assert.Equal("postgres", loaded.Engine); + Assert.Equal(6432, loaded.Port); + } + + private static async Task ReadServerIdByNameAsync( + NpgsqlDataSource dataSource, string name, System.Threading.CancellationToken ct) + { + await using var command = dataSource.CreateCommand( + "SELECT server_id FROM config_monitored_servers WHERE name = $1"); + command.Parameters.AddWithValue(name); + return Convert.ToInt32(await command.ExecuteScalarAsync(ct), System.Globalization.CultureInfo.InvariantCulture); + } + + /// Walks up to the directory holding PerformanceMonitor.sln — the same idiom as + /// CollectorStateContractTests.FindRepoRoot. + private static string? FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + for (var i = 0; i < 10 && directory is not null; i++) + { + if (File.Exists(Path.Combine(directory.FullName, "PerformanceMonitor.sln"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/Darling/Darling.Tests/ServerIdentitySurvivesAnEditTests.cs b/Darling/Darling.Tests/ServerIdentitySurvivesAnEditTests.cs new file mode 100644 index 000000000..450d23298 --- /dev/null +++ b/Darling/Darling.Tests/ServerIdentitySurvivesAnEditTests.cs @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2158: editing a server's address keeps its identity, so its collected history stays attached to it. +/// +/// The defect. The Add/Edit save re-derived server_id from host/database/read-only-intent on +/// every save. So an operator fixing a hostname typo produced a row under a NEW id and the old row was deleted — +/// which left the registry tidy and every collect.* row keyed to the old id orphaned, with nothing +/// pointing at it. The visible symptom is a server that reads as though it had never been monitored, which is +/// why it went unnoticed: nothing looks broken, the history is simply gone. +/// +/// Why identity must be assigned rather than derived, argued from consequences. Three issues pull +/// on this and they pull in opposite directions. #2158 says a config edit must NOT change the identity. +/// #2228 says two different configs resolving to one real database must not be two identities — which no +/// config-derived hash can decide, because the configs genuinely differ. #2218 says two instances on one host +/// need distinguishing, which wants MORE fields in the derivation and therefore makes #2158 strictly worse. +/// Only one shape satisfies all three: the identity is allocated once and never recomputed, the derived address +/// is a lookup key rather than the identity, and what the target actually IS comes from the connection. This +/// file pins the first of those three. +/// +public sealed class ServerIdentitySurvivesAnEditTests +{ + /// + /// THE FIX, pinned at the source: the row builder prefers the row's existing identity and only derives one + /// when there is none (an Add). + /// + /// Pinned textually because the alternative is a WPF dialog — reproducing it behaviourally means + /// standing up AddServerDialog with a live store and a real edit, which the suite cannot do. A + /// re-derivation here is invisible in every other test and silently discards history, so it is worth + /// holding at the only level available. + /// + [Fact] + public void TheEditSaveKeepsTheOriginalIdentity_AndOnlyAddDerivesOne() + { + var source = ReadDialogSource(); + + Assert.Contains( + "ServerId = _originalServerId ?? ViewerDataService.ComputeServerId(host, database, readOnlyIntent),", + source, StringComparison.Ordinal); + + /* And the delete-the-old-identity step is GONE: with the id preserved there is no second row to clean + up, and leaving the delete in would drop the row that was just written. */ + Assert.DoesNotContain("await _dataService.DeleteMonitoredServerAsync(original);", source, StringComparison.Ordinal); + } + + /// + /// The collision guard survived the change and now asks about the ADDRESS. + /// + /// This is the half that would have been easy to lose. The old guard compared a derived id against + /// the registry, which only works while every row's id equals the hash of its own address — precisely the + /// invariant this change gives up. Left as it was, an edit could point a second registration at an address + /// another server already monitors and the guard would never fire, because the derived id it looked up + /// belongs to nobody. That is #2228's shape reached from the registry side. + /// + [Fact] + public void TheCollisionGuardChecksTheAddress_NotADerivedId() + { + var source = ReadDialogSource(); + + Assert.Contains("GetMonitoredServerByAddressAsync(row.Host, row.Database, row.ReadOnlyIntent)", source, StringComparison.Ordinal); + /* Compared by id afterwards, which is what excludes "collided with myself" on an edit that leaves the + address alone (a rename, or new credentials). */ + Assert.Contains("occupant.ServerId != row.ServerId", source, StringComparison.Ordinal); + Assert.DoesNotContain("await _dataService.GetMonitoredServerAsync(row.ServerId) is not null", source, StringComparison.Ordinal); + } + + /// + /// The address lookup matches a NULL database with IS NOT DISTINCT FROM. A plain = never + /// matches NULL in SQL, so every server-scoped registration — the common case — would read as "address + /// free" and the guard would pass for all of them. + /// + [Fact] + public void TheAddressLookupMatchesANullDatabase() + { + var sql = ViewerDataService.MonitoredServerByAddressSql; + + Assert.Contains("WHERE host = $1", sql, StringComparison.Ordinal); + Assert.Contains("database IS NOT DISTINCT FROM $2", sql, StringComparison.Ordinal); + Assert.Contains("read_only_intent = $3", sql, StringComparison.Ordinal); + /* Secret-free, so a read-only seat gets an answer rather than 42501 on the column it is denied. */ + Assert.DoesNotContain("encrypted_password", sql, StringComparison.Ordinal); + } + + /// + /// The file-vs-store reconcile no longer calls an edited server "not monitored". + /// + /// That log line gives ADVICE — "add them with the Viewer's Add Server dialog" — so being wrong is + /// worse than being silent: after an address edit the file's derived id matches nothing, and an id-only + /// comparison would tell the operator to re-add the one server they had just fixed, on every start. + /// + [Fact] + public void AnEditedServerIsNotReportedAsFileOnly() + { + var file = new[] { Server("prod-01", host: "prod-01.old.example.com") }; + + /* The store row kept its identity through the edit, so its id is NOT the hash of the file's address. */ + var storeIds = new HashSet { 999_111 }; + var storeNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "prod-01" }; + + Assert.Empty(StoreConfigProvider.ServersOnlyInFile(file, storeIds, storeNames)); + + /* Without the name arm this is exactly the wrong answer the old comparison gave. */ + Assert.Single(StoreConfigProvider.ServersOnlyInFile(file, storeIds, new HashSet())); + } + + /// + /// A genuinely removed server is STILL reported. The Viewer's Remove hard-deletes the row, so it is absent + /// under both keys — the name arm must not turn this log line off altogether, which is the obvious way to + /// over-apply the fix. + /// + [Fact] + public void ADeletedServerIsStillReportedAsFileOnly() + { + var file = new[] { Server("gone-01", host: "gone-01.example.com") }; + + var missing = StoreConfigProvider.ServersOnlyInFile( + file, new HashSet { 999_111 }, new HashSet(StringComparer.OrdinalIgnoreCase) { "someone-else" }); + + Assert.Equal(new[] { "gone-01" }, missing); + } + + /// + /// Matching is either-or, not name-only. Nothing enforces display-name uniqueness, so a name-only + /// comparison would hide a genuinely unmonitored server behind a same-named sibling; and an id match alone + /// must still be enough, which is the path every unedited server takes. + /// + [Fact] + public void AnIdMatchAloneIsEnough_AndNameMatchingDoesNotReplaceIt() + { + var server = Server("prod-02", host: "prod-02.example.com"); + + /* Id present, name absent — the ordinary case for a server nobody has edited. */ + Assert.Empty(StoreConfigProvider.ServersOnlyInFile( + new[] { server }, new HashSet { server.ServerId }, new HashSet())); + + /* Neither present. */ + Assert.Single(StoreConfigProvider.ServersOnlyInFile( + new[] { server }, new HashSet(), new HashSet())); + } + + /// + /// Omitting the names keeps the old id-only behaviour, so a caller that has not been updated cannot start + /// silently suppressing the warning. + /// + [Fact] + public void WithNoNamesSuppliedTheComparisonIsIdOnly() + { + var server = Server("prod-03", host: "prod-03.example.com"); + + Assert.Single(StoreConfigProvider.ServersOnlyInFile(new[] { server }, new HashSet())); + Assert.Empty(StoreConfigProvider.ServersOnlyInFile(new[] { server }, new HashSet { server.ServerId })); + } + + private static MonitoredServer Server(string name, string host) => + new() { Name = name, Host = host, Auth = "integrated" }; + + private static string ReadDialogSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "AddServerDialog.xaml.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/StatementSplitTimingTests.cs b/Darling/Darling.Tests/StatementSplitTimingTests.cs new file mode 100644 index 000000000..812c10737 --- /dev/null +++ b/Darling/Darling.Tests/StatementSplitTimingTests.cs @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the open-vs-drain timing split (#2164). It exists because a single blended sql: number could +/// not answer the question a 5x payload cut raised on production: the byte budget moved bytes 5x and the +/// batch clock ~0%, so the cost is upstream of shipping — but WHICH statement was unprovable from the log, +/// and the next fix would have been a guess. Open time (everything before the first rowset) and drain time +/// (row streaming) have different fixes, so they must be separately visible. +/// +public sealed class StatementSplitTimingTests +{ + private static CollectorContext NewContext() => new() + { + ServerId = 1, + ServerName = "s", + CollectionTime = new DateTime(2026, 8, 10, 0, 0, 0, DateTimeKind.Utc), + Deltas = new CollectorDeltaCalculator(), + }; + + [Fact] + public void OpenMs_DefaultsToZero_SoAnUnmeasuredHostIsNotReadAsInstant() + { + /* Lite does not measure this today. Zero must mean "not measured", which is why the log only emits + the split when the value is positive rather than printing "open:0ms" and inviting the reader to + conclude the aggregate was free. */ + Assert.Equal(0, NewContext().PerItemOpenMs); + } + + [Theory] + /* An aggregate-bound pass: nearly all the batch is spent before the first row arrives, so no client + byte budget can shorten it — the query_store shape measured on the field server. */ + [InlineData(100_000L, 0L, 98_000L, 2_000L)] + /* A drain-bound pass: rows are cheap to produce and expensive to move, where the budget IS the lever. */ + [InlineData(100_000L, 0L, 3_000L, 97_000L)] + /* The watermark phase is a STORE round trip the driver's stopwatch already started before. It must come + out of drain, not inflate it — the review catch this arithmetic exists to prevent. */ + [InlineData(100_000L, 40_000L, 55_000L, 5_000L)] + /* Degenerate: phases exceeding the batch total (skew across separate stopwatches) must clamp at zero + rather than print a negative drain, which would read as a measurement bug in the field. */ + [InlineData(5_000L, 3_000L, 6_000L, 0L)] + public void DrainExcludesWatermarkAndOpen_AndNeverGoesNegative(long sqlMs, long watermarkMs, long openMs, long expectedDrain) + { + var context = NewContext(); + context.PerItemWatermarkMs = watermarkMs; + context.PerItemOpenMs = openMs; + + /* Calls the SHIPPED arithmetic (CollectorContext.DrainMsFrom) — the log line calls the same method, + so this cannot drift into pinning a copy of the formula the way the first cut did. */ + Assert.Equal(expectedDrain, context.DrainMsFrom(sqlMs)); + } + + [Fact] + public void EveryPhaseAccountedFor_ThePartsNeverExceedTheWhole() + { + /* The split's contract as a reader sees it: wm + open + drain == the sql: total, so nothing is + silently unattributed. Holds for any measurement where the phases fit inside the total. */ + var context = NewContext(); + context.PerItemWatermarkMs = 1_200; + context.PerItemOpenMs = 300_000; + const long sqlMs = 350_000; + + Assert.Equal(sqlMs, context.PerItemWatermarkMs + context.PerItemOpenMs + context.DrainMsFrom(sqlMs)); + } + + /// + /// #2312: the separate plan-XML and text fetches run INSIDE the driver's sql: stopwatch but are + /// their own queries against the Query Store catalogs — on ayr-01 a 0-row closed-only cycle still cost + /// 298s and the blended number could not say where. They must come out of drain exactly like the + /// watermark phase, or drain silently absorbs the one cost this investigation needs isolated. + /// + [Fact] + public void DrainExcludesTheSeparateFetchPhases() + { + var context = NewContext(); + context.PerItemWatermarkMs = 5_000; + context.PerItemOpenMs = 1_000; + context.PerItemPlanFetchMs = 200_000; + context.PerItemTextFetchMs = 90_000; + const long sqlMs = 300_000; + + Assert.Equal(4_000, context.DrainMsFrom(sqlMs)); + Assert.Equal(sqlMs, + context.PerItemWatermarkMs + context.PerItemOpenMs + + context.PerItemPlanFetchMs + context.PerItemTextFetchMs + context.DrainMsFrom(sqlMs)); + } + + /// Zero must mean "no separate fetch ran" — the log gates its long form on exactly that. + [Fact] + public void FetchPhases_DefaultToZero_SoAFetchlessCollectorIsNotReadAsMeasured() + { + var context = NewContext(); + Assert.Equal(0, context.PerItemPlanFetchMs); + Assert.Equal(0, context.PerItemTextFetchMs); + } +} diff --git a/Darling/Darling.Tests/StoreIsOnThisMachineTests.cs b/Darling/Darling.Tests/StoreIsOnThisMachineTests.cs new file mode 100644 index 000000000..ccde50e9d --- /dev/null +++ b/Darling/Darling.Tests/StoreIsOnThisMachineTests.cs @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2279: the signal behind the Add Server hint about a machine-bound credential. +/// +/// What it is for. A SQL-auth password is stored as a DPAPI LocalMachine blob, decryptable +/// only on the machine that wrote it — and the SERVICE is what has to decrypt it. So a credential saved from a +/// viewer on another PC can never be used, and the server fails to connect on every sweep afterwards. That is +/// the #2255 field report; #2273 made the resulting failure explain itself, and this warns before it happens. +/// +/// +/// Why loopback is the signal. The managed deploy builds its store connection on literal +/// 127.0.0.1ViewerSettings mirroring the service's own +/// DarlingManagedPostgres.BuildConnectionString — and the service runs where its managed store runs. So a +/// loopback store means viewer and service share a machine and a saved credential will work. It is a proxy +/// derived from the product's own architecture rather than a guess about deployment. +/// +/// And why it warns rather than refuses. A non-loopback store does NOT prove the viewer is remote: +/// a bring-your-own store on another host with the service local reads exactly the same. The signal is good +/// enough to decide whether to SAY something and not good enough to decide whether to BLOCK — inverting that +/// would refuse a legitimate first-run Add on the service host, which is worse than the problem. +/// +public sealed class StoreIsOnThisMachineTests +{ + /// + /// THE CASE THAT MUST STAY SILENT: the real managed connection string. This is the single-box deploy the + /// whole DPAPI design targets and the overwhelmingly common one — a hint that fires for everyone is a hint + /// nobody reads, so a false positive here would defeat the feature rather than merely annoy. + /// + [Theory] + [InlineData("Host=127.0.0.1;Port=5641;Username=darling;Password=x;Database=darling")] + [InlineData("Host=127.0.0.1;Port=5641;Database=darling")] + [InlineData("Host=localhost;Port=5641;Database=darling")] + [InlineData("Host=LOCALHOST;Database=darling")] + [InlineData("Host=::1;Database=darling")] + [InlineData("Host= 127.0.0.1 ;Database=darling")] + [InlineData("Server=127.0.0.1;Database=darling")] + public void ALoopbackStoreIsSilent(string connectionString) + { + Assert.True(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// + /// An OMITTED host counts as local, because that is what it means — Npgsql defaults to localhost, so a + /// string with no Host is a local store and warning about it would be wrong. + /// + [Theory] + [InlineData("Port=5641;Database=darling")] + [InlineData("Database=darling")] + [InlineData(";;;")] + public void AnOmittedHostCountsAsLocal(string connectionString) + { + Assert.True(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// Nothing at all is not evidence of a remote store. + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void AnAbsentConnectionStringIsSilent(string? connectionString) + { + Assert.True(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// THE CASE THAT WARNS: a store on another host, which is how the #2255 report was configured. + [Theory] + [InlineData("Host=prod-pos-use2-monitor-01;Port=5641;Database=darling")] + [InlineData("Host=10.149.55.242;Port=5432;Database=darling")] + [InlineData("Host=store.internal.example.com;Database=darling")] + [InlineData("Server=otherbox;Database=darling")] + public void ARemoteStoreWarns(string connectionString) + { + Assert.False(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// + /// A host that merely CONTAINS a loopback spelling is remote. Guards the obvious over-match, which would + /// silence the hint for exactly the hosts most likely to be typed by hand. + /// + [Theory] + [InlineData("Host=localhost.evil.example.com;Database=darling")] + [InlineData("Host=127.0.0.1.example.com;Database=darling")] + [InlineData("Host=mylocalhost;Database=darling")] + public void AHostThatMerelyLooksLoopbackStillWarns(string connectionString) + { + Assert.False(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// + /// An unparseable string fails toward SILENCE. This decides only whether to show a hint, so the wrong + /// direction is a warning the operator cannot act on attached to a string they cannot fix from here. + /// + /// These inputs genuinely throw — verified against DbConnectionStringBuilder rather than + /// assumed, because a catch for an exception the framework never raises is dead code pretending to be a + /// safeguard. Note "Host=a;;;=====bad" does NOT throw: it parses as Host=a, which is correctly + /// treated as remote. + /// + [Theory] + [InlineData("=")] + [InlineData("Host")] + [InlineData("{bad}")] + [InlineData("Host=a;b")] + [InlineData("Host==a")] + public void AnUnparseableConnectionStringIsSilent(string connectionString) + { + Assert.True(ViewerDataService.StoreHostIsLoopback(connectionString)); + } + + /// A string that parses to a real non-loopback host warns, even with trailing junk. + [Fact] + public void AParseableHostWinsOverTrailingJunk() + { + Assert.False(ViewerDataService.StoreHostIsLoopback("Host=a;;;=====bad")); + } + + /// + /// The dialog surfaces the hint only for SQL auth on a non-local store, and CLEARS its own message when the + /// mode changes away — the same self-clearing discipline the Azure arm uses, which is what stops a stale + /// hint sitting under an unrelated auth mode. + /// + /// Pinned at the source: the alternative is standing up a WPF dialog with a live store. + /// + [Fact] + public void TheDialogShowsTheHintOnlyForSqlAuthOnANonLocalStore() + { + var source = ReadDialogSource(); + + Assert.Contains("SqlAuthRadio.IsChecked == true && _dataService is { StoreIsOnThisMachine: false }", source, StringComparison.Ordinal); + Assert.Contains("StatusText.Text = SqlCredentialMachineBoundHint;", source, StringComparison.Ordinal); + /* Self-clearing, exactly like the Azure message above it. */ + Assert.Contains("else if (StatusText.Text == SqlCredentialMachineBoundHint)", source, StringComparison.Ordinal); + } + + /// + /// The hint has to be actionable, not just alarming: it names all three ways to produce a credential the + /// service can use, which is what turns it from a warning into an instruction. + /// + [Fact] + public void TheHintNamesEveryWayToProduceAUsableCredential() + { + var source = ReadDialogSource(); + + Assert.Contains("viewer on the service's host", source, StringComparison.Ordinal); + Assert.Contains("--add-server", source, StringComparison.Ordinal); + Assert.Contains("env:/file:", source, StringComparison.Ordinal); + } + + private static string ReadDialogSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "AddServerDialog.xaml.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/StoreSelfMetricsTests.cs b/Darling/Darling.Tests/StoreSelfMetricsTests.cs index 6c7a81f30..f8e24231d 100644 --- a/Darling/Darling.Tests/StoreSelfMetricsTests.cs +++ b/Darling/Darling.Tests/StoreSelfMetricsTests.cs @@ -7,9 +7,17 @@ */ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Storage; using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; using Xunit; namespace Darling.Tests; @@ -20,6 +28,10 @@ namespace Darling.Tests; /// whole design leans on — the table that MEASURES the compression/retention machinery is not in the /// collector catalog, so that machinery can never recurse onto it (no hypertable conversion, no /// compression policy, no catalog-driven purge; its retention is the sweep's own bounded DELETE). +/// +/// #1776 own-store — the live tests here mint their own scratch databases via +/// ScratchPostgres (they apply compression policies and drive run_job, which the shared fixture must +/// never inherit), so they cannot race the shared store and serializing them would be pure slowdown. /// public sealed class StoreSelfMetricsTests { @@ -29,8 +41,10 @@ public void V53_MigrationIdentity_AndStorageVersionTracksTheNewestRung() var v53 = PgMigrations.Scripts.Single(m => m.Version == 53); Assert.Equal("store-self-metrics", v53.Name); - Assert.Equal(54, PgMigrations.Scripts[^1].Version); - Assert.Equal(54, StorageVersion.SchemaVersion); + /* The invariant the test name states, with no literal to go stale: the build's schema version IS + the newest registered rung. Three in-flight branches bumping versions made the literal form a + recurring multi-test failure (#2210 round, again here at V62). */ + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); /* collect.-qualified like V44/V47/V49, and idempotent so a re-run is a no-op. */ Assert.Contains("CREATE TABLE IF NOT EXISTS collect.store_metrics (", v53.Sql, StringComparison.Ordinal); @@ -55,6 +69,67 @@ upgrade fails on a column fresh code writes and the upgraded store lacks. */ } } + [Fact] + public void V56_JobTelemetryColumns_MigrationAndSweepAgree_AndTheProbeKnowsTheRung() + { + /* #2136: every column the background-job sweep arm writes must exist in the V56 migration, or the + first hourly run after an upgrade fails on a column fresh code writes and the store lacks — + the exact failure class the V53 column pin above guards. */ + var v56 = PgMigrations.Scripts.Single(m => m.Version == 56); + Assert.Equal("store-metrics-background-jobs", v56.Name); + foreach (var column in new[] + { + "last_run_duration_ms", "schedule_interval_ms", "total_runs", "total_failures", + }) + { + Assert.Contains($"ADD COLUMN IF NOT EXISTS {column} bigint", v56.Sql, StringComparison.Ordinal); + Assert.Contains(column, StoreSelfMetrics.BackgroundJobInsertSql, StringComparison.Ordinal); + } + + /* The insert reads only TimescaleDB catalog surfaces, which is why the sweep gates it with the + hypertable arm — a plain-PG store skips it silently. schedule_interval rides along so + "duration vs cadence" — the tripwire that matters — is one division over the stored series. */ + Assert.Contains("FROM timescaledb_information.job_stats", StoreSelfMetrics.BackgroundJobInsertSql, StringComparison.Ordinal); + Assert.Contains("JOIN timescaledb_information.jobs", StoreSelfMetrics.BackgroundJobInsertSql, StringComparison.Ordinal); + Assert.Contains("'background_job'", StoreSelfMetrics.BackgroundJobInsertSql, StringComparison.Ordinal); + + /* The probe sentinel + arm: a fully-migrated V56 store maps to exactly the required version + (the connect-time-gate trap), and a V55 store without the columns caps at 55. */ + Assert.Contains("column_name = 'last_run_duration_ms'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + Assert.Equal(56, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true)); + Assert.Equal(55, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: false)); + } + + [Fact] + public void V57_CadenceKnob_MigrationSettingsAndProbeAgree() + { + /* #2136 (the alert half): the knob column the settings surfaces name must exist in the V57 + migration — the same first-run-after-upgrade failure class the V53/V56 column pins guard. */ + var v57 = PgMigrations.Scripts.Single(m => m.Version == 57); + Assert.Equal("store-job-cadence-knob", v57.Name); + Assert.Contains( + "ADD COLUMN IF NOT EXISTS store_job_cadence_warn_percent integer NOT NULL DEFAULT 25", + v57.Sql, StringComparison.Ordinal); + + /* The probe sentinel + arm: a fully-migrated V57 store maps to exactly the required version, + and a V56 store without the knob caps at 56. */ + Assert.Contains("column_name = 'store_job_cadence_warn_percent'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + Assert.Equal(57, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true, hasJobCadenceKnob: true)); + Assert.Equal(56, ViewerDataService.MapProbedSchemaVersion( + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, + true, true, true, true, true, true, hasJobMetricsColumns: true, hasJobCadenceKnob: false)); + } + [Fact] public void StoreMetrics_IsNotACollectorTable_SoTheMachineryItMeasuresCannotReachIt() { @@ -69,8 +144,9 @@ hand its retention to the policy path instead of the sweep's own 400-day DELETE. public void ViewerSchemaGate_KnowsV53_SoAFullyMigratedStoreIsNotRefused() { /* The trap a StorageVersion bump sets: a probe that cannot SEE the newest migration maps every - healthy store below RequiredStoreSchemaVersion and the connect-time gate refuses it permanently. */ - Assert.Equal(54, ViewerDataService.RequiredStoreSchemaVersion); + healthy store below RequiredStoreSchemaVersion and the connect-time gate refuses it permanently. + Invariant form, no literal to go stale: the gate always requires exactly the build's version. */ + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); Assert.Contains("table_name = 'store_metrics'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); /* The V53 arm: store_metrics present (and everything below it, but NOT V54's gz column — @@ -166,4 +242,418 @@ makes a run's rows join and keeps the timestamps naive UTC by the cross-store co Assert.DoesNotContain("now()", sql, StringComparison.Ordinal); Assert.Contains("$1", sql, StringComparison.Ordinal); } + + /// + /// The sweep END TO END against a real TimescaleDB (#2136) — the drift-catcher the SQL pins alone + /// cannot be: every INSERT the sweep runs must agree with the columns the migrations created, and the + /// failure class this guards (sweep writes a column the upgraded store lacks) only surfaces when the + /// statements actually execute. Mints its own scratch store (the #1776 own-store idiom), migrates it, + /// converts + applies compression policies so real background jobs exist, then asserts one run writes + /// hypertable, dimension, store, AND background_job rows — the job rows carrying a schedule interval, + /// because "duration vs cadence" is the series' whole point. + /// + [Fact] + public async Task Sweep_EndToEnd_WritesEveryObjectKind_IncludingBackgroundJobs_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live self-metrics sweep test (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await TimescaleSupport.ApplyCompressionPolicyAsync(connection, null, ct); + + var written = await StoreSelfMetrics.SweepAsync( + connection, timescaleAvailable: true, DateTime.UtcNow, null, ct); + Assert.True(written > 0, "the sweep wrote nothing"); + + await using var kinds = new NpgsqlCommand(@" +SELECT + count(*) FILTER (WHERE object_kind = 'hypertable'), + count(*) FILTER (WHERE object_kind = 'dimension'), + count(*) FILTER (WHERE object_kind = 'store'), + count(*) FILTER (WHERE object_kind = 'background_job'), + count(*) FILTER (WHERE object_kind = 'background_job' AND schedule_interval_ms > 0) +FROM collect.store_metrics", connection); + await using var reader = await kinds.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + + Assert.True(reader.GetInt64(0) > 0, "no hypertable rows"); + Assert.True(reader.GetInt64(1) > 0, "no dimension rows"); + Assert.Equal(1, reader.GetInt64(2)); + Assert.True(reader.GetInt64(3) > 0, "no background_job rows — the compression policies just applied guarantee jobs exist"); + Assert.True(reader.GetInt64(4) > 0, "background_job rows carry no schedule interval — duration-vs-cadence needs it"); + await reader.CloseAsync(); + + /* And the READ path carries the new fields end to end (the review catch: written but never read + back would leave get_store_metrics returning job rows with null metrics). */ + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + var latest = await PerformanceMonitor.Darling.Service.Mcp.DarlingStoreMetricsReader.GetLatestAsync(dataSource, ct); + var job = latest.FirstOrDefault(r => r.ObjectKind == "background_job"); + Assert.NotNull(job); + Assert.True(job!.ScheduleIntervalMs is > 0, "the reader dropped the job's schedule interval"); + } + + /* ---------------- #2136 synthetic scale test ---------------- */ + + /// + /// The #2136 capacity claim, proven end to end rather than asserted from one production observation: + /// job runtimes scale with raw volume, the V56 telemetry RECORDS that growth, and the #2141 alert + /// FIRES from real store readings. One throwaway hypertable with a compression policy that is PARKED + /// except when a measurement deliberately arms it (created parked in one transaction — the #1888 + /// discipline — so no background tick ever races a measurement, the #2143 class), driven at 1x and + /// then 10x row volume: + /// + /// a scheduler-driven run at each scale (arm, poll total_runs, park — foreground run_job does + /// NOT update this accounting, CI-proved); job_stats.last_run_duration must be measurable (the + /// premise the whole telemetry stands on) and must GROW with volume; + /// a self-metrics sweep after each run; the store_metrics series must carry both readings, in + /// order, growing — this is the series an operator (and the cadence alert's detail text) trends; + /// alter_job shrinks the schedule interval to half the measured 10x duration, and the REAL + /// evaluator, fed by the REAL against this + /// store, must fire the Critical tier under the storejob: key. + /// + /// Volumes (50k vs 500k rows in one closed chunk each, after a discarded warm-up run) are chosen so + /// the big run does strictly more compression work than the 1x run by a margin no runner jitter + /// plausibly inverts; the assertion is monotonicity, not a ratio, for exactly that reason. The + /// margin is a full order of magnitude because 4x was NOT enough (#2160): a fast runner's fixed + /// per-run cost plus cache warmth accumulating across the two measured runs inverted 50k-vs-200k + /// in the field (d1=279ms, d4=217ms). + /// Seeds are midday-anchored (#1972) so a run near midnight cannot split a chunk. + /// + [Fact] + public async Task ScaleTest_JobDurationGrowsWithVolume_TelemetryRecordsIt_AndTheAlertFires_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #2136 scale test (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + const string Table = "tick2136_scale"; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + + /* Throwaway hypertable + compression, policy created PARKED in one transaction (#1888): the + scheduler is a separate backend and must never see an armed job, or a background run races the + deterministic run_job calls below and the durations stop being ours. */ + await ExecAsync(connection, + $"CREATE TABLE collect.{Table} (collection_time timestamp NOT NULL, server_id integer NOT NULL, value bigint)", ct); + await ExecAsync(connection, TimescaleSupport.CreateHypertableSql($"collect.{Table}", "collection_time"), ct); + await ExecAsync(connection, TimescaleSupport.EnableCompressionSql($"collect.{Table}"), ct); + await using (var tx = await connection.BeginTransactionAsync(ct)) + { + await using (var create = new NpgsqlCommand(TimescaleSupport.AddCompressionPolicySql($"collect.{Table}"), connection, tx)) + { + await create.ExecuteNonQueryAsync(ct); + } + await using (var park = new NpgsqlCommand($@" +SELECT alter_job(job_id, scheduled => false) +FROM timescaledb_information.jobs +WHERE hypertable_schema = 'collect' AND hypertable_name = '{Table}' +AND (proc_name LIKE '%compression%' OR proc_name LIKE '%columnstore%')", connection, tx)) + { + await park.ExecuteNonQueryAsync(ct); + } + await tx.CommitAsync(ct); + } + + var jobId = Convert.ToInt64((await new NpgsqlCommand($@" +SELECT job_id +FROM timescaledb_information.jobs +WHERE hypertable_schema = 'collect' AND hypertable_name = '{Table}' +AND (proc_name LIKE '%compression%' OR proc_name LIKE '%columnstore%')", connection).ExecuteScalarAsync(ct))!); + + /* Warm-up: the first run of a policy pays one-time costs (worker spin-up, catalog warm-up) that + would inflate d1 and could invert the monotonicity assertion. Run once on a token chunk and + discard the measurement. Doubles as the canary that this scratch database HAS a scheduler: + if it never runs, the arm-and-poll below fails with its own diagnosis rather than a mystery. */ + await SeedTickRowsAsync(connection, Table, daysBack: 12, rows: 2_000, ct); + await RunJobViaSchedulerAsync(connection, jobId, ct); + + /* 1x: one closed chunk, 50k rows. */ + await SeedTickRowsAsync(connection, Table, daysBack: 10, rows: 50_000, ct); + await RunJobViaSchedulerAsync(connection, jobId, ct); + long d1 = await ReadJobDurationMsAsync(connection, jobId, ct); + var work1 = await DescribeJobWorkAsync(connection, Table, jobId, ct); /* #2266 */ + Assert.True(d1 > 0, + "a scheduler-driven run left job_stats.last_run_duration unmeasurable — the premise the " + + "V56 telemetry and the #2141 alert both stand on. (Foreground run_job is already known " + + "not to update this accounting — CI proved that on this test's first version — which is " + + "why the runs go through the real scheduler.)"); + await StoreSelfMetrics.SweepAsync(connection, timescaleAvailable: true, DateTime.UtcNow, null, ct); + + /* 10x: one closed chunk, 500k rows. */ + await SeedTickRowsAsync(connection, Table, daysBack: 8, rows: 500_000, ct); + await RunJobViaSchedulerAsync(connection, jobId, ct); + long d10 = await ReadJobDurationMsAsync(connection, jobId, ct); + var work10 = await DescribeJobWorkAsync(connection, Table, jobId, ct); /* #2266 */ + await StoreSelfMetrics.SweepAsync(connection, timescaleAvailable: true, DateTime.UtcNow.AddSeconds(2), null, ct); + + /* 1. The capacity claim itself: more volume, longer run. Monotonicity, not a ratio — runner + jitter owns the constant factor, the direction is ours. + + #2266: the failure message now reports what the job DID, not only how long it took. This test + has failed intermittently on diffs that cannot reach it, and the reading that mattered was + d1=689ms / d10=689ms — BYTE-IDENTICAL. Two independent sub-second timings of different + workloads do not land on the same millisecond by chance, so the earlier "runner jitter" + explanation cannot be right; something is making both runs do the same work. The scheduler + helper already rules out a stale read (it waits for last_successful_finish to ADVANCE), which + leaves "both runs compressed the same amount, plausibly none" — and that is invisible from a + duration alone. Chunk counts make it visible the first time it recurs, without a rig. */ + Assert.True(d10 > d1, + $"10x volume did not run longer than 1x (d1={d1}ms, d10={d10}ms) — job runtime is not " + + "scaling with volume, which invalidates the #2136 capacity model." + + $"\n after 1x ({50_000} rows seeded): {work1}" + + $"\n after 10x ({500_000} rows seeded): {work10}" + + "\n If the compressed-chunk counts are EQUAL, the two runs did the same work and this " + + "assertion was never measuring the capacity model — the volumes are not producing " + + "compressible chunks, which is a fixture defect rather than a timing tolerance one (#2266)."); + + /* 2. The telemetry recorded the growth: two series points for this job, in order, growing. */ + await using (var series = new NpgsqlCommand(@" +SELECT last_run_duration_ms +FROM collect.store_metrics +WHERE object_kind = 'background_job' AND object_name LIKE '%' || $1 || '%' +ORDER BY metric_time", connection)) + { + series.Parameters.AddWithValue(Table); + var points = new List(); + await using var reader = await series.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + points.Add(reader.GetInt64(0)); + } + + Assert.Equal(2, points.Count); + Assert.Equal(d1, points[0]); + Assert.Equal(d10, points[1]); + } + + /* 3. The alert fires from REAL readings: shrink the schedule interval to half the measured 10x + duration (percent ≈ 200), then run the real reader into the real evaluator. */ + await ExecAsync(connection, $@" +SELECT alter_job({jobId}::integer, schedule_interval => ( + SELECT last_run_duration / 2 FROM timescaledb_information.job_stats WHERE job_id = {jobId}))", ct); + + var readings = await TimescaleSupport.ReadJobCadenceReadingsAsync(connection, null, ct); + var tickReading = Assert.Single(readings, r => r.JobId == jobId); + Assert.True(tickReading.LastRunDurationMs is > 0 && tickReading.ScheduleIntervalMs > 0, + "the cadence reader dropped the duration or interval for the tick job"); + + var deliverer = new CadenceRecordingDeliverer(); + var evaluator = new DarlingSelfAlertEvaluator( + new CadenceFakeSettings(), deliverer, new CadenceFakeHistoryStore(), _ => false); + await evaluator.EvaluateStoreJobCadenceAsync(new[] { tickReading }, ct); + + var fired = Assert.Single(deliverer.Outcomes); + Assert.Equal(DarlingSelfAlertEvaluator.JobCadenceMetric, fired.MetricName); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + Assert.Equal($"storejob:{jobId}", fired.ServerKey); + } + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(ct); + } + + /// Seeds one closed, compression-eligible chunk: midday-anchored (#1972) N days back, + /// spreading rows across seconds inside the day so they stay in ONE chunk. + private static Task SeedTickRowsAsync( + NpgsqlConnection connection, string table, int daysBack, int rows, CancellationToken ct) => + ExecAsync(connection, $@" +INSERT INTO collect.{table} +SELECT date_trunc('day', now()::timestamp) - INTERVAL '{daysBack} days' + INTERVAL '12 hours' + + ((g % 40000) || ' milliseconds')::interval, + {8850}, + g +FROM generate_series(1, {rows}) AS g", ct); + + /// + /// Runs the job through the REAL scheduler — arm with next_start => now(), poll + /// total_runs until it increments, park again. Foreground run_job deliberately NOT + /// used: CI proved it does not update job_stats.last_run_duration (that accounting lives in + /// the scheduler path), and the scheduler path is the one production's telemetry actually reads — + /// so this is both the working mechanism and the honest one. Parking between measurements keeps + /// each run's chunks OURS (the #1888 concern, inverted: armed on purpose, once, per measurement; + /// the next background tick is an hour out, far beyond the test's lifetime). + /// + private static async Task RunJobViaSchedulerAsync(NpgsqlConnection connection, long jobId, CancellationToken ct) + { + /* Poll on last_successful_finish, NOT total_runs: total_runs increments when a run STARTS, and + job_stats reports last_run_duration as NULL while the run is in flight — CI proved it, by + catching the larger run mid-flight and reading 0ms (the 1x run had merely finished inside one + poll tick). last_successful_finish only advances at COMPLETION, so a read after it moves is + a read of a finished run's accounting. */ + var before = await ReadLastSuccessfulFinishAsync(connection, jobId, ct); + await ExecAsync(connection, $"SELECT alter_job({jobId}::integer, scheduled => true, next_start => now())", ct); + + var deadline = DateTime.UtcNow.AddSeconds(90); + while (await ReadLastSuccessfulFinishAsync(connection, jobId, ct) <= before) + { + Assert.True(DateTime.UtcNow < deadline, + $"the scheduler did not COMPLETE a run of job {jobId} within 90s of next_start => now() — " + + "either this scratch database has no scheduler, the cluster is out of background workers " + + "(see CiClusterWorkerSizingTests for the sizing this suite depends on), or the run failed " + + "(last_successful_finish never advances for a failed run — check job_stats.last_run_status)"); + await Task.Delay(500, ct); + } + + await ExecAsync(connection, $"SELECT alter_job({jobId}::integer, scheduled => false)", ct); + } + + private static async Task ReadLastSuccessfulFinishAsync(NpgsqlConnection connection, long jobId, CancellationToken ct) + { + /* -infinity (never finished) maps to DateTime.MinValue via Npgsql, which orders below every real + finish — exactly the "before" baseline a first run needs. */ + await using var command = new NpgsqlCommand( + "SELECT coalesce(last_successful_finish, '-infinity'::timestamptz) FROM timescaledb_information.job_stats WHERE job_id = $1", + connection); + command.Parameters.AddWithValue(jobId); + var value = await command.ExecuteScalarAsync(ct); + return value is DateTime finish ? finish : DateTime.MinValue; + } + + /// + /// What the compression job actually DID, as one line for a failure message (#2266). + /// + /// Added because a duration alone cannot distinguish "this run compressed ten times as much and the + /// machine was noisy" from "both runs compressed nothing and the cost is all fixed overhead" — and the + /// intermittent failures of this test have produced BYTE-IDENTICAL durations, which only the second story + /// explains. Reporting chunk counts turns the next recurrence into a diagnosis instead of another re-run. + /// + /// Deliberately best-effort and never throwing: it exists to explain a failure, so a fault here must + /// not replace the assertion's own message with its own — that is the #1902 mistake in miniature. A missing + /// Timescale view or a renamed column degrades to a note saying so. + /// + private static async Task DescribeJobWorkAsync( + NpgsqlConnection connection, string table, long jobId, CancellationToken ct) + { + try + { + await using var command = new NpgsqlCommand(@" +SELECT + (SELECT count(*) FROM timescaledb_information.chunks + WHERE hypertable_schema = 'collect' AND hypertable_name = $1) AS chunks_total, + (SELECT count(*) FROM timescaledb_information.chunks + WHERE hypertable_schema = 'collect' AND hypertable_name = $1 AND is_compressed) AS chunks_compressed, + (SELECT total_runs::bigint FROM timescaledb_information.job_stats WHERE job_id = $2) AS total_runs, + (SELECT last_run_status::text FROM timescaledb_information.job_stats WHERE job_id = $2) AS last_run_status, + (SELECT last_successful_finish::text FROM timescaledb_information.job_stats + WHERE job_id = $2) AS last_successful_finish", + connection); + command.Parameters.AddWithValue(table); + command.Parameters.AddWithValue(jobId); + + await using var reader = await command.ExecuteReaderAsync(ct); + if (!await reader.ReadAsync(ct)) + { + return "(job_stats returned no row)"; + } + + return $"chunks={reader.GetInt64(0)} compressed={reader.GetInt64(1)} " + + $"total_runs={(reader.IsDBNull(2) ? "?" : reader.GetInt64(2).ToString(CultureInfo.InvariantCulture))} " + + $"last_run_status={(reader.IsDBNull(3) ? "?" : reader.GetString(3))} " + + $"last_successful_finish={(reader.IsDBNull(4) ? "?" : reader.GetString(4))}"; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + /* Broad on purpose: see the summary. An explanation that throws is worse than no explanation, + because it replaces the failure being explained. */ + return $"(could not describe the job's work: {ex.GetType().Name}: {ex.Message})"; + } + } + + private static async Task ReadJobDurationMsAsync(NpgsqlConnection connection, long jobId, CancellationToken ct) + { + await using var command = new NpgsqlCommand( + "SELECT (EXTRACT(EPOCH FROM last_run_duration) * 1000)::bigint FROM timescaledb_information.job_stats WHERE job_id = $1", + connection); + command.Parameters.AddWithValue(jobId); + var value = await command.ExecuteScalarAsync(ct); + return value is null or DBNull ? 0L : Convert.ToInt64(value); + } + + /* Minimal local fakes: only AlertsEnabled + CooldownMinutes matter to the cadence path; the rest + satisfy the interface at inert defaults. Local copies rather than sharing DarlingSelfAlertTests' + private harness — the coupling worth having is the READING record, not the test scaffolding. */ + + private sealed class CadenceRecordingDeliverer : IAlertDeliverer + { + public List Outcomes { get; } = new(); + + public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + Outcomes.Add(outcome); + return Task.CompletedTask; + } + } + + private sealed class CadenceFakeHistoryStore : IAlertHistoryStore + { + public Task RecordAlertAsync(AlertHistoryRecord record) => Task.CompletedTask; + public Task GetLastEmailSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => + Task.FromResult(null); + public Task GetLastWebhookSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => + Task.FromResult(null); + public Task GetLastAlertTimeAsync(string serverId, string metricName) => + Task.FromResult(null); + } + + private sealed class CadenceFakeSettings : IAlertEngineSettings + { + public bool AlertsEnabled { get; set; } = true; + public bool CpuEnabled { get; set; } + public bool BlockingEnabled { get; set; } + public bool DeadlockEnabled { get; set; } + public bool PoisonWaitEnabled { get; set; } + public bool LongRunningQueryEnabled { get; set; } + public bool TempDbSpaceEnabled { get; set; } + public bool LowDiskEnabled { get; set; } + public bool LongRunningJobEnabled { get; set; } + public bool FailedJobEnabled { get; set; } + public bool PvsEnabled { get; set; } + public bool DatabaseStateEnabled { get; set; } + public bool ForcePlanFailureEnabled { get; set; } = true; + public int CpuThresholdPercent { get; set; } = 80; + public int BlockingCountThreshold { get; set; } = 1; + public int BlockingWaitSecondsThreshold { get; set; } + public int DeadlockCountThreshold { get; set; } = 1; + public int PoisonWaitThresholdMs { get; set; } = 500; + public int LongRunningQueryThresholdMinutes { get; set; } = 30; + public int LongRunningQueryMaxResults { get; set; } = 5; + public bool LongRunningQueryExcludeSpServerDiagnostics { get; set; } = true; + public bool LongRunningQueryExcludeWaitFor { get; set; } = true; + public bool LongRunningQueryExcludeBackups { get; set; } = true; + public bool LongRunningQueryExcludeMiscWaits { get; set; } = true; + public bool LongRunningQueryExcludeCdc { get; set; } = true; + public int TempDbSpaceThresholdPercent { get; set; } = 80; + public int LowDiskThresholdPercent { get; set; } = 10; + public int LowDiskThresholdGb { get; set; } = 5; + public int DiskCriticalFreePercent { get; set; } = 3; + public int DiskCriticalFreeGb { get; set; } = 2; + public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int CollectionStaleMinutes { get; set; } = 30; + public int CollectionFailureThreshold { get; set; } = 10; + public int PvsThresholdPercent { get; set; } = 40; + public int PvsFloorGb { get; set; } = 1; + public int LongRunningJobMultiplier { get; set; } = 3; + public int FailedJobLookbackMinutes { get; set; } = 60; + public int CooldownMinutes { get; set; } = 5; + public IReadOnlyList ExcludedDatabases { get; } = new List(); + public CpuAlertMode CpuAlertMode { get; set; } = CpuAlertMode.TotalServer; + } } diff --git a/Darling/Darling.Tests/StoreSelfMetricsTimeoutTests.cs b/Darling/Darling.Tests/StoreSelfMetricsTimeoutTests.cs new file mode 100644 index 000000000..6f7359dc4 --- /dev/null +++ b/Darling/Darling.Tests/StoreSelfMetricsTimeoutTests.cs @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The #2317 sweep timeout. The self-metrics sizing queries (hypertable_detailed_size across every +/// hypertable, pg_database_size over the whole store) outgrew Npgsql's default 30 seconds ~5x/day +/// on the dogfood fleet, and the cancel surfaces as "Exception while reading from stream" — a timeout +/// wearing a network-fault costume (the #2294 lesson one layer over). Every statement in the sweep must +/// carry the explicit timeout; a sixth statement added without one silently rides the 30s default and +/// reintroduces the fake fault, so this pins the count of constructions to the count of timeouts. +/// +public sealed class StoreSelfMetricsTimeoutTests +{ + [Fact] + public void EverySweepStatementCarriesTheTimeout() + { + var source = ReadStoreSelfMetricsSource(); + + var constructions = Regex.Matches(source, @"new NpgsqlCommand\(").Count; + var timeouts = Regex.Matches(source, @"CommandTimeout = SweepTimeoutSeconds").Count; + + Assert.True(constructions > 0, "the sweep no longer constructs commands where this pin expects them — re-point it"); + Assert.Equal(constructions, timeouts); + } + + /// Five minutes, matching DarlingRetention's destructive-statement budget — an hourly sweep + /// on its own connection can afford patience, and a sweep that cannot finish in five minutes should + /// skip the tick rather than retry into the same load. + [Fact] + public void TheTimeoutIsTheRetentionBudget() + => Assert.Equal(300, StoreSelfMetrics.SweepTimeoutSeconds); + + private static string ReadStoreSelfMetricsSource([System.Runtime.CompilerServices.CallerFilePath] string thisFile = "") + { + var dir = System.IO.Path.GetDirectoryName(thisFile)!; + var relative = System.IO.Path.Combine("Darling", "PerformanceMonitor.Darling.Storage", "StoreSelfMetrics.cs"); + while (dir is not null && !System.IO.File.Exists(System.IO.Path.Combine(dir, relative))) + { + dir = System.IO.Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return System.IO.File.ReadAllText(System.IO.Path.Combine(dir!, relative)); + } +} diff --git a/Darling/Darling.Tests/StoreTlsCertificateTests.cs b/Darling/Darling.Tests/StoreTlsCertificateTests.cs new file mode 100644 index 000000000..556ef1b17 --- /dev/null +++ b/Darling/Darling.Tests/StoreTlsCertificateTests.cs @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Net; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2117: the store's printed root must validate the served chain under the EXACT trust semantics +/// Npgsql applies to Root Certificate=… — an in +/// with the root in the custom store. The field +/// failure was platform-shaped: the old single self-signed end-entity cert (critical CA=false) +/// validated on macOS/Linux chain engines but Windows refused it as its own trust anchor, so the +/// exact connection string --print-viewer-connection printed failed VerifyFull on the +/// platform most viewers run on. These tests run on every CI OS, which is what makes them the +/// arbiter rather than another single-platform anecdote. +/// +public sealed class StoreTlsCertificateTests +{ + [Fact] + public void GeneratedChain_ValidatesUnderNpgsqlsCustomRootTrust_OnEveryPlatform() + { + var generated = StoreTlsCertificates.Create("testhost", IPAddress.Parse("192.0.2.10"), validityYears: 5); + + var served = X509Certificate2Collection(); + served.ImportFromPem(generated.ServerCertChainPem); + Assert.Equal(2, served.Count); + + using var root = X509Certificate2.CreateFromPem(generated.RootCertPem); + + Assert.True( + BuildsUnderCustomRootTrust(served, root), + "The freshly-generated chain must validate against its own printed root under Npgsql's " + + "custom-root trust — this is the exact verify-full path a remote viewer takes."); + } + + [Fact] + public void GeneratedLeaf_CarriesTheListenIpAndHostSans() + { + var listenIp = IPAddress.Parse("192.0.2.10"); + var generated = StoreTlsCertificates.Create("testhost", listenIp, validityYears: 5); + + var served = X509Certificate2Collection(); + served.ImportFromPem(generated.ServerCertChainPem); + using var leaf = served[0]; + + /* The reuse gate reads the served file's FIRST cert — the leaf must be first and must cover + the listen IP, or every restart would rotate the chain. */ + Assert.True(DarlingManagedPostgres.CertificateSanCoversIp(leaf, listenIp)); + Assert.Contains("CN=testhost", leaf.Subject, StringComparison.Ordinal); + Assert.Contains("Darling store root", served[1].Issuer, StringComparison.Ordinal); + } + + private static X509Certificate2Collection X509Certificate2Collection() => new(); + + /// Npgsql's Root Certificate validation, mirrored: custom-root trust with the operator's + /// root as the ONLY anchor, revocation off (a discarded-key local CA publishes no CRL), any extra + /// served certs available as intermediates. + private static bool BuildsUnderCustomRootTrust(X509Certificate2Collection served, X509Certificate2 root) + { + using var chain = new X509Chain(); + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(root); + chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + for (var i = 1; i < served.Count; i++) + { + chain.ChainPolicy.ExtraStore.Add(served[i]); + } + + return chain.Build(served[0]); + } +} diff --git a/Darling/Darling.Tests/SweepPressureClassifierTests.cs b/Darling/Darling.Tests/SweepPressureClassifierTests.cs new file mode 100644 index 000000000..bbc6b5fa6 --- /dev/null +++ b/Darling/Darling.Tests/SweepPressureClassifierTests.cs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// Decision-table pins for the shared (#2296) — the roll-up both +/// SKUs' get_collection_health serve so half-rate collection stops being visible only as a service-log +/// warning. This SAME table is pinned identically in Lite.Tests so the two SKUs cannot drift. +/// +/// The load-bearing case is the motivating measurement: prod-pos-use2-multi-01's four heavy +/// collectors averaged 22,141 + 16,590 + 13,544 + 8,437 ms against a 60s cadence — the body could not +/// fit, every relaunch was skipped (~50 warnings/hour), the server collected at half rate, and all 40 +/// collectors read HEALTHY, because from each one's own seat nothing was wrong. +/// +public sealed class SweepPressureClassifierTests +{ + private static (string, double, int) C(string name, double avgMs, int freqMin) => (name, avgMs, freqMin); + + /// The #2296 measurement verbatim: ~101% of the minute — SATURATED, not a warning-log easter egg. + [Fact] + public void TheMotivatingServerReadsSaturated() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("procedure_stats", 22_141, 1), + C("query_store", 16_590, 1), + C("plan_correction", 13_544, 1), + C("query_stats", 8_437, 1), + }); + + Assert.Equal(SweepPressureClassifier.Saturated, pressure.Verdict); + Assert.Equal(60_712, pressure.BusyMsPerMinute, 3); + Assert.True(pressure.BusyPercent > 100.0); + } + + /// An ordinary in-region profile sits far below every threshold. + [Fact] + public void AHealthyProfileReadsOk() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("wait_stats", 180, 1), + C("cpu_utilization", 95, 1), + C("query_stats", 2_400, 1), + C("database_size_stats", 1_200, 60), + }); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.True(pressure.BusyPercent < 5.0); + } + + /// + /// The band edges, both inclusive: 45,000 ms/min is exactly 75% (AT_RISK), 60,000 exactly 100% + /// (SATURATED). Inclusive because the average already smooths spikes — a body that AVERAGES the + /// boundary is over it half the time. + /// + [Fact] + public void TheBandEdgesAreInclusive() + { + Assert.Equal(SweepPressureClassifier.Ok, + SweepPressureClassifier.Compute(new[] { C("a", 44_999, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.AtRisk, + SweepPressureClassifier.Compute(new[] { C("a", 45_000, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.AtRisk, + SweepPressureClassifier.Compute(new[] { C("a", 59_999, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.Saturated, + SweepPressureClassifier.Compute(new[] { C("a", 60_000, 1) }).Verdict); + } + + /// + /// A non-recurring collector (frequency 0: on-load, unknown name) contributes nothing however long it + /// runs — it does not compete for the sweep. A zero-duration entry likewise adds nothing. + /// + [Fact] + public void OnLoadAndZeroDurationCollectorsAreExcluded() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("database_config", 500_000, 0), + C("trace_flags", 0, 1), + C("wait_stats", 300, 1), + }); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.Equal(300, pressure.BusyMsPerMinute, 3); + } + + /// + /// Amortization is by each collector's OWN cadence: an hourly collector averaging 30s costs 500 ms of + /// every minute, not 30,000 — the mistake this pin forbids is charging slow collectors at the fast + /// cadence, which would flag every server with a heavy daily job. + /// + [Fact] + public void SlowCollectorsAreAmortizedByTheirOwnCadence() + { + var pressure = SweepPressureClassifier.Compute(new[] { C("index_object_stats", 30_000, 60) }); + + Assert.Equal(500, pressure.BusyMsPerMinute, 3); + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + } + + /// No collectors — a server before first collection — is OK with zero demand, never a verdict from nothing. + [Fact] + public void AnEmptyWindowReadsOkWithZeroDemand() + { + var pressure = SweepPressureClassifier.Compute(Array.Empty<(string, double, int)>()); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.Equal(0, pressure.BusyMsPerMinute); + Assert.Equal(0, pressure.BusyPercent); + } +} diff --git a/Darling/Darling.Tests/TargetProviderTests.cs b/Darling/Darling.Tests/TargetProviderTests.cs new file mode 100644 index 000000000..83096fbd1 --- /dev/null +++ b/Darling/Darling.Tests/TargetProviderTests.cs @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using Microsoft.Data.SqlClient; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Targets; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the engine-execution seam: the right driver types come out of each provider, the collector +/// parameter mapping is total for both engines, and a driver failure is named the same way for both. +/// +public class TargetProviderTests +{ + private static CollectorQuery Query(params CollectorParameter[] parameters) + => new("SELECT 1", parameters); + + [Fact] + public void ResolvesAProviderForEveryDeclaredEngine() + { + foreach (CollectorTargetEngine engine in Enum.GetValues()) + { + var provider = TargetProviders.For(engine); + Assert.Equal(engine, provider.Engine); + } + } + + [Fact] + public void ProducesTheDriverTypesEachEngineNeeds() + { + Assert.IsType(SqlServerTargetProvider.Instance.CreateConnection("Server=nowhere")); + Assert.IsType(PostgresTargetProvider.Instance.CreateConnection("Host=nowhere")); + } + + /// + /// Every parameter type a definition can declare must map on BOTH engines. An unmapped type + /// throwing at runtime, inside a sweep, is the failure this prevents. + /// + [Fact] + public void MapsEveryCollectorParameterTypeOnBothEngines() + { + foreach (CollectorParameterType type in Enum.GetValues()) + { + var query = Query(new CollectorParameter("@p", Value(type), type)); + + using var sqlConnection = new SqlConnection("Server=nowhere"); + using var sqlCommand = SqlServerTargetProvider.Instance.CreateCommand(query, sqlConnection, 60); + Assert.Single(sqlCommand.Parameters); + + using var pgConnection = new NpgsqlConnection("Host=nowhere"); + using var pgCommand = PostgresTargetProvider.Instance.CreateCommand(query, pgConnection, 60); + Assert.Single(pgCommand.Parameters); + } + + static object Value(CollectorParameterType type) => type switch + { + CollectorParameterType.DateTime2 => new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Unspecified), + CollectorParameterType.Int32 => 1, + CollectorParameterType.BigInt => 1L, + _ => "x", + }; + } + + /// A null parameter value becomes DBNull, not a null reference, on both engines. + [Fact] + public void MapsNullParameterValuesToDbNull() + { + var query = Query(new CollectorParameter("@p", null, CollectorParameterType.NVarChar128)); + + using var sqlConnection = new SqlConnection("Server=nowhere"); + using var sqlCommand = SqlServerTargetProvider.Instance.CreateCommand(query, sqlConnection, 60); + Assert.Equal(DBNull.Value, sqlCommand.Parameters[0].Value); + + using var pgConnection = new NpgsqlConnection("Host=nowhere"); + using var pgCommand = PostgresTargetProvider.Instance.CreateCommand(query, pgConnection, 60); + Assert.Equal(DBNull.Value, pgCommand.Parameters[0].Value); + } + + [Fact] + public void RejectsAConnectionFromTheWrongEngine() + { + using var pgConnection = new NpgsqlConnection("Host=nowhere"); + Assert.Throws(() => + SqlServerTargetProvider.Instance.CreateCommand(Query(), pgConnection, 60)); + + using var sqlConnection = new SqlConnection("Server=nowhere"); + Assert.Throws(() => + PostgresTargetProvider.Instance.CreateCommand(Query(), sqlConnection, 60)); + } + + [Fact] + public void AppliesTheCommandTimeoutOnBothEngines() + { + using var sqlConnection = new SqlConnection("Server=nowhere"); + using var sqlCommand = SqlServerTargetProvider.Instance.CreateCommand(Query(), sqlConnection, 300); + Assert.Equal(300, sqlCommand.CommandTimeout); + + using var pgConnection = new NpgsqlConnection("Host=nowhere"); + using var pgCommand = PostgresTargetProvider.Instance.CreateCommand(Query(), pgConnection, 300); + Assert.Equal(300, pgCommand.CommandTimeout); + } + + /// + /// The Postgres SQLSTATEs here are the ones actually observed while probing our Aurora fleet: + /// 42501 from a function needing rds_replication, 42P01 from pg_stat_statements in a database + /// where the view was never created, 0A000 from pg_stat_wal (Aurora blocks it), and a 55-class + /// error from a feature that is switched off rather than empty. + /// + [Theory] + [InlineData("42501", CollectorTargetFault.Permissions)] + [InlineData("42P01", CollectorTargetFault.ObjectMissing)] + [InlineData("42883", CollectorTargetFault.ObjectMissing)] + [InlineData("0A000", CollectorTargetFault.FeatureDisabled)] + [InlineData("55000", CollectorTargetFault.FeatureDisabled)] + [InlineData("57014", CollectorTargetFault.CommandTimeout)] + [InlineData("08006", CollectorTargetFault.ConnectionFatal)] + [InlineData("08000", CollectorTargetFault.ConnectionFatal)] + [InlineData("57P01", CollectorTargetFault.ConnectionFatal)] + [InlineData("XX000", CollectorTargetFault.Unclassified)] + public void ClassifiesPostgresSqlStates(string sqlState, CollectorTargetFault expected) + { + var exception = new PostgresException("boom", "ERROR", "ERROR", sqlState); + Assert.Equal(expected, PostgresTargetProvider.Instance.Classify(exception, yieldsOnLockTimeout: false)); + } + + /// + /// A lock timeout is a yield only for a collector that deliberately set a short lock timeout. + /// Same rule, both engines — it is a property of the collector, not of the database. + /// + [Fact] + public void TreatsALockTimeoutAsAYieldOnlyForCollectorsThatOptIn() + { + var pgLockTimeout = new PostgresException("boom", "ERROR", "ERROR", "55P03"); + + Assert.Equal( + CollectorTargetFault.LockTimeoutYield, + PostgresTargetProvider.Instance.Classify(pgLockTimeout, yieldsOnLockTimeout: true)); + Assert.Equal( + CollectorTargetFault.Unclassified, + PostgresTargetProvider.Instance.Classify(pgLockTimeout, yieldsOnLockTimeout: false)); + } + + [Fact] + public void ClassifiesUnrecognizedExceptionsAsUnclassifiedSoTheyStayLoud() + { + var boom = new InvalidOperationException("something else entirely"); + + Assert.Equal(CollectorTargetFault.Unclassified, SqlServerTargetProvider.Instance.Classify(boom, false)); + Assert.Equal(CollectorTargetFault.Unclassified, PostgresTargetProvider.Instance.Classify(boom, false)); + } + + [Fact] + public void ClassifiesATimeoutExceptionAsACommandTimeoutOnPostgres() + { + Assert.Equal( + CollectorTargetFault.CommandTimeout, + PostgresTargetProvider.Instance.Classify(new TimeoutException(), false)); + } + + /// + /// Per-database fan-out must change ONLY the database. Every other setting — credentials, timeouts, + /// TLS posture — has to survive, or a per-database collector would silently connect on different terms + /// than the collector that probed the server. + /// + [Fact] + public void WithDatabase_ChangesOnlyTheDatabase() + { + var sql = SqlServerTargetProvider.Instance.WithDatabase( + "Server=sql1;Initial Catalog=master;User ID=mon;Password=p;Encrypt=Strict;Connect Timeout=15", "AdventureWorks"); + var sqlBuilder = new SqlConnectionStringBuilder(sql); + + Assert.Equal("AdventureWorks", sqlBuilder.InitialCatalog); + Assert.Equal("sql1", sqlBuilder.DataSource); + Assert.Equal("mon", sqlBuilder.UserID); + Assert.Equal(SqlConnectionEncryptOption.Strict, sqlBuilder.Encrypt); + Assert.Equal(15, sqlBuilder.ConnectTimeout); + + var pg = PostgresTargetProvider.Instance.WithDatabase( + "Host=aurora;Database=postgres;Username=mon;Password=p;SSL Mode=VerifyFull;Timeout=15", "appdb"); + var pgBuilder = new NpgsqlConnectionStringBuilder(pg); + + Assert.Equal("appdb", pgBuilder.Database); + Assert.Equal("aurora", pgBuilder.Host); + Assert.Equal("mon", pgBuilder.Username); + Assert.Equal(SslMode.VerifyFull, pgBuilder.SslMode); + Assert.Equal(15, pgBuilder.Timeout); + } + + /// + /// SQL Server enumerates from master — on an Azure SQL DB logical server the configured entry points + /// at one user database, where sys.databases lists only itself. PostgreSQL enumerates from wherever it + /// already is, because pg_database is a shared catalog. + /// + [Fact] + public void DatabaseListPlan_EnumeratesFromTheRightPlacePerEngine() + { + var (sqlConnectionString, sqlQuery) = SqlServerTargetProvider.Instance.BuildDatabaseListPlan( + "Server=sql1;Initial Catalog=CustomerDb;User ID=mon;Password=p", null); + + Assert.Equal("master", new SqlConnectionStringBuilder(sqlConnectionString).InitialCatalog); + Assert.Contains("sys.databases", sqlQuery.Text, StringComparison.Ordinal); + + const string pgConnectionString = "Host=aurora;Database=postgres;Username=mon;Password=p"; + var (pgConnection, pgQuery) = PostgresTargetProvider.Instance.BuildDatabaseListPlan(pgConnectionString, null); + + Assert.Equal(pgConnectionString, pgConnection); + Assert.Contains("pg_database", pgQuery.Text, StringComparison.Ordinal); + } + + /// + /// Both filters keep the fan-out from attempting connections that cannot succeed. template0 is frozen + /// and refuses connections outright, so including it would guarantee one failed connection per cycle + /// forever; datallowconn = false is a database an administrator has deliberately closed. + /// + [Fact] + public void PostgresDatabaseList_SkipsTemplatesAndClosedDatabases() + { + var (_, query) = PostgresTargetProvider.Instance.BuildDatabaseListPlan("Host=aurora;Database=postgres", null); + + Assert.Contains("datallowconn", query.Text, StringComparison.Ordinal); + Assert.Contains("NOT datistemplate", query.Text, StringComparison.Ordinal); + } + + /// + /// The exclusion list has to reach the right column name on each engine, parameterized rather than + /// interpolated — a database named with a quote must not be able to alter the enumeration query. + /// + [Fact] + public void DatabaseListPlan_AppliesTheExclusionListPerEngineColumn() + { + var excluded = new[] { "tempdb_clone", "scratch" }; + + var (_, sqlQuery) = SqlServerTargetProvider.Instance.BuildDatabaseListPlan("Server=sql1", excluded); + Assert.Contains("name NOT IN (@excl_db_0, @excl_db_1)", sqlQuery.Text, StringComparison.Ordinal); + Assert.Equal(2, sqlQuery.Parameters.Count); + + var (_, pgQuery) = PostgresTargetProvider.Instance.BuildDatabaseListPlan("Host=aurora", excluded); + Assert.Contains("datname NOT IN (@excl_db_0, @excl_db_1)", pgQuery.Text, StringComparison.Ordinal); + Assert.Equal(2, pgQuery.Parameters.Count); + + /* Values travel as parameters, never as text in the query. */ + Assert.DoesNotContain("scratch", pgQuery.Text, StringComparison.Ordinal); + Assert.DoesNotContain("scratch", sqlQuery.Text, StringComparison.Ordinal); + } + + /// + /// An empty exclusion list must produce no clause and no parameters, not a dangling AND. + /// Each engine gets a connection string in ITS OWN dialect. A single string for both looks tidier + /// and does not work: BuildDatabaseListPlan hops to another database via the provider's + /// WithDatabase, which parses through that engine's real builder — so + /// SqlConnectionStringBuilder rejects Host= outright ("Keyword not supported"), and the + /// test fails on its own fixture rather than on the thing it is checking. + /// + [Theory] + [InlineData(CollectorTargetEngine.SqlServer, "Server=sql1;Database=master;Integrated Security=true")] + [InlineData(CollectorTargetEngine.PostgreSql, "Host=pg1;Database=postgres;Username=monitor")] + public void DatabaseListPlan_WithNoExclusionsIsClean(CollectorTargetEngine engine, string connectionString) + { + var (_, query) = TargetProviders.For(engine).BuildDatabaseListPlan(connectionString, Array.Empty()); + + Assert.Empty(query.Parameters); + Assert.DoesNotContain("NOT IN", query.Text, StringComparison.Ordinal); + Assert.DoesNotContain("@excl_db_", query.Text, StringComparison.Ordinal); + } + + /// + /// Every engine in the enum has a provider. Split out from the case above, which needs an + /// engine-specific connection string and so cannot loop the enum — this is the part that genuinely + /// must cover all of them, and it fails when a new engine is added without one. + /// + [Fact] + public void EveryEngineHasAProvider() + { + foreach (CollectorTargetEngine engine in Enum.GetValues()) + { + Assert.NotNull(TargetProviders.For(engine)); + } + } + + /// + /// The enumeration query has to be executable through each provider's own command factory — the same + /// parameter mapping every collector query goes through, so an exclusion parameter cannot be mapped + /// one way here and another way there. + /// + [Fact] + public void DatabaseListPlan_ParametersMapThroughTheProvidersOwnCommandFactory() + { + var excluded = new[] { "scratch" }; + + var (_, sqlQuery) = SqlServerTargetProvider.Instance.BuildDatabaseListPlan("Server=sql1", excluded); + using var sqlConnection = new SqlConnection("Server=nowhere"); + using var sqlCommand = SqlServerTargetProvider.Instance.CreateCommand(sqlQuery, sqlConnection, 60); + Assert.Single(sqlCommand.Parameters); + + var (_, pgQuery) = PostgresTargetProvider.Instance.BuildDatabaseListPlan("Host=aurora", excluded); + using var pgConnection = new NpgsqlConnection("Host=nowhere"); + using var pgCommand = PostgresTargetProvider.Instance.CreateCommand(pgQuery, pgConnection, 60); + Assert.Single(pgCommand.Parameters); + } +} diff --git a/Darling/Darling.Tests/TimescaleSupportTests.cs b/Darling/Darling.Tests/TimescaleSupportTests.cs index be3d9de4a..7a84d8ed4 100644 --- a/Darling/Darling.Tests/TimescaleSupportTests.cs +++ b/Darling/Darling.Tests/TimescaleSupportTests.cs @@ -152,14 +152,33 @@ managed store compact (#1458). */ [Fact] public void IsCompressionJobStuck_NextStartNegativeInfinity_IsStuck() { - /* The dominant failure mode: next_start = -infinity, so the scheduler never re-fires it. Stuck - regardless of status/last-run — it will never run again. */ + /* The dominant failure mode: next_start = -infinity on a job that is NOT running — the scheduler + abandoned it and never re-fires it. */ Assert.True(TimescaleSupport.IsCompressionJobStuck( nextStartIsNegativeInfinity: true, jobStatus: "Scheduled", lastRunStartedAtUtc: null, scheduleInterval: TimeSpan.FromHours(12), nowUtc: s_now, out var reason)); Assert.Contains("-infinity", reason, StringComparison.Ordinal); } + [Fact] + public void IsCompressionJobStuck_NegativeInfinityWhileRunning_IsTheMidRunMarker_NotStuck() + { + /* Measured live on TimescaleDB 2.x: from scheduler pickup to run completion, next_start reads + -infinity WITH job_status = 'Running' — the engine only computes the real next start when the + run finishes. An unconditioned -infinity arm flagged every healthy job caught mid-run (the + field's transient stuck→self-healed alert noise, and the CI flake where the live test caught + its own re-arm-triggered run). Mid-run belongs to the elapsed-bound arm: */ + Assert.False(TimescaleSupport.IsCompressionJobStuck( + nextStartIsNegativeInfinity: true, jobStatus: "Running", lastRunStartedAtUtc: s_now.AddMinutes(-3), + scheduleInterval: TimeSpan.FromHours(12), nowUtc: s_now, out _)); + + /* ...which still catches a genuinely HUNG run that carries the mid-run marker. */ + Assert.True(TimescaleSupport.IsCompressionJobStuck( + nextStartIsNegativeInfinity: true, jobStatus: "Running", lastRunStartedAtUtc: s_now.AddHours(-30), + scheduleInterval: TimeSpan.FromHours(12), nowUtc: s_now, out var reason)); + Assert.Contains("Running", reason, StringComparison.Ordinal); + } + [Fact] public void IsCompressionJobStuck_HealthyScheduled_IsNotStuck() { @@ -1519,24 +1538,43 @@ BEFORE TimescaleDB's background scheduler assigns its first real next run — an CORRECT to flag -infinity — so asserting immediately after ApplyCompressionPolicy raced that window and intermittently false-failed on a slow CI runner. Deterministically settle the job into the healthy state the assertion is actually about: give it a real FUTURE next_start (via the same - alter_job the self-heal uses), then wait for the catalog to reflect a non-(-infinity) next_start. */ + alter_job the self-heal uses), then wait for the detector itself to report it healthy. + + The wait's RESULT is what the assertion below reads, and that is the whole point rather than a + convenience: a wait that only proves "healthy at some instant", followed by a fresh read, asserts + against a DIFFERENT observation than the one it validated, and the job can leave the healthy state + in the gap between them (the scheduler picking it up reads next_start = -infinity with status + Running mid-run — see leg 3). That gap is this test's third flake in the same class, after #1760 + polled a copy of one detector arm and after the wait was introduced to close it; it survived + because the helper's guarantee stops at the moment it returns. Consuming the returned snapshot + makes settled-according-to-the-wait and settled-according-to-the-assertion the same observation + by construction, which is what the helper's contract claimed all along. */ using (var arm = new NpgsqlCommand("SELECT alter_job($1::integer, next_start => now() + interval '1 hour')", connection)) { arm.Parameters.Add(new NpgsqlParameter { Value = jobId }); await arm.ExecuteNonQueryAsync(ct); } - await WaitUntilDetectorReportsHealthyAsync(connection, jobId, ct); + var healthy = await WaitUntilDetectorReportsHealthyAsync(connection, jobId, ct); /* The SQL really is valid against the live catalog, and this job really is in its result set. ReadStuckCompressionJobsAsync is failure-isolated (a broken query is swallowed and returns an EMPTY list), so DoesNotContain ALONE would pass just as happily against SQL that never compiled — the one thing this leg claims to prove. Run the production const directly, where a syntax or column error throws, and require the job to be present: only then does "not flagged" mean the detector looked at - this job and judged it healthy. */ + this job and judged it healthy. + + This one keeps its OWN read, which is safe where the health assertion is not: StuckCompressionJobsSql + filters on proc_name alone, so it returns every compression job whatever state it is in, and "this job + is in the result set" cannot race. Flagging is the C# predicate applied on top of those rows, and that + is the only part that moves. */ var observed = await ReadObservedJobIdsAsync(connection, ct); Assert.Contains(jobId, observed); - var healthy = await TimescaleSupport.ReadStuckCompressionJobsAsync(connection, DateTime.UtcNow, null, ct); + /* Deliberately tautological, and kept for what it documents rather than what it can catch: `healthy` is + the snapshot the wait already found clean, so this cannot fail today. It states the property leg (1) + exists to assert, at the place a reader looks for it, and it fails loudly if the helper is ever + changed to return something other than the satisfying poll's own result. The load-bearing check is + the wait's bounded loop, which fails carrying the detector's reason string. */ Assert.DoesNotContain(healthy, s => s.JobId == jobId); /* (2) The #1586 REGRESSION GUARD: the production re-arm runs the real alter_job against TimescaleDB and @@ -1553,9 +1591,12 @@ this job and judged it healthy. */ DETECTION logic is covered by the pure IsCompressionJobStuck unit tests. */ Assert.True(await TimescaleSupport.TryRearmJobAsync(connection, jobId, null, ct)); - /* (3) After a real re-arm (next_start => now()) the job is scheduled/running within bound, not stuck. */ - var afterRearm = await TimescaleSupport.ReadStuckCompressionJobsAsync(connection, DateTime.UtcNow, null, ct); - Assert.DoesNotContain(afterRearm, s => s.JobId == jobId); + /* (3) After a real re-arm (next_start => now()) the job settles healthy. SETTLES, not "reads healthy + on one snapshot": next_start => now() makes the job immediately due, the scheduler picks it up, and + from pickup to completion job_stats reads next_start = -infinity with status Running — the mid-run + marker (measured live; the detector now defers that state to its elapsed-bound arm). A single + un-settled read raced the very run the re-arm triggered, which was this test's own flake. */ + await WaitUntilDetectorReportsHealthyAsync(connection, jobId, ct); } /// @@ -1571,8 +1612,15 @@ DETECTION logic is covered by the pure IsCompressionJobStuck unit tests. */ /// drift, so settled-according-to-the-wait IS settled-according-to-the-assertion. Bounded, and it fails /// loudly carrying the detector's OWN reason string — a job that never settles is genuinely stuck and must /// not silently pass, and the reason names which arm held it rather than assuming next_start. + /// + /// RETURNS the flagged list from the poll that satisfied it, and callers must assert against THAT + /// rather than issuing a fresh read. The guarantee above holds only at the instant this returns: the + /// scheduler is free to pick the job up immediately afterward, and mid-run it reads + /// next_start = -infinity with status Running, which the detector flags and is right to flag. A + /// caller that re-queries is therefore asserting on an observation this method never validated — which is + /// exactly how the race came back after being closed once. /// - private static async Task WaitUntilDetectorReportsHealthyAsync( + private static async Task> WaitUntilDetectorReportsHealthyAsync( NpgsqlConnection connection, long jobId, System.Threading.CancellationToken ct) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); @@ -1582,7 +1630,7 @@ private static async Task WaitUntilDetectorReportsHealthyAsync( var mine = flagged.FirstOrDefault(s => s.JobId == jobId); if (mine is null) { - return; + return flagged; } Assert.True(DateTime.UtcNow < deadline, diff --git a/Darling/Darling.Tests/ViewerCollectorCoverageTests.cs b/Darling/Darling.Tests/ViewerCollectorCoverageTests.cs index d694db741..03720b15c 100644 --- a/Darling/Darling.Tests/ViewerCollectorCoverageTests.cs +++ b/Darling/Darling.Tests/ViewerCollectorCoverageTests.cs @@ -45,9 +45,29 @@ public sealed class ViewerCollectorCoverageTests /// private static readonly HashSet KnownStoreOnlyOrUnbuiltTables = new(StringComparer.OrdinalIgnoreCase) { - // Empty: every collector table now has a Darling viewer reader. database_states is read by - // ViewerDataService.DatabaseStates.cs (the override editor's backing store), so it is covered - // by the reader-layer scan and needs no allow-list entry. + // database_states is read by ViewerDataService.DatabaseStates.cs (the override editor's backing + // store), so it is covered by the reader-layer scan and needs no allow-list entry. + + // UNBUILT UI (parity board Tier 1) -- remove each when the PostgreSQL tab ships. + // The eight PostgreSQL collector tables are read through MCP today, not the WPF viewer: each has a + // reader class + MCP tool (get_pg_wait_stats, get_pg_top_queries, get_pg_wraparound_risk, + // get_pg_xmin_horizon, get_pg_replication_slots, get_pg_autovacuum_health, get_pg_io_stats, + // get_pg_blocking) and is exposed over /api/read, so the data is not invisible — it is just not on a + // WPF tab, because the viewer's surfaces are SQL-Server-shaped and a PostgreSQL target's signals do + // not slot into them. Deliberately listed one per line so removing one at a time is a one-line diff. + "pg_wait_stats", + "pg_statement_stats", + "pg_wraparound_stats", + "pg_xmin_horizon", + "pg_replication_slot_stats", + "pg_autovacuum_stats", + "pg_io_stats", + // pg_blocking_edges is the one with a genuine SQL Server counterpart on the Blocking tab, and it + // still cannot share it: that tab is built on blocked_process_report XML (an engine-recorded event + // with a deadlock graph and a threshold), while this table holds periodic samples of an edge list. + // Rendering samples through a surface labelled as an event log would misrepresent coverage, which is + // a worse outcome than the tab not existing yet. + "pg_blocking_edges", }; [Fact] diff --git a/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs b/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs index 9534d1966..c28ccdf5e 100644 --- a/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs +++ b/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs @@ -217,7 +217,7 @@ public void UpdateFlagsSql_SetsOnlyCapturePlansMcp_OnTheSingleRow_NeverPaused() [Fact] public void SelectSql_ReadsPausedAndTheViewerOwnedFlags() { - Assert.Contains("SELECT paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port FROM config_service WHERE id = 1", + Assert.Contains("SELECT paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port, query_store_backfill_enabled, query_store_text_budget_mb, max_concurrent_sweeps FROM config_service WHERE id = 1", ViewerDataService.ServiceConfigSelectSql, StringComparison.Ordinal); } diff --git a/Darling/Darling.Tests/ViewerDataServiceTests.cs b/Darling/Darling.Tests/ViewerDataServiceTests.cs index 79902ccdc..16df2c96c 100644 --- a/Darling/Darling.Tests/ViewerDataServiceTests.cs +++ b/Darling/Darling.Tests/ViewerDataServiceTests.cs @@ -9,6 +9,7 @@ using System; using System.Globalization; using System.IO; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; using System.Windows.Media; @@ -204,6 +205,84 @@ public void Parse_ManagedMode_MissingCredential_ThrowsAReadableFirstRunHint() var ex = Assert.Throws(() => ViewerSettings.Parse(json)); Assert.Contains("service", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("Start the PerformanceMonitor Darling service once", ex.Message, StringComparison.Ordinal); + /* #2197: even the first-run voice carries the one sentence for the operator who HAS already + started it — this string is what the main window shows, so it cannot be a dead end either. */ + Assert.Contains("ALREADY started it", ex.Message, StringComparison.Ordinal); + } + + /// + /// #2197, the viewer half of the CLI's DarlingStoreBootstrapEvidence. This message is rendered by the + /// main window, so it reaches the same operator with the same absence — and before this it gave the + /// same first-run advice to a store whose bootstrap had already failed. + /// + [Fact] + public void Parse_ManagedMode_MissingCredentialAfterAFailedBootstrap_PointsAtTheServiceLogInstead() + { + var root = Directory.CreateTempSubdirectory("darling-viewer-failedboot-"); + try + { + /* The #2185 shape: the service wrote the store's own credential immediately before initdb, and + initdb died — so the admin role credential the viewer wants was never provisioned. */ + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + Directory.CreateDirectory(Path.Combine(root.FullName, "store")); + var storeCredential = PerformanceMonitor.Darling.Service.DarlingManagedPostgres.CredentialPathFor(dataDirectory); + File.WriteAllText(storeCredential, "not-a-real-credential"); + + var json = $$"""{ "postgres": { "managed": true, "dataDirectory": {{JsonSerializer.Serialize(dataDirectory)}} } }"""; + var ex = Assert.Throws(() => ViewerSettings.Parse(json)); + + Assert.DoesNotContain("Start the PerformanceMonitor Darling service once", ex.Message, StringComparison.Ordinal); + Assert.Contains("NOT a first run", ex.Message, StringComparison.Ordinal); + Assert.Contains(storeCredential, ex.Message, StringComparison.Ordinal); + Assert.Contains("darling-service_yyyyMMdd.log", ex.Message, StringComparison.Ordinal); + Assert.Contains("Nothing in darling.json produces this", ex.Message, StringComparison.Ordinal); + } + finally + { + root.Delete(recursive: true); + } + } + + /// + /// The viewer does not reference the Service project, so the file names it probes for bootstrap evidence + /// are DUPLICATED under the file's sliver rule — the same rule the DPAPI entropy and the role/credential + /// names already live under. Pin them, or a rename on the service side silently turns the viewer's + /// sharper branch off and nothing fails. + /// + [Fact] + public void ManagedDerivation_BootstrapEvidenceFileNames_MatchTheServiceConstants() + { + var root = Directory.CreateTempSubdirectory("darling-viewer-sliver-"); + try + { + var dataDirectory = Path.Combine(root.FullName, "store", "pg"); + var storeFolder = Path.Combine(root.FullName, "store"); + Directory.CreateDirectory(storeFolder); + + /* Each service-side name, one at a time, must be the one the viewer's probe recognises. */ + foreach (var evidenceFile in new[] + { + PerformanceMonitor.Darling.Service.DarlingManagedPostgres.CredentialFileName, + PerformanceMonitor.Darling.Service.DarlingManagedPostgres.McpCredentialFileName, + PerformanceMonitor.Darling.Service.DarlingManagedPostgres.ServerLogFileName, + }) + { + var path = Path.Combine(storeFolder, evidenceFile); + File.WriteAllText(path, "x"); + + var json = $$"""{ "postgres": { "managed": true, "dataDirectory": {{JsonSerializer.Serialize(dataDirectory)}} } }"""; + var ex = Assert.Throws(() => ViewerSettings.Parse(json)); + Assert.Contains("NOT a first run", ex.Message, StringComparison.Ordinal); + Assert.Contains(path, ex.Message, StringComparison.Ordinal); + + File.Delete(path); + } + } + finally + { + root.Delete(recursive: true); + } } [Fact] @@ -543,10 +622,68 @@ public void RequiredStoreSchemaVersion_TracksTheBuildSchemaVersion_AndTheProbeCo /* Pin: a fully-migrated store (all sentinels present) must map to exactly the required version. If a future migration bumps StorageVersion, this fails until a matching sentinel + map arm is added — the guard against the probe silently under-reporting a newer store as skewed, which would make - the connect-time gate refuse to open the viewer against a perfectly healthy store. */ - Assert.Equal( - ViewerDataService.RequiredStoreSchemaVersion, - ViewerDataService.MapProbedSchemaVersion(true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true)); + the connect-time gate refuse to open the viewer against a perfectly healthy store. + + Built by REFLECTION so the call's arity tracks the signature: the literal-true form silently + defaults every newly added sentinel parameter to false, maps one version low, and fails this test + on every probe extension — it broke on the V61 bump and again on the V61+V62 merge. All-true IS + the contract here (a fully-migrated store has every sentinel), so the arity is the only thing the + literals ever expressed. */ + var map = typeof(ViewerDataService).GetMethod( + nameof(ViewerDataService.MapProbedSchemaVersion), + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + var allTrue = Enumerable.Repeat((object)true, map.GetParameters().Length).ToArray(); + Assert.Equal(ViewerDataService.RequiredStoreSchemaVersion, (int)map.Invoke(null, allTrue)!); + + /* Probe row width vs mapper arity is not checked HERE, and the reason it was left to the live probes + was that one ordinal is legitimately a compound (two EXISTS OR-ed), so counting the token EXISTS + lies — it reads one high. That is true of token counting and not of the seam: counting top-level + select items by PAREN DEPTH is immune to the compound, because the nested EXISTS sit inside + parentheses. StoreSchemaProbe_ColumnCount_MatchesTheMapArity below does that, so the seam is now + pinned at build time rather than only on a real store. */ + } + + /// + /// The probe SQL's column count must equal 's + /// parameter count, because GetStoreSchemaVersionAsync reads them positionally. + /// + /// Nothing else catches this. Add a sentinel to the SQL and forget the parameter and the + /// last column is silently ignored — a fully-migrated store reports one rung short and the viewer refuses + /// a healthy store. Add the parameter and forget the SQL column and + /// reader.GetBoolean(n) throws IndexOutOfRange at connect time. Both are runtime-only against a + /// live store, both are invisible to every other test here (the arity test above builds its arguments + /// from the signature, so it agrees with itself either way), and the second one is the same class of + /// ordinal-drift defect that a live-target review had to find by hand. + /// + /// Top-level select items are counted by paren depth rather than by counting the string + /// EXISTS: the V22/V23 composite column contains two nested EXISTS of its own, so a naive + /// substring count reads one HIGHER than the select list actually is. Paren depth is immune, because the + /// nested pair sits inside parentheses — which is why this can be pinned at build time at all. + /// + [Fact] + public void StoreSchemaProbe_ColumnCount_MatchesTheMapArity() + { + var sql = ViewerDataService.StoreSchemaProbeSql; + var selectAt = sql.IndexOf("SELECT", StringComparison.Ordinal); + Assert.True(selectAt >= 0, "the probe must be a SELECT"); + + var depth = 0; + var columns = 1; + foreach (var c in sql.AsSpan(selectAt + "SELECT".Length)) + { + if (c == '(') depth++; + else if (c == ')') depth--; + else if (c == ',' && depth == 0) columns++; + } + + var parameters = typeof(ViewerDataService) + .GetMethod( + nameof(ViewerDataService.MapProbedSchemaVersion), + System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic + | System.Reflection.BindingFlags.Public)! + .GetParameters().Length; + + Assert.Equal(parameters, columns); } } @@ -592,6 +729,29 @@ public void ViewerStoreUnreachableException_AsksWhetherTheServiceIsRunning_AndNa Assert.Contains("Darling service", ex.Message, StringComparison.Ordinal); Assert.Contains("darling.json", ex.Message, StringComparison.Ordinal); Assert.Equal("connection refused", ex.InnerException?.Message); + + /* #2117 finding 2: the message itself carries the underlying error's first line — the swallowed + detail cost a field operator hours, and the fixed prose alone reads identically for a TLS chain + rejection, a wrong password, a pg_hba refusal, and a dead host. */ + Assert.Contains("Underlying error: connection refused", ex.Message, StringComparison.Ordinal); + } + + /// + /// The surfaced first line is trimmed of the CR a CRLF-raised exception leaves behind (Windows is + /// exactly where this fix is aimed), and only the FIRST line is taken — a multi-line inner message + /// must not flood the one-line status surface. + /// + [Fact] + public void ViewerStoreUnreachableException_SurfacesTheFirstLineOnly_WithNoTrailingCarriageReturn() + { + var ex = new ViewerStoreUnreachableException(new InvalidOperationException("boom\r\nmore detail\r\neven more")); + + Assert.Contains("Underlying error: boom", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("boom\r", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("more detail", ex.Message, StringComparison.Ordinal); + + /* And a null inner message degrades to the explicit placeholder, never a crash. */ + Assert.Contains("Underlying error: (none)", new ViewerStoreUnreachableException(null!).Message, StringComparison.Ordinal); } [Fact] diff --git a/Darling/Darling.Tests/ViewerFinOpsTests.cs b/Darling/Darling.Tests/ViewerFinOpsTests.cs index c59f3a530..3eabadd65 100644 --- a/Darling/Darling.Tests/ViewerFinOpsTests.cs +++ b/Darling/Darling.Tests/ViewerFinOpsTests.cs @@ -309,16 +309,65 @@ public void ServerInventorySql_JoinsLatestServerPropertiesToRegistry_WithInvento } } + /// + /// The inventory grid read now projects the verdict INPUTS and classifies in C# through the shared + /// ProvisioningVerdict, rather than deciding inline in SQL. + /// + /// It used to carry its own CASE ... memory_ratio > 0.95 THEN UNDER_PROVISIONED — copies + /// 5 and 6 of the bug in #2246, and on the screen the field report was actually looking at, so the grid + /// disagreed with the drill-down for the same server. The absence assertions are the drift guard: a SQL + /// verdict here can never come back without failing this test. + /// [Fact] - public void ServerMetricsSql_ReadsCpuStorageIdleProvisioning() + public void ServerMetricsSql_ProjectsTheVerdictInputs_AndDoesNotDecideInSql() { var sql = ViewerDataService.ServerMetricsSql; Assert.Contains("FROM v_cpu_utilization_stats", sql, StringComparison.Ordinal); Assert.Contains("FROM v_database_size_stats", sql, StringComparison.Ordinal); Assert.Contains("FROM v_query_stats", sql, StringComparison.Ordinal); Assert.Contains("EXCEPT", sql, StringComparison.Ordinal); - Assert.Contains("OVER_PROVISIONED", sql, StringComparison.Ordinal); - Assert.Contains("UNDER_PROVISIONED", sql, StringComparison.Ordinal); + + /* The pressure inputs the shared predicate needs, which this read did not fetch before. */ + Assert.Contains("FROM v_memory_grant_stats", sql, StringComparison.Ordinal); + Assert.Contains("waiter_count", sql, StringComparison.Ordinal); + Assert.Contains("timeout_error_count_delta", sql, StringComparison.Ordinal); + Assert.Contains("forced_grant_count_delta", sql, StringComparison.Ordinal); + Assert.Contains("max_workers_count", sql, StringComparison.Ordinal); + + /* And the verdict itself must NOT be decided here any more. */ + Assert.DoesNotContain("OVER_PROVISIONED", sql, StringComparison.Ordinal); + Assert.DoesNotContain("UNDER_PROVISIONED", sql, StringComparison.Ordinal); + Assert.DoesNotContain("RIGHT_SIZED", sql, StringComparison.Ordinal); + } + + /// + /// The drift guard, applied uniformly to all THREE reads that feed the provisioning verdict rather than + /// only the one that used to decide in SQL (#2246). + /// + /// Darling has no live-Postgres harness for these, so a source pin is the whole safety net: if a + /// future edit drops the grants CTE, the reader keeps consuming ordinals the SELECT list no longer + /// produces and the verdict silently falls back to "no pressure anywhere" — the same shape of silent + /// wrongness this issue is about. The SQL was executed against the live store when it was written; this + /// is what keeps it honest afterwards. + /// + [Theory] + [InlineData(nameof(ViewerDataService.UtilizationEfficiencySql))] + [InlineData(nameof(ViewerDataService.ProvisioningTrendSql))] + [InlineData(nameof(ViewerDataService.ServerMetricsSql))] + public void EveryVerdictRead_FetchesThePressureInputs(string sqlName) + { + var sql = (string)typeof(ViewerDataService).GetField(sqlName)!.GetValue(null)!; + + Assert.Contains("FROM v_memory_grant_stats", sql, StringComparison.Ordinal); + Assert.Contains("waiter_count", sql, StringComparison.Ordinal); + Assert.Contains("timeout_error_count_delta", sql, StringComparison.Ordinal); + Assert.Contains("forced_grant_count_delta", sql, StringComparison.Ordinal); + Assert.Contains("max_workers_count", sql, StringComparison.Ordinal); + + /* And none of them may decide the verdict, which is now the shared predicate's job alone. */ + Assert.DoesNotContain("OVER_PROVISIONED", sql, StringComparison.Ordinal); + Assert.DoesNotContain("UNDER_PROVISIONED", sql, StringComparison.Ordinal); + Assert.DoesNotContain("RIGHT_SIZED", sql, StringComparison.Ordinal); } // ── PG dialect guard across every FinOps read ── diff --git a/Darling/Darling.Tests/ViewerPerfmonRunningJobsTests.cs b/Darling/Darling.Tests/ViewerPerfmonRunningJobsTests.cs index 1c1c17214..3e8494ee7 100644 --- a/Darling/Darling.Tests/ViewerPerfmonRunningJobsTests.cs +++ b/Darling/Darling.Tests/ViewerPerfmonRunningJobsTests.cs @@ -41,15 +41,20 @@ public void DistinctPerfmonCountersSql_ListsEveryCounter_OverTheWindow() } [Fact] - public void PerfmonTrendsSql_SumsValueAndDelta_GroupedByCounterAndTime() + public void PerfmonTrendsSql_SumsValueAndDelta_ButTakesTheIntervalAsMax() { var sql = ViewerDataService.PerfmonTrendsSql(3); Assert.Contains("FROM v_perfmon_stats", sql, StringComparison.Ordinal); - /* The two aggregates are CAST to bigint for the typed GetInt64 reader (Postgres SUM(bigint) + /* Every aggregate is CAST to bigint for the typed GetInt64 reader (Postgres SUM(bigint) returns numeric). */ Assert.Contains("CAST(SUM(cntr_value) AS bigint)", sql, StringComparison.Ordinal); Assert.Contains("CAST(SUM(delta_cntr_value) AS bigint)", sql, StringComparison.Ordinal); + /* The interval is NOT additive across a counter's instance rows — it is one measured sweep gap + repeated per instance, so SUM would multiply the denominator by the instance count (12-17 for + Transactions/sec on the fleet). Same pin as the MCP read carries, #2234. */ + Assert.Contains("CAST(MAX(sample_interval_seconds) AS bigint)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(sample_interval_seconds)", sql, StringComparison.Ordinal); Assert.Contains("GROUP BY counter_name, collection_time", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY counter_name, collection_time", sql, StringComparison.Ordinal); } diff --git a/Darling/Darling.Tests/XamlStaticResourceHygieneTests.cs b/Darling/Darling.Tests/XamlStaticResourceHygieneTests.cs new file mode 100644 index 000000000..0e6ebf488 --- /dev/null +++ b/Darling/Darling.Tests/XamlStaticResourceHygieneTests.cs @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Xunit; + +namespace Darling.Tests; + +/// +/// #2114 / #2181 / #2331: a {StaticResource Key} whose key is not resolvable from the file that +/// references it is not a style nit — inside a DataGrid cell template it throws +/// XamlParseException during measure, and WPF's re-attempted template realization stack-overflows +/// the process (0xc00000fd, uncatchable, no error dialog). The first version of this scan modeled +/// the failure as CROSS-APP (a key defined nowhere in the same app), because that was #2114's shape — +/// and #2331 proved that model too generous: the Darling Viewer's own Query Store grid referenced +/// DarkButton, a key that IS defined in the app… in MainWindow.xaml's window resources, +/// which a UserControl's templates cannot see. StaticResource resolves LEXICALLY at load (own file, then +/// merged dictionaries, then App.xaml) — never through the runtime element tree, which is +/// DynamicResource's job. It shipped in 3.4.0 and stayed invisible on the dogfood box only because an +/// EMPTY grid never applies its cell template. +/// +/// So the model here is per-FILE, matching WPF's actual lookup: a reference in file F must resolve +/// from F's own definitions, dictionaries F merges (transitively, via +/// <ResourceDictionary Source="…"/>), or the app scope (App.xaml plus everything IT merges). +/// The scan proved exact before it was adopted: run over both apps it flagged exactly the one real crash +/// and zero false positives. A key that must be shared across files belongs in App.xaml or a merged +/// dictionary — moving it there is the fix this test demands, never widening the model back. +/// +public sealed class XamlStaticResourceHygieneTests +{ + /* Each app scope: its XAML subtrees. Shared control libraries would join the scope of every + app that references them; today neither app consumes XAML from outside its own tree. */ + private static readonly (string App, string[] Roots)[] Scopes = + { + ("Lite", new[] { "Lite" }), + ("Darling.Viewer", new[] { Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer") }), + }; + + private static readonly Regex Reference = new( + @"\{StaticResource\s+(?[A-Za-z0-9_.]+)\s*\}", RegexOptions.Compiled); + + private static readonly Regex Definition = new( + @"x:Key\s*=\s*""(?[A-Za-z0-9_.]+)""", RegexOptions.Compiled); + + private static readonly Regex MergeSource = new( + @"[^""]+)""", RegexOptions.Compiled); + + [Fact] + public void EveryStaticResourceKey_ResolvesFromTheFileThatReferencesIt() + { + var root = FindRepoRoot(); + Assert.True(root is not null, + "Could not locate the repository root (walked up from the test binary looking for " + + "PerformanceMonitor.sln). This test scans the source tree, so it cannot run without it — fix the " + + "walk-up rather than skipping, or the rule stops being enforced without anyone noticing."); + + var offenders = new List(); + foreach (var (app, roots) in Scopes) + { + var files = roots + .Select(r => Path.Combine(root!, r)) + .Where(Directory.Exists) + .SelectMany(r => Directory.EnumerateFiles(r, "*.xaml", SearchOption.AllDirectories)) + .Where(f => !f.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}")) + .ToList(); + Assert.NotEmpty(files); + + /* App scope: App.xaml's own keys plus everything it merges — visible everywhere in the app, + because Application resources are the last stop of every StaticResource lookup. */ + var appXaml = files.FirstOrDefault(f => Path.GetFileName(f) == "App.xaml"); + var appScope = appXaml is null + ? new HashSet(StringComparer.Ordinal) + : TransitiveDefinitions(appXaml, new HashSet(StringComparer.OrdinalIgnoreCase)); + + /* System-supplied keys referenced by name, never defined in app XAML. */ + appScope.Add("SystemParameters.VerticalScrollBarWidthKey"); + + foreach (var file in files) + { + var visible = TransitiveDefinitions(file, new HashSet(StringComparer.OrdinalIgnoreCase)); + visible.UnionWith(appScope); + + foreach (Match m in Reference.Matches(File.ReadAllText(file))) + { + var key = m.Groups["key"].Value; + if (!visible.Contains(key)) + offenders.Add($"{Path.GetRelativePath(root!, file)}: StaticResource {key} ({app} scope)"); + } + } + } + + Assert.True(offenders.Count == 0, + "StaticResource keys that do not resolve from the file referencing them (own definitions + " + + "merged dictionaries + App.xaml scope). StaticResource is LEXICAL — a key defined in another " + + "window's or control's resources is invisible no matter who hosts whom at runtime, and inside a " + + "cell template the miss is the #2114/#2331 uncatchable stack-overflow crash. Define the key in " + + "the same file, move it to App.xaml / a merged dictionary, or drop the explicit Style:\n" + + string.Join("\n", offenders)); + } + + /// The keys defined in a file plus, transitively, in every dictionary it merges via + /// Source= (relative paths, and ;component/ pack paths with the prefix stripped and the + /// remainder resolved against the referencing file — correct for same-assembly sources, the only kind + /// this repo uses; a cross-assembly pack URI would not resolve here). A Source that cannot be resolved + /// contributes nothing, which only ever makes the scan stricter. + private static HashSet TransitiveDefinitions(string file, HashSet seenFiles) + { + var keys = new HashSet(StringComparer.Ordinal); + if (!seenFiles.Add(file) || !File.Exists(file)) + { + return keys; + } + + var text = File.ReadAllText(file); + foreach (Match m in Definition.Matches(text)) + { + keys.Add(m.Groups["key"].Value); + } + + foreach (Match m in MergeSource.Matches(text)) + { + var src = m.Groups["src"].Value; + var componentIndex = src.IndexOf(";component/", StringComparison.OrdinalIgnoreCase); + if (componentIndex >= 0) + { + src = src[(componentIndex + ";component/".Length)..]; + } + + var candidate = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(file)!, src.Replace('/', Path.DirectorySeparatorChar))); + keys.UnionWith(TransitiveDefinitions(candidate, seenFiles)); + } + + return keys; + } + + /// Same walk-up idiom as DocCommentHygieneTests.FindRepoRoot. + private static string? FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + for (var i = 0; i < 10 && directory is not null; i++) + { + if (File.Exists(Path.Combine(directory.FullName, "PerformanceMonitor.sln"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} diff --git a/Darling/Darling.Tests/packages.lock.json b/Darling/Darling.Tests/packages.lock.json index 7447cc6b2..d870e712d 100644 --- a/Darling/Darling.Tests/packages.lock.json +++ b/Darling/Darling.Tests/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0-windows7.0": { "Microsoft.NET.Test.Sdk": { @@ -27,16 +27,6 @@ "xunit.v3.mtp-v1": "[3.2.2]" } }, - "CredentialManagement": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" - }, - "Hardcodet.NotifyIcon.Wpf": { - "type": "Transitive", - "resolved": "2.0.1", - "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" - }, "HarfBuzzSharp": { "type": "Transitive", "resolved": "8.3.1.1", @@ -81,21 +71,6 @@ "resolved": "18.8.1", "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" }, - "Microsoft.Data.SqlClient": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", - "dependencies": { - "Microsoft.Bcl.Cryptography": "9.0.13", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", - "Microsoft.Extensions.Caching.Memory": "9.0.13", - "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", - "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" - } - }, "Microsoft.Data.SqlClient.Extensions.Abstractions": { "type": "Transitive", "resolved": "7.0.2", @@ -139,15 +114,6 @@ "Microsoft.Extensions.Primitives": "9.0.13" } }, - "Microsoft.Extensions.Configuration": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", - "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Primitives": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -195,17 +161,6 @@ "Microsoft.Extensions.Primitives": "10.0.10" } }, - "Microsoft.Extensions.Configuration.Json": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.UserSecrets": { "type": "Transitive", "resolved": "10.0.10", @@ -272,35 +227,6 @@ "resolved": "10.0.10", "contentHash": "jSOCVxEwCd4Aq925kJVz1kSO1EpX2OHYKL04qVREXkDU7Ce3pVDdHPYm+fEy8y/th2kJf/DAstRHpJAqoNWP8w==" }, - "Microsoft.Extensions.Hosting": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.Binder": "10.0.10", - "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", - "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.Configuration.Json": "10.0.10", - "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", - "Microsoft.Extensions.Diagnostics": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", - "Microsoft.Extensions.FileProviders.Physical": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging.Configuration": "10.0.10", - "Microsoft.Extensions.Logging.Console": "10.0.10", - "Microsoft.Extensions.Logging.Debug": "10.0.10", - "Microsoft.Extensions.Logging.EventLog": "10.0.10", - "Microsoft.Extensions.Logging.EventSource": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -313,34 +239,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.10" } }, - "Microsoft.Extensions.Hosting.WindowsServices": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "qY7XhE2ljtqCwDKexJf3uG4E7r0teE/DU0eEUDbFqGN2HAIz0WK7P3u4RnnnqOy/3Jz5vdTfTuDMFsrwxrcmeg==", - "dependencies": { - "Microsoft.Extensions.Hosting": "10.0.10", - "Microsoft.Extensions.Logging.EventLog": "10.0.10", - "System.ServiceProcess.ServiceController": "10.0.10" - } - }, - "Microsoft.Extensions.Logging": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", "resolved": "10.0.10", @@ -528,41 +426,15 @@ "resolved": "5.0.0", "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" }, - "ModelContextProtocol": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "X/KDNZDP9Zgs7YXXxxpKiDMWWPXYrs764lVgc6vEM4pCg87bYz4VaceeiKW91XDTGgUvIbmRYXgWqyNpV32xRg==", - "dependencies": { - "Microsoft.Extensions.Caching.Abstractions": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "ModelContextProtocol.Core": "[2.0.0]" - } - }, - "ModelContextProtocol.AspNetCore": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "dXrB7sBpQjUQU0UcdyFPJbOTFw7yaceD+OgAZVAeBveRzbiBlg89jEygAtcgOwd/L+O+YpM98zfwaXz067NFDQ==", - "dependencies": { - "ModelContextProtocol": "[2.0.0]" - } - }, "ModelContextProtocol.Core": { "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "piFR0HtA/2Oc1tgk96EE5Tye6qA2sg3WGRAXBhUqo/BWikdEYEs2UuqtmwLrQZJUge1nUOPgGHYsb15VIBK8iw==", + "resolved": "2.1.0", + "contentHash": "cU/urrhRxE4/iSyBIJI7QOaFqSP1FOEnwEHsct9n6t6/XluCAFD9iqnrPkBAsEYr+f/G4tVQ21U+6wN/6fQvOg==", "dependencies": { "Microsoft.Extensions.AI.Abstractions": "10.8.3", "Microsoft.Extensions.Logging.Abstractions": "10.0.10" } }, - "Npgsql": { - "type": "Transitive", - "resolved": "10.0.3", - "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==", - "dependencies": { - "Microsoft.Extensions.Logging.Abstractions": "10.0.0" - } - }, "OpenTK": { "type": "Transitive", "resolved": "4.9.4", @@ -670,17 +542,6 @@ "SkiaSharp.NativeAssets.Linux.NoDependencies": "3.119.0" } }, - "ScottPlot.WPF": { - "type": "Transitive", - "resolved": "5.1.59", - "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", - "dependencies": { - "OpenTK": "4.9.4", - "OpenTK.GLWpfControl": "4.3.3", - "ScottPlot": "5.1.59", - "SkiaSharp.Views.WPF": "3.119.0" - } - }, "SkiaSharp": { "type": "Transitive", "resolved": "3.119.0", @@ -747,16 +608,6 @@ "Microsoft.IdentityModel.Tokens": "8.16.0" } }, - "System.IO.FileSystem.AccessControl": { - "type": "Transitive", - "resolved": "5.0.0", - "contentHash": "SxHB3nuNrpptVk+vZ/F+7OHEpoHUIKKMl02bUmYHQr1r+glbZQxs7pRtsf4ENO29TVm2TH3AEeep2fJcy92oYw==" - }, - "System.Security.Cryptography.ProtectedData": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "BKt0SQgq2lq3ESE68jkeLwv95ypANrPDtkTOIFGcnhg2aRUeUUBDrQlkVDZctrg1WenVcvn5P5XZnuYI7q6rFQ==" - }, "System.ServiceProcess.ServiceController": { "type": "Transitive", "resolved": "10.0.10", @@ -765,11 +616,6 @@ "System.Diagnostics.EventLog": "10.0.10" } }, - "Velopack": { - "type": "Transitive", - "resolved": "1.2.0", - "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" - }, "xunit.analyzers": { "type": "Transitive", "resolved": "1.27.0", @@ -854,7 +700,7 @@ "dependencies": { "CredentialManagement": "[1.0.2, )", "Microsoft.Extensions.Logging.Abstractions": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )" + "ModelContextProtocol": "[2.1.0, )" } }, "performancemonitor.darling.analysis": { @@ -874,8 +720,8 @@ "Microsoft.Data.SqlClient": "[7.0.2, )", "Microsoft.Extensions.Hosting": "[10.0.10, )", "Microsoft.Extensions.Hosting.WindowsServices": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )", - "ModelContextProtocol.AspNetCore": "[2.0.0, )", + "ModelContextProtocol": "[2.1.0, )", + "ModelContextProtocol.AspNetCore": "[2.1.0, )", "PerformanceMonitor.Alerting": "[1.0.0, )", "PerformanceMonitor.Collectors": "[1.0.0, )", "PerformanceMonitor.Common": "[1.0.0, )", @@ -930,6 +776,176 @@ "PerformanceMonitor.PlanAnalysis": "[1.0.0, )", "ScottPlot.WPF": "[5.1.59, )" } + }, + "CredentialManagement": { + "type": "CentralTransitive", + "requested": "[1.0.2, )", + "resolved": "1.0.2", + "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" + }, + "Hardcodet.NotifyIcon.Wpf": { + "type": "CentralTransitive", + "requested": "[2.0.1, )", + "resolved": "2.0.1", + "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" + }, + "Microsoft.Data.SqlClient": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "9.0.13", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", + "Microsoft.Extensions.Caching.Memory": "9.0.13", + "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", + "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" + } + }, + "Microsoft.Extensions.Hosting": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.Binder": "10.0.10", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.Configuration.Json": "10.0.10", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Diagnostics": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", + "Microsoft.Extensions.FileProviders.Physical": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging.Configuration": "10.0.10", + "Microsoft.Extensions.Logging.Console": "10.0.10", + "Microsoft.Extensions.Logging.Debug": "10.0.10", + "Microsoft.Extensions.Logging.EventLog": "10.0.10", + "Microsoft.Extensions.Logging.EventSource": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Hosting.WindowsServices": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "qY7XhE2ljtqCwDKexJf3uG4E7r0teE/DU0eEUDbFqGN2HAIz0WK7P3u4RnnnqOy/3Jz5vdTfTuDMFsrwxrcmeg==", + "dependencies": { + "Microsoft.Extensions.Hosting": "10.0.10", + "Microsoft.Extensions.Logging.EventLog": "10.0.10", + "System.ServiceProcess.ServiceController": "10.0.10" + } + }, + "Microsoft.Extensions.Logging": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" + } + }, + "ModelContextProtocol": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "Oa4rU7EL9C2qyFjQj1dx+ysGMzfWDRpM8RRaUMmLGs5vPvfJ9xyz4ZtyF4ychY+Nx1b/auGCqIQLqSz/IpPkKA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "ModelContextProtocol.Core": "[2.1.0]" + } + }, + "ModelContextProtocol.AspNetCore": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "yhJ8bBXIgrX0mAgRYRgzcbH3bLdv3MDSkG52utRW9EAAtQrPw/g7Q/T6EurxKV+L+Zefv8VVUYcNbXEOd9GgfA==", + "dependencies": { + "ModelContextProtocol": "[2.1.0]" + } + }, + "Npgsql": { + "type": "CentralTransitive", + "requested": "[10.0.3, )", + "resolved": "10.0.3", + "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "ScottPlot.WPF": { + "type": "CentralTransitive", + "requested": "[5.1.59, )", + "resolved": "5.1.59", + "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", + "dependencies": { + "OpenTK": "4.9.4", + "OpenTK.GLWpfControl": "4.3.3", + "ScottPlot": "5.1.59", + "SkiaSharp.Views.WPF": "3.119.0" + } + }, + "System.IO.FileSystem.AccessControl": { + "type": "CentralTransitive", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "SxHB3nuNrpptVk+vZ/F+7OHEpoHUIKKMl02bUmYHQr1r+glbZQxs7pRtsf4ENO29TVm2TH3AEeep2fJcy92oYw==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "BKt0SQgq2lq3ESE68jkeLwv95ypANrPDtkTOIFGcnhg2aRUeUUBDrQlkVDZctrg1WenVcvn5P5XZnuYI7q6rFQ==" + }, + "Velopack": { + "type": "CentralTransitive", + "requested": "[1.2.0, )", + "resolved": "1.2.0", + "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" } } } diff --git a/Darling/Dockerfile b/Darling/Dockerfile index 7dfb98036..af9cb9d96 100644 --- a/Darling/Dockerfile +++ b/Darling/Dockerfile @@ -13,7 +13,16 @@ # Secrets never land in darling.json — use env:/file: references (#1804 stage 1), which are compose # `secrets:`-friendly. -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +# Pinned to the SDK feature band global.json asks for, NOT the floating :10.0 tag. global.json requests +# 10.0.302 with rollForward "latestPatch", which rolls only inside the 3xx band — so the moment the floating +# tag advanced to an SDK 10.0.400 image, every container build died with "A compatible .NET SDK was not +# found ... Requested SDK version: 10.0.302 / Installed SDKs: 10.0.400" and a bare exit code 155. It broke +# dev and both open PRs at once, and it broke on a Microsoft image refresh rather than on any commit here. +# +# The runner-side build never saw it because setup-dotnet resolves its SDK FROM global.json; only the +# container had an independent opinion about which SDK to use. Pinning removes that second opinion: bump +# this line and global.json together, deliberately, instead of being bumped by an upstream tag move. +FROM mcr.microsoft.com/dotnet/sdk:10.0.302 AS build WORKDIR /src COPY . . RUN dotnet publish Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj \ diff --git a/Darling/PerformanceMonitor.Darling.Analysis/AnalysisShutdown.cs b/Darling/PerformanceMonitor.Darling.Analysis/AnalysisShutdown.cs new file mode 100644 index 000000000..da9d4cf3c --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Analysis/AnalysisShutdown.cs @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using Npgsql; + +namespace PerformanceMonitor.Darling.Analysis; + +/// +/// Classifies whether an analysis-pass failure is the residue of the host shutting down, so the +/// catch sites can tell unfinished because we asked it to stop from unfinished because +/// something broke (#2299). Before this, a clean Stop-Service logged seven ERRORs from +/// work still in flight after "collection loop stopped" — the loop's data source is disposed at +/// method scope exit and the managed postmaster is then pg_ctl stop -m fast-ed, so the +/// abandoned pass's next store read throws (or the server +/// kills its open connection with 57P01), and those seven lines were 7 of the day's 9 ERRORs, +/// burying the two that meant something. +/// +public static class AnalysisShutdown +{ + /// + /// True when this failure should be ABANDONED quietly because the host is stopping: the + /// stopping token has fired AND the exception is a shape shutdown produces. Both halves are + /// load-bearing — the same exceptions with the token NOT signalled mean a data source was + /// disposed (or a connection administratively killed) while the service was meant to be + /// running, which is a real bug whose only evidence is exactly this text, so it must stay + /// an ERROR. Catch sites use this in a when filter so shutdown residue propagates + /// (unwinding the pass to one Information line) instead of being swallowed per-metric. + /// + public static bool IsShutdownAbandon(Exception ex, CancellationToken stoppingToken) => + stoppingToken.IsCancellationRequested && IsShutdownResidue(ex); + + /// + /// The exception shapes a stop produces, detected structurally (the + /// discipline — never message matching): + /// is the token observed properly; + /// (bare or wrapped one level, Npgsql surfaces both) is + /// the loop's data source disposed underneath an in-flight read; SQLSTATE 57P01/57P02/57P03 + /// are the postmaster going away server-side — the same trio + /// PostgresTargetProvider classifies as connection-fatal. A + /// is deliberately NOT residue: a command timeout coinciding + /// with shutdown still means the query outgrew its deadline, and relabelling it would hide + /// the growth signal #2294 made visible. + /// + internal static bool IsShutdownResidue(Exception ex) => + ex is OperationCanceledException + || ex is ObjectDisposedException + || ex.InnerException is ObjectDisposedException + || ex is PostgresException { SqlState: "57P01" or "57P02" or "57P03" }; +} diff --git a/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs b/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs index 147236ab1..afab53daf 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Npgsql; @@ -115,7 +116,8 @@ public DarlingAnalysisService(NpgsqlDataSource postgres, IPlanFetcher? planFetch /// Default time range is the last 4 hours. Host-UTC window (Lite's clock semantics — /// Darling's collectors stamp rows with the service host's UTC clock). /// - public async Task> AnalyzeAsync(int serverId, string serverName, int hoursBack = 4) + public async Task> AnalyzeAsync( + int serverId, string serverName, int hoursBack = 4, CancellationToken cancellationToken = default) { var timeRangeEnd = DateTime.UtcNow; var timeRangeStart = timeRangeEnd.AddHours(-hoursBack); @@ -125,7 +127,8 @@ public async Task> AnalyzeAsync(int serverId, string serve ServerId = serverId, ServerName = serverName, TimeRangeStart = timeRangeStart, - TimeRangeEnd = timeRangeEnd + TimeRangeEnd = timeRangeEnd, + CancellationToken = cancellationToken }; return await AnalyzeAsync(context); @@ -146,7 +149,7 @@ public async Task> AnalyzeAsync(AnalysisContext context) { // 0. Check minimum data span — total history, not the analysis window. // A server with 100h of total history can be analyzed over a 4h window. - var dataSpanHours = await GetTotalDataSpanHoursAsync(context.ServerId); + var dataSpanHours = await GetTotalDataSpanHoursAsync(context.ServerId, context.CancellationToken); if (dataSpanHours < MinimumDataHours) { var needed = MinimumDataHours >= 24 @@ -168,6 +171,14 @@ public async Task> AnalyzeAsync(AnalysisContext context) return []; } + /* #2299: abandon BETWEEN the expensive store stages when the host is stopping. The + fact collector's per-query catches are deliberately silent, so a stop mid-collect + cannot unwind from inside it — these boundary checks are what turn the token into + an exit. The post-enrichment tail (action build + insert) carries no check: by + then the expensive work is done, and finishing preserves it when the store is + still up, while a store already gone classifies quietly. */ + context.CancellationToken.ThrowIfCancellationRequested(); + // 1. Collect facts from the Postgres store var facts = await _collector.CollectFactsAsync(context); @@ -177,6 +188,8 @@ public async Task> AnalyzeAsync(AnalysisContext context) return []; } + context.CancellationToken.ThrowIfCancellationRequested(); + // 1.5. Detect anomalies (compare analysis window against baseline) var anomalies = await _anomalyDetector.DetectAnomaliesAsync(context); facts.AddRange(anomalies); @@ -257,6 +270,17 @@ public async Task> AnalyzeAsync(AnalysisContext context) return findings; } + catch (Exception ex) when (AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) + { + /* #2299: the ONE line a stop is allowed to cost. The component catches let shutdown + residue propagate instead of logging it per-metric, so seven ERRORs collapse to + this Information — and it states the loss honestly: whatever this pass would have + written is gone, and the next scheduled pass recomputes it from the store. */ + _logger?.LogInformation( + "[DarlingAnalysisService] Analysis abandoned at shutdown for {Server} — this pass's findings are lost by design; the next pass recomputes them ({Detail})", + context.ServerName, ex.Message); + return []; + } catch (Exception ex) { _logger?.LogError("[DarlingAnalysisService] Analysis failed for {Server}: {Message}", @@ -395,23 +419,26 @@ FROM wait_stats /// the analysis window. A server with 100 hours of total history can safely /// be analyzed over a 4-hour window without dilution. /// - private async Task GetTotalDataSpanHoursAsync(int serverId) + private async Task GetTotalDataSpanHoursAsync(int serverId, CancellationToken cancellationToken) { try { - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); using var cmd = new NpgsqlCommand(TotalDataSpanSql, connection); cmd.Parameters.AddWithValue(serverId); - var result = await cmd.ExecuteScalarAsync(); + var result = await cmd.ExecuteScalarAsync(cancellationToken); if (result == null || result is DBNull) return 0; return Convert.ToDouble(result); } - catch + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken)) { + /* Probe failure reads as "no data yet" — EXCEPT shutdown residue, which must not be + allowed to masquerade as a 0-hour history (#2299): it propagates to the pass's + shutdown catch instead of producing a bogus insufficient-data skip. */ return 0; } } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PerformanceMonitor.Darling.Analysis.csproj b/Darling/PerformanceMonitor.Darling.Analysis/PerformanceMonitor.Darling.Analysis.csproj index 302aa629c..ddc4f7476 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PerformanceMonitor.Darling.Analysis.csproj +++ b/Darling/PerformanceMonitor.Darling.Analysis/PerformanceMonitor.Darling.Analysis.csproj @@ -16,7 +16,7 @@ - + @@ -33,6 +33,9 @@ + + diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs index b4dc279e0..e9e82e7da 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Npgsql; @@ -105,7 +106,7 @@ public async Task> DetectAnomaliesAsync(AnalysisContext context) var anomalies = new List(); // Check if baseline period has any data at all — if not, skip all anomaly detection. - if (!await HasBaselineDataAsync(context.ServerId)) + if (!await HasBaselineDataAsync(context.ServerId, context.CancellationToken)) return anomalies; // Existing detection methods (upgraded to time-bucketed baselines) @@ -288,7 +289,7 @@ private async Task DetectObjectStatsAnomalies(AnalysisContext context, List - private async Task HasBaselineDataAsync(int serverId) + private async Task HasBaselineDataAsync(int serverId, CancellationToken cancellationToken) { try { - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); using var cmd = new NpgsqlCommand(HasBaselineDataSql, connection); cmd.Parameters.AddWithValue(serverId); @@ -386,10 +387,17 @@ private async Task HasBaselineDataAsync(int serverId) made Kind-Unspecified for the naive-UTC timestamp columns. */ cmd.Parameters.AddWithValue(AsNaive(DateTime.UtcNow.AddDays(-30))); - var count = Convert.ToInt64(await cmd.ExecuteScalarAsync() ?? 0); + var count = Convert.ToInt64(await cmd.ExecuteScalarAsync(cancellationToken) ?? 0); return count > 0; } - catch { return false; } + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken)) + { + /* Silent on a genuine fault BY DESIGN (Lite's gate posture: an unreadable canary reads + as "no baseline data" and detection just sits out the pass) — but shutdown residue is + excluded (#2299), or a stop mid-gate would masquerade as an empty baseline instead of + unwinding to the pass's single Information line like every other read here. */ + return false; + } } /// @@ -400,22 +408,22 @@ private async Task DetectCpuAnomalies(AnalysisContext context, List anomal try { var baseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.Cpu, context.TimeRangeStart); + context.ServerId, MetricNames.Cpu, context.TimeRangeStart, context.CancellationToken); if (baseline.SampleCount == 0) return; // No effectiveStdDev<=0 early return — an untrustworthy/zero-dispersion baseline falls // back to the absolute bar (below) rather than going silent. var effectiveStdDev = baseline.EffectiveStdDev; - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var cmd = new NpgsqlCommand(CpuWindowSql, connection); cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var reader = await cmd.ExecuteReaderAsync(); - if (!await reader.ReadAsync()) return; + using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) return; var peakCpu = reader.IsDBNull(0) ? 0.0 : Convert.ToDouble(reader.GetValue(0)); var avgCpu = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); @@ -454,7 +462,7 @@ private async Task DetectCpuAnomalies(AnalysisContext context, List anomal Metadata = metadata }); } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] CPU anomaly detection failed: {Message}", ex.Message); } @@ -474,9 +482,9 @@ private async Task DetectWaitAnomalies(AnalysisContext context, List anoma try { var baseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.WaitMsPerSec, context.TimeRangeStart); + context.ServerId, MetricNames.WaitMsPerSec, context.TimeRangeStart, context.CancellationToken); - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); // Current window: all-types wait ms/sec per collection (interval via LAG), then PEAK. double peakRate; @@ -488,8 +496,8 @@ private async Task DetectWaitAnomalies(AnalysisContext context, List anoma rateCmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); rateCmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var rateReader = await rateCmd.ExecuteReaderAsync(); - if (!await rateReader.ReadAsync()) return; + using var rateReader = await rateCmd.ExecuteReaderAsync(context.CancellationToken); + if (!await rateReader.ReadAsync(context.CancellationToken)) return; peakRate = rateReader.IsDBNull(0) ? 0.0 : Convert.ToDouble(rateReader.GetValue(0)); totalWaitMs = rateReader.IsDBNull(1) ? 0.0 : Convert.ToDouble(rateReader.GetValue(1)); collectionCount = rateReader.IsDBNull(2) ? 0L : Convert.ToInt64(rateReader.GetValue(2)); @@ -548,8 +556,8 @@ floor is what was measured WITH the 5.0 cutoff. The ratio still rides the metada contribCmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); contribCmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var contribReader = await contribCmd.ExecuteReaderAsync(); - while (await contribReader.ReadAsync()) + using var contribReader = await contribCmd.ExecuteReaderAsync(context.CancellationToken); + while (await contribReader.ReadAsync(context.CancellationToken)) { var waitType = contribReader.GetString(0); metadata[$"contrib_{waitType}"] = Convert.ToDouble(contribReader.GetValue(1)); @@ -565,7 +573,7 @@ floor is what was measured WITH the 5.0 cutoff. The ratio still rides the metada Metadata = metadata }); } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] Wait anomaly detection failed: {Message}", ex.Message); } @@ -580,19 +588,19 @@ private async Task DetectBlockingAnomalies(AnalysisContext context, List a try { var blockingBaseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.Blocking, context.TimeRangeStart); + context.ServerId, MetricNames.Blocking, context.TimeRangeStart, context.CancellationToken); var deadlockBaseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.Deadlock, context.TimeRangeStart); + context.ServerId, MetricNames.Deadlock, context.TimeRangeStart, context.CancellationToken); - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var cmd = new NpgsqlCommand(BlockingWindowSql, connection); cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var reader = await cmd.ExecuteReaderAsync(); - if (!await reader.ReadAsync()) return; + using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) return; var currentBlocking = Convert.ToInt64(reader.GetValue(0)); var currentDeadlocks = Convert.ToInt64(reader.GetValue(1)); @@ -663,7 +671,7 @@ so normalize them to per-hour before the ratio — otherwise the ratio scales wi }); } } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] Blocking anomaly detection failed: {Message}", ex.Message); } @@ -677,20 +685,20 @@ private async Task DetectIoAnomalies(AnalysisContext context, List anomali try { var baseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.IoLatency, context.TimeRangeStart); + context.ServerId, MetricNames.IoLatency, context.TimeRangeStart, context.CancellationToken); if (baseline.SampleCount == 0) return; var effectiveStdDev = baseline.EffectiveStdDev; - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var cmd = new NpgsqlCommand(IoWindowSql, connection); cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var reader = await cmd.ExecuteReaderAsync(); - if (!await reader.ReadAsync()) return; + using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) return; var currentReadLat = reader.IsDBNull(0) ? 0.0 : Convert.ToDouble(reader.GetValue(0)); var currentWriteLat = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); @@ -755,7 +763,7 @@ private async Task DetectIoAnomalies(AnalysisContext context, List anomali }); } } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] I/O anomaly detection failed: {Message}", ex.Message); } @@ -769,20 +777,20 @@ private async Task DetectBatchRequestAnomalies(AnalysisContext context, List an try { var baseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.SessionCount, context.TimeRangeStart); + context.ServerId, MetricNames.SessionCount, context.TimeRangeStart, context.CancellationToken); if (baseline.SampleCount == 0) return; var effectiveStdDev = baseline.EffectiveStdDev; - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var cmd = new NpgsqlCommand(SessionWindowSql, connection); cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var reader = await cmd.ExecuteReaderAsync(); - if (!await reader.ReadAsync()) return; + using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) return; var avgConnections = reader.IsDBNull(0) ? 0.0 : Convert.ToDouble(reader.GetValue(0)); var peakConnections = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); @@ -883,7 +891,7 @@ private async Task DetectSessionAnomalies(AnalysisContext context, List an Metadata = metadata }); } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] Session anomaly detection failed: {Message}", ex.Message); } @@ -898,20 +906,20 @@ private async Task DetectQueryDurationAnomalies(AnalysisContext context, List ano try { var baseline = await _baselineProvider.GetBaselineAsync( - context.ServerId, MetricNames.Memory, context.TimeRangeStart); + context.ServerId, MetricNames.Memory, context.TimeRangeStart, context.CancellationToken); if (baseline.SampleCount == 0) return; var effectiveStdDev = baseline.EffectiveStdDev; - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var cmd = new NpgsqlCommand(MemoryWindowSql, connection); cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); - using var reader = await cmd.ExecuteReaderAsync(); - if (!await reader.ReadAsync()) return; + using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) return; var avgPressure = reader.IsDBNull(0) ? 0.0 : Convert.ToDouble(reader.GetValue(0)); var peakPressure = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); @@ -1014,7 +1022,7 @@ private async Task DetectMemoryAnomalies(AnalysisContext context, List ano Metadata = metadata }); } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgAnomalyDetector] Memory anomaly detection failed: {Message}", ex.Message); } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs index a388a08d9..96534f1f7 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs @@ -10,6 +10,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Npgsql; @@ -73,12 +74,12 @@ public PgBaselineProvider(NpgsqlDataSource postgres, ILogger? logger = null) /// Returns the most specific bucket available, collapsing as needed. /// public async Task GetBaselineAsync( - int serverId, string metricName, DateTime analysisTime) + int serverId, string metricName, DateTime analysisTime, CancellationToken cancellationToken = default) { var hourOfDay = analysisTime.Hour; var dayOfWeek = (int)analysisTime.DayOfWeek; // Sunday=0 — matches EXTRACT(DOW) in both engines - var baselines = await GetOrComputeBaselinesAsync(serverId, metricName, analysisTime); + var baselines = await GetOrComputeBaselinesAsync(serverId, metricName, analysisTime, cancellationToken); if (baselines == null || baselines.Count == 0) return BaselineBucket.Empty; @@ -97,7 +98,7 @@ public void InvalidateCache(int serverId) public void ClearCache() => _cache.Clear(); private async Task?> GetOrComputeBaselinesAsync( - int serverId, string metricName, DateTime analysisTime) + int serverId, string metricName, DateTime analysisTime, CancellationToken cancellationToken) { var cacheKey = $"{serverId}:{metricName}"; var roundedHour = new DateTime(analysisTime.Year, analysisTime.Month, analysisTime.Day, analysisTime.Hour, 0, 0); @@ -109,7 +110,7 @@ public void InvalidateCache(int serverId) return cached.Buckets; } - var buckets = await ComputeBaselinesAsync(serverId, metricName, analysisTime); + var buckets = await ComputeBaselinesAsync(serverId, metricName, analysisTime, cancellationToken); _cache[cacheKey] = new CachedBaseline { @@ -122,17 +123,47 @@ public void InvalidateCache(int serverId) } private async Task?> ComputeBaselinesAsync( - int serverId, string metricName, DateTime analysisTime) + int serverId, string metricName, DateTime analysisTime, CancellationToken cancellationToken) { var query = GetBaselineQuery(metricName); if (query == null) return null; + return await ComputeBucketsAsync(serverId, metricName, analysisTime, query, cancellationToken); + } + + /// + /// Did this failure mean "the statement ran out of time" rather than "the connection broke"? + /// + /// Worth a named predicate because the two are indistinguishable in the message Npgsql produces. + /// Npgsql enforces its command timeout by CANCELLING the statement, so the server logs + /// canceling statement due to user request and the client is left holding a torn stream, which it + /// reports as "Exception while reading from stream". Read literally that says the network failed; what + /// actually happened is a query outgrowing its deadline on a store that grew. On the dogfood box the two + /// log lines sat 267 ms apart in different files, and correlating them by hand is not a diagnosis the + /// next person should have to repeat. + /// + /// Structural, not message matching: 57014 is query_canceled, the server saying it + /// cancelled; a anywhere in the chain is Npgsql's own deadline. Everything + /// else stays "failed", because labelling a genuine connection fault a timeout is the same defect aimed + /// the other way. + /// + internal static bool IsCommandTimeout(Exception ex) => + ex is PostgresException { SqlState: "57014" } + || ex is TimeoutException + || ex.InnerException is TimeoutException; + + private async Task?> ComputeBucketsAsync( + int serverId, string metricName, DateTime analysisTime, string query, CancellationToken cancellationToken) + { var absStdDevFloor = BaselineMath.AbsStdDevFloorFor(metricName); var windowStart = analysisTime.AddDays(-BaselineMath.BaselineWindowDays); + /* Timed so the failure path can say how long it got, not just that it failed — see the catch. */ + var elapsed = System.Diagnostics.Stopwatch.StartNew(); + try { - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); using var cmd = new NpgsqlCommand(query, connection); cmd.Parameters.AddWithValue(serverId); @@ -143,13 +174,13 @@ public void InvalidateCache(int serverId) var buckets = new Dictionary<(int, int), BaselineBucket>(); - using var reader = await cmd.ExecuteReaderAsync(); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); /* #1743: the robust-scaffold metrics return eight columns (…, median_val, mad_val) and carry sentinel tier rows; the two event-family metrics (blocking, deadlock) keep the six-column classical shape — detected by column count, so their buckets read Median=0/Mad=0 and the robust path degrades for them. */ var hasRobustColumns = reader.FieldCount >= 8; - while (await reader.ReadAsync()) + while (await reader.ReadAsync(cancellationToken)) { var hour = Convert.ToInt32(reader.GetValue(0)); var dow = Convert.ToInt32(reader.GetValue(1)); @@ -183,9 +214,33 @@ public void InvalidateCache(int serverId) return buckets; } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, cancellationToken)) { - _logger?.LogError("[PgBaselineProvider] Failed to compute baselines for {MetricName}: {Message}", metricName, ex.Message); + /* A command TIMEOUT and a genuine connection fault are the same message here, and that cost real + diagnosis time on the dogfood box: Npgsql surfaces its own client-side timeout as + "Exception while reading from stream" — it cancels the statement, the server logs + `canceling statement due to user request`, and the client only sees the torn stream. Read + literally, that says "the network broke"; what actually happened is that this query outgrew its + timeout on a store that had grown. Correlating the two logs by timestamp (267 ms apart) is not + something the next person should have to redo, so the distinction is reported here. + + Detected structurally rather than by message text: 57014 is the server telling us it cancelled, + and a TimeoutException anywhere in the chain is Npgsql's own deadline. Anything else keeps the + old wording, because calling a real connection fault a timeout would be the same defect + pointing the other way. */ + if (IsCommandTimeout(ex)) + { + _logger?.LogError( + "[PgBaselineProvider] Baseline query for {MetricName} did not finish within its command timeout — gave up after {Seconds:F1}s, so this metric has NO baseline this pass and its anomaly detection is silent (the collected data is unaffected). The store side logs this as 'canceling statement due to user request'. If it repeats, the window this query scans has outgrown the timeout: {Message}", + metricName, elapsed.Elapsed.TotalSeconds, ex.Message); + } + else + { + _logger?.LogError( + "[PgBaselineProvider] Failed to compute baselines for {MetricName} after {Seconds:F1}s: {Message}", + metricName, elapsed.Elapsed.TotalSeconds, ex.Message); + } + return null; } } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs index b0bc40650..dc2b980dc 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs @@ -280,7 +280,45 @@ private async Task CollectParameterSensitiveQueries(AnalysisFinding finding, Ana } public const string RegressedQueriesSql = @" -WITH deduped AS +WITH psp_signature AS +( + -- #2138 gap 3: the PARAMETER_SENSITIVITY detector's EXACT firing signature (same floors, same + -- ratio, same analysis window) reduced to the (database, query_hash) set it would report. Using + -- the detector's own thresholds is what keeps the flag honest: a query flagged here IS one the + -- detector counts when it fires, never a looser lookalike. Grant/spill divergence stay metadata + -- on the PSP side — they do not fire the detector alone, so they do not fire this flag alone. + SELECT DISTINCT + database_name, + query_hash + FROM + ( + SELECT + database_name, + query_hash, + query_plan_hash, + execution_count, + creation_time, + min_worker_time, + max_worker_time, + ROW_NUMBER() OVER + ( + PARTITION BY database_name, query_hash, query_plan_hash + ORDER BY collection_time DESC + ) AS rn + FROM v_query_stats + WHERE server_id = $1 + AND collection_time >= $3 + AND collection_time <= $4 + AND delta_execution_count > 0 + ) AS latest_cache + WHERE rn = 1 + AND min_worker_time >= 10000 + AND max_worker_time >= 250000 + AND execution_count >= 20 + AND creation_time <= $3 + AND max_worker_time::DOUBLE PRECISION / NULLIF(min_worker_time, 0) >= 10 +), +deduped AS ( -- LOAD-BEARING (correctness, not just perf): query_store_stats rows are CUMULATIVE per-Query-Store- -- interval snapshots. The QueryStoreCollector is incremental and re-fetches the OPEN interval every @@ -305,6 +343,7 @@ WITH deduped AS plan_id, replica_role, query_plan_hash, + query_hash, execution_count, avg_cpu_time_us, avg_duration_us, @@ -338,6 +377,7 @@ plan_dedup AS replica_role, query_plan_hash, MAX(plan_id) AS plan_id, + any_value(query_hash) AS query_hash, any_value(query_text) AS query_text, SUM(execution_count) AS execs, SUM(avg_cpu_time_us * execution_count)::DOUBLE PRECISION / NULLIF(SUM(execution_count), 0) AS cpu_per_exec, @@ -366,39 +406,81 @@ cheapest AS SELECT DISTINCT ON (database_name, query_id, replica_role) * FROM plan_dedup ORDER BY database_name, query_id, replica_role, cpu_per_exec ASC +), +scored AS +( + SELECT + l.database_name, + l.query_id, + l.query_plan_hash AS latest_plan_hash, + l.cpu_per_exec AS latest_cpu, + l.dur_per_exec AS latest_dur, + b.query_plan_hash AS best_plan_hash, + b.plan_id AS best_plan_id, + b.cpu_per_exec AS best_cpu, + b.dur_per_exec AS best_dur, + -- #2138: the SAME CPU-primary scoring as the PLAN_REGRESSION fact (PgFactCollector.QueryPerf.cs, + -- where the rationale lives). The drill-down must agree with the fact that displays it: under the + -- old GREATEST a duration-only regression could appear here that the fact never counted. + CASE + WHEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 2 + THEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) + WHEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) >= 4 + AND l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 1.25 + THEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) / 2 + END AS regression_factor, + -- #2150: text lives in collect.query_store_text now, keyed on (server, database, query_id) — the + -- grain `latest` is already at — so it resolves with the keyed join below rather than being carried + -- up through any_value(). l.query_text stays as the fallback: it is where text lived before the + -- cutover, so history collected earlier still shows a statement instead of an empty drill-down. + LEFT(COALESCE(x.query_sql_text, l.query_text), 500) AS query_text, + l.replica_role, + l.execs * l.cpu_per_exec AS latest_total_cpu_us, + -- #2138 gap 3: does this regressed query ALSO carry the parameter-sensitivity signature in the + -- plan cache? Keyed on (database, query_hash) — the hash bridges Query Store and the cache. + -- Steers the force-plan remediation's caution text; the future bot never auto-forces on true. + EXISTS + ( + SELECT 1 + FROM psp_signature AS p + WHERE p.database_name = l.database_name + AND p.query_hash = l.query_hash + ) AS parameter_sensitivity_cofired + FROM latest AS l + JOIN cheapest AS b + ON b.database_name = l.database_name + AND b.query_id = l.query_id + -- IS NOT DISTINCT FROM, never = (and never USING, which is an equi-join): replica_role is NULL on + -- every standalone server, every non-AG server and everything below SQL Server 2022, and NULL = NULL + -- is UNKNOWN — matching on it with = would join nothing and silently empty this drill-down for the + -- overwhelming majority of installs. The NULL-safe operator groups those rows as DISTINCT ON does. + AND b.replica_role IS NOT DISTINCT FROM l.replica_role + -- #2150 text resolution (see the projection). LEFT, so a query whose text has not been fetched yet + -- still reports its regression; one row per key by primary key, so no fan-out and none of the + -- aggregates above are affected. + LEFT JOIN query_store_text AS x + ON x.server_id = $1 + AND x.database_name = l.database_name + AND x.query_id = l.query_id + WHERE l.query_plan_hash <> b.query_plan_hash ) SELECT - l.database_name, - l.query_id, - l.query_plan_hash AS latest_plan_hash, - l.cpu_per_exec AS latest_cpu, - l.dur_per_exec AS latest_dur, - b.query_plan_hash AS best_plan_hash, - b.plan_id AS best_plan_id, - b.cpu_per_exec AS best_cpu, - b.dur_per_exec AS best_dur, - GREATEST - ( - l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0), - l.dur_per_exec / NULLIF(b.dur_per_exec, 0) - ) AS regression_factor, - LEFT(l.query_text, 500) AS query_text, - l.replica_role -FROM latest AS l -JOIN cheapest AS b - ON b.database_name = l.database_name - AND b.query_id = l.query_id - -- IS NOT DISTINCT FROM, never = (and never USING, which is an equi-join): replica_role is NULL on - -- every standalone server, every non-AG server and everything below SQL Server 2022, and NULL = NULL - -- is UNKNOWN — matching on it with = would join nothing and silently empty this drill-down for the - -- overwhelming majority of installs. The NULL-safe operator groups those rows as DISTINCT ON does. - AND b.replica_role IS NOT DISTINCT FROM l.replica_role -WHERE l.query_plan_hash <> b.query_plan_hash -AND GREATEST - ( - l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0), - l.dur_per_exec / NULLIF(b.dur_per_exec, 0) - ) >= 2 + database_name, + query_id, + latest_plan_hash, + latest_cpu, + latest_dur, + best_plan_hash, + best_plan_id, + best_cpu, + best_dur, + regression_factor, + query_text, + replica_role, + parameter_sensitivity_cofired +FROM scored +WHERE regression_factor >= 2 +AND latest_total_cpu_us >= 10000000 ORDER BY regression_factor DESC LIMIT 5"; @@ -416,6 +498,11 @@ private async Task CollectRegressedQueries(AnalysisFinding finding, AnalysisCont cmd.CommandTimeout = DrillDownCommandTimeoutSeconds; cmd.Parameters.AddWithValue(context.ServerId); cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart.AddDays(-14))); + /* $3/$4: the STANDARD analysis window for the psp_signature CTE — deliberately not the 14-day + comparison window above, so the flag matches what the PARAMETER_SENSITIVITY detector itself + would report for this run. */ + cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); + cmd.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); var items = new List(); using var reader = await cmd.ExecuteReaderAsync(); @@ -438,8 +525,11 @@ private async Task CollectRegressedQueries(AnalysisFinding finding, AnalysisCont standalone/non-AG/pre-2022 server, which is the overwhelming majority; it is only populated on an AG primary with Query Store for secondary replicas enabled, where two rows for the same query are now legitimately distinct rather than one silently dropped. - Last in the row so the existing reader ordinals are untouched. */ - replica_role = reader.IsDBNull(11) ? "" : reader.GetString(11) + Appended after the older columns so the existing reader ordinals are untouched. */ + replica_role = reader.IsDBNull(11) ? "" : reader.GetString(11), + /* #2138 gap 3: the plan-cache PSP signature co-fired for this query's hash. Steers the + force-plan caution text; the future bot never auto-forces a flagged target. */ + parameter_sensitivity_cofired = !reader.IsDBNull(12) && reader.GetBoolean(12) }); } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.cs index 4af698cc8..38f4f930a 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.cs @@ -106,6 +106,11 @@ public async Task EnrichFindingsAsync(List findings, AnalysisCo { foreach (var finding in findings) { + /* #2299: between findings is the natural abandon point — the per-finding catch below + deliberately does NOT swallow shutdown residue, so this throw (and any residue from + a drill-down mid-read) unwinds the pass to the service's single Information line. */ + context.CancellationToken.ThrowIfCancellationRequested(); + try { finding.DrillDown = new Dictionary(); @@ -187,7 +192,7 @@ 0.5 display gate. */ if (finding.DrillDown.Count == 0) finding.DrillDown = null; } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgDrillDownCollector] Drill-down failed for {StoryPath}: {ExceptionType}: {Message}", finding.StoryPath, ex.GetType().Name, ex.Message); diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs index cf548e175..ef05508e5 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs @@ -302,11 +302,22 @@ compared AS l.force_failure_count AS force_failure_count, b.cpu_per_exec AS best_cpu, b.dur_per_exec AS best_dur, - GREATEST - ( - l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0), - l.dur_per_exec / NULLIF(b.dur_per_exec, 0) - ) AS regression_factor + -- #2138: CPU is the PRIMARY signal — duration alone is confounded by blocking, IO waits, and + -- machine contention that no plan choice caused, so it must not fire a plan-regression verdict + -- by itself. A CPU regression scores at its own ratio; a duration-dominant one fires only when + -- EXTREME (>= 4x) AND corroborated by at least mild CPU worsening (>= 1.25x), scored at half + -- the duration ratio so it competes honestly with CPU-detected rows. NULL when neither path + -- fires — the >= 2 gate below drops it. + CASE + WHEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 2 + THEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) + WHEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) >= 4 + AND l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 1.25 + THEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) / 2 + END AS regression_factor, + -- The resource-expenditure half of the importance gate (#2138): total CPU the LATEST plan burned + -- over the window. The exec-count floor above only counts; this weighs. + l.execs * l.cpu_per_exec AS latest_total_cpu_us FROM ranked AS l JOIN ranked AS b ON b.database_name = l.database_name @@ -327,6 +338,10 @@ AND l.query_plan_hash <> b.query_plan_hash regression_factor FROM compared WHERE regression_factor >= 2 +-- 10 CPU-seconds across the window: a NOISE floor, not an importance ranking — it exists to exclude +-- near-zero-cost queries whose ratios are all sampling jitter; magnitude ranking stays with +-- regression_factor and the scorer. +AND latest_total_cpu_us >= 10000000 ORDER BY regression_factor DESC LIMIT 20"; @@ -365,18 +380,18 @@ private async Task CollectPlanRegressionFactsAsync(AnalysisContext context, List { worstQueryId = reader.IsDBNull(0) ? 0L : ToInt64(reader.GetValue(0)); var latestCpu = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); - var latestDur = reader.IsDBNull(2) ? 0.0 : Convert.ToDouble(reader.GetValue(2)); worstLatestForced = (!reader.IsDBNull(3) && Convert.ToBoolean(reader.GetValue(3))) ? 1 : 0; worstForceFailures = reader.IsDBNull(4) ? 0L : ToInt64(reader.GetValue(4)); var bestCpu = reader.IsDBNull(5) ? 0.0 : Convert.ToDouble(reader.GetValue(5)); - var bestDur = reader.IsDBNull(6) ? 0.0 : Convert.ToDouble(reader.GetValue(6)); worstFactor = reader.IsDBNull(7) ? 0.0 : Convert.ToDouble(reader.GetValue(7)); worstLatestCpu = latestCpu; worstBestCpu = bestCpu; + // Which CASE branch fired, not which raw ratio is larger (review catch on #2138): + // CPU has PRECEDENCE in the scoring, so a row with cpu 2.5x and duration 10x is a + // CPU-detected regression at 2.5 — comparing magnitudes would mislabel it duration. var cpuRatio = bestCpu > 0 ? latestCpu / bestCpu : 0.0; - var durRatio = bestDur > 0 ? latestDur / bestDur : 0.0; - worstDimension = cpuRatio >= durRatio ? 1 : 2; // 1 = cpu, 2 = duration + worstDimension = cpuRatio >= 2 ? 1 : 2; // 1 = cpu, 2 = duration } offenderCount++; } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs index a933ac4b9..acb2995e7 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs @@ -166,7 +166,7 @@ public async Task> FilterMutedFindingsAsync( try { - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); var mutedHashes = await GetMutedHashesAsync(connection, context.ServerId); foreach (var story in stories) @@ -208,7 +208,7 @@ public async Task> FilterMutedFindingsAsync( }); } } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgFindingStore] FilterMutedFindingsAsync failed: {Message}", ex.Message); } @@ -239,14 +239,14 @@ public async Task> InsertFindingsAsync( try { - await using var connection = await _postgres.OpenConnectionAsync(); + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); foreach (var finding in findings) { await InsertFindingAsync(connection, finding); } } - catch (Exception ex) + catch (Exception ex) when (!AnalysisShutdown.IsShutdownAbandon(ex, context.CancellationToken)) { _logger?.LogError("[PgFindingStore] InsertFindingsAsync failed: {Message}", ex.Message); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs index c9869ddcf..d48c2f166 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs @@ -666,23 +666,82 @@ public async Task GetAnomalousJobsAsync( /// /// Seeds a first-observation baseline for any database in the latest snapshot that has none - /// (insert-if-absent; never overwrites an existing baseline or user override). A CRITICAL first - /// observation (SUSPECT / RECOVERY_PENDING / EMERGENCY) is deliberately NOT baselined: onboarding a - /// server mid-outage must not learn the bad state as expected — such a database stays pending (no - /// row) and the deviation read alerts on it until it recovers or an operator sets an expected state. + /// (insert-if-absent; never overwrites an existing baseline or user override). A first observation in + /// an integrity or transient state is deliberately NOT baselined + /// (): onboarding a server mid-outage or + /// mid-restore must not learn that state as expected — such a database stays pending (no row) until it + /// settles into a steady state, and the deviation read alerts meanwhile only if the state is critical. /// config schema qualified explicitly; database_states resolves to collect through the search_path. /// $1 server_id. /// - public const string SeedDatabaseStateExpectedSql = @" + public const string SeedDatabaseStateExpectedSql = $@" INSERT INTO config.database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at) SELECT $1, ds.database_name, CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END, false, (now() AT TIME ZONE 'UTC') FROM database_states ds WHERE ds.server_id = $1 AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) AND ds.state_desc IS NOT NULL -AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) NOT IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY') +AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) NOT IN ({DatabaseStateTokens.NeverBaselinedSqlList}) ON CONFLICT (server_id, database_name) DO NOTHING"; + /// + /// #2189: re-learns an ILLEGITIMATE inferred baseline as ONLINE once the database reaches ONLINE. The + /// rule is the seed's own, applied after the fact — an expectation recording a state the seed would + /// refuse to learn () is not a baseline at all, + /// it is a snapshot of a database mid-something, and the moment that database is demonstrably healthy + /// the honest move is to learn the steady state rather than page about the improvement forever. + /// + /// This is what heals the rows the old seed already poisoned, which the widened exclusion above + /// cannot: a database baselined RESTORING mid-restore deviated by being healthy forever, and the only + /// escape was an operator noticing and re-baselining by hand. It also covers the route that is still + /// open and always will be — "reset to current" pressed during a restore, or during an outage, records + /// whatever it sees with no state filter at all, and this un-writes it on the next sweep. + /// + /// Two gates, and both matter more than they look. + /// + /// is_user_override = false: an operator who declared an expected state MEANT it, and + /// #2166's composition contract depends on that — a database parked at expected OFFLINE stays silent + /// while parked and still alerts the moment it comes back ONLINE. Only the machine's own inference is + /// second-guessed, never the operator's. + /// + /// The state list, which is deliberately NOT "anything that is not ONLINE". OFFLINE and STANDBY + /// are steady states the seed is happy to learn, and leaving one is real news that must still fire. + /// A STANDBY secondary that turns up ONLINE has stopped being a secondary — somebody recovered it, log + /// shipping is broken, and healing it would replace that alert with silence and then fire the moment + /// the operator FIXED it. An auto-baselined OFFLINE database brought up for an hour and re-parked would + /// come back deviating forever against a baseline it never had. Both are the reported bug's own shape, + /// which is why the heal only ever touches states that were never a legitimate baseline. + /// + /// Reads the EFFECTIVE state, not state_desc — the same CASE as the seed and the deviation + /// read. That is load-bearing rather than cosmetic: a standby log-shipping secondary reports + /// state_desc = 'ONLINE' with is_in_standby set, so matching on the raw column would + /// re-baseline every such secondary to ONLINE and then alert it forever for being STANDBY — the very + /// bug being fixed, re-created for the one database family #1986 went out of its way to keep quiet. + /// + /// The alerted-state memory is dropped with the baseline it described (#2166). A memory saying + /// "the operator was told about ONLINE" only meant anything against the stale expectation; carried + /// past it, it would judge the next episode against an announcement about a baseline that no longer + /// exists. Clearing is the safe direction — it can cost an extra alert, never a missed one. $1 + /// server_id. + /// + public const string HealDatabaseStateBaselineToOnlineSql = $@" +UPDATE config.database_state_expected e +SET expected_state = 'ONLINE', + updated_at = (now() AT TIME ZONE 'UTC'), + last_alerted_state = NULL, + last_alerted_at = NULL +WHERE e.server_id = $1 +AND e.is_user_override = false +AND e.expected_state IN ({DatabaseStateTokens.NeverBaselinedSqlList}) +AND EXISTS ( + SELECT 1 + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND ds.database_name = e.database_name + AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) = 'ONLINE' +)"; + /// /// Tidies auto-baselines for databases no longer in the newest snapshot (dropped/renamed); user /// overrides are kept. $1 server_id. @@ -698,6 +757,42 @@ SELECT 1 FROM database_states ds AND ds.database_name = e.database_name )"; + /// + /// #2166: clears the alerted-state memory for any database the store now shows back AT its expected + /// state. Runs beside the seed and the prune, on the same connection, for the same reason they do — it + /// is store maintenance derived from what the store holds, not from anything a process observed. + /// + /// That distinction is the whole point. The engine also clears on the falling edge it witnesses, + /// but that path is reachable only through its in-memory active set, which empties on every restart. A + /// service restart landing between an alert and the recovery therefore left the persisted + /// last_alerted_state sticky forever: the database was never in active to be noticed as + /// recovered, so the next parking read as already-announced and was swallowed. This statement cannot + /// have that gap, because it asks the store rather than remembering. The engine's clear stays as the + /// immediate path — a recovery inside one process should not wait for the next cycle's sweep — and this + /// is what actually owns the invariant. + /// + /// One sample at expected is enough, deliberately, where the DEVIATION rule needs two: clearing is + /// the safe direction (it can only cause an extra alert, never a missed one), and a flap cannot exploit + /// it because a flap does not survive the two-sample deviation test to alert in the first place. The + /// "(ignore)" sentinel clears too — an operator silencing a database should not leave a memory behind + /// that outlives the silence. $1 server_id. + /// + public const string ClearRecoveredDatabaseStateAlertsSql = @" +UPDATE config.database_state_expected e +SET last_alerted_state = NULL, + last_alerted_at = NULL +WHERE e.server_id = $1 +AND e.last_alerted_state IS NOT NULL +AND (e.expected_state = '(ignore)' + OR EXISTS ( + SELECT 1 + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND ds.database_name = e.database_name + AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) = e.expected_state + ))"; + /// /// The databases whose state deviates from their expected state in BOTH of the two most recent /// collections (a two-sample rule that absorbs restart transients — RECOVERY_PENDING / RECOVERING — and @@ -707,7 +802,7 @@ SELECT 1 FROM database_states ds /// sentinel; each row carries current + expected (expected is empty for a pending row). Lite's DuckDB /// read ported to Postgres. $1 server_id. /// - public const string DatabaseStateDeviationsSql = @" + public const string DatabaseStateDeviationsSql = $@" WITH newest AS ( SELECT MAX(collection_time) AS t FROM database_states WHERE server_id = $1 ), @@ -725,7 +820,7 @@ previous AS ( FROM database_states ds WHERE ds.server_id = $1 AND ds.collection_time = (SELECT t FROM prev) ) -SELECT l.database_name, l.eff, COALESCE(e.expected_state, '') +SELECT l.database_name, l.eff, COALESCE(e.expected_state, ''), COALESCE(e.last_alerted_state, '') FROM latest l JOIN previous p ON p.database_name = l.database_name @@ -733,8 +828,8 @@ LEFT JOIN config.database_state_expected e ON e.server_id = $1 AND e.database_name = l.database_name WHERE (e.expected_state IS NULL - AND l.eff IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY') - AND p.eff IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY')) + AND l.eff IN ({DatabaseStateTokens.CriticalSqlList}) + AND p.eff IN ({DatabaseStateTokens.CriticalSqlList})) OR (e.expected_state IS NOT NULL AND e.expected_state <> '(ignore)' AND l.eff IS DISTINCT FROM e.expected_state AND p.eff IS DISTINCT FROM e.expected_state) @@ -754,12 +849,32 @@ public async Task> GetDatabaseStatesAsync( await seed.ExecuteNonQueryAsync(cancellationToken); } + /* Beside the seed because it is the same job from the other end (#2189): the seed learns a baseline + for a database that has none, this un-learns one the database has since outgrown. Both run before + the read, so a poisoned expectation is corrected on the cycle that notices it rather than firing + once more first. */ + using (var heal = new NpgsqlCommand(HealDatabaseStateBaselineToOnlineSql, connection)) + { + heal.Parameters.AddWithValue(serverId); + await heal.ExecuteNonQueryAsync(cancellationToken); + } + using (var prune = new NpgsqlCommand(PruneDatabaseStateExpectedSql, connection)) { prune.Parameters.AddWithValue(serverId); await prune.ExecuteNonQueryAsync(cancellationToken); } + /* Before the read, so this cycle judges against a memory the store has already healed rather than + one carried over from a restart (#2166). A database cleared here is one that is back at its + expected state, so it cannot appear in the deviation read below either way — the ordering matters + for the NEXT deviation, not this one. */ + using (var clearRecovered = new NpgsqlCommand(ClearRecoveredDatabaseStateAlertsSql, connection)) + { + clearRecovered.Parameters.AddWithValue(serverId); + await clearRecovered.ExecuteNonQueryAsync(cancellationToken); + } + using (var command = new NpgsqlCommand(DatabaseStateDeviationsSql, connection)) { command.Parameters.AddWithValue(serverId); @@ -770,7 +885,8 @@ public async Task> GetDatabaseStatesAsync( { DatabaseName = reader.IsDBNull(0) ? "" : reader.GetString(0), StateDesc = reader.IsDBNull(1) ? "" : reader.GetString(1), - ExpectedState = reader.IsDBNull(2) ? "" : reader.GetString(2) + ExpectedState = reader.IsDBNull(2) ? "" : reader.GetString(2), + LastAlertedState = reader.IsDBNull(3) ? "" : reader.GetString(3) }); } } @@ -778,6 +894,88 @@ public async Task> GetDatabaseStatesAsync( return items; } + /// + /// Forced plans whose failure counter ROSE between the two most recent collections that carried the + /// plan (#2157). $1 server_id. + /// + /// Shape notes: query_store_stats holds one row per plan PER INTERVAL per collection, and the + /// forcing columns are plan-level attributes repeated across those rows — so the CTE collapses each + /// (plan, collection_time) to one value with MAX before any comparison. The two-hour window bounds + /// the hypertable scan; a plan not collected within it is by definition not failing right now, and + /// Query Store's own flush cadence (900s) means an active plan appears several times inside it. + /// + /// The > comparison is what makes this a delta read: equal counters are silence, and a + /// LOWER counter (unforce/re-force reset) is silence too rather than a negative delta. + /// + public const string ForcePlanFailuresSql = @" +WITH per_collection AS ( + SELECT + qs.database_name, + qs.query_id, + qs.plan_id, + qs.collection_time, + MAX(COALESCE(qs.force_failure_count, 0)) AS failures, + MAX(CASE WHEN qs.is_forced_plan THEN 1 ELSE 0 END) AS forced, + MAX(COALESCE(qs.plan_forcing_type, '')) AS forcing_type, + MAX(COALESCE(qs.last_force_failure_reason, '')) AS reason + FROM query_store_stats AS qs + WHERE qs.server_id = $1 + AND qs.collection_time > now() - interval '2 hours' + GROUP BY qs.database_name, qs.query_id, qs.plan_id, qs.collection_time +), +ranked AS ( + SELECT + pc.*, + ROW_NUMBER() OVER (PARTITION BY pc.database_name, pc.query_id, pc.plan_id ORDER BY pc.collection_time DESC) AS rn + FROM per_collection AS pc +) +SELECT + n.database_name, + n.query_id, + n.plan_id, + n.forcing_type, + n.reason, + n.failures - p.failures AS failure_delta, + n.failures AS total_failures +FROM ranked AS n +JOIN ranked AS p + ON p.database_name = n.database_name + AND p.query_id = n.query_id + AND p.plan_id = n.plan_id + AND p.rn = 2 +WHERE n.rn = 1 +AND n.forced = 1 +AND n.failures > p.failures +ORDER BY n.database_name, n.query_id, n.plan_id"; + + public async Task> GetForcePlanFailuresAsync( + string serverKey, CancellationToken cancellationToken = default) + { + var serverId = ParseServerKey(serverKey); + + var items = new List(); + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + using var command = new NpgsqlCommand(ForcePlanFailuresSql, connection); + command.Parameters.AddWithValue(serverId); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + items.Add(new ForcePlanFailureInfo + { + DatabaseName = reader.IsDBNull(0) ? "" : reader.GetString(0), + QueryId = reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + PlanId = reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + ForcingType = reader.IsDBNull(3) ? "" : reader.GetString(3), + FailureReason = reader.IsDBNull(4) ? "" : reader.GetString(4), + FailureDelta = reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + TotalFailures = reader.IsDBNull(6) ? 0 : reader.GetInt64(6) + }); + } + + return items; + } + private int ResolveRunningJobsCadence(int serverId) => ResolveCadence(_runningJobsCadenceMinutes, serverId, "running_jobs"); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index 8f6837945..81df67e99 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -49,6 +49,10 @@ public DarlingAlertSettings(DarlingConfig config) public bool PvsEnabled => _config.Alerts.PvsEnabled; public bool DatabaseStateEnabled => _config.Alerts.DatabaseStateEnabled; + /* #2157: ON with no store column yet — see AppAlertEngineSettings for the reasoning, including why a + darling.json-only flag would not survive a store reload (ApplyToConfig swaps Alerts wholesale). */ + public bool ForcePlanFailureEnabled => true; + public int CpuThresholdPercent => _config.Alerts.CpuThresholdPercent; public int BlockingCountThreshold => _config.Alerts.BlockingCountThreshold; @@ -62,6 +66,21 @@ public DarlingAlertSettings(DarlingConfig config) public int LowDiskThresholdPercent => Math.Clamp(_config.Alerts.LowDiskThresholdPercent, 0, 100); public int LowDiskThresholdGb => Math.Max(0, _config.Alerts.LowDiskThresholdGb); + /* #2107: the previously-hardcoded thresholds, clamped on read like their siblings so a + hand-edited store value can't drive a nonsense threshold. The critical floors keep low-disk's + 0-100 percent clamp and 0-floor GB shape; the staleness window and failure fast-path get + floors that keep the self-alerts meaningful (a 0-minute window would fire on every sweep). */ + public int DiskCriticalFreePercent => Math.Clamp(_config.Alerts.DiskCriticalFreePercent, 0, 100); + public int DiskCriticalFreeGb => Math.Max(0, _config.Alerts.DiskCriticalFreeGb); + public int SelfDiskFreeWarnPercent => Math.Clamp(_config.Alerts.SelfDiskFreeWarnPercent, 0, 100); + public int CollectionStaleMinutes => Math.Clamp(_config.Alerts.CollectionStaleMinutes, 5, 1440); + + /// #2136: the Store Job Over Cadence warning percent. Clamped [5, 100] — below 5 would fire + /// on healthy jobs (the production worst runs ~7% of cadence), and at 100 the Warning tier merges + /// into the fixed Critical tier, so higher values would only disable the warning silently. + public int StoreJobCadenceWarnPercent => Math.Clamp(_config.Alerts.StoreJobCadenceWarnPercent, 5, 100); + public int CollectionFailureThreshold => Math.Clamp(_config.Alerts.CollectionFailureThreshold, 1, 1000); + /* #1984: percent clamped like low-disk's (0 = off); the GB floor merely floored at 0 — unlike the percent it has no meaningful upper bound. */ public int PvsThresholdPercent => Math.Clamp(_config.Alerts.PvsThresholdPercent, 0, 100); @@ -209,5 +228,7 @@ the sibling channels use. */ 1) read through the by-reference config seam — a store reload reflects it immediately; clamped 0–2 like Lite/Dashboard. The re-notify cooldown stays Lite's hardcoded default (not a knob). */ public double AnalysisNotifySeverity => Math.Clamp(_config.Analysis.NotifySeverity, 0.0, 2.0); - public int AnalysisNotifyCooldownMinutes => 360; + /* #2107: was a hardcoded 360 while the shared engine accepts a clamped [30, 10080] value and + Lite always passed a configured one through — the Darling parity gap gotqn called out. */ + public int AnalysisNotifyCooldownMinutes => Math.Clamp(_config.Alerts.AnalysisNotifyCooldownMinutes, 30, 10080); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingCliCommands.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingCliCommands.cs index 7bc981d4e..64f6ee93c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingCliCommands.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingCliCommands.cs @@ -21,6 +21,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Collectors; using PerformanceMonitor.Darling.Service.Hosting; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; @@ -121,6 +122,14 @@ public static bool IsCollapseLegacySlicesVerb(string arg) => public static bool IsRecompressPlanDimVerb(string arg) => string.Equals(arg, "--recompress-plan-dim", StringComparison.OrdinalIgnoreCase); + /// The verb handles — register monitored server(s) in the store from a + /// JSON array on STDIN (#2256). The store is authoritative after the first seed, so on a headless host with no + /// GUI and no MCP client this was previously impossible: the web surface excludes the write tools by design + /// and darling.json is a one-time bootstrap. + public static bool IsAddServerVerb(string arg) => + string.Equals(arg, "--add-server", StringComparison.OrdinalIgnoreCase) + || string.Equals(arg, "--add-servers", StringComparison.OrdinalIgnoreCase); + /// --version/-v — print the product version and exit. public static bool IsVersionVerb(string arg) => string.Equals(arg, "--version", StringComparison.OrdinalIgnoreCase) @@ -151,7 +160,8 @@ public static bool IsKnownVerb(string arg) => || IsDisableWebVerb(arg) || IsBackfillRollupsVerb(arg) || IsCollapseLegacySlicesVerb(arg) - || IsRecompressPlanDimVerb(arg); + || IsRecompressPlanDimVerb(arg) + || IsAddServerVerb(arg); /// /// Classifies the exe's command line from its FIRST argument (#1581): no args → run the host; a recognized @@ -224,6 +234,7 @@ public static string UsageText() => " PerformanceMonitor.Darling.Service.exe --backfill-rollups Materialize the retention rollups back over existing history, after a disk preflight." + Environment.NewLine + " PerformanceMonitor.Darling.Service.exe --collapse-legacy-slices Repair Query Store rows collected before the split-slice fix, then re-materialize the rollups they fed." + Environment.NewLine + " PerformanceMonitor.Darling.Service.exe --recompress-plan-dim Convert the plan dimension's pre-V54 text rows to gzip in batches while the service runs, then VACUUM FULL to return the space to the volume (--no-vacuum-full to skip; --vacuum-full to compact an already-converted store)." + Environment.NewLine + + " PerformanceMonitor.Darling.Service.exe --add-server, --add-servers Register monitored server(s) from a JSON array on stdin (the add_servers shape); the running service picks them up without a restart." + Environment.NewLine + " PerformanceMonitor.Darling.Service.exe --backfill-rollups --dry-run Show the plan, the disk estimate and the time budget, and change nothing."; /// @@ -285,11 +296,7 @@ public static string FormatProbeLine(string serverName, ConnectionProbeResult pr return $" [FAIL] {serverName}: {probe.Error}"; } - var edition = string.IsNullOrEmpty(probe.EngineEditionDescription) - ? DarlingServerConnector.DescribeEngineEdition(probe.EngineEdition) - : probe.EngineEditionDescription; - var msdb = probe.HasMsdbAccess ? "msdb access: yes" : "msdb access: NO (failed-job alerts unavailable)"; - return $" [PASS] {serverName}: SQL major version {probe.MajorVersion}, {edition}, {msdb}"; + return $" [PASS] {serverName}: {DarlingServerConnector.DescribeProbeFacts(probe)}"; } /// @@ -338,8 +345,18 @@ VIEWER machine (a bare filename resolves against the folder holding the viewer's /* Read the cert BEFORE anything reaches STDOUT: every STDERR line — including the missing-cert NOTE — must be emitted ahead of the payload (#1953 item 3). The field report watched the live password scroll past and only THEN saw the redirect advice, which is exactly backwards for a warning. */ - var certificate = File.Exists(handoff.CertificatePath) - ? (await File.ReadAllTextAsync(handoff.CertificatePath, cancellationToken)).Trim() + /* #2117: prefer the distributable ROOT (the CA that signed the served leaf) when the store + carries the fixed chain shape — that is what verify-full's Root Certificate must anchor + on. A legacy store has no root.crt, and its single self-signed server.crt remains the + right (if Windows-hostile) thing to print. */ + var distributableCertPath = DarlingManagedPostgres.RootCertificatePathFor(handoff.CertificatePath); + if (!File.Exists(distributableCertPath)) + { + distributableCertPath = handoff.CertificatePath; + } + + var certificate = File.Exists(distributableCertPath) + ? (await File.ReadAllTextAsync(distributableCertPath, cancellationToken)).Trim() : null; /* Guidance + the live-secret warning go to STDERR, so redirecting STDOUT to a file or the clipboard @@ -383,7 +400,9 @@ must be emitted ahead of the payload (#1953 item 3). The field report watched th /* Emit the server cert PEM so the operator can copy it to the viewer machine. */ if (certificate is not null) { - output.WriteLine($"# Server TLS certificate ({DarlingManagedPostgres.ServerCertFileName}) — save as '{clientCertificatePath}' on the viewer machine:"); + /* #2117 review catch: name the file whose CONTENT is actually below — root.crt on a + chain-shaped store, server.crt only on a legacy one. */ + output.WriteLine($"# Server TLS certificate ({Path.GetFileName(distributableCertPath)}) — save as '{clientCertificatePath}' on the viewer machine:"); output.WriteLine(certificate); } @@ -469,10 +488,13 @@ The cert lives in the same directory as the credential (ParentOf(dataDirectory)) if (!File.Exists(credentialPath)) { - error.WriteLine( - $"The '{role}' role credential ({credentialPath}) does not exist yet. Start the PerformanceMonitor " + - "Darling service once so its first run provisions the least-privilege roles and their credentials, " + - "then re-run this command."); + /* #2197: which of the two things this means is decided from the store's own files, not assumed. + A bootstrap that has already failed produces this same absence, and telling THAT operator to + start the service again is the dead end the field report walked into. */ + error.WriteLine(DarlingStoreBootstrapEvidence.MissingCredentialMessage( + $"The '{role}' role credential ({credentialPath})", + "provisions the least-privilege roles and their credentials", + dataDirectory)); return null; } @@ -644,13 +666,20 @@ that fails later has not thrown away the previous export. */ return 1; } + /* #2117: prefer the distributable ROOT on chain-shaped stores — the print verb's rule. */ + var exportCertPath = DarlingManagedPostgres.RootCertificatePathFor(handoff.CertificatePath); + if (!File.Exists(exportCertPath)) + { + exportCertPath = handoff.CertificatePath; + } + try { - certificate = (await File.ReadAllTextAsync(handoff.CertificatePath, cancellationToken)).Trim(); + certificate = (await File.ReadAllTextAsync(exportCertPath, cancellationToken)).Trim(); } catch (Exception ex) { - error.WriteLine($"Could not read the server TLS certificate ({handoff.CertificatePath}): {ex.Message}"); + error.WriteLine($"Could not read the server TLS certificate ({exportCertPath}): {ex.Message}"); return 1; } @@ -2427,9 +2456,7 @@ managed concerns. In BYO the operator's own PostgreSQL holds config_service — var connectionString = DarlingManagedPostgres.TryBuildConnectionStringFromStoredCredential(postgres); if (connectionString is null) { - error.WriteLine( - "The managed store credential does not exist yet — start the PerformanceMonitor Darling service once " + - "so its first run initializes the store, then re-run this command."); + error.WriteLine(DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres)); return 1; } @@ -2927,7 +2954,7 @@ public static async Task CollapseLegacySlicesAsync( if (string.IsNullOrWhiteSpace(connectionString)) { error.WriteLine(postgres.Managed - ? "The managed store credential does not exist yet — start the PerformanceMonitor Darling service once so its first run initializes the store, then re-run this command." + ? DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres) : "postgres.connectionString is empty, so there is no store to repair."); return 1; } @@ -2981,42 +3008,118 @@ public static async Task CollapseLegacySlicesAsync( return 0; } - /* SLICED PER DAY, not one call over the whole span. CollapseSliceAsync runs each slice in ONE + /* SLICED, not one call over the whole span. CollapseSliceAsync runs each slice in ONE transaction, and that transaction takes locks on the raw chunks it touches — which the compression policy also wants. Handing it the entire survey span would make one long transaction sitting across however much history the store keeps, which is exactly the lock-duration family that has bitten this - repo before (#1564/#1567). On a default 4-day raw tier this is a handful of slices; on a store with - a widened retention it is the protection the method's own doc promises. + repo before (#1564/#1567). + + Slice width is ADAPTIVE (#2105 round three): a day is the fast default, but on the field store + that motivated this the FIRST day-wide stage aggregation blew through the 15-minute statement + timeout — the operator watched it die at minute ~15 with the bare stream exception, three walls + deep. A failed slice now halves the window and retries the SAME start (the shared + QueryStoreBackfillState.AdaptiveSpan schedule the backfill worker uses, 24h base → 22.5m floor), + a completed slice resets to full width, and only a slice that fails AT the floor gives up to the + existing re-run message. Narrowing is announced so the operator sees progress, not a hang. The half-open upper bound includes the newest collapsed row — the survey reports that instant itself, not a bound past it — hence the final slice's one-second nudge. */ long removed = 0; var sliceStart = survey.OldestUtc!.Value.Date; + var spanStart = sliceStart; var collapseEnd = survey.NewestUtc!.Value.AddSeconds(1); + var fullWidth = TimeSpan.FromDays(1); + var consecutiveFailures = 0; + var retriedAtWidth = false; - try + while (sliceStart < collapseEnd) { - while (sliceStart < collapseEnd) + var span = QueryStoreBackfillState.AdaptiveSpan(fullWidth, consecutiveFailures); + var sliceEnd = sliceStart + span; + if (sliceEnd > collapseEnd) { - var sliceEnd = sliceStart.AddDays(1); - if (sliceEnd > collapseEnd) - { - sliceEnd = collapseEnd; - } + sliceEnd = collapseEnd; + } - removed += await QueryStoreSliceRepair.CollapseSliceAsync( + /* The width the slice ACTUALLY covers — the final slice clamps to the range end, so the + nominal AdaptiveSpan width can overstate it, and both the retry decision and the operator + messages must speak in real terms (review catch). */ + var actualWidth = sliceEnd - sliceStart; + + try + { + var sliceRemoved = await QueryStoreSliceRepair.CollapseSliceAsync( connection, sliceStart, sliceEnd, cancellationToken); + removed += sliceRemoved; + + /* Per-slice progress (#2105 operator feedback): the run used to be SILENT between the + survey banner and DONE — on a big backlog that is an hour-plus of blank console that + reads as a hang, on the exact stores where trust in this verb is already bruised. + Percent is of the survey's own span, so it always ends at 100. */ + var pctDone = 100.0 * (sliceEnd - spanStart).Ticks / (collapseEnd - spanStart).Ticks; + output.WriteLine($" [OK] {sliceStart:yyyy-MM-dd HH:mm} +{actualWidth.TotalMinutes:F0}m — {sliceRemoved:N0} removed ({pctDone:F0}% of span, {removed:N0} total)"); + consecutiveFailures = 0; + retriedAtWidth = false; sliceStart = sliceEnd; } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - /* Each slice is its own transaction, so earlier slices are already committed and are not lost — - and the collapse is idempotent, so re-running picks up where this stopped. */ - error.WriteLine($" The collapse failed after {removed:N0} row(s); the failing slice was rolled back: {ex.Message}"); - error.WriteLine(" Slices already committed are safe. Re-run to continue — the repair is idempotent."); - return 1; + catch (Exception ex) when (ex is not OperationCanceledException) + { + var next = NextNarrowingFailureCount(fullWidth, consecutiveFailures, actualWidth); + + /* A slice already at/below the adaptive floor (usually the clamped final tail — nothing + says the leftover is ≥ the floor) can't be narrowed, but its likeliest failure is the + transient/connection kind the fresh-connection retry exists for — so it earns ONE + same-width retry before the give-up (review catch: giving up on the tail's first + failure silently exempted the run's usual last slice from the retry mechanism). */ + var sameWidthRetry = next is null && !retriedAtWidth; + + if (next is int || sameWidthRetry) + { + /* The statement-timeout failure this loop exists to survive surfaces as a broken + STREAM, not a clean server-side cancel — the connection underneath is very likely + dead, and retrying on it would fail instantly through every halving step (review + catch). Cycle it: close is safe on a broken connection, and reopen draws a fresh + physical connection. Session state doesn't matter — the slice's SET LOCAL and + per-command timeouts are transaction/command scoped. A failed REOPEN degrades to + the same clean idempotent-rerun message as every other failure here, never an + unhandled crash (review catch — this verb has no caller safety net). */ + try + { + await connection.CloseAsync(); + await connection.OpenAsync(cancellationToken); + } + catch (Exception reopenEx) when (reopenEx is not OperationCanceledException) + { + error.WriteLine($" The collapse failed after {removed:N0} row(s); the slice at {sliceStart:yyyy-MM-dd HH:mm} failed ({FirstLineOf(ex.Message)}) and the store connection could not be reopened: {FirstLineOf(reopenEx.Message)}"); + error.WriteLine(" Slices already committed are safe. Re-run to continue — the repair is idempotent."); + return 1; + } + + if (next is int narrowerFailures) + { + consecutiveFailures = narrowerFailures; + retriedAtWidth = false; + var narrower = QueryStoreBackfillState.AdaptiveSpan(fullWidth, narrowerFailures); + output.WriteLine($" [RETRY] slice {sliceStart:yyyy-MM-dd HH:mm} +{actualWidth.TotalMinutes:F0}m failed ({FirstLineOf(ex.Message)}); narrowing to {narrower.TotalMinutes:F0}m and retrying."); + } + else + { + retriedAtWidth = true; + output.WriteLine($" [RETRY] slice {sliceStart:yyyy-MM-dd HH:mm} +{actualWidth.TotalMinutes:F0}m failed ({FirstLineOf(ex.Message)}); already at the narrowest width — retrying once on a fresh connection."); + } + + continue; + } + + /* Narrowing exhausted AND the same-width retry spent — this range cannot be repaired + unattended. Each slice is its own transaction, so earlier slices are already committed + and are not lost — and the collapse is idempotent, so re-running picks up where this + stopped. */ + error.WriteLine($" The collapse failed after {removed:N0} row(s); the failing {actualWidth.TotalMinutes:F0}m slice at {sliceStart:yyyy-MM-dd HH:mm} was rolled back: {ex.Message}"); + error.WriteLine(" Slices already committed are safe. Re-run to continue — the repair is idempotent."); + return 1; + } } output.WriteLine($" Collapsed. Rows removed: {removed:N0}"); @@ -3064,6 +3167,37 @@ widened to whole buckets so a partially-covered bucket is recomputed rather than private static DateTime Floor(DateTime value, TimeSpan bucket) => bucket <= TimeSpan.Zero ? value : new DateTime(value.Ticks - (value.Ticks % bucket.Ticks), value.Kind); + /// + /// The collapse loop's narrowing decision, pure so it pins without a live timeout: the smallest + /// failure count whose width actually narrows a + /// slice that COVERED (a clamped final slice can be narrower than + /// several nominal halving steps, and re-running an identical window just re-hits the same wall), or + /// null when no step can — the slice already sits at/below the adaptive floor, where the caller's + /// one same-width fresh-connection retry is the only move left. + /// + internal static int? NextNarrowingFailureCount(TimeSpan fullWidth, int consecutiveFailures, TimeSpan actualWidth) + { + var next = consecutiveFailures + 1; + var narrower = QueryStoreBackfillState.AdaptiveSpan(fullWidth, next); + while (narrower >= actualWidth) + { + var evenNarrower = QueryStoreBackfillState.AdaptiveSpan(fullWidth, next + 1); + if (evenNarrower >= narrower) + { + return null; + } + + next++; + narrower = evenNarrower; + } + + return next; + } + + /// An exception message's first line, CR-trimmed — one-line operator output must stay one line. + private static string FirstLineOf(string message) + => message.Split('\n')[0].TrimEnd('\r'); + /// How --recompress-plan-dim handles the closing VACUUM FULL (#2076). public enum RecompressVacuumMode { @@ -3120,6 +3254,30 @@ public static (string? ConfigPath, bool DryRun, RecompressVacuumMode Mode, strin return (configPath, dryRun, mode, null); } + /// + /// #2171: whether the store's plan_xml_compression setting is 'none' — the mode where the live + /// writer stores plans as plain text and recompression would fight it forever. Reads defensively: + /// the column arrives at V62, and the verb must keep working against the older stores it exists + /// to convert, so a missing column (42703) is "no mode to conflict with". Public-for-tests via + /// the live suite; the verb is its only production caller. + /// + internal static async Task StoreIsSetToPlainTextPlansAsync( + NpgsqlConnection connection, CancellationToken cancellationToken) + { + try + { + await using var codec = new NpgsqlCommand( + "SELECT plan_xml_compression FROM config_service WHERE id = 1", connection); + return await codec.ExecuteScalarAsync(cancellationToken) is string mode + && string.Equals(mode.Trim(), "none", StringComparison.OrdinalIgnoreCase); + } + catch (PostgresException ex) when (ex.SqlState == "42703") + { + /* Pre-V62 store: no column, no mode to conflict with — proceed. */ + return false; + } + } + /// /// --recompress-plan-dim (#2076): convert the plan dimension's pre-V54 text rows to the gzip form /// V54's write path produces (#2069), in bounded batches, while the service keeps running. @@ -3177,7 +3335,7 @@ public static async Task RecompressPlanDimAsync( if (string.IsNullOrWhiteSpace(connectionString)) { error.WriteLine(postgres.Managed - ? "The managed store credential does not exist yet — start the PerformanceMonitor Darling service once so its first run initializes the store, then re-run this command." + ? DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres) : "postgres.connectionString is empty, so there is no store to convert."); return 1; } @@ -3197,6 +3355,20 @@ public static async Task RecompressPlanDimAsync( output.WriteLine("PerformanceMonitor Darling — plan-dimension recompression (--recompress-plan-dim)"); output.WriteLine(); + /* #2171: a store configured plan_xml_compression = 'none' WANTS text rows — the operator chose + direct-SQL readability, and this verb would convert exactly the rows the live writer keeps + producing, the two fighting forever. Refuse with the way out rather than silently churning. */ + if (await StoreIsSetToPlainTextPlansAsync(connection, cancellationToken)) + { + error.WriteLine( + "This store is configured plan_xml_compression = 'none' (plans deliberately stored as " + + "plain text for direct-SQL consumers, #2171). Recompressing would convert rows the live " + + "writer keeps producing as text - the two would fight forever. If you want gzip storage " + + "back, set plan_xml_compression = 'gzip' in the store's service settings first, then " + + "re-run this verb."); + return 1; + } + PlanDimRecompression.Survey survey; try { @@ -3415,6 +3587,208 @@ private static async Task CompactPlanDimAsync( return 0; } + /// + /// What --add-server prints when stdin carries nothing — to STDOUT, per the [#2097] lesson that a + /// prompt or error on STDERR is invisible in the ISE and some integrated terminals, so a verb that writes + /// only there reads as hung. + /// + public static string AddServerUsageText() => + "Nothing arrived on stdin, so no server was added." + Environment.NewLine + + Environment.NewLine + + "--add-server (or --add-servers) reads a JSON ARRAY of servers from stdin — the same shape the" + Environment.NewLine + + "add_servers MCP tool takes." + Environment.NewLine + + "The password is read from stdin rather than the command line on purpose: an argument is visible in the" + Environment.NewLine + + "process list and in shell history." + Environment.NewLine + + Environment.NewLine + + " PowerShell: Get-Content servers.json | .\\PerformanceMonitor.Darling.Service.exe --add-server" + Environment.NewLine + + " cmd: type servers.json | PerformanceMonitor.Darling.Service.exe --add-server" + Environment.NewLine + + Environment.NewLine + + "servers.json, SQL Server and PostgreSQL:" + Environment.NewLine + + " [" + Environment.NewLine + + " {\"host\":\"sql01\",\"auth\":\"integrated\"}," + Environment.NewLine + + " {\"host\":\"aurora.cluster-abc.us-east-1.rds.amazonaws.com\",\"engine\":\"postgres\"," + Environment.NewLine + + " \"auth\":\"SQL\",\"username\":\"darling_monitor\",\"password\":\"...\"}" + Environment.NewLine + + " ]"; + + /// + /// Renders the add_servers result JSON as operator lines plus an exit code. PURE, so the formatting and + /// the exit-code policy pin without a store — the same split uses. + /// + /// Exit 0 requires that something landed and nothing failed. A batch of pure duplicates exits 0: re-running + /// the same file is idempotent, not an error. Nothing at all landed (an empty array, or every entry rejected) + /// exits 1, because a verb that changed nothing must not report success to a deployment script. + /// + internal static (IReadOnlyList Lines, int ExitCode) FormatAddServerOutcome(string resultJson) + { + var lines = new List(); + try + { + using var document = JsonDocument.Parse(resultJson); + var root = document.RootElement; + + /* The whole-payload rejection shape — {status, message} with no per-server results. */ + if (!root.TryGetProperty("results", out var results) || results.ValueKind != JsonValueKind.Array) + { + var status = root.TryGetProperty("status", out var s) ? s.GetString() : "error"; + var message = root.TryGetProperty("message", out var m) ? m.GetString() : resultJson; + lines.Add($" [{status?.ToUpperInvariant()}] {message}"); + return (lines, 1); + } + + foreach (var result in results.EnumerateArray()) + { + var server = result.TryGetProperty("server", out var sv) ? sv.GetString() : null; + var status = (result.TryGetProperty("status", out var st) ? st.GetString() : null) ?? string.Empty; + var detail = result.TryGetProperty("detail", out var dt) ? dt.GetString() : null; + var tag = status switch + { + "added" => "ADDED", + "duplicate" => "SKIP", + "connection_failed" => "FAIL", + "invalid" => "INVALID", + _ => status.ToUpperInvariant(), + }; + lines.Add(string.IsNullOrWhiteSpace(detail) + ? $" [{tag}] {server ?? "(unnamed)"}" + : $" [{tag}] {server ?? "(unnamed)"}: {detail}"); + } + + var added = root.TryGetProperty("added", out var a) ? a.GetInt32() : 0; + var skipped = root.TryGetProperty("skipped", out var k) ? k.GetInt32() : 0; + var failed = root.TryGetProperty("failed", out var f) ? f.GetInt32() : 0; + + lines.Add(string.Empty); + lines.Add(string.Format( + CultureInfo.InvariantCulture, + "{0} added, {1} already registered, {2} failed.", + added, + skipped, + failed)); + + if (added > 0) + { + /* The registry write bumps config_version through trg_bump_monitored_servers, which the worker + polls every sweep — so say the restart is unnecessary rather than leaving them to wonder. */ + lines.Add("The running service picks these up on its next config poll; no restart is needed."); + } + + return (lines, failed > 0 || (added == 0 && skipped == 0) ? 1 : 0); + } + catch (JsonException) + { + /* Not every failure arrives as JSON. AddServersAsync's catch-all returns McpHelpers.FormatError, + which is PLAIN TEXT ("Error during add_servers: ..."), so a genuine store failure that happens + AFTER the request parsed — a dropped connection mid-batch, a constraint violation — lands here. + That text IS the message the operator needs; wrapping it in "could not parse" buries the one line + that explains the failure, precisely when the verb is being used as a deployment gate. Only + something that looked like JSON and was not gets the parse wrapper. */ + var text = resultJson?.Trim() ?? string.Empty; + lines.Add(text.StartsWith('{') || text.StartsWith('[') + ? $" Could not parse the result: {text}" + : $" {text}"); + return (lines, 1); + } + } + + /// + /// --add-server (#2256): registers monitored server(s) in the store from a JSON array on stdin, through + /// the SAME path the MCP tool uses — so validation, dedupe, + /// the in-process connection probe, password encryption and the identity computation are shared rather than + /// reimplemented. server_id in particular is a hash of the storage name, which is exactly the part an + /// operator cannot safely produce by hand. + /// + /// Why this exists: the store is authoritative after the first seed, so darling.json edits are ignored, + /// and the web surface deliberately excludes the write tools. A headless host — the field report ran Windows + /// Server 2012, which cannot run the Viewer at all — had no supported path. + /// + public static async Task AddServerAsync( + string? configPath, TextReader input, TextWriter output, TextWriter error, CancellationToken cancellationToken) + { + var json = input is null ? null : await input.ReadToEndAsync(cancellationToken); + if (string.IsNullOrWhiteSpace(json)) + { + output.WriteLine(AddServerUsageText()); + return 1; + } + + DarlingConfig config; + try + { + config = DarlingConfig.Load(configPath); + } + catch (Exception ex) + { + error.WriteLine($"Could not load configuration: {ex.Message}"); + return 1; + } + + var postgres = config.Postgres; + if (postgres is null) + { + error.WriteLine("postgres section is required."); + return 1; + } + + string? connectionString; + if (postgres.Managed) + { + /* The managed store credential is DPAPI, so it can only be read on Windows. Bring-your-own needs no + such guard, which is why this is scoped to the managed branch rather than the whole verb — a Linux + host pointed at its own Postgres can register servers. */ + if (!OperatingSystem.IsWindows()) + { + error.WriteLine("A managed Postgres store keeps its credential in DPAPI, so --add-server needs Windows. " + + "A bring-your-own store (postgres.connectionString) works on any platform."); + return 1; + } + + connectionString = DarlingManagedPostgres.TryBuildConnectionStringFromStoredCredential(postgres); + if (string.IsNullOrWhiteSpace(connectionString)) + { + /* Emitted HERE, inside the branch the guard above proved is Windows, rather than from a shared + check below keyed on postgres.Managed. The sibling verbs can write it below because they carry + [SupportedOSPlatform("windows")] on the whole method; this one deliberately does not, and a + bool is not something the platform analyzer can correlate with an earlier OS guard — so the + call has to sit where Windows is provable rather than where it merely happens to hold. */ + error.WriteLine(DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres)); + return 1; + } + } + else + { + connectionString = postgres.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + { + error.WriteLine("postgres.connectionString is empty, so there is no store to register a server in."); + return 1; + } + } + + output.WriteLine(); + output.WriteLine("PerformanceMonitor Darling — register monitored server(s) (--add-server)"); + output.WriteLine(); + + string resultJson; + try + { + await using var dataSource = NpgsqlDataSource.Create(connectionString); + resultJson = await DarlingMcpServerAdminTools.AddServers(dataSource, json); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + error.WriteLine($"Could not reach the store: {ex.Message}"); + return 1; + } + + var (lines, exitCode) = FormatAddServerOutcome(resultJson); + foreach (var line in lines) + { + output.WriteLine(line); + } + + return exitCode; + } + /// /// Materializes the query-acceleration rollups back over pre-existing history so the held raw retention /// policies can arm themselves (#1759 Phase 2). Runs while the service is UP. @@ -3465,7 +3839,7 @@ public static async Task BackfillRollupsAsync( if (string.IsNullOrWhiteSpace(connectionString)) { error.WriteLine(postgres.Managed - ? "The managed store credential does not exist yet — start the PerformanceMonitor Darling service once so its first run initializes the store, then re-run this command." + ? DarlingStoreBootstrapEvidence.MissingStoreCredentialMessage(postgres) : "postgres.connectionString is empty, so there is no store to back fill."); return 1; } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs index 7ba11ec65..17ab1fa73 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs @@ -10,6 +10,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; +using System.Data.Common; using System.Diagnostics; using System.Linq; using System.Threading; @@ -18,6 +19,7 @@ using Microsoft.Extensions.Logging; using Npgsql; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Targets; using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Storage; @@ -60,6 +62,13 @@ capture_plans is honored on the NEXT cycle without reconstructing the runner. */ collecting Object DDL. Read through a provider (not a captured bool) for symmetry with _capturePlans, so a future live reload is honored on the NEXT cycle without rebuilding. */ private readonly Func _collectSchemaChanges; + private readonly Func _compressPlanContent; + + /* Feeds CollectorContext.TextByteBudgetOverride on every cycle (#2164) — the query_store collector's + per-database text budget in MB (config_service.query_store_text_budget_mb, V59). Provider-read for + the same reason as the two above: a store reload takes effect on the NEXT cycle without rebuilding + the runner. Lite has no equivalent and keeps the collector's compile-time constant. */ + private readonly Func _textBudgetMb; /* Azure SQL DB logins without master access fall back to single-database mode, throttled per server so master isn't retried every cycle (#857 — mirrors Lite). @@ -70,6 +79,53 @@ capture_plans is honored on the NEXT cycle without reconstructing the runner. */ could permanently demote a healthy server to single-database collection (#1506). */ private readonly ConcurrentDictionary _azureMasterInaccessibleSince = new(); + /// + /// When a server's live query_store collection last failed a per-database item — the backfill + /// worker's yield-to-live signal (#2111), read through + /// and judged by . Stamped only for + /// query_store (the one collector with a backfill worker to yield); in-memory on purpose — a + /// service restart forgetting the stamps just means one backfill slice races one live cycle once. + /// + private readonly ConcurrentDictionary _lastQueryStoreItemFailureUtc = new(); + + /// The #2111 yield-to-live read side: null when the server has never failed a live + /// query_store item this process lifetime. + public DateTime? LastQueryStoreItemFailureUtc(int serverId) + => _lastQueryStoreItemFailureUtc.TryGetValue(serverId, out var failure) ? failure : null; + + /// + /// Consecutive live query_store failures per DATABASE — the adaptive-shrink signal (#2111 + /// promoted from reserve): a member whose window keeps exceeding the command timeout gets a + /// progressively narrower catch-up window () + /// until one fits, and the skipped range rides the same hole records the clamp already writes. + /// Reset on the database's next successful item; in-memory like the yield stamps and for the + /// same reason — a restart forgetting the count costs one full-width attempt. + /// + private readonly ConcurrentDictionary<(int ServerId, string Database), int> _consecutiveQueryStoreItemFailures = new(); + + private int ConsecutiveQueryStoreItemFailures(int serverId, string database) + => _consecutiveQueryStoreItemFailures.TryGetValue((serverId, database), out var count) ? count : 0; + + private void OnQueryStoreItemFailed(int serverId, string database) + { + _lastQueryStoreItemFailureUtc[serverId] = DateTime.UtcNow; + _consecutiveQueryStoreItemFailures.AddOrUpdate((serverId, database), 1, static (_, current) => current + 1); + } + + private void OnQueryStoreItemSucceeded(int serverId, string database) + => _consecutiveQueryStoreItemFailures.TryRemove((serverId, database), out _); + + /// + /// Per-DATABASE observed plan-XML size estimate for the plan fetch's candidate sizing (#2312 + /// Finding 1): + /// was designed to learn each database's real average from its own shipped passes — the 11x fleet + /// spread is the whole argument for it — and the call site passed null, so every pass on every + /// database sized its decompression window from the 160KB first-contact seed. In-memory like the + /// failure counters and for the same reason: a restart forgetting the estimate costs exactly one + /// first-contact-sized pass. + /// + private readonly ConcurrentDictionary<(int ServerId, string Database), QueryStorePlanXmlState.PlanSizeEstimate> _observedPlanSize = new(); + private static readonly TimeSpan AzureMasterRecheckInterval = TimeSpan.FromMinutes(15); public const int CommandTimeoutSeconds = 60; @@ -84,13 +140,19 @@ capture_plans is honored on the NEXT cycle without reconstructing the runner. */ /// behavior). The worker passes () => config.CollectSchemaChangeEvents so a noisy/benchmark box /// can suppress the default-trace Object:Created/Deleted flood; tests pass a constant lambda. /// - public DarlingCollectorRunner(NpgsqlDataSource postgres, CollectorDeltaCalculator deltas, ILogger? logger = null, Func? capturePlans = null, Func? collectSchemaChanges = null) + public DarlingCollectorRunner(NpgsqlDataSource postgres, CollectorDeltaCalculator deltas, ILogger? logger = null, Func? capturePlans = null, Func? collectSchemaChanges = null, Func? textBudgetMb = null, Func? compressPlanContent = null) { _postgres = postgres ?? throw new ArgumentNullException(nameof(postgres)); _deltas = deltas ?? throw new ArgumentNullException(nameof(deltas)); _logger = logger; _capturePlans = capturePlans ?? (() => true); + /* Null provider = keep the collector's own compile-time budget (what Lite and every test does). */ + _textBudgetMb = textBudgetMb ?? (() => 0); _collectSchemaChanges = collectSchemaChanges ?? (() => true); + /* #2171: plan_xml_compression provider — true = gzip into query_plan_gz (the default), + false = 'none': plain text into query_plan_xml so direct-SQL consumers read it bare. + The worker passes () => config.PlanXmlCompression == "gzip"; tests pass a constant. */ + _compressPlanContent = compressPlanContent ?? (() => true); } public async Task RunAsync( @@ -101,8 +163,10 @@ public async Task RunAsync( var collectionTime = DateTime.UtcNow; /* Some collectors don't exist on some targets (e.g. ring buffers on Azure SQL DB) — - skip the cycle entirely, matching Lite. */ - if (!definition.AppliesTo(server.Target)) + skip the cycle entirely, matching Lite. CollectorCatalog.AppliesTo composes the + engine-dialect check over the definition's own gate, so a T-SQL definition can never be + dispatched at a non-SQL-Server target. */ + if (!CollectorCatalog.AppliesTo(definition, server.Target)) { return new CollectorRunResult(0, 0, 0); } @@ -135,6 +199,111 @@ declared keys (every other collector) means no query runs. Mirrors Lite. */ ? null : await GetCollectorStateAsync(server.ServerId, definition.Name, cancellationToken); + /* #2188: retire the per-database state rows of databases that no longer exist, BEFORE the load + below, so this cycle also works from a cleaned dictionary rather than one carrying names the + server dropped. Runs for query_store regardless of plan capture, because the backfill worker's + per-database keys orphan the same way and are pruned in the same pass. + + Gated on the SAME AppliesTo that decides whether database_states is collected at all, rather than + leaning on the statement's own empty-snapshot guard to no-op: on Azure SQL DB there is no snapshot + by design, so this would otherwise run three guaranteed-no-op deletes on every cycle forever and + #2191's boundary would be emergent rather than stated. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal) + && DatabaseStateCollector.Instance.AppliesTo(server.Target)) + { + await PruneOrphanedQueryStoreDatabaseStateAsync(server.ServerId, cancellationToken); + } + else if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal) + && server.Target.IsAzureSqlDb) + { + /* #2191's boundary, now crossable. Azure SQL DB has no database_states snapshot by design, which + is why this was a stated no-op — but after #2220 a registration that names a database sweeps + only that database, so its one legitimate key is the connection string's own catalog. No + master read, nothing filtered, nothing that can go stale. A registration naming NO database is + still skipped: it is a registration of the logical SERVER, and a single-name prune there would + delete every live watermark it legitimately has. */ + var ownDatabase = new SqlConnectionStringBuilder(server.ConnectionString).InitialCatalog; + if (AzureSweepScope.OwnDatabaseOrEmpty(ownDatabase).Count > 0) + { + await PruneForeignQueryStoreDatabaseStateAsync(server.ServerId, ownDatabase, cancellationToken); + } + } + + /* #2164: the per-database plan-XML watermarks, owned by the HOST under its own state collector name + rather than declared by the definition — the QueryStoreBackfillState seam. The definition cannot + declare these: the keys are one per DATABASE and only known at runtime, and declaring a prefix + would make query_store a second state-declaring collector, which is a two-host contract change + (CollectorStateContractTests) rather than the local one this is. Loaded only when plan capture is + on, because that is the only case where anything reads or writes them. */ + if (collectorState is null + && string.Equals(definition.Name, "query_store", StringComparison.Ordinal) + && _capturePlans()) + { + collectorState = await GetCollectorStateAsync( + server.ServerId, QueryStorePlanXmlState.StateCollectorName, cancellationToken); + } + + /* #2150: the text watermark lives under its OWN state owner, so it is a second read merged into the + same dictionary — the two prefixes (planwm: / textwm:) cannot collide, and the definition still + sees one flat State. Read unconditionally for query_store rather than behind _capturePlans(), + because the text fetch is not gated on plan capture: a host that turned plans off still needs its + statement text. Merged rather than replacing, so a store that has plan state but no text state + yet (every store before this rung) keeps working. */ + if (string.Equals(definition.Name, "query_store", StringComparison.Ordinal)) + { + var textState = await GetCollectorStateAsync( + server.ServerId, QueryStoreTextState.StateCollectorName, cancellationToken); + + if (textState is { Count: > 0 }) + { + var merged = new Dictionary(StringComparer.Ordinal); + if (collectorState is not null) + { + foreach (var entry in collectorState) + { + merged[entry.Key] = entry.Value; + } + } + + foreach (var entry in textState) + { + merged[entry.Key] = entry.Value; + } + + collectorState = merged; + } + } + + /* #2312: the open-interval refresh stamps, the third owner merged into the same flat State — + qsowm: cannot collide with planwm:/textwm:. Read unconditionally for query_store like the + text watermark (the skip applies regardless of plan capture), and merged the same way so a + store predating this state keeps working: absent keys read as "include the open interval", + which is today's behavior exactly. */ + if (string.Equals(definition.Name, "query_store", StringComparison.Ordinal)) + { + var openIntervalState = await GetCollectorStateAsync( + server.ServerId, QueryStoreOpenIntervalState.StateCollectorName, cancellationToken); + + if (openIntervalState is { Count: > 0 }) + { + var merged = new Dictionary(StringComparer.Ordinal); + if (collectorState is not null) + { + foreach (var entry in collectorState) + { + merged[entry.Key] = entry.Value; + } + } + + foreach (var entry in openIntervalState) + { + merged[entry.Key] = entry.Value; + } + + collectorState = merged; + } + } + var context = new CollectorContext { ServerId = server.ServerId, @@ -150,6 +319,20 @@ declared keys (every other collector) means no query runs. Mirrors Lite. */ ExcludedDatabases = server.Config.ExcludedDatabases?.ToArray() ?? Array.Empty(), PerfmonCounterOverride = null, CapturePlanXml = _capturePlans(), + /* #2150: ON. query_sql_text is no longer carried on every runtime-stats row — it is fetched once + per query_id into collect.query_store_text (FetchAndStoreQueryTextAsync, below) and resolved + back by the readers, all six of which now prefer that table and fall back to the fact row's + own column. + Why this had to be one change and not two: text is immutable per query_id, but the runtime + rows are re-collected every cycle, so the inline column re-shipped the same statement text on + every snapshot — that is what made query_store_stats the largest table in the store. Flipping + nulls the inline column, so a reader that had not been converted would have shown blank text + for new rows while looking perfectly healthy. Rows collected BEFORE this flip still carry + their text inline, which is why the readers keep the fallback instead of switching over. */ + FetchQueryTextSeparately = true, + /* #2164: 0 from the default provider means "no override" — the collector keeps its own + constant. Converted MB -> bytes here so the store knob stays operator-friendly. */ + TextByteBudgetOverride = _textBudgetMb() > 0 ? _textBudgetMb() * 1024 * 1024 : null, CollectSchemaChangeEvents = _collectSchemaChanges(), }; @@ -166,6 +349,16 @@ declared keys (every other collector) means no query runs. Mirrors Lite. */ items WERE found and merely some of their probes failed. Lite's twin is _lastCollectionNote. */ string? collectionNote = null; + /* The engine's provider, resolved ONCE for both branches. It used to be resolved only inside the + per-database branch, and the branch below opened a hardcoded SqlConnection — so every collector + that does NOT fan out per database was handed a SQL Server connection whatever the target was. + Six of the seven PostgreSQL collectors take that path (only pg_autovacuum_stats fans out), and + SqlClient rejects Npgsql's keywords while parsing the connection string, before any query runs: + "Keyword not supported: 'host'". Worse, an ArgumentException is neither SqlException nor + PostgresException, so it missed BOTH classification arms in DarlingWorker and recorded a raw + ERROR every sweep forever — including for all three Tier 0 outage predictors. */ + var targetProvider = TargetProviders.For(server.Target); + if (definition.RunsPerDatabase(context.Target)) { /* Azure SQL DB scopes some DMVs to the connected database — run the query once per @@ -186,7 +379,16 @@ have timed out at 60s. */ ? definition.BuildQuery(context) : null; var perDbTimeout = definition.CommandTimeoutSecondsOverride ?? CommandTimeoutSeconds; - var databases = await GetAzureDatabaseListAsync(server, cancellationToken); + var perDbProvider = targetProvider; + + /* Two enumeration paths because the FAILURE semantics genuinely differ, not the SQL. On + Azure SQL DB an inaccessible master has a real fallback (collect the one connected + database) and a re-probe throttle to stop hammering it; on PostgreSQL a login that + cannot read pg_database cannot monitor the server at all, so inventing a fallback would + turn a permissions problem into a silent one-database collection. */ + var databases = server.Target.Engine == CollectorTargetEngine.PostgreSql + ? await GetPostgresDatabaseListAsync(server, cancellationToken) + : await GetAzureDatabaseListAsync(server, cancellationToken); var attempted = 0; var failed = 0; @@ -205,6 +407,19 @@ have timed out at 60s. */ { cancellationToken.ThrowIfCancellationRequested(); attempted++; + + /* #2150: THE path the field report is on — Azure SQL DB collects query_store per database + here, not through the enumerated driver, so the wall-clock ceiling has to be applied on + both. Null for every collector that declares none, in which case dbToken IS + cancellationToken and this loop is byte-for-byte what it was. */ + using var dbBudget = EnumeratedCollectorDriver.StartItemBudget( + definition.PerItemWallClockBudget, cancellationToken); + var dbToken = dbBudget?.Token ?? cancellationToken; + + /* #2312: this database's open-interval stamp, staged at decision time and landed only + after its read and flush succeed — per iteration, so a fault cannot leak a stamp + into a sibling database's landing. */ + string? stagedOpenIntervalStamp = null; try { /* The authoritative database_name for XE rows read on this path — see @@ -222,9 +437,67 @@ where flooring a stale watermark would WRONGLY truncate legitimate catch-up branch on Azure SQL DB (#1836) and does need the bound, so it applies WatermarkPolicy.ClampCatchup inside its own cutoff computation: the clamp travels with the collector that needs it instead of with the path. */ + /* dbToken throughout this branch (#2150 review catch): the interface contract says the + budget covers "the watermark refresh, the command, and the whole drain", and the + enumerated path's perItemWatermark delegate already honours that. Leaving these three + store round-trips on cancellationToken made THIS loop — the one the field report is + actually on — the only place the promise was not kept, and a store that has stopped + answering is exactly the stall the budget exists to bound. Safe for the hole records + specifically: a budget expiry abandons the whole pass, so the watermark does not + advance, the clamp is re-derived next cycle, and the hole is re-recorded (merged wider + with any already pending) rather than lost. */ context.Watermark = await GetLastCollectedTimeForDatabaseAsync( server.ServerId, definition.TargetTable, definition.WatermarkColumn!, - definition.PerDatabaseWatermarkColumn!, databaseName, cancellationToken); + definition.PerDatabaseWatermarkColumn!, databaseName, dbToken); + + /* #2111 adaptive shrink, Azure arm — tighten BEFORE BuildQuery: the + definition's own clamp only floors OLDER watermarks, so a tighter one + passes through untouched. The skipped range is recorded as a hole here + (wider than the clamp's own record would be, so the block below firing + too would merge, not conflict). */ + var azureFailures = ConsecutiveQueryStoreItemFailures(server.ServerId, databaseName); + if (azureFailures > 0 + && string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var adaptiveSpan = QueryStoreBackfillState.AdaptiveSpan(WatermarkPolicy.MaxCatchup, azureFailures); + var tighterFloor = collectionTime - adaptiveSpan; + if (context.Watermark is DateTime azureRaw) + { + if (azureRaw < tighterFloor) + { + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive catch-up shrink: {Failures} consecutive failed cycles — window narrowed to {Minutes:F0}m; the skipped range rides the backfill hole.", + server.Config.DisplayName, databaseName, azureFailures, adaptiveSpan.TotalMinutes); + await RecordQueryStoreBackfillHoleAsync(server.ServerId, databaseName, azureRaw, tighterFloor, dbToken); + context.Watermark = tighterFloor; + } + } + else + { + /* Never-succeeded database: tighten the first-run fallback too (the + review catch); no hole — pre-watermark history is the tail's job. */ + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive first-contact shrink: {Failures} consecutive failed cycles — first-run window narrowed to {Minutes:F0}m.", + server.Config.DisplayName, databaseName, azureFailures, adaptiveSpan.TotalMinutes); + context.Watermark = tighterFloor; + } + } + + /* #2312, Azure arm: same per-database open-interval decision as the enumerated + delegate, BEFORE BuildQuery bakes the predicate. Staged into the local, landed + only in the post-flush success block below — a per-database fault this loop + tolerates must re-include next cycle, not spend the refresh window. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var includeOpen = QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + context.State, databaseName, collectionTime); + context.IncludeOpenInterval = includeOpen; + if (includeOpen) + { + stagedOpenIntervalStamp = QueryStoreOpenIntervalState.Format(collectionTime); + } + } + dbPlan = definition.BuildQuery(context); /* The definition clamped its own cutoff — surface the same WARNING the @@ -248,18 +521,21 @@ branch from growing backfill state they have no worker for. */ && WatermarkPolicy.ClampCatchup(context.Watermark, collectionTime) is DateTime azureClampedFloor) { await RecordQueryStoreBackfillHoleAsync( - server.ServerId, databaseName, context.Watermark.Value, azureClampedFloor, cancellationToken); + server.ServerId, databaseName, context.Watermark.Value, azureClampedFloor, dbToken); } } } var sqlSlice = Stopwatch.StartNew(); List batch; - using (var dbConnection = await OpenAzureDatabaseConnectionAsync(server, databaseName, cancellationToken)) - using (var dbCommand = CreateCollectorCommand(dbPlan, dbConnection, perDbTimeout)) - using (var dbReader = await dbCommand.ExecuteReaderAsync(cancellationToken)) + /* dbToken, not cancellationToken (#2150): connect, execute and drain are the phases the + budget bounds. The FLUSH below deliberately stays on cancellationToken — abandoning a + write already in flight would trade a slow cycle for a partially-written one. */ + using (var dbConnection = await OpenDatabaseConnectionAsync(perDbProvider, server, databaseName, dbToken)) + using (var dbCommand = CreateCollectorCommand(perDbProvider, dbPlan, dbConnection, perDbTimeout)) + using (var dbReader = await dbCommand.ExecuteReaderAsync(dbToken)) { - batch = await definition.ReadAsync(dbReader, context, cancellationToken); + batch = await definition.ReadAsync(dbReader, context, dbToken); /* #1875: the payload path's probe-failure contract, on the path that used to ignore it. blocked_process_report is the declaring collector that also runs per @@ -270,7 +546,7 @@ set and the loop simply never advanced the reader to it — the rows were built if (definition.EmitsProbeFailures) { cycleProbeFailures.Add( - await EnumeratedCollectorDriver.ReadPayloadProbeFailuresAsync(dbReader, cancellationToken)); + await EnumeratedCollectorDriver.ReadPayloadProbeFailuresAsync(dbReader, dbToken)); } } sqlMs += sqlSlice.ElapsedMilliseconds; @@ -301,6 +577,58 @@ context signal stays this database's until the next read resets it. */ context.PerItemTextBytesShipped / (1024.0 * 1024.0), context.PerItemShippedBoundary?.ToString("o") ?? "n/a"); } + + /* #2111: success resets the adaptive-shrink count on the Azure arm too. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemSucceeded(server.ServerId, databaseName); + + /* #2312: read and flush both landed — the staged open-interval stamp may too. */ + if (stagedOpenIntervalStamp is not null) + { + context.PendingState[QueryStoreOpenIntervalState.KeyFor(databaseName)] = stagedOpenIntervalStamp; + } + } + } + catch (OutOfMemoryException) + { + /* AHEAD of the budget arm, because ItemBudgetExpired classifies on the TOKENS and never + looks at the exception type (review catch). Without this, an OOM thrown while the + budget's timer had already fired — materializing a large batch, or inside the store + write — would be caught by that arm and logged as a routine per-database timeout, + silently breaking the invariant the generic catch below states outright. The shared + EnumeratedCollectorDriver already orders it this way; these two loops did not. */ + throw; + } + catch (Exception ex) when (EnumeratedCollectorDriver.ItemBudgetExpired(dbBudget, cancellationToken)) + { + /* #2150: this database ran out of wall clock. Counted as a per-database failure so the + cycle moves on — one database must not be able to starve the rest, which is the harm + the field report describes. Ahead of the generic catch because a cancelled command + does not reliably arrive as an OperationCanceledException, so that filter cannot be + trusted to claim it; the token check is what keeps a real shutdown out of this arm. + The provider's own cancellation exception is dropped in favour of the budget message: + it describes HOW the read was stopped, not why. */ + _ = ex; + var budgetFailure = EnumeratedCollectorDriver.ItemBudgetException( + definition.PerItemWallClockBudget!.Value); + failed++; + firstFailure ??= budgetFailure; + + /* Same #2111 stamp the generic arm makes, and it MATTERS more here: this is what turns + the bound from a cut that repeats forever into one that converges. The consecutive + count narrows this database's next catch-up window, so a database that cannot finish + in the budget keeps halving until it can. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(server.ServerId, databaseName); + } + + /* WARNING, not Debug, unlike the routine per-database skip beside it: an offline + database is ordinary and this is a collector that could not finish its work. */ + _logger?.LogWarning( + "{Collector} on '{Server}' database [{Database}] {Message}", + definition.Name, server.Config.DisplayName, databaseName, budgetFailure.Message); } catch (Exception ex) when (ex is not OperationCanceledException and not OutOfMemoryException) { @@ -308,6 +636,17 @@ context signal stays this database's until the next read resets it. */ routine one-database miss. */ failed++; firstFailure ??= ex; + + /* #2111: the yield-to-live stamp + adaptive-shrink count for the Azure SQL DB + arm — query_store reaches THIS per-database loop there, not the enumeration + path's onItemError, and without the stamp the backfill worker would never + yield on an Azure target (the review catch on #2112). Same query_store-only + guard as the hole recording above. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(server.ServerId, databaseName); + } + _logger?.LogDebug("Skipping database '{Database}' for {Collector}: {Error}", databaseName, definition.Name, ex.Message); } } @@ -335,8 +674,8 @@ Rethrow the first failure so RunOneAsync classifies it (SESSION_MISSING / PERMIS } else { - using var sqlConnection = new SqlConnection(server.ConnectionString); - await sqlConnection.OpenAsync(cancellationToken); + using var targetConnection = CreateTargetConnection(server); + await targetConnection.OpenAsync(cancellationToken); var enumerationPlan = definition.BuildEnumerationQuery(context); if (enumerationPlan is not null) @@ -346,7 +685,7 @@ Rethrow the first failure so RunOneAsync classifies it (SESSION_MISSING / PERMIS with a warning, matching Lite. */ var listSlice = Stopwatch.StartNew(); EnumerationOutcome enumeration; - using (var enumerationCommand = CreateCollectorCommand(enumerationPlan, sqlConnection, CommandTimeoutSeconds)) + using (var enumerationCommand = CreateCollectorCommand(targetProvider, enumerationPlan, targetConnection, CommandTimeoutSeconds)) using (var enumerationReader = await enumerationCommand.ExecuteReaderAsync(cancellationToken)) { /* Shared read (#1837): the item list, then the OPTIONAL second result set of items the @@ -377,7 +716,7 @@ were simply quiet (#1837). Mirrors Lite's _lastCollectionNote. */ { try { - using var probeCommand = CreateCollectorCommand(probePlan, sqlConnection, 10); + using var probeCommand = CreateCollectorCommand(targetProvider, probePlan, targetConnection, 10); var probeResult = await probeCommand.ExecuteScalarAsync(cancellationToken); if (probeResult is not null && probeResult != DBNull.Value) { @@ -398,6 +737,14 @@ were simply quiet (#1837). Mirrors Lite's _lastCollectionNote. */ database on it, flushing each before reading the next. */ await using var pgConnection = await _postgres.OpenConnectionAsync(cancellationToken); + /* #2312: open-interval stamps STAGED at decision time (perItemWatermark, below), landed + into PendingState only from onItemComplete — which the driver invokes solely after the + item's read AND flush succeeded. Staging them straight into PendingState would let a + per-item fault the driver tolerates still "spend" the 15-minute refresh window for a + cycle that captured nothing (the review catch): PendingState flushes as long as the + whole run survives. Keyed per item, so one database's decision cannot land on another. */ + var stagedOpenIntervalStamps = new Dictionary(StringComparer.Ordinal); + var driverResult = await EnumeratedCollectorDriver.RunAsync( items, /* Per-database watermark refresh + the 24h catch-up clamp, computed INSIDE the loop — @@ -408,6 +755,12 @@ Only query_store (the sole enumeration collector with a per-database timestamp ? null : async (item, ct) => { + /* #2164: the driver's per-item stopwatch starts BEFORE this delegate, so the + watermark refresh — a STORE read, plus a store write on the clamp path below — + would otherwise be silently counted as row-streaming time. Measured here so + DrainMsFrom can subtract it; the whole point of the split is that each number + names one real phase. */ + var watermarkWatch = Stopwatch.StartNew(); var raw = await GetLastCollectedTimeForDatabaseAsync( server.ServerId, definition.TargetTable, definition.WatermarkColumn!, definition.PerDatabaseWatermarkColumn!, item, ct); @@ -430,27 +783,180 @@ it has no worker for. */ await RecordQueryStoreBackfillHoleAsync(server.ServerId, item, raw.Value, clamped.Value, ct); } } + + /* #2111 adaptive shrink (promoted from reserve on field evidence — a member + whose 1h window intermittently exceeds the command timeout stays stuck for + hours): after N consecutive live failures the window halves per failure + toward 15 minutes, and the range the tighter floor skips rides the SAME + hole records the clamp writes — deferred to the trickle, never dropped. + Success resets the count, so a recovered member is back at full width + next cycle. */ + var failures = ConsecutiveQueryStoreItemFailures(server.ServerId, item); + if (failures > 0 + && string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var span = QueryStoreBackfillState.AdaptiveSpan(WatermarkPolicy.MaxCatchup, failures); + var tighterFloor = collectionTime - span; + if (clamped is DateTime current) + { + if (current < tighterFloor) + { + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive catch-up shrink: {Failures} consecutive failed cycles — window narrowed to {Minutes:F0}m; the skipped range rides the backfill hole.", + server.Config.DisplayName, item, failures, span.TotalMinutes); + await RecordQueryStoreBackfillHoleAsync(server.ServerId, item, current, tighterFloor, ct); + clamped = tighterFloor; + } + } + else + { + /* Never-succeeded database (null watermark): the definition's 60-minute + first-run fallback is MaxCatchup-sized, so it can be exactly the window + that cannot fit — tighten it the same way (the review catch on the first + cut, which gated shrink on a non-null watermark and left first contact + retrying the full width forever). No hole record: pre-watermark history + is the backfill TAIL's job by design. */ + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive first-contact shrink: {Failures} consecutive failed cycles — first-run window narrowed to {Minutes:F0}m.", + server.Config.DisplayName, item, failures, span.TotalMinutes); + clamped = tighterFloor; + } + } + context.Watermark = clamped; + + /* #2312: decide per database whether this cycle reads the OPEN interval. The + stamp is only STAGED here — it lands in PendingState from onItemComplete, + after this item's read and flush actually succeeded, so a per-item fault + (which this driver swallows by design) re-includes next time instead of + spending the refresh window on a cycle that captured nothing. Name-guarded + like the hole records: only query_store's payload reads the flag. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var includeOpen = QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + context.State, item, collectionTime); + context.IncludeOpenInterval = includeOpen; + if (includeOpen) + { + stagedOpenIntervalStamps[QueryStoreOpenIntervalState.KeyFor(item)] = + QueryStoreOpenIntervalState.Format(collectionTime); + } + } + + context.PerItemWatermarkMs = watermarkWatch.ElapsedMilliseconds; }, readItem: async (item, ct) => { var batch = new List(); - using var itemCommand = CreateCollectorCommand(definition.BuildPerItemQuery(item, context), sqlConnection, itemTimeout); + using var itemCommand = CreateCollectorCommand(targetProvider, definition.BuildPerItemQuery(item, context), targetConnection, itemTimeout); + /* #2164: time the OPEN separately from the drain. ExecuteReaderAsync returns only + when the first rowset is available, so for query_store's staged batch this is the + #pm_qs_slice aggregate plus time-to-first-row — the part no client-side budget can + shorten. Everything after is streaming, which the budget does govern. The blended + sql: number could not tell those apart, which is why a 5x payload cut looked like + it did nothing. */ + /* Cleared BEFORE the open so an item whose open faults cannot log the previous + item's split as its own — a stale timing is worse than no timing. The watermark + phase is NOT cleared here: it ran already, for THIS item, and clearing it would + hand its milliseconds to drain. The fetch phases clear on the same rule. */ + context.PerItemOpenMs = 0; + context.PerItemPlanFetchMs = 0; + context.PerItemTextFetchMs = 0; + var openWatch = Stopwatch.StartNew(); using var itemReader = await itemCommand.ExecuteReaderAsync(ct); + context.PerItemOpenMs = openWatch.ElapsedMilliseconds; await definition.ReadItemAsync(item, itemReader, batch, context, ct); + /* #2210: this database's plan-XML fetch, right after its runtime-stats read. A separate + query on purpose — it ships in plan_id order, so a budget cut truncates a SUFFIX, + which is the only reason the watermark can advance from a cut pass at all. */ + /* `is SqlConnection` rather than a bare cast, and it does two jobs (merge resolution + against #2213's provider seam): the connection here is a provider-neutral + DbConnection now, and this fetch is Query-Store-only, so the pattern narrows the + type the signature needs AND gates the engine in one expression that cannot drift + from either. The enumerated path serves PostgreSQL targets since #2213; query_store + declares TargetEngine = SqlServer so it never reaches here for one, but relying on + the catalog for that would be an invariant held somewhere else. */ + if (context.CapturePlanXml && targetConnection is SqlConnection planFetchConnection) + { + /* #2312 investigation: timed so the log split can say whether the invariant + per-cycle cost lives HERE rather than in the payload — a 0-row cycle's + blended sql: could not distinguish them. */ + var planFetchWatch = Stopwatch.StartNew(); + await FetchAndStorePlansAsync(planFetchConnection, server, item, context, itemTimeout, ct); + context.PerItemPlanFetchMs = planFetchWatch.ElapsedMilliseconds; + } + + /* #2150: and this database's statement-text fetch, for the same reason and with the + same shape — the payload no longer carries query_sql_text, because selecting it + inside the shipping TOP/ORDER BY made a Top-N Sort materialize nvarchar(max) text + for the whole qualifying set (measured 4.67s vs 0.45s time-to-first-row). Ships in + query_id order so a budget cut is a suffix, which is what lets the watermark + advance from a cut pass. + + Gated on the same flag the payload branches on, so the two can never disagree + about who owns the text: if the column is nulled, this runs. */ + if (context.FetchQueryTextSeparately && targetConnection is SqlConnection textFetchConnection) + { + /* #2312 investigation: same split as the plan fetch above. */ + var textFetchWatch = Stopwatch.StartNew(); + await FetchAndStoreQueryTextAsync(textFetchConnection, server, item, context, itemTimeout, ct); + context.PerItemTextFetchMs = textFetchWatch.ElapsedMilliseconds; + } + return batch; }, writeBatch: (batch, ct) => WriteBatchAsync(pgConnection, definition, batch, server, collectionTime, context, ct), onItemComplete: (item, batchCount, itemSqlMs, itemStorageMs) => { + /* #2111: a completed item resets the adaptive-shrink count — recovery returns + the member to the full catch-up width on its next cycle. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemSucceeded(server.ServerId, item); + + /* #2312: NOW the open-interval stamp may land — this hook only fires after + the item's read and flush both succeeded. Remove, not read: a stamp left + staged (read faulted) must not leak into a later run's landing. */ + if (stagedOpenIntervalStamps.Remove(QueryStoreOpenIntervalState.KeyFor(item), out var landedStamp)) + { + context.PendingState[QueryStoreOpenIntervalState.KeyFor(item)] = landedStamp; + } + } + /* Per-DATABASE line for non-empty batches (#1565): the per-server summary blends every database into one number, which hid a single busy database's 50s burst behind four quiet siblings. Quiet databases (0 rows — the 2-of-3 cycles between Query Store's 900s flushes) stay silent. */ if (batchCount > 0) { - _logger?.LogInformation(" [{Server}] {Collector} [{Database}] => {Rows} rows (sql:{SqlMs}ms, pg:{PgMs}ms)", - server.Config.DisplayName, definition.Name, item, batchCount, itemSqlMs, itemStorageMs); + /* #2164: open vs drain, because they have different fixes. A pass that is nearly + all OPEN is bound by server-side work before the first row (for query_store, + the #pm_qs_slice aggregate) and no client-side budget or payload trimming will + touch it; a pass that is mostly drain is bound by moving rows, where the byte + budget and the link are the levers. Only emitted when the host measured it. */ + if (context.PerItemOpenMs > 0) + { + /* #2312: the fetch phases print only when a separate fetch actually ran, + so every other collector's line is byte-identical to before. */ + if (context.PerItemPlanFetchMs > 0 || context.PerItemTextFetchMs > 0) + { + _logger?.LogInformation(" [{Server}] {Collector} [{Database}] => {Rows} rows (sql:{SqlMs}ms = wm:{WatermarkMs}ms + open:{OpenMs}ms + drain:{DrainMs}ms + plan_fetch:{PlanFetchMs}ms + text_fetch:{TextFetchMs}ms, pg:{PgMs}ms)", + server.Config.DisplayName, definition.Name, item, batchCount, itemSqlMs, + context.PerItemWatermarkMs, context.PerItemOpenMs, context.DrainMsFrom(itemSqlMs), + context.PerItemPlanFetchMs, context.PerItemTextFetchMs, itemStorageMs); + } + else + { + _logger?.LogInformation(" [{Server}] {Collector} [{Database}] => {Rows} rows (sql:{SqlMs}ms = wm:{WatermarkMs}ms + open:{OpenMs}ms + drain:{DrainMs}ms, pg:{PgMs}ms)", + server.Config.DisplayName, definition.Name, item, batchCount, itemSqlMs, + context.PerItemWatermarkMs, context.PerItemOpenMs, context.DrainMsFrom(itemSqlMs), itemStorageMs); + } + } + else + { + _logger?.LogInformation(" [{Server}] {Collector} [{Database}] => {Rows} rows (sql:{SqlMs}ms, pg:{PgMs}ms)", + server.Config.DisplayName, definition.Name, item, batchCount, itemSqlMs, itemStorageMs); + } } var capHit = definition.PerItemRowCountWarnThreshold is int cap && batchCount >= cap; @@ -465,9 +971,22 @@ behind four quiet siblings. Quiet databases (0 rows — the 2-of-3 cycles betwee } }, onItemError: (item, ex) => + { + /* #2111: stamp the yield-to-live signal (any database's live failure vouches + for the whole replica being contended) + the per-database adaptive-shrink + count. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(server.ServerId, item); + } + _logger?.LogWarning("Failed to collect {Collector} from [{Database}] on '{Server}': {Message}", - definition.Name, item, server.Config.DisplayName, ex.Message), - cancellationToken); + definition.Name, item, server.Config.DisplayName, ex.Message); + }, + cancellationToken, + /* #2150: the per-database wall-clock ceiling. Null for every collector but + query_store, so this argument leaves every other cycle untouched. */ + perItemBudget: definition.PerItemWallClockBudget); rowsWritten = driverResult.Rows; sqlMs += driverResult.SqlMs; @@ -481,7 +1000,7 @@ so all three paths share one writer. */ var sqlSlice = Stopwatch.StartNew(); var plan = definition.BuildQuery(context); List rows; - using (var command = CreateCollectorCommand(plan, sqlConnection, definition.CommandTimeoutSecondsOverride ?? CommandTimeoutSeconds)) + using (var command = CreateCollectorCommand(targetProvider, plan, targetConnection, definition.CommandTimeoutSecondsOverride ?? CommandTimeoutSeconds)) using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { rows = await definition.ReadAsync(reader, context, cancellationToken); @@ -510,7 +1029,7 @@ exactly what they were. */ { try { - using var supplementalCommand = CreateCollectorCommand(supplementalPlan, sqlConnection, CommandTimeoutSeconds); + using var supplementalCommand = CreateCollectorCommand(targetProvider, supplementalPlan, targetConnection, CommandTimeoutSeconds); using var supplementalReader = await supplementalCommand.ExecuteReaderAsync(cancellationToken); await definition.ApplySupplementalAsync(rows, supplementalReader, context, cancellationToken); } @@ -534,7 +1053,53 @@ exactly what they were. */ path. Outside the storage-phase timer: this is host bookkeeping, not collected data. */ if (context.PendingState.Count > 0) { - await SaveCollectorStateAsync(server.ServerId, definition.Name, context.PendingState, cancellationToken); + /* #2164: query_store's pending state is the plan-XML watermark set, which belongs to the host's + own state owner, NOT to the definition's name — the definition declares no state keys, so a row + written under "query_store" would never be read back and the watermark would silently never + apply. Everything else keeps writing under its definition. */ + var stateOwner = string.Equals(definition.Name, "query_store", StringComparison.Ordinal) + ? QueryStorePlanXmlState.StateCollectorName + : definition.Name; + + /* #2150/#2312: query_store's pending state now carries THREE watermark families with three + owners, so it is split by prefix on the way out. Writing one under another's owner would + still read back (the load above merges all three), but it would then never be pruned: the + shared prune set pairs each prefix with its owner, and a prefix pruned under the wrong + owner deletes nothing — which is indistinguishable from having nothing to prune. */ + var textKeys = context.PendingState + .Where(entry => entry.Key.StartsWith(QueryStoreTextState.WatermarkKeyPrefix, StringComparison.Ordinal)) + .ToDictionary(entry => entry.Key, entry => entry.Value, StringComparer.Ordinal); + var openIntervalKeys = context.PendingState + .Where(entry => entry.Key.StartsWith(QueryStoreOpenIntervalState.WatermarkKeyPrefix, StringComparison.Ordinal)) + .ToDictionary(entry => entry.Key, entry => entry.Value, StringComparer.Ordinal); + + if (textKeys.Count > 0 || openIntervalKeys.Count > 0) + { + var others = context.PendingState + .Where(entry => !textKeys.ContainsKey(entry.Key) && !openIntervalKeys.ContainsKey(entry.Key)) + .ToDictionary(entry => entry.Key, entry => entry.Value, StringComparer.Ordinal); + + if (textKeys.Count > 0) + { + await SaveCollectorStateAsync( + server.ServerId, QueryStoreTextState.StateCollectorName, textKeys, cancellationToken); + } + + if (openIntervalKeys.Count > 0) + { + await SaveCollectorStateAsync( + server.ServerId, QueryStoreOpenIntervalState.StateCollectorName, openIntervalKeys, cancellationToken); + } + + if (others.Count > 0) + { + await SaveCollectorStateAsync(server.ServerId, stateOwner, others, cancellationToken); + } + } + else + { + await SaveCollectorStateAsync(server.ServerId, stateOwner, context.PendingState, cancellationToken); + } } _logger?.LogDebug("Collected {RowCount} {Collector} rows for server '{Server}'", @@ -658,7 +1223,8 @@ dimension table instead of inline onto every row. Derived from the same schema if (transaction is not null) { await PayloadDimensionWriter.FlushAsync( - pgConnection, transaction, dimensions, storedCollectionTime, cancellationToken); + pgConnection, transaction, dimensions, storedCollectionTime, cancellationToken, + compressPlanContent: _compressPlanContent()); await transaction.CommitAsync(cancellationToken); } @@ -685,7 +1251,7 @@ public async Task> FetchRowsAsync( int commandTimeoutSeconds, CancellationToken cancellationToken) { - if (!definition.AppliesTo(server.Target)) + if (!CollectorCatalog.AppliesTo(definition, server.Target)) { return new List(); } @@ -704,14 +1270,27 @@ public async Task> FetchRowsAsync( ExcludedDatabases = server.Config.ExcludedDatabases?.ToArray() ?? Array.Empty(), PerfmonCounterOverride = null, CapturePlanXml = _capturePlans(), + /* #2150: stays FALSE here, and NOT because the readers are behind — this is FetchRowsAsync, the + on-demand live fetch. It returns the rows straight to the caller and writes nothing: no store + insert, no text fetch, no watermark. Turning it on would null query_sql_text in rows that have + no side table to be resolved from, so text would simply be gone. Today's only caller is the + active-queries snapshot, which has no Query Store text at all, so this is a guard for the next + caller rather than a live behaviour. */ + FetchQueryTextSeparately = false, + /* #2164: 0 from the default provider means "no override" — the collector keeps its own + constant. Converted MB -> bytes here so the store knob stays operator-friendly. */ + TextByteBudgetOverride = _textBudgetMb() > 0 ? _textBudgetMb() * 1024 * 1024 : null, CollectSchemaChangeEvents = _collectSchemaChanges(), }; var plan = definition.BuildQuery(context); - using var connection = new SqlConnection(server.ConnectionString); + /* Engine-neutral: a Postgres target gets an NpgsqlConnection here and the definition's + ReadAsync never knows the difference — it reads a DbDataReader either way. */ + var provider = TargetProviders.For(server.Target); + using var connection = provider.CreateConnection(server.ConnectionString); await connection.OpenAsync(cancellationToken); - using var command = CreateCollectorCommand(plan, connection, commandTimeoutSeconds); + using var command = CreateCollectorCommand(provider, plan, connection, commandTimeoutSeconds); using var reader = await command.ExecuteReaderAsync(cancellationToken); return await definition.ReadAsync(reader, context, cancellationToken); } @@ -779,6 +1358,219 @@ public async Task> GetCollectorStateAsync( return state; } + /// + /// Fetches one database's un-stored plan XML in plan_id order, lands it into the shared plan dimension + /// plus the map, and advances that database's watermark to what actually LANDED (#2210). + /// + /// Failure-isolated, and that is load-bearing rather than defensive: plan XML is an enrichment on top + /// of runtime statistics, so a fetch that throws must not cost the database its runtime stats. It logs and + /// returns with the watermark untouched, which is safe by construction — the watermark only ever advances to + /// content already written, so the next pass simply re-selects the same plans. + /// + /// The candidate window is seeded conservatively rather than adapted, DELIBERATELY, and this is the one + /// piece of the ratified design not yet wired: the adaptive input is the previous pass's own + /// bytes-per-plan, and there is nowhere to keep it. CollectorContext is shared with Lite, so adding a + /// field is a two-host contract change — the same reasoning that put the watermark under its own state owner + /// rather than on the definition — and the state VALUE is a parsed planId:stamp pair that cannot carry + /// a third field without a format change and a migration for readers. Passing null means K comes from + /// FirstContactAvgPlanBytes, which over-estimates plan size and therefore under-sizes the window: it + /// fetches fewer plans per pass than it could, and never more than it should. Slower convergence, never + /// unsafe. + /// + private async Task FetchAndStorePlansAsync( + SqlConnection sqlConnection, + ServerRuntime server, + string databaseName, + CollectorContext context, + int itemTimeout, + CancellationToken cancellationToken) + { + try + { + var watermark = QueryStorePlanXmlState.Resolve(context.State, databaseName, context.CollectionTime); + var budget = context.TextByteBudgetOverride ?? 12 * 1024 * 1024; + + /* #2312 Finding 1: size the window from THIS database's learned average instead of the + 160KB seed every pass — zero AvgBytes means never learned, which is the seed's job. */ + var estimate = _observedPlanSize.TryGetValue((server.ServerId, databaseName), out var carried) + ? carried + : default; + var candidates = QueryStorePlanXmlState.CandidatePlanCount( + estimate.AvgBytes > 0 ? estimate.AvgBytes : null, budget, estimate.CatchUpInProgress, out var clamped); + + if (clamped) + { + _logger?.LogInformation( + "query_store plan fetch on '{Server}' database [{Database}]: candidate window clamped to {K} — a bound sized this pass, not a measurement.", + server.Config.DisplayName, databaseName, candidates); + } + + var query = QueryStoreCollector.Instance.BuildPlanFetchQuery( + databaseName, context, watermark, candidates, budget); + + var fetched = new List(); + using (var command = CreateCollectorCommand(query, sqlConnection, itemTimeout)) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + fetched.Add(new FetchedPlan( + reader.GetInt64(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + PlanHash: null)); + } + } + + /* Learn from what this pass actually decompressed and shipped — BEFORE the empty-pass + early return, because an empty pass is the one that proves the walk caught up (nvarchar + length * 2 is DATALENGTH exactly, no server round-trip needed). NULL-XML rows count for + the window (they shipped, the watermark passes them) but not for the average's divisor + (they carried no bytes to average — the review catch). */ + var shippedBytes = 0L; + var plansMeasured = 0; + foreach (var plan in fetched) + { + if (plan.PlanXml is not null) + { + shippedBytes += (long)plan.PlanXml.Length * 2; + plansMeasured++; + } + } + _observedPlanSize[(server.ServerId, databaseName)] = + QueryStorePlanXmlState.Learn(estimate, shippedBytes, fetched.Count, plansMeasured, candidates, budget); + + if (fetched.Count == 0) + { + return; + } + + await using var pgConnection = await _postgres.OpenConnectionAsync(cancellationToken); + var landed = await QueryStorePlanWriter.WriteAsync( + pgConnection, server.ServerId, databaseName, fetched, context.CollectionTime, cancellationToken); + + var advance = QueryStorePlanXmlState.AdvanceWatermark(watermark, landed); + if (!advance.ArrivedInPlanIdOrder) + { + /* Loud rather than swallowed: the fetch's ORDER BY is what makes a budget cut a suffix, so + out-of-order arrival means that safety argument no longer holds and the pass earns nothing. */ + _logger?.LogWarning( + "query_store plan fetch on '{Server}' database [{Database}]: plans arrived OUT OF plan_id order — watermark held at {Watermark}. The ORDER BY is what makes a cut safe, so this pass earned no advance.", + server.Config.DisplayName, databaseName, watermark); + return; + } + + if (advance.Watermark > watermark) + { + /* Same stamp discipline as the runtime write-back: carried FORWARD across an advance, stamped + fresh only when the standing watermark was 0 (this pass WAS the full fetch). Re-stamping on + every advance would push the sweep period out forever on any database that keeps compiling. */ + var stamp = watermark > 0 + ? QueryStorePlanXmlState.ResolveStamp(context.State, databaseName) ?? context.CollectionTime + : context.CollectionTime; + + context.PendingState[QueryStorePlanXmlState.KeyFor(databaseName)] = + QueryStorePlanXmlState.Format(advance.Watermark, stamp); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger?.LogWarning(ex, + "query_store plan fetch failed on '{Server}' database [{Database}] — runtime statistics are unaffected and the watermark is unchanged, so the next pass re-selects the same plans.", + server.Config.DisplayName, databaseName); + } + } + + /// + /// One database's statement-text fetch (#2150), the sibling of . + /// + /// Exists because the runtime-stats payload stopped carrying query_sql_text: selecting it + /// inside the shipping TOP ... ORDER BY made a Top-N Sort materialize nvarchar(max) text + /// for the entire qualifying set before emitting row one (measured 4.67s against 0.45s + /// time-to-first-row, and neither the row cap nor the byte budget could bound it). + /// + /// A failure here is text-only. Runtime statistics are already written by the time this + /// runs, and the watermark only advances on what LANDED — so a throw leaves the rows in place with their + /// text unresolved and the next pass re-selects the same statements. That is why this is a warning + /// rather than a failure of the collector. + /// + /// Known property of a first fill, stated rather than discovered. The walk is ASCENDING by + /// query_id, because that is what makes a byte-budget cut a resumable suffix. On a store whose + /// watermark is still 0 that means the OLDEST statements resolve first, while the rows being collected + /// right now reference the newest ids — so a fresh store shows missing text for recent statements until + /// the walk catches up. Steady state is the opposite and is the case that matters: the watermark sits + /// near the top, so a newly-seen statement is fetched on the next pass. + /// + private async Task FetchAndStoreQueryTextAsync( + SqlConnection sqlConnection, + ServerRuntime server, + string databaseName, + CollectorContext context, + int itemTimeout, + CancellationToken cancellationToken) + { + try + { + var watermark = QueryStoreTextState.Resolve(context.State, databaseName, context.CollectionTime); + var budget = context.TextByteBudgetOverride ?? 12 * 1024 * 1024; + + var query = QueryStoreCollector.Instance.BuildTextFetchQuery( + databaseName, context, watermark, QueryStoreTextState.CandidateTexts, budget); + + var fetched = new List(); + using (var command = CreateCollectorCommand(query, sqlConnection, itemTimeout)) + await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + fetched.Add(new FetchedQueryText( + reader.GetInt64(0), + reader.IsDBNull(1) ? null : reader.GetString(1))); + } + } + + if (fetched.Count == 0) + { + return; + } + + await using var pgConnection = await _postgres.OpenConnectionAsync(cancellationToken); + var landed = await QueryStoreTextWriter.WriteAsync( + pgConnection, server.ServerId, databaseName, fetched, context.CollectionTime, cancellationToken); + + var advance = QueryStoreTextState.AdvanceWatermark(watermark, landed); + if (!advance.ArrivedInQueryIdOrder) + { + /* Loud rather than swallowed, same as the plan fetch: the ORDER BY is what makes a budget cut + a suffix, so out-of-order arrival means that safety argument no longer holds and the pass + earns no advance. */ + _logger?.LogWarning( + "query_store text fetch on '{Server}' database [{Database}]: statements arrived OUT OF query_id order — watermark held at {Watermark}. The ORDER BY is what makes a cut safe, so this pass earned no advance.", + server.Config.DisplayName, databaseName, watermark); + return; + } + + if (advance.Watermark > watermark) + { + /* Stamp carried FORWARD across an advance and stamped fresh only when the standing watermark + was 0 (this pass WAS the full walk). Re-stamping on every advance would push the refresh + horizon out forever on any database that keeps seeing new statements — which is exactly + where a Query Store reset, the thing the horizon exists to recover from, would hurt most. */ + var stamp = watermark > 0 + ? QueryStoreTextState.ResolveStamp(context.State, databaseName) ?? context.CollectionTime + : context.CollectionTime; + + context.PendingState[QueryStoreTextState.KeyFor(databaseName)] = + QueryStoreTextState.Format(advance.Watermark, stamp); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger?.LogWarning(ex, + "query_store text fetch failed on '{Server}' database [{Database}] — runtime statistics are already written and the watermark is unchanged, so those rows keep unresolved text and the next pass re-selects the same statements.", + server.Config.DisplayName, databaseName); + } + } + /// /// Upserts what the definition observed this cycle (), /// after the cycle completed — so a cycle that collected zero rows still records what it saw, which is @@ -878,6 +1670,216 @@ public async Task DeleteCollectorStateKeyAsync( } } + /// + /// Retires one collector's per-database collector_state rows for databases the server no longer + /// has (#2188). One statement per (owner, prefix) pair — $1 server_id, $2 collector_name, + /// $3 the key prefix, which is also what reconstructs each live database's key for the anti-join. + /// + /// The existence list is database_states, not the collector's enumeration. That is + /// the whole design. query_store's enumeration is a heavily FILTERED list — ONLINE only, AG primaries + /// only, the excluded-database filter, the vendor-name screen, HAS_DBACCESS, and a per-database + /// probe that can fail — so a database missing from one cycle's items is far more often offline, + /// excluded or unprobeable than dropped, and pruning on that absence would delete live watermarks on + /// exactly the servers that have such databases. database_states is an unfiltered + /// SELECT ... FROM sys.databases, so it answers the only question being asked here: does this + /// name still exist on the instance. + /// + /// Guarded on the snapshot existing (newest IS NOT NULL): a server that has never + /// collected database_states, or whose rows have aged out, produces an empty snapshot, and an unguarded + /// anti-join against nothing deletes EVERY row. The subselect always yields one row (MAX over zero rows + /// is NULL), so the guard is what turns "no snapshot" into "prune nothing" instead of "prune all". + /// + /// And guarded on the snapshot being NEWER than the state row + /// (s.updated_at < snapshot.newest), which is the stronger of the two and the one that makes + /// this correct rather than merely usually-correct. Existing is not the same as CURRENT: if + /// database_states stops collecting for a server — a per-server schedule change, a failing collector, + /// anything — the newest snapshot freezes, and every database created after that instant is missing from + /// it while being perfectly alive. Presence alone would prune such a database's watermark on EVERY cycle + /// forever, silently paying a full plan-XML refetch each time: the exact cost #2164 exists to remove, + /// with a log line confidently calling a live database dropped. A snapshot cannot judge a row written + /// after it was taken. This holds regardless of the two collectors' relative cadences, so nothing here + /// depends on database_states being scheduled more often than query_store; both stamps are the SERVICE + /// clock's naive UTC (collectionTime and both read + /// DateTime.UtcNow), never the monitored server's. A genuinely dropped database is still pruned: + /// its last state write necessarily precedes any snapshot taken after the drop. + /// + /// The two guards overlap — < against a NULL newest is already NULL, so the freshness + /// test alone would cover the empty snapshot. The explicit NULL check stays anyway: "no snapshot prunes + /// nothing" is a promise worth reading off the statement instead of deriving from three-valued + /// logic. + /// + /// Bounded consequence, either way. Deleting a watermark that should have stayed costs one + /// full plan-XML refetch for that database and nothing else — the same conservative path an absent or + /// expired watermark already takes (), which is why racing + /// an in-flight cycle is safe: the write-back is an upsert, so a cycle that had already loaded the state + /// simply restores the row it is still using, and a row deleted for a genuinely dropped database has no + /// cycle to race. + /// + /// Not reached on Azure SQL DB, where DatabaseStateCollector.AppliesTo is false and + /// there is therefore no snapshot to check — the guard makes it a no-op rather than a mass delete. Those + /// orphans stay (#2191 tracks the Azure arm, including why that path's own database list cannot be used + /// as the existence check); the accumulation is bounded to one ~100-byte row per database name ever + /// seen. + /// + /// Best-effort like every sibling here: a failed prune leaves the rows and the next cycle retries. + /// Nothing downstream reads them — an orphan is a row nobody asks about, which is why this is hygiene + /// rather than a correctness fix. + /// + internal const string PruneOrphanedDatabaseStateKeysSql = @" +DELETE FROM collector_state s +USING (SELECT MAX(collection_time) AS newest FROM database_states WHERE server_id = $1) snapshot +WHERE s.server_id = $1 +AND s.collector_name = $2 +AND starts_with(s.state_key, $3) +AND snapshot.newest IS NOT NULL +AND s.updated_at < snapshot.newest +AND NOT EXISTS + ( + SELECT 1 + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = snapshot.newest + AND s.state_key = $3 || ds.database_name + ) +RETURNING s.state_key"; + + /// + /// The Azure SQL DB variant (#2191): prune every per-database state key that is not the ONE database this + /// registration names. $1 server_id, $2 collector_name, $3 key prefix, $4 the + /// registration's own database. + /// + /// Why this needs no snapshot, and no freshness guard. The on-prem statement anti-joins + /// database_states because the question there is "does this name still exist on the instance", and + /// it needs the two guards because a SNAPSHOT can be empty or stale. Here there is no snapshot: after + /// #2220 a registration that names a database sweeps only that database, so its one legitimate key is + /// derivable from the connection string's own catalog — which is current by construction and cannot go + /// stale, be empty, or be filtered. That is why #2191 looked unfixable when it was filed and is not now: + /// it asked for "an authoritative unfiltered sys.databases read from master, used only on the success + /// path", and #2220 removed the need for any master read on this path at all. + /// + /// What it actually deletes, today. Mostly #2220's residue. Before that fix each Azure + /// registration swept every sibling database on the logical server and wrote a watermark for each, all + /// under its own server_id — so these keys are the state half of that contamination. collector_state + /// carries no retention (it is state, not facts), so unlike the collected rows those orphans would + /// otherwise persist forever rather than ageing out. + /// + /// Bounded consequence, exactly as on the on-prem path: deleting a watermark that should have + /// stayed costs one full plan-XML refetch for that database and nothing else. + /// + internal const string PruneForeignDatabaseStateKeysSql = @" +DELETE FROM collector_state s +WHERE s.server_id = $1 +AND s.collector_name = $2 +AND starts_with(s.state_key, $3) +AND s.state_key <> $3 || $4 +RETURNING s.state_key"; + + /// + /// Runs for every owner/prefix in the SHARED + /// — the same set Lite's DuckDB twin + /// (RemoteCollectorService.PruneOrphanedQueryStoreDatabaseStateAsync) iterates, so a prefix + /// cannot end up pruned on one SKU and orphaning on the other. Once per query_store cycle for one + /// server. Separate statements rather than one combined predicate because Npgsql's positional + /// parameters cannot span a multi-statement batch, and three narrow deletes down the primary key are + /// easier to read than one that ORs three prefixes together. + /// + internal async Task PruneOrphanedQueryStoreDatabaseStateAsync(int serverId, CancellationToken cancellationToken) + { + try + { + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + var pruned = new List(); + + foreach (var (owner, prefix) in QueryStorePerDatabaseState.PrunableKeys) + { + using var command = new NpgsqlCommand(PruneOrphanedDatabaseStateKeysSql, connection); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(owner); + command.Parameters.AddWithValue(prefix); + + /* RETURNING rather than a rows-affected count: the only symptom of a WRONG delete here is a + silent refetch, so a bare number would leave nothing to diagnose it with. The keys name + the databases, which is what makes a mistaken prune visible in the log. */ + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + pruned.Add(reader.GetString(0)); + } + } + + if (pruned.Count > 0) + { + /* Information, like DarlingObservability's orphaned-server sweep: rare, and it names a + database lifecycle event the operator may not know the monitor noticed. */ + _logger?.LogInformation( + "[server_id {ServerId}] pruned {Count} query_store state row(s) for database(s) no longer on the server: {Keys}", + serverId, pruned.Count, string.Join(", ", pruned)); + } + } + catch (Exception ex) + { + _logger?.LogDebug(ex, "Pruning orphaned query_store database state failed; next cycle retries"); + } + } + + /// + /// The Azure SQL DB arm of the #2188 prune (#2191): retire every per-database state key that does not + /// belong to the one database this registration names. + /// + /// Runs the same shared set as the on-prem + /// path, so a prefix cannot be pruned on one target type and left orphaning on the other. + /// + /// + /// The registration's own database — the connection string's initial catalog. Callers must only reach + /// here when that is non-empty: a registration naming no database (or naming master) is a + /// registration of the logical SERVER, whose legitimate database set is everything on it, and pruning + /// against a single name there would delete every live watermark it has. + /// + internal async Task PruneForeignQueryStoreDatabaseStateAsync( + int serverId, string ownDatabase, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(ownDatabase)) + { + return; + } + + try + { + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + var pruned = new List(); + + foreach (var (owner, prefix) in QueryStorePerDatabaseState.PrunableKeys) + { + using var command = new NpgsqlCommand(PruneForeignDatabaseStateKeysSql, connection); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(owner); + command.Parameters.AddWithValue(prefix); + command.Parameters.AddWithValue(ownDatabase); + + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + pruned.Add(reader.GetString(0)); + } + } + + if (pruned.Count > 0) + { + /* Names the keys rather than counting them, like the on-prem twin: the only symptom of a + wrong delete here is a silent refetch, so a bare number would leave nothing to diagnose + with. On an Azure server upgraded past #2220 this fires ONCE and clears that fix's state + residue, which is worth saying plainly rather than looking like a recurring anomaly. */ + _logger?.LogInformation( + "[server_id {ServerId}] pruned {Count} query_store state row(s) belonging to databases other than this registration's [{Database}]: {Keys}", + serverId, pruned.Count, ownDatabase, string.Join(", ", pruned)); + } + } + catch (Exception ex) + { + _logger?.LogDebug(ex, "Pruning foreign query_store database state failed; next cycle retries"); + } + } + /// /// The #2022 backfill write entry: the SAME private COPY writer every live path routes through /// (dimension diversion, positional contract, naive-UTC stamp), on its own store connection. @@ -992,45 +1994,71 @@ public void OnServerReconnected(int serverId) } /// - /// Lists databases on an Azure SQL DB logical server, mirroring Lite's #857 behavior: try - /// master enumeration first (with the per-server exclusion filter), and on a master-access - /// error fall back to the connection's own database, throttling re-probes per server. + /// The databases one Azure SQL DB registration's per-database sweep covers. + /// + /// A registration that names a database sweeps that database, and nothing else (#2220) — + /// which is the common case, since server_id hashes host[:database][:RO] and registering + /// each database separately is how you get separate identities. That path returns immediately and never + /// touches master. + /// + /// Only a registration naming NO database — or naming master, where a catalog-less Azure + /// connection lands — is a registration of the logical SERVER, and only that one enumerates: master + /// first with the per-server exclusion filter, and on a master-access error a fallback that has nothing + /// to fall back to and therefore throws (#857's shape, now the exceptional path rather than the default). + /// The re-probe throttle is deliberately NOT consulted there; see the comment at the call site. + /// + /// It read master unconditionally before #2220, sweeping every online database on the logical + /// server into whichever registration ran the sweep — N registrations of N databases meant N² collection + /// with every registration's history contaminated by its siblings'. /// internal async Task> GetAzureDatabaseListAsync(ServerRuntime server, CancellationToken cancellationToken) { var targetDb = new SqlConnectionStringBuilder(server.ConnectionString).InitialCatalog; - /* Skip the throttle when there is nothing to fall back TO — see Lite's twin. With no target - database the fallback can only throw, so honouring the throttle would guarantee 15 minutes - of failure without ever attempting to recover. */ - var hasFallback = SingleDbOrEmpty(targetDb).Count > 0; - - if (hasFallback && IsMasterProbeThrottled(server.ServerId)) + /* #2220: a registration that NAMES a database is a registration OF that database, so its sweep + covers exactly that one and never touches master. Before this, EVERY database-scoped collector + enumerated master and swept every online database on the logical server, storing all of it under + the one server_id of whichever registration ran the sweep — N registrations of N databases on one + server meant N² collection with every registration's history contaminated by its siblings'. + + This also subsumes the #857 case it looks like it bypasses, and improves on it: a login granted + access to one user database but not to master HAS a named database, so it now returns here without + probing master at all, rather than probing, failing, forming a verdict and falling back. Master is + reached only by a registration that names no database — the logical-server registration, which has + nothing else to enumerate from. */ + var ownDatabase = AzureSweepScope.OwnDatabaseOrEmpty(targetDb); + if (ownDatabase.Count > 0) { - return FallbackDatabaseList(server, targetDb, reason: "master previously inaccessible", quiet: true); + return ownDatabase; } - var masterConnectionString = new SqlConnectionStringBuilder(server.ConnectionString) - { - InitialCatalog = "master", - }.ConnectionString; - - var (exclusionClause, exclusionParameters) = DatabaseExclusionFilter.Build( - server.Config.ExcludedDatabases, "name"); + /* NO throttle check here, and that is deliberate rather than an omission — restoring what the + `hasFallback &&` guard used to achieve. This branch is reached ONLY when the registration names no + database, so there is nothing to fall back TO: honouring the throttle would return + FallbackDatabaseList, which throws immediately without probing, and would keep throwing for the + whole recheck interval while never attempting the one thing that could recover. Probing master + every cycle is the cheaper failure. (Review caught me reintroducing exactly this: I read + `hasFallback &&` as a redundant condition when it was there to DISABLE the throttle.) + + The throttle machinery itself is left alone. It is tested behaviour from #857/#1506, and it is now + unreachable in production for a different reason than this one: its whole purpose was to stop + re-probing master for a registration that HAS a fallback, and such a registration no longer probes + master at all. Retiring it is its own change, with those tests. */ + + /* The query and the hop to master both come from the provider, so the enumeration set is defined + in exactly one place per engine. What stays here is the failure policy below, which is the + part that is genuinely Azure-specific. */ + var (masterConnectionString, enumerationQuery) = SqlServerTargetProvider.Instance.BuildDatabaseListPlan( + server.ConnectionString, server.Config.ExcludedDatabases); var databases = new List(); try { using var connection = new SqlConnection(masterConnectionString); await connection.OpenAsync(cancellationToken); - using var command = new SqlCommand( - $"SELECT name FROM sys.databases WHERE state_desc = N'ONLINE' AND database_id > 0 {exclusionClause} ORDER BY name;", - connection) - { CommandTimeout = CommandTimeoutSeconds }; - foreach (var parameter in exclusionParameters) - { - command.Parameters.Add(ToSqlParameter(parameter)); - } + /* Azure master enumeration is SQL-Server-only, but it goes through the same parameter + mapping as every other command so a type cannot be mapped two ways. */ + using var command = CreateCollectorCommand(enumerationQuery, connection, CommandTimeoutSeconds); using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -1106,26 +2134,89 @@ private List FallbackDatabaseList(ServerRuntime server, string? targetDb } internal async Task OpenAzureDatabaseConnectionAsync(ServerRuntime server, string databaseName, CancellationToken cancellationToken) + => (SqlConnection)await OpenDatabaseConnectionAsync( + SqlServerTargetProvider.Instance, server, databaseName, cancellationToken); + + /// + /// The connection for a collector that reads the server as a whole — engine-resolved from the probed + /// target, never constructed directly. + /// Extracted so it can be PINNED by test. This is the exact seam that broke: the non-per-database + /// branch built a SqlConnection literally, so six of the seven PostgreSQL collectors got a SQL + /// Server connection and failed in the connection-string parser before running a query. Both engines' + /// providers were already correct and individually tested — nothing asserted that the RUNNER asked them. + /// A test that opens nothing and only checks the returned TYPE is enough to catch it, which is why it is + /// worth having. + /// + internal static DbConnection CreateTargetConnection(ServerRuntime server) { - var connectionString = new SqlConnectionStringBuilder(server.ConnectionString) - { - InitialCatalog = databaseName, - }.ConnectionString; + ArgumentNullException.ThrowIfNull(server); - var connection = new SqlConnection(connectionString); - await connection.OpenAsync(cancellationToken); - return connection; + return TargetProviders.For(server.Target).CreateConnection(server.ConnectionString); } - private static List SingleDbOrEmpty(string? targetDb) + /// + /// The engine-neutral per-database connection: same monitored server, one specific database. + /// PostgreSQL has no alternative to this. A SQL Server collector can reach another database + /// without reconnecting (EXECUTE [db].sys.sp_executesql), but a PostgreSQL connection is + /// bound to one database for its lifetime, so a per-database collector there is necessarily one + /// connection per database per cycle. That is the cost of reading pg_stat_user_tables and + /// friends at all, and it is why per-database PostgreSQL collectors get slow cadences. + /// + internal static async Task OpenDatabaseConnectionAsync( + ITargetProvider provider, ServerRuntime server, string databaseName, CancellationToken cancellationToken) { - if (string.IsNullOrEmpty(targetDb) || string.Equals(targetDb, "master", StringComparison.OrdinalIgnoreCase)) + var connection = provider.CreateConnection( + provider.WithDatabase(server.ConnectionString, databaseName)); + + try + { + await connection.OpenAsync(cancellationToken); + return connection; + } + catch { - return new List(); + /* The caller only disposes what it receives, so a connection that fails to open must be + disposed HERE or it leaks — once per database per cycle, on exactly the unreachable + database the per-database loop is designed to skip and keep going past. */ + await connection.DisposeAsync(); + throw; } - return new List { targetDb }; } + /// + /// Lists the databases to fan out over on a PostgreSQL target. + /// No master-inaccessible fallback and no re-probe throttle, unlike the Azure twin, because + /// neither has a meaning here: pg_database is a shared catalog readable from the connected + /// database, so a failure means the login or the server is broken rather than that one catalog is + /// out of reach. Falling back to the connected database would convert a permissions problem into a + /// quiet partial collection, which is the failure mode that fallback exists to avoid elsewhere. + /// + internal async Task> GetPostgresDatabaseListAsync(ServerRuntime server, CancellationToken cancellationToken) + { + var provider = TargetProviders.For(server.Target); + var (connectionString, query) = provider.BuildDatabaseListPlan( + server.ConnectionString, server.Config.ExcludedDatabases); + + var databases = new List(); + + using var connection = provider.CreateConnection(connectionString); + await connection.OpenAsync(cancellationToken); + using var command = CreateCollectorCommand(provider, query, connection, CommandTimeoutSeconds); + using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + databases.Add(reader.GetString(0)); + } + + return databases; + } + + /* #2220: delegates to the shared rule. Both runners carried their own copy of this predicate, and a + sweep-scoping rule that disagrees between Lite and Darling is the same class of defect as the one + #2220 fixes. */ + private static List SingleDbOrEmpty(string? targetDb) => + AzureSweepScope.OwnDatabaseOrEmpty(targetDb); + /// /// Whether master enumeration failed in a way that means database-scoped collectors should fall back /// to the connection's own catalog (#857). Deliberately broader than "this login cannot read master": @@ -1140,26 +2231,20 @@ internal static bool ShouldFallBackToSingleDatabaseError(int errorNumber) => SqlErrorClassification.ShouldFallBackToSingleDatabase(errorNumber); /* Internal, not private: QueryStoreBackfill (#2022) builds its slice commands through the same - parameter mapping so the two paths cannot drift on a type. */ - internal static SqlCommand CreateCollectorCommand(CollectorQuery plan, SqlConnection connection, int commandTimeoutSeconds) - { - var command = new SqlCommand(plan.Text, connection) { CommandTimeout = commandTimeoutSeconds }; - - foreach (var parameter in plan.Parameters) - { - command.Parameters.Add(ToSqlParameter(parameter)); - } + parameter mapping so the two paths cannot drift on a type. - return command; - } + Still SqlCommand-typed and still SQL-Server-only, because every caller of THIS overload is: + Query Store backfill and the Azure per-database/master paths are SQL Server features by + definition. The engine-neutral path goes through CreateCollectorCommand(ITargetProvider, ...) + below, and both end up in the same parameter mapping inside SqlServerTargetProvider, so a + parameter type cannot be mapped two ways. */ + internal static SqlCommand CreateCollectorCommand(CollectorQuery plan, SqlConnection connection, int commandTimeoutSeconds) + => (SqlCommand)SqlServerTargetProvider.Instance.CreateCommand(plan, connection, commandTimeoutSeconds); - private static SqlParameter ToSqlParameter(CollectorParameter parameter) => parameter.Type switch - { - CollectorParameterType.DateTime2 => new SqlParameter(parameter.Name, SqlDbType.DateTime2) { Value = parameter.Value ?? DBNull.Value }, - CollectorParameterType.NVarChar128 => new SqlParameter(parameter.Name, SqlDbType.NVarChar, 128) { Value = parameter.Value ?? DBNull.Value }, - CollectorParameterType.NVarChar260 => new SqlParameter(parameter.Name, SqlDbType.NVarChar, 260) { Value = parameter.Value ?? DBNull.Value }, - CollectorParameterType.Int32 => new SqlParameter(parameter.Name, SqlDbType.Int) { Value = parameter.Value ?? DBNull.Value }, - CollectorParameterType.BigInt => new SqlParameter(parameter.Name, SqlDbType.BigInt) { Value = parameter.Value ?? DBNull.Value }, - _ => throw new ArgumentOutOfRangeException(nameof(parameter), parameter.Type, "Unmapped collector parameter type"), - }; + /// + /// The engine-neutral command factory: same collector query, whichever engine the target is. + /// + private static DbCommand CreateCollectorCommand( + ITargetProvider provider, CollectorQuery plan, DbConnection connection, int commandTimeoutSeconds) + => provider.CreateCommand(plan, connection, commandTimeoutSeconds); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingCommandExecutor.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingCommandExecutor.cs index ea14113b5..f6821fcaa 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingCommandExecutor.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingCommandExecutor.cs @@ -467,6 +467,9 @@ internal static (string ResultStatus, string ResultJson) MapProbeResult(Connecti { if (probe.Success) { + /* The PostgreSQL facts ride alongside rather than replacing anything: an existing consumer + reading majorVersion/engineEdition keeps working, and one that knows about engine can tell + why those are 0 on a Postgres target instead of reading it as a half-failed probe. */ var json = JsonSerializer.Serialize(new { success = true, @@ -477,6 +480,12 @@ internal static (string ResultStatus, string ResultJson) MapProbeResult(Connecti isAzureManagedInstance = probe.IsAzureManagedInstance, isAwsRds = probe.IsAwsRds, hasMsdbAccess = probe.HasMsdbAccess, + engine = probe.Engine.ToString(), + postgresMajorVersion = probe.PostgresMajorVersion, + postgresVersionNum = probe.PostgresVersionNum, + isAurora = probe.IsAurora, + isInRecovery = probe.IsInRecovery, + facts = DarlingServerConnector.DescribeProbeFacts(probe), }); return ("connected", json); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs index 7431ee9fa..5fa83dd29 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs @@ -12,6 +12,7 @@ using System.Net; using System.Net.Sockets; using System.Text.Json; +using PerformanceMonitor.Collectors; using System.Text.Json.Serialization; using PerformanceMonitor.Notifications; @@ -53,6 +54,58 @@ public sealed class DarlingConfig [JsonPropertyName("capturePlans")] public bool CapturePlans { get; set; } = true; + /// + /// Whether the query_store backfill loop runs at all (#2167). Store-backed (config_service, V58) and + /// read live by the worker's loop, so an operator can stop a runaway drain (a freshly restored catalog + /// against a cross-region server) without a restart and without touching plan capture. Default on. + /// + [JsonPropertyName("queryStoreBackfillEnabled")] + public bool QueryStoreBackfillEnabled { get; set; } = true; + + /// + /// Per-database text byte budget for the query_store collector, in MEGABYTES (#2164). Store-backed + /// (config_service, V59), clamped [4,256] on read, default 64 = the previous compile-time constant. + /// Lower it when the monitored fleet is a network hop away: the budget bounds memory, but it also + /// sets how long one collector query holds the monitored server open draining to this client, which + /// over a cross-region link is the tenant-visible cost. A cut is always resumable (#1960), so a + /// smaller budget trades catch-up latency for shorter statements — never data. + /// + [JsonPropertyName("queryStoreTextBudgetMb")] + public int QueryStoreTextBudgetMb { get; set; } = 64; + + /// + /// #2316: how many days a stored plan XML outlives its last sighting before the dimension GC may + /// take it — the bound the fact-coupled horizon cannot provide on a store younger than the fact + /// retention (measured: 127 GB of parameter-sniffing plan churn in the dim's first 22 days, with + /// the coupled GC unable to fire until a month after projected disk-full). Facts keep their full + /// retention; a plan older than this renders as a missing plan, which every reader handles. + /// 0 disables (fact-coupled horizon alone); enabled values clamp to [7,365] on read. + /// + [JsonPropertyName("planContentRetentionDays")] + public int PlanContentRetentionDays { get; set; } = 21; + + /// + /// The plan-XML storage codec (#2171). Store-backed (config_service, V62), normalized to 'gzip' or + /// 'none' on read. 'gzip' (default, unchanged): plans live as gzip bytes in query_plan_gz - 14.0x + /// measured, readable only through the apps/MCP. 'none': plans written as plain text into + /// query_plan_xml - lz4 TOAST compresses ~8.9x, and anything reading the store directly over SQL + /// (Grafana, report tooling) gets plan XML back with no extension and no UDF. Flipping it affects + /// NEW rows only; the readers' text-first-else-gz resolution covers every mix. The dimension is + /// content-addressed either way, so the digest and dedup are codec-independent. + /// + [JsonPropertyName("planXmlCompression")] + public string PlanXmlCompression { get; set; } = "gzip"; + + /// + /// How many per-server collection bodies may hold a SQL connection at once (#2170) — the #1553 fleet + /// gate, previously hardcoded to 4. Store-backed (config_service, V59), clamped [1,16], default 4. + /// Raise it on a host with headroom watching a large fleet, where 4-wide serialization is what makes + /// sweeps queue and the Fleet Health screen report staleness while every collector is healthy. + /// Peak transient memory is roughly this × . + /// + [JsonPropertyName("maxConcurrentSweeps")] + public int MaxConcurrentSweeps { get; set; } = 4; + /// /// Whether the default_trace_events collector records Object:Created/Altered/Deleted schema-change /// (DDL) events. Default TRUE (today's behavior). Set false on a noisy or benchmark box where a @@ -241,6 +294,23 @@ public IReadOnlyList Validate() { problems.Add($"{label}: auth must be 'integrated' or 'sql' (got '{server.Auth}')."); } + + /* Caught here, in the pre-flight, rather than only where the connection string is built. + MonitoredServerConnection throws on this too — it has to, since it is what actually + knows the driver can't honour it — but that throw happens at first connect, which for a + service means the misconfiguration surfaces in a log after deployment instead of in + --test-connection before it. */ + if (server.IsPostgres && !server.UsesSqlAuth) + { + problems.Add( + $"{label}: a PostgreSQL target requires auth 'sql' with a username and password " + + "(integrated/Kerberos auth is not supported for PostgreSQL targets)."); + } + + if (server.Port is not 0 && server.Port is < 1 or > 65535) + { + problems.Add($"{label}: port must be between 1 and 65535 (got {server.Port})."); + } } return problems; @@ -434,6 +504,41 @@ public sealed class AlertsConfig [JsonPropertyName("pvsFloorGb")] public int PvsFloorGb { get; set; } = 1; + /// #2107: the store volume's self-alert warning percent (was a compile-time 10.0; + /// 0 disables the check — percent is its only trigger). + [JsonPropertyName("selfDiskFreeWarnPercent")] + public int SelfDiskFreeWarnPercent { get; set; } = 10; + + /// #2107: how long collection may go quiet before Collection Stopped / Agent Not + /// Running fire (was a compile-time 30 minutes). + [JsonPropertyName("collectionStaleMinutes")] + public int CollectionStaleMinutes { get; set; } = 30; + + /// #2107: the Collection Stopped fast path — consecutive failures with zero successes + /// that fire without waiting out the staleness window (was a compile-time 10). + [JsonPropertyName("collectionFailureThreshold")] + public int CollectionFailureThreshold { get; set; } = 10; + + /// #2107: the low-disk CRITICAL severity tier's percent floor (#1136 — grades the + /// target-volume alert; was a compile-time 3.0). + [JsonPropertyName("diskCriticalFreePercent")] + public int DiskCriticalFreePercent { get; set; } = 3; + + /// #2107: the low-disk CRITICAL severity tier's GB floor (was a compile-time 2.0). + [JsonPropertyName("diskCriticalFreeGb")] + public int DiskCriticalFreeGb { get; set; } = 2; + + /// #2107: the analysis notification cooldown — the shared engine clamps [30, 10080] + /// and Lite always passed a configured value through; Darling hardcoded 360. + [JsonPropertyName("analysisNotifyCooldownMinutes")] + public int AnalysisNotifyCooldownMinutes { get; set; } = 360; + + /// #2136: the Store Job Over Cadence warning threshold — a store background job whose + /// last run reaches this percent of its own schedule interval fires the Warning tier. The + /// Critical tier is fixed at 100 (a job outrunning its cadence compounds refresh lag). + [JsonPropertyName("storeJobCadenceWarnPercent")] + public int StoreJobCadenceWarnPercent { get; set; } = 25; + [JsonPropertyName("longRunningJobEnabled")] public bool LongRunningJobEnabled { get; set; } = true; @@ -974,9 +1079,29 @@ public sealed class MonitoredServer [JsonPropertyName("name")] public string Name { get; set; } = ""; + /// + /// Which database engine this target runs: "sqlserver" (default) or "postgres" + /// (accepted spellings: postgres, postgresql, pg, aurora-postgresql). + /// This is configuration rather than something probed, because it has to be known BEFORE + /// connecting — it decides which driver builds the connection string and which detection query + /// runs. An omitted or unrecognized value means SQL Server, so every existing darling.json keeps + /// its exact present behaviour. + /// + [JsonPropertyName("engine")] + public string Engine { get; set; } = "sqlserver"; + [JsonPropertyName("host")] public string Host { get; set; } = ""; + /// + /// TCP port, for PostgreSQL targets on a non-default port. 0 (the default) means "use the + /// driver's default", which is 5432. + /// Unused for SQL Server, which carries a non-default port in the host itself as + /// host,1433 — that convention is left alone rather than migrated. + /// + [JsonPropertyName("port")] + public int Port { get; set; } + /// Azure SQL Database: the one database this entry monitors (feeds the storage-name identity). [JsonPropertyName("database")] public string? Database { get; set; } @@ -1036,14 +1161,74 @@ public sealed class MonitoredServer [JsonIgnore] public bool UsesSqlAuth => string.Equals(Auth, "sql", StringComparison.OrdinalIgnoreCase); + /// + /// parsed. Anything unrecognized resolves to + /// rather than throwing: a typo in one server entry + /// must not stop the service from starting and monitoring everything else. The mismatch surfaces + /// immediately anyway — the SQL Server detection query fails against a Postgres target. + /// + [JsonIgnore] + public CollectorTargetEngine TargetEngine => Engine?.Trim().ToLowerInvariant() switch + { + "postgres" or "postgresql" or "pg" or "aurora-postgresql" or "aurora" => CollectorTargetEngine.PostgreSql, + _ => CollectorTargetEngine.SqlServer, + }; + + /// True when this entry targets PostgreSQL, so the SQL Server-only config is inapplicable. + [JsonIgnore] + public bool IsPostgres => TargetEngine == CollectorTargetEngine.PostgreSql; + /// Display name falls back to the host. [JsonIgnore] public string DisplayName => string.IsNullOrWhiteSpace(Name) ? Host : Name; /// - /// The canonical storage identity (host[:database][:RO]) — hashed to server_id via the shared - /// ServerIdHelper, so this Darling entry derives the same id Lite would for the same server. + /// The canonical storage identity (host[:database][:pg][:port][:RO]) — hashed to server_id via the + /// shared ServerIdHelper, so this Darling entry derives the same id Lite would for the same server. + /// + /// #2218: engine and port are passed so a PostgreSQL instance cannot collide with a SQL Server on the + /// same host, and two PostgreSQL instances on one host cannot collide with each other. Both are inert for a + /// SQL Server entry and the resulting name is byte-identical to what it was before — Engine folds to + /// no token for SQL Server, and Port is a PostgreSQL-only field that stays 0 there, because SQL + /// Server carries a non-default port inside Host as host,1433 and is therefore already + /// discriminated by the host string. That is why no conditional is needed here: the defaults ARE the + /// backwards-compatible case. + /// + [JsonIgnore] + public string StorageName => PerformanceMonitor.Common.ServerIdHelper.BuildStorageName( + Host, Database, ReadOnlyIntent, Engine, Port); + + /// + /// config_monitored_servers.server_id as READ FROM THE STORE, or null for an entry that has no + /// store row yet — a darling.json bootstrap entry before the first seed. + /// + /// Not settable from the file () on purpose. The registry is + /// authoritative for identity once seeded, so letting an operator pin a server_id in + /// darling.json would create a second authority that could disagree with it — and disagree + /// silently, since nothing downstream re-checks. + /// + [JsonIgnore] + public int? StoredServerId { get; set; } + + /// + /// This server's server_id: the stored value when there is one, otherwise derived from + /// . + /// + /// This is the single place a monitored server's identity is decided (#2218, #2158). It used + /// to be recomputed at twelve call sites — every operator-command lookup, the reconcile, the self-alert + /// stamps, the schedule resolution — which is what makes identity-derived-from-mutable-config expensive + /// to change: a stored surrogate is only useful if nothing re-derives it behind the store's back. + /// + /// Today the two are always equal, because the seed and the Viewer both write exactly this + /// hash, so reading the stored value changes no behaviour and no data moves. The point is that the + /// FALLBACK is now the only derivation: when identity stops being derivable, this property is what + /// changes, and the twelve call sites do not. + /// + /// The store is preferred over the derivation rather than merely agreeing with it, because that is + /// the ordering that makes a stored id which no longer matches its host keep working — which is the + /// whole point of storing it. /// [JsonIgnore] - public string StorageName => PerformanceMonitor.Common.ServerIdHelper.BuildStorageName(Host, Database, ReadOnlyIntent); + public int ServerId => + StoredServerId ?? PerformanceMonitor.Common.ServerIdHelper.GetDeterministicHashCode(StorageName); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs index 014abf0ce..61dc23017 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs @@ -204,6 +204,10 @@ public sealed class DarlingManagedPostgres /* Process budgets. pg_ctl start/stop get -w -t 60 of their own, so the outer budget only has to outlive them; initdb on a cold disk can take tens of seconds. */ private static readonly TimeSpan s_initDbTimeout = TimeSpan.FromSeconds(180); + + /* #2185: `--version` prints one line and exits, so this bounds a diagnostic probe, not real work. Short + on purpose — it runs while a startup failure is already being reported. */ + private static readonly TimeSpan s_versionProbeTimeout = TimeSpan.FromSeconds(5); private static readonly TimeSpan s_pgCtlTimeout = TimeSpan.FromSeconds(90); private static readonly TimeSpan s_statusTimeout = TimeSpan.FromSeconds(30); private const int PgCtlWaitSeconds = 60; @@ -779,8 +783,9 @@ public async Task StopIfStartedByThisProcessAsync() try { var binDirectory = Path.Combine(_runtimeRoot, "pgsql", "bin"); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"stop -D \"{_dataDirectory}\" -m fast -w -t {PgCtlWaitSeconds}", s_pgCtlTimeout, CancellationToken.None); @@ -791,7 +796,13 @@ public async Task StopIfStartedByThisProcessAsync() } else { - _logger.LogWarning("Managed Postgres stop reported exit code {ExitCode}: {Output}", exitCode, output); + /* {ExitCode} stays the raw int so structured sinks keep a numeric field to filter on; + the decoded meaning rides its own field. */ + _logger.LogWarning( + "Managed Postgres stop reported exit code {ExitCode} ({ExitCodeMeaning}): {Output}", + exitCode, + DarlingToolExitCode.Describe(exitCode), + DarlingToolExitCode.FormatOutput(output, exitCode)); } } catch (Exception ex) @@ -879,16 +890,25 @@ holds the superuser password in the clear. */ TryHardenCredentialFile(passwordFile, allowInteractiveRead: false); try { + var initDb = Path.Combine(binDirectory, "initdb.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "initdb.exe"), + initDb, $"-D \"{_dataDirectory}\" -U {UserName} -A scram-sha-256 --pwfile=\"{passwordFile}\" -E UTF8 --locale=C --data-checksums", s_initDbTimeout, cancellationToken); if (exitCode != 0) { + /* #2185: on a LOADER status, gather the evidence ourselves rather than asking the operator + to run two commands and report back. That thread took four exchanges and the decisive fact + — `initdb --version` working while the bootstrap died — only ever existed in the reporter's + shell. Two extra process launches on a path that has already failed fatally is free. */ + var runtimeProbe = DarlingToolExitCode.IsLoaderStatus(exitCode) + ? await ProbeRuntimeBinariesAsync(binDirectory, cancellationToken) + : string.Empty; + throw new InvalidOperationException( - $"initdb failed (exit code {exitCode}) for {_dataDirectory}. Output:\n{output}"); + BuildInitDbFailureMessage(exitCode, initDb, _dataDirectory, output, runtimeProbe)); } } finally @@ -906,6 +926,56 @@ holds the superuser password in the clear. */ _logger.LogInformation("Managed Postgres cluster initialized (scram-sha-256, data checksums, UTF8/C locale)"); } + /// + /// The first-run initdb failure, as an operator reads it (#2186). The leading clause is unchanged on + /// purpose — it is what the existing field reports and the issue tracker are searchable by — and + /// everything the raw form withheld follows it: the exit code decoded, the loader diagnosis when + /// Windows set that code, and an Output: field that says it is empty BECAUSE the process was + /// killed before it could write, rather than looking like data that failed to arrive. + /// + internal static string BuildInitDbFailureMessage( + int exitCode, string exePath, string dataDirectory, string output, string runtimeProbe = "") + => $"initdb failed (exit code {DarlingToolExitCode.Describe(exitCode)}) for {dataDirectory}." + + DarlingToolExitCode.Diagnose(exitCode, exePath) + + runtimeProbe + + $"\nOutput:\n{DarlingToolExitCode.FormatOutput(output, exitCode)}"; + + /// + /// Asks each of the two binaries for its version and reports which one could not load (#2185). + /// + /// Never throws and never blocks meaningfully. This runs on a path that has ALREADY failed + /// fatally, and its only job is to add a sentence to an exception that is about to be raised. A probe that + /// threw would replace a precise "initdb failed, here is why" with whatever the probe hit; a probe that + /// hung would turn a fast failure into a service that appears wedged at startup. So every fault mode — + /// missing file, unreadable directory, cancellation, timeout — resolves to the empty string, which + /// composes to the exact message the product produced before this existed. + /// + /// --version is the right probe because it is the ONE invocation that loads the binary and + /// its full dependency chain without touching the data directory, the port, or the cluster: it is the + /// loader test with no side effect. A five-second budget is generous for a process that prints one line. + /// + private static async Task ProbeRuntimeBinariesAsync(string binDirectory, CancellationToken cancellationToken) + { + try + { + /* The order matters for the reader, not the logic: initdb is what failed, postgres is the + hypothesis. Both are probed even when the first one loads, because "both loaded" is itself a + finding — it rules out a permanently missing dependency and redirects to the event log. */ + var (initDbCode, _) = await RunToolAsync( + Path.Combine(binDirectory, "initdb.exe"), "--version", s_versionProbeTimeout, cancellationToken); + var (postgresCode, _) = await RunToolAsync( + Path.Combine(binDirectory, "postgres.exe"), "--version", s_versionProbeTimeout, cancellationToken); + + return DarlingToolExitCode.DescribeRuntimeProbe(initDbCode, postgresCode); + } + catch (Exception) + { + /* Deliberately unfiltered. See the summary: the caller is composing a fatal message and there is + no fault here worth surfacing over the failure that is already being reported. */ + return string.Empty; + } + } + /// /// Marker-guarded conf append, re-checked on EVERY start — heals the crash window between /// initdb and the first append, which would otherwise silently cost TimescaleDB @@ -1094,7 +1164,7 @@ private async Task EnsureDataDirectoryMajorAsync(string binDirectory, Cancellati { var dataMajor = DarlingStoreUpgrade.ParseDataDirectoryMajor( await File.ReadAllTextAsync(Path.Combine(_dataDirectory, "PG_VERSION"), cancellationToken)); - var bundledMajor = await ReadRuntimeMajorAsync(binDirectory, cancellationToken); + var (bundledMajor, probeExitCode) = await ReadRuntimeMajorAsync(binDirectory, cancellationToken); if (DarlingStoreUpgrade.MustRefuseUnidentifiableRuntime(dataMajor, bundledMajor)) { @@ -1105,12 +1175,15 @@ DARLING01 as "skipping the runtime version check. The store starts normally." an is the strongest possible evidence they must not be used, not a reason to wave them through. Refusing here costs nothing that proceeding would have saved — the start was going to fail regardless — and it converts a cryptic Win32 status code into a message naming the rescued - runtime to restore. */ + runtime to restore. #2186 finished the thought: the refusal used to assert that the binaries + did not run without ever saying HOW it knew, so the one piece of evidence it held — the probe's + own exit code — died in a local variable. It is now quoted, decoded, and diagnosed. */ throw new InvalidOperationException( - $"The store's data directory {_dataDirectory} is PostgreSQL {dataMajor}, but the runtime at {binDirectory} could not be identified — its binaries did not run. " + + $"The store's data directory {_dataDirectory} is PostgreSQL {dataMajor}, but the runtime at {binDirectory} could not be identified — pg_ctl --version exited {DarlingToolExitCode.Describe(probeExitCode)} instead of reporting a version. " + "A runtime that cannot report its own version cannot start this store either, so the service is stopping here rather than failing deeper with a Win32 error code. " + $"This usually means the wrong package was deployed. Restore the previous runtime from {PreviousRuntimeHint()} over {Path.GetDirectoryName(binDirectory)}, or redeploy a package whose PostgreSQL major is {dataMajor} or newer, then restart the service. " + - "The data directory has not been touched."); + "The data directory has not been touched." + + DarlingToolExitCode.Diagnose(probeExitCode, Path.Combine(binDirectory, "pg_ctl.exe"))); } if (dataMajor is null || bundledMajor is null) @@ -1172,7 +1245,7 @@ runtime to restore. */ /* The revert put the PREVIOUS runtime back behind the same bin path, so the identity read above now describes binaries that are no longer there. Re-read it, or the post-start completion would try to move the extension to a version this runtime does not ship. */ - _bundledMajor = await ReadRuntimeMajorAsync(binDirectory, cancellationToken) ?? 0; + _bundledMajor = (await ReadRuntimeMajorAsync(binDirectory, cancellationToken)).Major ?? 0; _bundledTimescaleVersion = ReadBundledTimescaleVersion(binDirectory); } } @@ -1181,12 +1254,14 @@ private string PreviousRuntimeHint() => Path.Combine(DarlingStoreUpgrade.PreviousRuntimeRootFor(_runtimeRoot), "pgsql"); /// The bundled runtime's PostgreSQL major, from the binaries themselves rather than from a - /// manifest that could disagree with what is on disk. - private static async Task ReadRuntimeMajorAsync(string binDirectory, CancellationToken cancellationToken) + /// manifest that could disagree with what is on disk. The probe's exit code rides out alongside it + /// (#2186): when the answer is "unidentifiable", that code is the ONLY evidence of why, and the + /// refusal that consumes it used to have to assert the reason instead of showing it. + private static async Task<(int? Major, int ExitCode)> ReadRuntimeMajorAsync(string binDirectory, CancellationToken cancellationToken) { var (exitCode, output) = await RunToolAsync( Path.Combine(binDirectory, "pg_ctl.exe"), "--version", s_statusTimeout, cancellationToken); - return exitCode == 0 ? DarlingStoreUpgrade.ParsePostgresMajor(output) : null; + return (exitCode == 0 ? DarlingStoreUpgrade.ParsePostgresMajor(output) : null, exitCode); } /// @@ -1218,8 +1293,9 @@ private string PreviousRuntimeHint() /// pg_ctl status: 0 = a postmaster is running on this data directory, 3 = not running, 4 = bad/inaccessible data directory. private async Task IsRunningAsync(string binDirectory, CancellationToken cancellationToken) { + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"status -D \"{_dataDirectory}\"", s_statusTimeout, cancellationToken); @@ -1228,11 +1304,25 @@ private async Task IsRunningAsync(string binDirectory, CancellationToken c { 0 => true, 3 => false, - _ => throw new InvalidOperationException( - $"pg_ctl status reported exit code {exitCode} for {_dataDirectory} — the data directory is not usable. Output:\n{output}"), + _ => throw new InvalidOperationException(BuildStatusFailureMessage(exitCode, pgCtl, _dataDirectory, output)), }; } + /// + /// The pg_ctl status failure (#2186). The data-directory verdict is CONDITIONAL: pg_ctl's own codes + /// (4 = bad or inaccessible data directory) do say the directory is unusable, but a Windows status + /// says only that pg_ctl never ran, and blaming the data directory for that sends an operator to + /// delete a perfectly good store over a missing DLL. + /// + internal static string BuildStatusFailureMessage(int exitCode, string exePath, string dataDirectory, string output) + { + var diagnosis = DarlingToolExitCode.Diagnose(exitCode, exePath); + return $"pg_ctl status reported exit code {DarlingToolExitCode.Describe(exitCode)} for {dataDirectory}" + + (diagnosis.Length == 0 ? " — the data directory is not usable." : ".") + + diagnosis + + $"\nOutput:\n{DarlingToolExitCode.FormatOutput(output, exitCode)}"; + } + /// /// pg_ctl start, windowed (-w): returns only when the server accepts connections. The -o /// runtime override carries the port (authoritative over the conf line), listen_addresses (always @@ -1263,8 +1353,9 @@ legacy file stays bounded too. Going forward the v6 logging collector owns the s server is down (this method only runs when nothing is listening), so nothing holds the file. */ CapLegacyServerLog(_serverLogPath, LegacyServerLogCapBytes, _logger); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var exitCode = await RunDetachingToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"-D \"{_dataDirectory}\" -o \"{runtimeOptions}\" -l \"{_serverLogPath}\" -w -t {PgCtlWaitSeconds} start", s_pgCtlTimeout, cancellationToken); @@ -1272,13 +1363,24 @@ server is down (this method only runs when nothing is listening), so nothing hol if (exitCode != 0) { throw new InvalidOperationException( - $"pg_ctl start failed (exit code {exitCode}) for {_dataDirectory}.\n" + - $"Server log tail:\n{ReadServerLogTail()}"); + BuildStartFailureMessage(exitCode, pgCtl, _dataDirectory, ReadServerLogTail())); } _logger.LogInformation("Managed Postgres started"); } + /// + /// The pg_ctl start failure (#2186). Its Server log tail has the same trap the initdb message's + /// Output had: a loader status means pg_ctl died before it could start a postmaster, so the tail + /// reads "(no server log written)" — accurate, and completely misleading about where to look. The + /// diagnosis says which situation this is before the tail invites an operator to read a log that was + /// never going to exist. + /// + internal static string BuildStartFailureMessage(int exitCode, string exePath, string dataDirectory, string serverLogTail) + => $"pg_ctl start failed (exit code {DarlingToolExitCode.Describe(exitCode)}) for {dataDirectory}." + + DarlingToolExitCode.Diagnose(exitCode, exePath) + + $"\nServer log tail:\n{serverLogTail}"; + /// The one-time cap on the legacy pre-rotation pg.log (#1652): past this size it is /// rolled to pg.log.old (replacing any previous roll) before the next start. Two files, bounded /// forever; small files are left alone so a healthy post-rotation pg.log is never churned. @@ -1731,6 +1833,8 @@ private static bool ContainsWhitespace(string value) /// internal void EnsureServerCertificate(IPAddress listenIp, string certPath, string keyPath) { + var rootPath = RootCertificatePathFor(certPath); + if (File.Exists(certPath) && File.Exists(keyPath)) { try @@ -1741,6 +1845,25 @@ internal void EnsureServerCertificate(IPAddress listenIp, string certPath, strin /* Present + loads + the SAN covers this listen IP -> reuse (delete-to-rotate). Re-harden the key every start (self-healing), same discipline as the credential files. */ TryHardenCredentialFile(keyPath, allowInteractiveRead: false); + + /* #2117: a cert pair WITHOUT root.crt beside it is the legacy single self-signed + end-entity shape, whose critical CA=false Basic Constraints Windows' chain engine + refuses as its own trust anchor under Npgsql's Root Certificate custom-root trust — + verify-full with the printed cert fails on exactly the machines viewers run on. + Deliberately NOT auto-rotated: operators who worked around it via the OS trust + store have a WORKING setup a silent regeneration would break. Advise instead. */ + if (!File.Exists(rootPath)) + { + _logger.LogWarning( + "The store TLS cert at {Cert} is the legacy single self-signed shape — remote viewers using " + + "SSL Mode=VerifyFull with Root Certificate fail certificate-chain validation on Windows " + + "(#2117). To rotate to the fixed chain shape: stop the service, delete {Cert} and {Key}, " + + "start the service, then re-run --print-viewer-connection and redistribute the new root " + + "certificate to viewer machines. Viewers that imported the old cert into the OS trust " + + "store keep working until you rotate.", + certPath, certPath, keyPath); + } + return; } @@ -1755,36 +1878,29 @@ the key every start (self-healing), same discipline as the credential files. */ certPath, ex.Message); } - /* Fall through to regenerate — overwrites both files (the service account owns them). */ + /* Fall through to regenerate — overwrites the files (the service account owns them). */ } - using var rsa = RSA.Create(2048); - var request = new CertificateRequest( - $"CN={Environment.MachineName}", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + /* #2117: a real two-cert chain — throwaway local CA signs the leaf, the CA key is discarded + inside Create(), postgres serves leaf+CA, and root.crt is what the operator distributes. + See StoreTlsCertificates for why the old single self-signed shape failed verify-full. */ + var generated = StoreTlsCertificates.Create(Environment.MachineName, listenIp, ServerCertValidityYears); - var sanBuilder = new SubjectAlternativeNameBuilder(); - sanBuilder.AddIpAddress(listenIp); - sanBuilder.AddDnsName(Environment.MachineName); - request.CertificateExtensions.Add(sanBuilder.Build()); - request.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, true)); - request.CertificateExtensions.Add( - new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, true)); - request.CertificateExtensions.Add( - new X509EnhancedKeyUsageExtension(new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") /* serverAuth */ }, false)); - - var notBefore = DateTimeOffset.UtcNow.AddDays(-1); - var notAfter = notBefore.AddYears(ServerCertValidityYears); - using var certificate = request.CreateSelfSigned(notBefore, notAfter); - - File.WriteAllText(certPath, certificate.ExportCertificatePem()); - File.WriteAllText(keyPath, rsa.ExportPkcs8PrivateKeyPem()); + File.WriteAllText(certPath, generated.ServerCertChainPem); + File.WriteAllText(keyPath, generated.ServerKeyPem); + File.WriteAllText(rootPath, generated.RootCertPem); TryHardenCredentialFile(keyPath, allowInteractiveRead: false); _logger.LogInformation( - "Generated a self-signed store TLS cert (CN/DNS SAN {Host}, IP SAN {Ip}, ~{Years}yr) at {Cert}", - Environment.MachineName, listenIp, ServerCertValidityYears, certPath); + "Generated the store TLS chain (CN/DNS SAN {Host}, IP SAN {Ip}, ~{Years}yr): leaf+CA at {Cert}, distributable root at {Root}", + Environment.MachineName, listenIp, ServerCertValidityYears, certPath, rootPath); } + /// The distributable root's path — always beside the served cert (#2117). Public-key + /// material only, so it is deliberately not hardened like the key. + internal static string RootCertificatePathFor(string certPath) + => Path.Combine(Path.GetDirectoryName(certPath) ?? ".", "root.crt"); + /// /// Whether carries an iPAddress SAN equal to /// — the reuse gate for the store TLS cert (verify-full pins the IP SAN). Reads the SAN extension @@ -2001,16 +2117,20 @@ private async Task ReconcileNetworkAsync( if (changed) { await File.WriteAllTextAsync(hbaPath, updated, cancellationToken); + var reloadPgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (reloadCode, reloadOutput) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + reloadPgCtl, $"reload -D \"{_dataDirectory}\"", s_statusTimeout, cancellationToken); if (reloadCode != 0) { _logger.LogCritical( - "pg_ctl reload failed (exit {ExitCode}) after updating pg_hba.conf — the network access change may not be live: {Output}", - reloadCode, reloadOutput); + "pg_ctl reload failed (exit {ExitCode}, {ExitCodeMeaning}) after updating pg_hba.conf — the network access change may not be live: {Output}{Diagnosis}", + reloadCode, + DarlingToolExitCode.Describe(reloadCode), + DarlingToolExitCode.FormatOutput(reloadOutput, reloadCode), + DarlingToolExitCode.Diagnose(reloadCode, reloadPgCtl)); } } @@ -2351,7 +2471,6 @@ style of full-pathing every PG tool. */ } } - /// /// /// Applies the optional per-invocation environment and working directory shared by both process /// runners. Values are ADDED to the inherited environment rather than replacing it — a PG tool still diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs index abedd7a9e..f3557b8c4 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedRoles.cs @@ -114,6 +114,11 @@ public static class DarlingManagedRoles "trust_server_certificate", "read_only_intent", "multi_subnet_failover", "excluded_databases", "monthly_cost_usd", "capture_plans", "is_enabled", "created_at", "modified_at", "alert_delivery_mode_override", + /* V68. Non-secret: which engine a target is, and its port, are exactly as sensitive as its + host — which is already readable. The fail-closed design is why they have to be named at + all: an unclassified column stays invisible to `viewer` rather than being exposed by + default, so the live security gate fails until someone decides which side it is on. */ + "engine", "port", }, SecretColumns: new[] { "encrypted_password" }), diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs new file mode 100644 index 000000000..646791596 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Alerting; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// Darling's — the three Tier 0 predictors read out of the store. +/// Every query takes the LATEST reading per subject rather than an aggregate, because all three are +/// levels rather than rates: the question is "where does this stand now", not "how much accumulated". +/// +public sealed class DarlingPostgresAlertReadAdapter : IPostgresAlertReadAdapter +{ + private readonly NpgsqlDataSource _postgres; + + /// + /// How far back a reading may be and still count as current. Two hours covers the slowest of the three + /// cadences (wraparound at 5 minutes) with room for a missed sweep, while still refusing to alert off a + /// stale row after collection has stopped — an alert fired from yesterday's number is worse than none. + /// + internal static readonly TimeSpan Freshness = TimeSpan.FromHours(2); + + public DarlingPostgresAlertReadAdapter(NpgsqlDataSource postgres) => _postgres = postgres; + + /// + /// Latest row per database, carrying the server's own autovacuum_freeze_max_age so the evaluator + /// can scale its thresholds to this cluster's configuration instead of to a constant. + /// The two ages are kept separate rather than pre-maxed: which counter is worse decides which + /// remedy the alert names, and MultiXact exhaustion is a different (and less familiar) problem than XID + /// exhaustion. + /// + internal const string WraparoundSql = """ + SELECT DISTINCT ON (database_name) + database_name, + frozen_xid_age, + min_multixid_age, + autovacuum_freeze_max_age, + autovacuum_multixact_freeze_max_age + FROM pg_wraparound_stats + WHERE server_id = $1 + AND collection_time >= $2 + ORDER BY database_name, collection_time DESC + """; + + /// + /// The current winning holder, plus how persistently THAT HOLDER has won across the window. + /// Persistence is computed here rather than left to the evaluator because it needs the whole + /// window, and one pass over the store is cheaper than shipping every row up to be counted. The + /// observation total counts DISTINCT collection times, not rows: several sources are recorded per + /// collection, so counting rows would inflate the denominator and make every holder look transient. + /// Held is counted per (source, holder), not per source. Counting by source alone answered a + /// different question than the alert asks: sixty different sessions each winning once rendered as "pid X + /// held the horizon 60/60 observations", which is the exact shape of a chronic holder and the opposite of + /// the truth — sixty short transactions are normal, one that will not end is the incident. The alert names + /// a specific pid or slot, so persistence has to be that thing's persistence. + /// The denominator counts collections that recorded ANY holder. The collector emits no rows when the + /// horizon is unheld, so counting only holder-bearing collections is what makes the ratio mean "of the + /// times something held it, how often was it this one" — which is the question. Note this became reachable + /// only once the collector stopped attributing its own backend: while Darling's own snapshot was always a + /// session holder, every collection had a holder and the distinction was invisible. + /// + internal const string XminSql = """ + WITH latest AS ( + SELECT source, holder, xmin_age, detail + FROM pg_xmin_horizon + WHERE server_id = $1 + AND collection_time >= $2 + AND is_winner + ORDER BY collection_time DESC, xmin_age DESC + LIMIT 1 + ), + window_stats AS ( + SELECT + COUNT(DISTINCT collection_time) AS observations_total, + COUNT(DISTINCT collection_time) FILTER ( + WHERE is_winner + AND source = (SELECT source FROM latest) + AND holder IS NOT DISTINCT FROM (SELECT holder FROM latest) + ) AS observations_held + FROM pg_xmin_horizon + WHERE server_id = $1 + AND collection_time >= $2 + ) + SELECT + l.source, + l.holder, + l.xmin_age, + w.observations_held, + w.observations_total, + l.detail + FROM latest AS l + CROSS JOIN window_stats AS w + """; + + /// + /// Latest state per slot plus its earliest retained figure in the window, so the evaluator can see + /// whether the pile is still growing — the difference between a consumer that is behind and a volume + /// filling in front of you. + /// + internal const string SlotSql = """ + WITH latest AS ( + SELECT DISTINCT ON (slot_name) + slot_name, wal_status, is_active, retained_wal_bytes, inactive_since + FROM collect.pg_replication_slot_stats + WHERE server_id = $1 + AND collection_time >= $2 + ORDER BY slot_name, collection_time DESC + ), + earliest AS ( + SELECT DISTINCT ON (slot_name) + slot_name, retained_wal_bytes AS first_retained + FROM collect.pg_replication_slot_stats + WHERE server_id = $1 + AND collection_time >= $2 + ORDER BY slot_name, collection_time ASC + ) + SELECT + l.slot_name, + l.wal_status, + l.is_active, + l.retained_wal_bytes, + GREATEST(l.retained_wal_bytes - e.first_retained, 0) AS growth_bytes, + l.inactive_since + FROM latest AS l + JOIN earliest AS e ON e.slot_name = l.slot_name + """; + + public async Task> GetWraparoundRiskAsync( + int serverId, CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = _postgres.CreateCommand(WraparoundSql); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(NaiveUtcNow() - Freshness); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PostgresWraparoundAlertInfo( + reader.IsDBNull(0) ? "(unknown)" : reader.GetString(0), + reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + /* ordinal 4: autovacuum_multixact_freeze_max_age. The collector has stored it since V63 and + this adapter simply never selected it, so the evaluator graded MultiXact age against the + XID setting — half the size by default, hence warnings at 2.2x premature. 0 reads as + "cannot judge that counter", which is the correct fail-quiet. */ + reader.IsDBNull(4) ? 0 : reader.GetInt64(4))); + } + + return rows; + } + + public async Task GetXminHorizonAsync( + int serverId, CancellationToken cancellationToken = default) + { + await using var command = _postgres.CreateCommand(XminSql); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(NaiveUtcNow() - Freshness); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return null; + } + + return new PostgresXminHorizonAlertInfo( + reader.IsDBNull(0) ? "(unknown)" : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : (int)reader.GetInt64(3), + reader.IsDBNull(4) ? 0 : (int)reader.GetInt64(4), + reader.IsDBNull(5) ? null : reader.GetString(5)); + } + + public async Task> GetReplicationSlotRiskAsync( + int serverId, CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = _postgres.CreateCommand(SlotSql); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(NaiveUtcNow() - Freshness); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PostgresSlotAlertInfo( + reader.IsDBNull(0) ? "(unknown)" : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + !reader.IsDBNull(2) && reader.GetBoolean(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + reader.IsDBNull(5) ? null : reader.GetDateTime(5))); + } + + return rows; + } + + /// Naive-UTC now, Kind-Unspecified - the product's PG timestamp discipline (the same + /// helper DarlingAlertReadAdapter carries). Kind=Utc binds infer timestamptz and shift the freshness + /// window by the store session's zone offset: east of UTC the three Tier 0 alerts silently never + /// fire. The doc blocks on the window SQL always said naive UTC; now the binds do too. + private static DateTime NaiveUtcNow() => + DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingRetention.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingRetention.cs index 7cc4b5275..8b75c5535 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingRetention.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingRetention.cs @@ -138,10 +138,22 @@ keeps a large first purge from ever hitting a timeout at all — a single unboun /// A : how many tables were touched and the coarse activity count (DELETE rows /// plus dropped chunks). The daily caller discards it; the on-demand purge_now command reports it. /// + /// + /// The V75 plan-content horizon (#2316): days a payload-dimension row outlives its last sighting + /// before the GC may take it, independent of the fact-coupled horizon. 0 (the default here, for + /// callers and tests that predate the knob) disables it — the fact-coupled horizon stands alone. + /// public static async Task PurgeAsync( NpgsqlDataSource postgres, bool timescaleAvailable, ILogger? logger, CancellationToken cancellationToken, - Func? retentionDaysFor = null) + Func? retentionDaysFor = null, int planContentRetentionDays = 0) { + /* Clamp at the destructive sink, like retentionDaysFor's clamp below (review catch): the value + arrives pre-clamped only when a store read succeeded and ApplyToConfig ran. On a + store-unreachable boot the worker passes darling.json's RAW value, and a file value of 1-6 + would prune plan content below the [7,365] contract — the failure direction is data loss, so + the sink does not trust its callers. */ + planContentRetentionDays = StoreConfigProvider.ClampPlanContentRetentionDays(planContentRetentionDays); + var sw = Stopwatch.StartNew(); var tablesPurged = 0; var totalRowsDeleted = 0; @@ -368,6 +380,7 @@ table and why. */ else { var dimensionCutoff = ComputeDimensionCutoff(utcNow, widestFactRetentionDays, oldestSurvivingDigestFact); + var planDimensionCutoff = ComputeDimensionCutoff(utcNow, widestFactRetentionDays, oldestSurvivingDigestFact, planContentRetentionDays); if (dimensionCutoff < utcNow.AddDays(-(widestFactRetentionDays + TimescaleSupport.ChunkIntervalDays + 1))) { /* Fixed string, same reasoning as the defer line: the greppable signature of a store @@ -376,11 +389,85 @@ whose GC is bounded by held history (clamped or failed purges) rather than by th logger?.LogInformation("dimension GC bounded by surviving facts: dimension content newer than the oldest digest-carrying fact row is retained"); } + /* #2210: the Query Store plan map goes FIRST, and its cutoff is deliberately LATER than the + dimension's — it prunes more aggressively, so the dim always outlives the map it points into. + The two bad end-states are not symmetric. A pruned map row whose dim row survives renders a + plan as "not collected" and leaves bytes unreclaimed until the dim's own horizon passes: + visible, self-correcting, no wrong answers. A pruned DIM row whose map row survives is a + reader resolving a live fact to absent content, silently, weeks after the cause. Ordering the + cutoffs makes the recoverable end-state the only reachable one, and + QueryStorePlanMap.MarginOrderingHolds is pinned against ChunkIntervalDays so shrinking that + constant cannot invert it unnoticed. */ + /* #2219: PostgreSQL statement text, on the same principle as the plan map below but with the + margin in the OPPOSITE direction, because the asymmetry is the other way round. Text is what a + fact row points AT, so it must OUTLIVE the statistics that reference it: text kept past its + facts is a few dead bytes, whereas facts kept past their text is a top-queries answer that + reads as a list of integers — the exact failure this table was added to fix. Hence a margin + ADDED to the fact horizon rather than subtracted from it. */ + var textCutoff = utcNow.AddDays(-(widestFactRetentionDays + PgStatementText.PruneMarginDays)); + var textDeleted = await PurgeOneAsync( + postgres, PgStatementText.TableName, + PgStatementText.PruneSql(TimescaleSupport.ChunkIntervalDays), + textCutoff, logger, cancellationToken); + if (textDeleted is not null) + { + tablesPurged++; + totalRowsDeleted += textDeleted.Value; + } + else + { + tablesFailed++; + } + + /* #2316 review catch: the map must learn the plan-content horizon too, or the dedicated + dim cutoff overtakes this one and a live map row can point at deleted content — the + silent-missing-plans failure the margin ordering exists to prevent. ComputeMapCutoff + keeps the map's cutoff strictly NEWER than the plan dim's under every knob value, so + the only reachable end-state stays the recoverable one (map row gone first, plan + renders as not-collected). The visible consequence is deliberate: a Query Store plan + fetch for an interval older than the knob misses, exactly like the dim itself. */ + var mapCutoff = ComputeMapCutoff(utcNow, widestFactRetentionDays, planContentRetentionDays); + var mapDeleted = await PurgeOneAsync( + postgres, QueryStorePlanMap.TableName, + QueryStorePlanMap.PruneSql(TimescaleSupport.ChunkIntervalDays), + mapCutoff, logger, cancellationToken); + if (mapDeleted is not null) + { + tablesPurged++; + totalRowsDeleted += mapDeleted.Value; + } + else + { + tablesFailed++; + } + + /* #2150: query_store statement text, same shape and same direction of margin as the two + above — it must outlive the facts that reference it, because text retired early reads as a + statement that never had text rather than as one whose text expired. */ + var queryTextCutoff = utcNow.AddDays(-(widestFactRetentionDays + QueryStoreTextStore.PruneMarginDays)); + var queryTextDeleted = await PurgeOneAsync( + postgres, QueryStoreTextStore.TableName, + QueryStoreTextStore.PruneSql(TimescaleSupport.ChunkIntervalDays), + queryTextCutoff, logger, cancellationToken); + if (queryTextDeleted is not null) + { + tablesPurged++; + totalRowsDeleted += queryTextDeleted.Value; + } + else + { + tablesFailed++; + } + foreach (var dimTable in PayloadDimensions.DimTables) { + /* #2316 review catch: the dedicated horizon applies to PLAN content only. query_text_dim + is ~40 MB against the plan dim's 127 GB — shortening it buys nothing and would quietly + break "text stays analyzable for the facts' full retention", which is half the knob's + own justification. The router is pure so the scoping is pinned by tests. */ var dimDeleted = await PurgeOneAsync( postgres, dimTable, TimeSlicedDeleteSql(dimTable, PayloadDimensions.LastSeenColumn), - dimensionCutoff, logger, cancellationToken); + ComputeDimTableCutoff(dimTable, dimensionCutoff, planDimensionCutoff), logger, cancellationToken); if (dimDeleted is not null) { tablesPurged++; @@ -587,16 +674,68 @@ internal static string TimeSlicedDeleteSql(string table, string timeColumn, stri /// digest-carrying facts anywhere — a fresh or fully-aged store) leaves the assumed horizon alone: /// with no facts, nothing can dangle, and last_seen still bounds what is old enough to take. /// - internal static DateTime ComputeDimensionCutoff(DateTime utcNow, int widestFactRetentionDays, DateTime? oldestSurvivingDigestFact) + internal static DateTime ComputeDimensionCutoff(DateTime utcNow, int widestFactRetentionDays, DateTime? oldestSurvivingDigestFact, int planContentRetentionDays = 0) { var assumed = utcNow.AddDays(-(widestFactRetentionDays + TimescaleSupport.ChunkIntervalDays + 1)); - if (oldestSurvivingDigestFact is null) + var coupled = assumed; + if (oldestSurvivingDigestFact is not null) + { + var measured = oldestSurvivingDigestFact.Value.AddDays(-1); + coupled = measured < assumed ? measured : assumed; + } + + /* #2316: the dedicated plan-content horizon DELIBERATELY overrides both safeties above for + content past its window — that is its entire point. The coupled horizon guarantees no fact + ever references deleted content, which also means a store younger than the fact retention + has an UNBOUNDED dimension (measured: 127 GB in the dim's first 22 days, with the coupled + GC unable to fire until a month after projected disk-full). With the knob enabled, a fact + older than the window keeps its metrics, hashes and text but renders a MISSING plan — the + null every reader already handles — in exchange for a bounded store. The same one-day + margin as the measured side covers the hourly last_seen refresh guard. Disabled (0 or + below) returns the coupled cutoff before any dedicated value is computed, so the old + behavior is reproduced exactly rather than approximated through a comparison. */ + if (planContentRetentionDays <= 0) + { + return coupled; + } + + var dedicated = utcNow.AddDays(-(planContentRetentionDays + 1)); + return dedicated > coupled ? dedicated : coupled; + } + + /// + /// Routes each payload dimension to its cutoff (#2316 review catch): the dedicated plan-content + /// horizon governs query_plan_dim ONLY — every other dimension (query text today) keeps the + /// fact-coupled cutoff, so text stays resolvable for the facts' full retention. Pure so the scoping + /// decision is pinned by tests rather than living as an inline ternary nothing exercises. + /// + internal static DateTime ComputeDimTableCutoff(string dimTable, DateTime coupledCutoff, DateTime planDimensionCutoff) => + string.Equals(dimTable, PayloadDimensions.QueryPlanDimTable, StringComparison.Ordinal) + ? planDimensionCutoff + : coupledCutoff; + + /// + /// The Query Store plan map's prune cutoff, knob-aware (#2316 review catch). The invariant + /// (): the DIMENSION must outlive the MAP, so the + /// only reachable end-state is the recoverable one — a map row pruned while its content survives + /// renders "not collected" and self-corrects; content pruned while a map row survives is a live fact + /// resolving to absent XML, silently. The coupled pair keeps that gap at ChunkIntervalDays; the + /// dedicated pair keeps it at one day (map at knob, dim at knob + 1 — the same one-day stamp-skew + /// margin as everywhere else, because TouchSql refreshes the map's stamp eagerly while the + /// dim's refresh is hourly-guarded, so the dim's stamp can trail). Both components are strictly + /// ordered, so the max-of-newer composition preserves the ordering under every knob value — + /// pinned in PlanContentRetentionTests across the full age sweep. + /// + internal static DateTime ComputeMapCutoff(DateTime utcNow, int widestFactRetentionDays, int planContentRetentionDays = 0) + { + var coupled = utcNow.AddDays(-(widestFactRetentionDays + QueryStorePlanMap.PruneMarginDays)); + if (planContentRetentionDays <= 0) { - return assumed; + return coupled; } - var measured = oldestSurvivingDigestFact.Value.AddDays(-1); - return measured < assumed ? measured : assumed; + var dedicated = utcNow.AddDays(-planContentRetentionDays); + return dedicated > coupled ? dedicated : coupled; } /// @@ -635,14 +774,13 @@ internal static string DropChunksSqlFor(string table, int retentionDays) /// (warned; the caller falls back to DELETE for that table). drop_chunks returns one row per /// dropped chunk, so the count comes from reading the result set. /// - private static async Task DropChunksOneAsync( + private static Task DropChunksOneAsync( NpgsqlDataSource postgres, string tableName, string dropChunksSql, ILogger? logger, CancellationToken cancellationToken) - { - try + => ExecuteDropChunksWithDeadlockRetryAsync(async () => { await using var connection = await postgres.OpenConnectionAsync(cancellationToken); using var command = new NpgsqlCommand(dropChunksSql, connection) { CommandTimeout = DeleteTimeoutSeconds }; @@ -655,13 +793,40 @@ internal static string DropChunksSqlFor(string table, int retentionDays) } return chunksDropped; - } - catch (Exception ex) when (ex is not OperationCanceledException) + }, tableName, logger); + + /// + /// Runs one table's drop_chunks with a SINGLE immediate retry on deadlock (#2143). 40P01 is transient + /// by definition — the deadlock partner (in the field: a TimescaleDB background job holding chunk + /// locks, caught live by the nightly's purge e2e) commits or aborts within milliseconds of the abort, + /// so one retry converts a wasted purge cycle into a completed one. Exactly ONE retry: a second + /// deadlock in a row means the contention is standing, and the DELETE fallback plus next cycle's + /// sweep — the behavior this wraps — is the right posture, not a retry loop camped on a lock queue. + /// Any non-deadlock failure keeps the original single-shot behavior. Internal, delegate-seamed, so + /// the retry/give-up/no-retry arms pin without a store. + /// + internal static async Task ExecuteDropChunksWithDeadlockRetryAsync( + Func> dropChunks, string tableName, ILogger? logger) + { + for (var attempt = 1; ; attempt++) { - /* Failure-isolated per table — warned here, then the caller's DELETE fallback runs. */ - logger?.LogWarning("Retention purge (drop_chunks) failed for {Table} — falling back to DELETE: {Message}", - tableName, ex.Message); - return null; + try + { + return await dropChunks(); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.DeadlockDetected && attempt == 1) + { + logger?.LogWarning( + "Retention purge (drop_chunks) deadlocked for {Table} — retrying once (the partner clears in milliseconds): {Message}", + tableName, ex.Message); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + /* Failure-isolated per table — warned here, then the caller's DELETE fallback runs. */ + logger?.LogWarning("Retention purge (drop_chunks) failed for {Table} — falling back to DELETE: {Message}", + tableName, ex.Message); + return null; + } } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingSecrets.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingSecrets.cs index 1dea89269..869840c23 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingSecrets.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingSecrets.cs @@ -38,6 +38,35 @@ public static string Protect(string plaintext) return Convert.ToBase64String(protectedBytes); } + /// + /// What a DPAPI decrypt failure actually means, in the operator's terms (#2255). + /// + /// Why this exists. ProtectedData.Unprotect throws + /// CryptographicException: Key not valid for use in specified state, and that message went into the + /// log verbatim, once every 60 seconds, forever. The field report shows exactly how it lands: the operator + /// read it as SQL Server rejecting the login and went looking at the server's credentials, because nothing + /// in it says DPAPI, names what failed to decrypt, or mentions that a MACHINE boundary is involved. + /// + /// The cause it points at. These blobs are , so + /// any user on the machine that wrote one can decrypt it and NO other machine ever can. That makes the + /// overwhelmingly likely cause a credential saved by a Viewer running on a DIFFERENT PC — which is the + /// documented single-box limitation of the Viewer's write path, not a permissions problem on the service + /// account. The remedies are therefore all "encrypt it on this host", which is what the message says. + /// + /// Kept as a function rather than a literal at each throw site so the three surfaces that can hit + /// this — a monitored server's password, the store credential, a network token — cannot drift into + /// explaining the same failure three different ways. + /// + internal static string DescribeDecryptFailure(string what) => + $"Could not DPAPI-decrypt {what}. This is a Windows Data Protection failure on THIS host, not SQL Server " + + "rejecting a login — no credential was ever sent to the server. These blobs are encrypted with " + + "LocalMachine scope, so they can only be decrypted on the machine that wrote them (any user on it, but " + + "no other machine). The usual cause is a credential saved by a Viewer running on a DIFFERENT PC: the " + + "Viewer encrypts on the machine it runs on, so a remotely-added server's password is unreadable here. " + + "Fix it on this host, any one of: re-add the server from a Viewer running on this machine; run " + + "'--add-server' here; run '--encrypt-password' here and paste the blob; or store the password as an " + + "'env:' / 'file:' reference, which is not machine-bound."; + public static string Unprotect(string base64Blob) { if (string.IsNullOrWhiteSpace(base64Blob)) @@ -75,7 +104,20 @@ public static string ResolvePassword(MonitoredServer server, out bool usedPlaint return DarlingSecretSource.Resolve(server.EncryptedPassword, $"servers['{server.DisplayName}'].encryptedPassword"); } - return Unprotect(server.EncryptedPassword); + /* #2255: the raw CryptographicException ("Key not valid for use in specified state") reached the + worker's connect-retry warning verbatim and repeated every 60s with no way to act on it. Server + identity is only known HERE, so this is where it gets attached. */ + try + { + return Unprotect(server.EncryptedPassword); + } + catch (CryptographicException ex) + { + throw new InvalidOperationException( + DescribeDecryptFailure($"the stored password for server '{server.DisplayName}' " + + "(servers[].encryptedPassword)"), + ex); + } } if (!string.IsNullOrWhiteSpace(server.Password)) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs index ea279bcdb..d4809bb53 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs @@ -154,6 +154,13 @@ the per-server conditions use. */ private readonly ConcurrentDictionary _activeDiskPressure = new(); private readonly ConcurrentDictionary _lastDiskPressureAlert = new(); + /// + /// The free-percent level the last Store Disk Pressure alert reported (#2101) — the worsening + /// watermark compares against, exactly the engine's + /// _lastAlertedLowDiskPercent idiom. Cleared on recovery so the next breach is fresh. + /// + private readonly ConcurrentDictionary _lastAlertedDiskPressurePercent = new(); + /// The fixed key for the fleet-level Store Disk Pressure edge (not a real server). private const string DiskKey = "store"; @@ -177,6 +184,25 @@ private enum CompressionJobHealth { ReArmed, Escalated } /// Prefixes the fleet-level compression-job alert serverKey so it never parses as a server_id. private const string CompressionKeyPrefix = "compressjob:"; + /* Store Job Over Cadence edge state (#2136). FLEET-level like disk pressure, MULTI-keyed by job_id like + the compression machine, but a STANDING condition (the AG Sync Fell Behind idiom): active flag + + cooldown re-fire while a job's last run keeps breaching its share of the schedule interval, one + "Store Job Cadence Recovered" resolution when a later run comes back under. */ + private readonly ConcurrentDictionary _activeJobOverCadence = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _lastJobOverCadenceAlert = new(StringComparer.Ordinal); + + /// The #2136 alert metric name — the Warning and Critical tiers share it (severity carries the + /// tier), so the deliverer's per-metric cooldown and the recovery resolution correlate cleanly. + internal const string JobCadenceMetric = "Store Job Over Cadence"; + + /// Prefixes the fleet-level cadence alert serverKey so it never parses as a server_id. + private const string JobCadenceKeyPrefix = "storejob:"; + + /// #2136: the Warning tier's percent-of-cadence threshold, read live through the same + /// by-reference settings seam as the AG thresholds (the clamp lives on DarlingAlertSettings). + /// The Critical tier is FIXED at 100: a job outrunning its own cadence compounds refresh lag. + private readonly Func _storeJobCadenceWarnPercent; + /* Availability Group edge state (#991), keyed by a COMPOSITE of serverId + the AG grain — an AG condition is per replica (ag + replica) or per database (ag + database + replica), not per server, so one server's two lagging databases must track (and recover) independently. Only the alert HISTORY is per server: every @@ -243,7 +269,8 @@ public DarlingSelfAlertEvaluator( Func? notifyAgHealth = null, Func? agLagAlertSeconds = null, Func? agRedoQueueAlertKb = null, - Func? agDisconnectRefireMinutes = null) + Func? agDisconnectRefireMinutes = null, + Func? storeJobCadenceWarnPercent = null) { _settings = settings ?? throw new ArgumentNullException(nameof(settings)); _deliverer = deliverer ?? throw new ArgumentNullException(nameof(deliverer)); @@ -260,6 +287,9 @@ public DarlingSelfAlertEvaluator( _agLagAlertSeconds = agLagAlertSeconds ?? (() => 300); _agRedoQueueAlertKb = agRedoQueueAlertKb ?? (() => 0); _agDisconnectRefireMinutes = agDisconnectRefireMinutes ?? (() => 0); + /* Unsupplied falls back to the V57 DDL default, so an evaluator built without the seam behaves + like a store at its shipped defaults (the AG-seam discipline). */ + _storeJobCadenceWarnPercent = storeJobCadenceWarnPercent ?? (() => 25); } private enum ConnectionState @@ -296,9 +326,13 @@ first fresh collection lands. */ { try { + /* #2107: store-backed window/threshold (clamped on read); the constants remain + only as the shipped defaults. */ var (lastSuccess, recentRuns, recentSuccess) = - await ReadCollectionSignalsAsync(postgres, serverId, ConsecutiveFailureThreshold, cancellationToken); - bool stopped = IsCollectionStopped(lastSuccess, recentRuns, recentSuccess, _utcNow(), out var reason); + await ReadCollectionSignalsAsync(postgres, serverId, _settings.CollectionFailureThreshold, cancellationToken); + bool stopped = IsCollectionStopped( + lastSuccess, recentRuns, recentSuccess, _utcNow(), + SettingsStaleWindow, _settings.CollectionFailureThreshold, out var reason); await ApplyCollectionStoppedAsync(serverId, serverName, stopped, reason, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -338,7 +372,7 @@ first fresh collection lands. */ var (agentCollectionTimeUtc, agentRunning) = await ReadLatestAgentStatusAsync(postgres, serverId, cancellationToken); bool? freshRunning = agentRunning.HasValue && agentCollectionTimeUtc.HasValue - && _utcNow() - agentCollectionTimeUtc.Value < StaleWindow + && _utcNow() - agentCollectionTimeUtc.Value < SettingsStaleWindow ? agentRunning : null; @@ -410,9 +444,13 @@ otherwise ask the collected history once and memoize the positive. */ /* A snapshot only judges while it is fresh; a missing one (no AGs, or the collector has never run) and a stale one are both "no signal", exactly as agent_status is treated above. */ bool IsFresh(DateTime? snapshotUtc) => - snapshotUtc.HasValue && _utcNow() - snapshotUtc.Value < StaleWindow; + snapshotUtc.HasValue && _utcNow() - snapshotUtc.Value < SettingsStaleWindow; } + /// #2107: the staleness window the sweep actually uses — store-backed, clamped on + /// read; remains only as the shipped default. + private TimeSpan SettingsStaleWindow => TimeSpan.FromMinutes(_settings.CollectionStaleMinutes); + /// /// Pure collection-stopped decision from the three store signals — no I/O, so it pins directly. /// A NEVER-succeeded server ( null) is deliberately NOT flagged by @@ -422,16 +460,23 @@ bool IsFresh(DateTime? snapshotUtc) => /// internal static bool IsCollectionStopped( DateTime? lastSuccessUtc, int recentRunCount, int recentSuccessCount, DateTime nowUtc, out string reason) + => IsCollectionStopped(lastSuccessUtc, recentRunCount, recentSuccessCount, nowUtc, StaleWindow, ConsecutiveFailureThreshold, out reason); + + /// #2107: the configurable form — the sweep passes the store-backed window and + /// threshold; the constant overload keeps the shipped defaults for the tests pinning them. + internal static bool IsCollectionStopped( + DateTime? lastSuccessUtc, int recentRunCount, int recentSuccessCount, DateTime nowUtc, + TimeSpan staleWindow, int consecutiveFailureThreshold, out string reason) { /* Fast path: the most-recent N runs all failed. */ - if (recentRunCount >= ConsecutiveFailureThreshold && recentSuccessCount == 0) + if (recentRunCount >= consecutiveFailureThreshold && recentSuccessCount == 0) { reason = $"The last {recentRunCount.ToString(CultureInfo.InvariantCulture)} collector runs all failed — no data is landing."; return true; } /* Backstop: a server that HAS collected before but hasn't succeeded within the staleness window. */ - if (lastSuccessUtc.HasValue && nowUtc - lastSuccessUtc.Value >= StaleWindow) + if (lastSuccessUtc.HasValue && nowUtc - lastSuccessUtc.Value >= staleWindow) { int minutes = (int)(nowUtc - lastSuccessUtc.Value).TotalMinutes; reason = $"No successful collection in {minutes.ToString(CultureInfo.InvariantCulture)} minutes — the collectors are failing or the server is unreachable."; @@ -920,7 +965,8 @@ await FireAsync( $"suspended ({suspendReason})", /* suspend_reason_desc against "SYNCHRONIZING". */ numericCurrentValue: StateOnlyValue, numericThresholdValue: StateOnlyValue, - cancellationToken); + cancellationToken, + context: AgDatabaseContext(database, ("Suspend Reason", suspendReason))); } else if (suspension == AgSuspensionDecision.Resumed) { @@ -965,7 +1011,8 @@ kilobytes on others — and on an AG or database whose name carries a digit ("Sales2024"), neither. #1846 already classified this metric state-only for exactly that reason; this is the write side finally agreeing with it. */ numericCurrentValue: StateOnlyValue, numericThresholdValue: StateOnlyValue, - cancellationToken); + cancellationToken, + context: AgDatabaseContext(database)); } } } @@ -989,6 +1036,11 @@ await RecordResolutionAsync(new AlertResolution( } } + /// The #2109 discrete-facts context for a database-scoped AG alert — the shared + /// builder keyed off the reading, so the fact names cannot drift from Lite's. + private static AlertContext AgDatabaseContext(AgDatabaseReading database, params (string, string)[] extras) + => AgAlertContexts.ForDatabase(database.DatabaseName, database.AgName, database.ReplicaServerName, extras); + /// The prefix every AG state key for one server starts with — the scope for /// and for the per-server recovery sweep. /* AG edge state is keyed by the AG GRAIN ALONE, deliberately without the serverId (#1696). An AG is one @@ -1059,6 +1111,12 @@ private static string DescribeAgDatabaseKey(string key) /// the one dangerous ambiguity this metric must never have back into the signature. /// internal static bool IsDiskPressure(long freeBytes, long totalBytes, out string reason, out double percentFree) + => IsDiskPressure(freeBytes, totalBytes, DiskFreeWarnPercent, out reason, out percentFree); + + /// #2107: the configurable form — the sweep passes the store-backed + /// SelfDiskFreeWarnPercent; the constant-threshold overload keeps the shipped default + /// for the tests pinning it. + internal static bool IsDiskPressure(long freeBytes, long totalBytes, double warnPercent, out string reason, out double percentFree) { if (totalBytes <= 0) { @@ -1068,7 +1126,7 @@ internal static bool IsDiskPressure(long freeBytes, long totalBytes, out string } percentFree = (double)freeBytes / totalBytes * 100.0; - if (percentFree < DiskFreeWarnPercent) + if (percentFree < warnPercent) { reason = $"The monitor store's disk volume has only {percentFree.ToString("0.#", CultureInfo.InvariantCulture)}% free ({FormatGb(freeBytes)} of {FormatGb(totalBytes)})."; return true; @@ -1130,18 +1188,34 @@ internal async Task ApplyDiskPressureAsync( } var now = _utcNow(); - bool pressure = IsDiskPressure(free, total, out var reason, out var percentFree); + /* #2107: store-backed threshold (clamped on read); the constant remains only as the + shipped default. */ + double warnPercent = _settings.SelfDiskFreeWarnPercent; + bool pressure = IsDiskPressure(free, total, warnPercent, out var reason, out var percentFree); if (pressure) { _activeDiskPressure[DiskKey] = true; - if (CooldownElapsed(_lastDiskPressureAlert, DiskKey, now)) + + /* #2101: a standing breach at an UNCHANGED level must not re-notify every cooldown — a + store volume parked at 7% free is one condition, not a condition per 15 minutes. The + same #754 worsening gate the target-server volume alert runs behind: fire on entry, + re-fire only when free% has dropped at least the margin below the last-alerted level + (still cooldown-limited), one resolution on recovery. This is THE self-alert with a + real measurement, which is what makes the gate fit here and deliberately NOT on the + state-only siblings (Collection Stopped / Agent Not Running / Capture Down) — those + have no level to worsen, and their per-cooldown "still broken" reminder is wanted. */ + double? lastAlertedPercent = + _lastAlertedDiskPressurePercent.TryGetValue(DiskKey, out var lastPct) ? lastPct : (double?)null; + if (LowDiskAlertGate.ShouldAlert(percentFree, lastAlertedPercent) + && CooldownElapsed(_lastDiskPressureAlert, DiskKey, now)) { _lastDiskPressureAlert[DiskKey] = now; + _lastAlertedDiskPressurePercent[DiskKey] = percentFree; var storeText = storeSizeBytes is long size ? $" The store currently holds {FormatGb(size)}." : ""; await FireAsync( DiskKey, "Monitor Store", "Store Disk Pressure", reason, - $"{DiskFreeWarnPercent.ToString("0.#", CultureInfo.InvariantCulture)}% free", + $"{warnPercent.ToString("0.#", CultureInfo.InvariantCulture)}% free", detail: reason + storeText + " When the store volume fills, collection and every write stop " + "for the WHOLE fleet, and a headless service has no dashboard to warn you. Free space on the " + "store volume, shorten retention (config_collector_schedules), enable TimescaleDB compression, " + @@ -1155,12 +1229,13 @@ AlertMetricClassifier.IsStateOnly must never list — percent-free is genuinely explicitly means an operator's volume path ("D2:\\") can no longer get there first, and the stored value stops depending on prose word order. The threshold is a real bound too, which is what separates this metric from every sibling above. */ - numericCurrentValue: percentFree, numericThresholdValue: DiskFreeWarnPercent, + numericCurrentValue: percentFree, numericThresholdValue: warnPercent, cancellationToken); } } else if (_activeDiskPressure.TryRemove(DiskKey, out var was) && was) { + _lastAlertedDiskPressurePercent.TryRemove(DiskKey, out _); await RecordResolutionAsync(new AlertResolution( DiskKey, "Monitor Store", "Store Disk Pressure", "Store Disk Pressure Resolved", "Monitor store volume free space recovered"), cancellationToken); @@ -1311,6 +1386,102 @@ public async Task EvaluateCompressionJobsAsync( } } + /// + /// The isolating entry point for the #2136 Store Job Over Cadence check — rides the worker's hourly + /// compression-job health sweep (same connection, same Timescale gate). Same failure isolation as + /// ; cancellation still propagates. + /// + public async Task EvaluateStoreJobCadenceAsync( + IReadOnlyList jobs, CancellationToken cancellationToken) + { + try + { + await ApplyStoreJobCadenceAsync(jobs, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger?.LogError("Store-job cadence self-alert failed: {Message}", ex.Message); + } + } + + /// + /// Applies the fleet-level Store Job Over Cadence condition (#2136) from the latest job readings: a + /// background job whose last SUCCESSFUL run consumed at least the warning share of its own schedule + /// interval is living too close to its ceiling — these runtimes scale SERIALLY with raw volume (the + /// finalize hash-aggregate runs in one process), so an onboarding wave moves them first, and a job + /// that outgrows its cadence compounds refresh lag silently. Tiers: WARNING at the store-backed knob + /// (V57, default 25), CRITICAL fixed at 100 — past 100 the job is still running when its next run is + /// due, which is no longer "close to" the ceiling but through it. A STANDING condition (the AG Sync + /// Fell Behind idiom): fire once on breach, re-fire only after the alert cooldown while it persists, + /// one "Store Job Cadence Recovered" resolution row when a later run comes back under the warning + /// threshold. A job with no schedule interval or no completed run yet has no cadence to breach and is + /// skipped without touching its standing state (no signal, the agent-status discipline). Gated on the + /// master alerts switch. Internal so it pins directly with a recording deliverer + controllable clock. + /// + internal async Task ApplyStoreJobCadenceAsync( + IReadOnlyList jobs, CancellationToken cancellationToken) + { + if (!_settings.AlertsEnabled) + { + return; + } + + var now = _utcNow(); + int warnPercent = _storeJobCadenceWarnPercent(); + + foreach (var job in jobs) + { + if (job.ScheduleIntervalMs <= 0 || job.LastRunDurationMs is not long durationMs) + { + continue; + } + + var key = job.JobId.ToString(CultureInfo.InvariantCulture); + double percent = 100.0 * durationMs / job.ScheduleIntervalMs; + var label = string.IsNullOrEmpty(job.JobName) ? $"job {key}" : $"{job.JobName} [{key}]"; + + if (percent >= warnPercent) + { + _activeJobOverCadence[key] = true; + if (CooldownElapsed(_lastJobOverCadenceAlert, key, now)) + { + _lastJobOverCadenceAlert[key] = now; + bool critical = percent >= 100.0; + await FireAsync( + JobCadenceKeyPrefix + key, "Monitor Store", JobCadenceMetric, + $"{percent:F0}% of schedule interval", $"{warnPercent}%", + detail: $"Store background {label} last ran for {durationMs / 1000.0:F0}s against a " + + $"{job.ScheduleIntervalMs / 1000.0:F0}s schedule interval ({percent:F0}%). " + + (critical + ? "The job now takes at least as long as its own cadence, so runs back up behind each " + + "other and everything it maintains (continuous-aggregate freshness, compression, " + + "retention) falls further behind every cycle. " + : "These runtimes scale with raw data volume, so this is the early warning that the " + + "store is outgrowing its job schedule — an onboarding wave moves this number first. ") + + "Compare the job's duration series in collect.store_metrics (object_kind = " + + "'background_job') to see the trend, and either reduce raw volume, extend the job's " + + "schedule_interval deliberately, or scale the store host.", + severity: critical ? AlertSeverityLevel.Critical : AlertSeverityLevel.Warning, + shortMessage: $"{label} ran {percent:F0}% of its schedule interval", + numericCurrentValue: Math.Round(percent, 1), + numericThresholdValue: critical ? 100 : warnPercent, + cancellationToken); + } + } + else if (_activeJobOverCadence.TryRemove(key, out var was) && was) + { + await RecordResolutionAsync(new AlertResolution( + JobCadenceKeyPrefix + key, "Monitor Store", JobCadenceMetric, + "Store Job Cadence Recovered", + $"Monitor Store: {label} is back under {warnPercent}% of its schedule interval"), cancellationToken); + } + } + } + /// /// Edge-applies the fleet-level compression-job self-heal machine (#1581) from the set of currently-stuck /// compression jobs the worker detected. Per job, in one check: @@ -1777,10 +1948,14 @@ FROM ag_database_replica_states /// The bound behind , on the same /// terms. Almost every self-alert's threshold is an English phrase ("collecting", "Online", "running /// on schedule"), not a bound. + /* The optional context TRAILS the cancellation token so the dozens of existing positional call + sites stay untouched — only the callers that have discrete facts to carry (#2109: the AG + database alerts) name it. */ private async Task FireAsync( string serverKey, string serverName, string metricName, string currentValue, string thresholdValue, string detail, AlertSeverityLevel? severity, string shortMessage, - double? numericCurrentValue, double? numericThresholdValue, CancellationToken cancellationToken) + double? numericCurrentValue, double? numericThresholdValue, CancellationToken cancellationToken, + AlertContext? context = null) { /* Same mute treatment as the engine: a muted self-alert is still recorded (flagged muted) but its channels are skipped — the deliverer honors AlertOutcome.Muted. */ @@ -1801,7 +1976,7 @@ so the service log showed "… Recovered" with nothing before it — which reads await _deliverer.DeliverAsync(new AlertOutcome( serverKey, serverName, metricName, currentValue, thresholdValue, - Context: null, DetailText: detail, + Context: context, DetailText: detail, NumericCurrentValue: numericCurrentValue, NumericThresholdValue: numericThresholdValue, Muted: muted, Severity: severity, ShortMessage: shortMessage), cancellationToken); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs index 9c692564a..8ecd55bad 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs @@ -7,10 +7,12 @@ */ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Npgsql; using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; @@ -34,6 +36,19 @@ public sealed class ServerRuntime public required int ServerId { get; init; } + /// + /// The database this connection ACTUALLY landed in — DB_NAME(), or current_database() on + /// PostgreSQL — or null when the probe did not return it (#2228). + /// + /// Not the same thing as Config.Database, which is what the registration ASKED for. An + /// Initial Catalog that is absent, misspelled, or overridden by the server lands somewhere else, and every + /// collected row is then stored under this registration's identity while describing a different database. + /// Nothing detected that: identity is registration-derived and never checked against the connection, so N + /// registrations that silently resolve to one database produce N identities and N full copies of the same + /// rows — the shape #2220 reported as byte-identical deadlock graphs under six ids. + /// + public string? ConnectedDatabase { get; init; } + public bool HasMsdbAccess { get; init; } public bool IsAwsRds { get; init; } @@ -60,7 +75,17 @@ public static class DarlingServerConnector depend on it (#1535). sqlserver_start_time - the one column that needs the DMV - is not read here (the service never surfaces a start time), so unlike Lite/Dashboard no best-effort start-time read is needed. Columns: 0 sql_version, 1 major_version, 2 utc_offset, - 3 engine_edition, 4 is_aws_rds, 5 has_msdb_access. */ + 3 engine_edition, 4 is_aws_rds, 5 has_msdb_access, 6 connected_database (#2228). + + #2228 — connected_database is DB_NAME(): the database this connection ACTUALLY reached, which is not + necessarily the one the registration names. An Initial Catalog that is absent, misspelled or overridden + lands somewhere else, and every collected row is then stored under this registration's identity while + describing a different database. DB_NAME() keeps this query's no-permission property: it needs no DMV, + so it does not reintroduce the VIEW DATABASE STATE dependency #1535 removed. APPENDED, because every + read above is positional and inserting a column mid-list shifts five other fields onto wrong values. + + Comments inside these probe strings stay to one short line each: the text is sent to the monitored + server on every connect, so the reasoning belongs here rather than on the wire. */ public const string DetectionQueryText = @" SELECT @@VERSION AS sql_version, @@ -68,7 +93,50 @@ depend on it (#1535). sqlserver_start_time - the one column that needs the DMV - DATEDIFF(MINUTE, GETUTCDATE(), GETDATE()) AS utc_offset_minutes, CONVERT(integer, SERVERPROPERTY('EngineEdition')) AS engine_edition, CASE WHEN DB_ID('rdsadmin') IS NOT NULL THEN 1 ELSE 0 END AS is_aws_rds, - HAS_DBACCESS(N'msdb') AS has_msdb_access"; + HAS_DBACCESS(N'msdb') AS has_msdb_access, + -- #2228: which database this connection actually landed in. Appended; see the comment above. + DB_NAME() AS connected_database"; + + /// + /// The tripwire's verdict: the message to raise when a registration is connected to a database it does not + /// name, or null when there is nothing to say (#2228). + /// + /// Silent unless BOTH sides name a database. A registration with no database is + /// server-scoped by design — it is meant to land wherever the login defaults and enumerate from there — so + /// comparing it to whatever that default turned out to be would fire on every correctly-configured + /// server-scoped registration in the fleet. That is the failure mode that gets a tripwire ignored, and an + /// ignored tripwire is worse than none: it trains the operator past the one line that matters. + /// + /// Case-insensitive because SQL Server database names are, under every collation the product + /// supports, and a registration that differs from the server only in case is not a misconfiguration. + /// PostgreSQL is case-sensitive in principle, but a registration whose case differs there fails to connect + /// rather than landing elsewhere, so the looser comparison costs nothing and avoids a false positive on + /// the engine where it would be wrong. + /// + /// Names what is WRONG and what to change, in that order, because the log line is the whole + /// diagnosis: the operator has to be able to act on it without reading the source. Deliberately does not + /// say "N copies" — this function sees one registration and cannot know whether a sibling collides with + /// it; claiming otherwise would be a guess dressed as a finding. + /// + public static string? DescribeDatabaseMismatch(string? registeredDatabase, string? connectedDatabase, string displayName) + { + if (string.IsNullOrWhiteSpace(registeredDatabase) || string.IsNullOrWhiteSpace(connectedDatabase)) + { + return null; + } + + if (string.Equals(registeredDatabase.Trim(), connectedDatabase.Trim(), StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + return $"Registration '{displayName}' is registered for database '{registeredDatabase}' but its connection " + + $"landed in '{connectedDatabase}'. Everything collected under this registration describes " + + $"'{connectedDatabase}', stored under this registration's identity — so if another registration " + + $"names '{connectedDatabase}', both are collecting the same database and its history is duplicated " + + "under two identities. Check this server's Initial Catalog / database setting in the Viewer's " + + "Manage Servers, or the 'database' field for it in darling.json."; + } public static string ResolveConnectionString(MonitoredServer config, ILogger? logger = null) { @@ -113,9 +181,37 @@ public static string ResolveConnectionString(MonitoredServer config, ILogger? lo return MonitoredServerConnection.BuildConnectionString(config, password); } + /* The PostgreSQL detection query. Deliberately built only from surfaces a pg_monitor-grade login + can read on Amazon Aurora, verified against live 16.11 and 17.7 clusters: + + current_setting('server_version_num') -> 160011 / 170007, so the major is a division rather + than string parsing (version() text formatting has changed across releases). + pg_is_in_recovery() -> reader vs writer. On Aurora every reader endpoint is + its own instance with its own statistics, so this is identity, not a routing hint. + aurora_version() -> present only on Aurora. Wrapped: on stock PostgreSQL + the function does not exist, and a missing function must read as "not Aurora" rather than + failing the whole probe. + + No timezone offset column: unlike SQL Server's DATEDIFF-on-GETDATE idiom, Postgres timestamps + here are read as-is and the store's convention is naive UTC either way. */ + public const string PostgresDetectionQueryText = @" +SELECT + version() AS server_version_text, + current_setting('server_version_num')::int / 10000 AS major_version, + pg_is_in_recovery() AS is_in_recovery, + (SELECT count(*) FROM pg_proc WHERE proname = 'aurora_version') > 0 AS has_aurora_marker, + current_setting('server_version_num')::int AS server_version_num, + -- #2228: which database this connection actually landed in. Appended; see the comment above. + current_database() AS connected_database"; + /// Connects, probes, and returns the runtime state for one configured server. public static async Task ConnectAsync(MonitoredServer config, ILogger? logger, CancellationToken cancellationToken) { + if (config.IsPostgres) + { + return await ConnectPostgresAsync(config, logger, cancellationToken); + } + var connectionString = ResolveConnectionString(config, logger); var storageName = config.StorageName; @@ -127,6 +223,7 @@ public static async Task ConnectAsync(MonitoredServer config, ILo int majorVersion = 0, engineEdition = 0; bool isAwsRds = false, hasMsdbAccess = true; + string? connectedDatabase = null; if (await reader.ReadAsync(cancellationToken)) { // Column indices per DetectionQueryText: 1 major_version, 3 engine_edition, @@ -135,6 +232,7 @@ public static async Task ConnectAsync(MonitoredServer config, ILo engineEdition = reader.IsDBNull(3) ? 0 : reader.GetInt32(3); isAwsRds = !reader.IsDBNull(4) && reader.GetInt32(4) == 1; hasMsdbAccess = reader.IsDBNull(5) || reader.GetInt32(5) == 1; + connectedDatabase = reader.IsDBNull(6) ? null : reader.GetString(6); /* #2228 */ } return new ServerRuntime @@ -153,10 +251,76 @@ so Darling attempted running_jobs/job_history/agent_status every cycle on a no-m HasMsdbAccess = hasMsdbAccess, }, StorageName = storageName, - ServerId = ServerIdHelper.GetDeterministicHashCode(storageName), + /* #2218: the STORED identity, not a fresh hash of storageName. This is the runtime's only + identity stamp — DarlingCollectorRunner copies it onto every CollectorContext, so every + collected row keys on whatever this says — which is why it has to read the registry rather + than re-derive from the connection fields the operator can edit. */ + ServerId = config.ServerId, HasMsdbAccess = hasMsdbAccess, IsAwsRds = isAwsRds, EngineEdition = engineEdition, + ConnectedDatabase = connectedDatabase, + }; + } + + /// + /// The PostgreSQL connect-and-probe. Same contract as the SQL Server path — open, probe, return a + /// whose is what the collectors' gate + /// reads — with the SQL Server-only facts left at their defaults. + /// HasMsdbAccess stays true and the Azure flags stay false because they are + /// meaningless here; no Postgres definition consults them, and the engine check in + /// keeps every + /// T-SQL definition away from this target regardless of their values. + /// + private static async Task ConnectPostgresAsync( + MonitoredServer config, ILogger? logger, CancellationToken cancellationToken) + { + var connectionString = ResolveConnectionString(config, logger); + var storageName = config.StorageName; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken); + + using var command = new NpgsqlCommand(PostgresDetectionQueryText, connection) { CommandTimeout = 30 }; + using var reader = await command.ExecuteReaderAsync(cancellationToken); + + int majorVersion = 0, versionNum = 0; + bool isInRecovery = false, isAurora = false; + string versionText = ""; + string? connectedDatabase = null; + if (await reader.ReadAsync(cancellationToken)) + { + versionText = reader.IsDBNull(0) ? "" : reader.GetString(0); + majorVersion = reader.IsDBNull(1) ? 0 : reader.GetInt32(1); + isInRecovery = !reader.IsDBNull(2) && reader.GetBoolean(2); + isAurora = !reader.IsDBNull(3) && reader.GetBoolean(3); + versionNum = reader.IsDBNull(4) ? 0 : reader.GetInt32(4); + connectedDatabase = reader.IsDBNull(5) ? null : reader.GetString(5); /* #2228 */ + } + + logger?.LogInformation( + "Connected to PostgreSQL target '{Server}': major {Major} (server_version_num {Num}), {Role}, Aurora: {Aurora} — {VersionText}", + config.DisplayName, majorVersion, versionNum, isInRecovery ? "reader (in recovery)" : "writer", isAurora, + versionText); + + /* A Postgres target reached through the SQL Server path would have failed on the detection + query, so an engine mismatch is loud. The reverse — a SQL Server host configured as + "postgres" — fails at connect, which is equally loud. */ + return new ServerRuntime + { + Config = config, + ConnectionString = connectionString, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = majorVersion, + PostgresVersionNum = versionNum, + IsAurora = isAurora, + IsInRecovery = isInRecovery, + }, + StorageName = storageName, + ServerId = config.ServerId, + ConnectedDatabase = connectedDatabase, }; } @@ -178,16 +342,26 @@ public static async Task ProbeAsync(MonitoredServer confi try { var runtime = await ConnectAsync(config, logger, cancellationToken); + var isPostgres = runtime.Target.Engine == CollectorTargetEngine.PostgreSql; return new ConnectionProbeResult( Success: true, MajorVersion: runtime.Target.SqlMajorVersion, EngineEdition: runtime.EngineEdition, - EngineEditionDescription: DescribeEngineEdition(runtime.EngineEdition), + /* No edition on a PostgreSQL target, and DescribeEngineEdition(0) would say + "Unknown (0)" — which reads as a probe that half-failed rather than one that + succeeded against a different engine. */ + EngineEditionDescription: isPostgres ? null : DescribeEngineEdition(runtime.EngineEdition), IsAzureSqlDb: runtime.Target.IsAzureSqlDb, IsAzureManagedInstance: runtime.Target.IsAzureManagedInstance, IsAwsRds: runtime.IsAwsRds, HasMsdbAccess: runtime.HasMsdbAccess, - Error: null); + Error: null, + ConnectedDatabase: runtime.ConnectedDatabase, + Engine: runtime.Target.Engine, + PostgresMajorVersion: runtime.Target.PostgresMajorVersion, + PostgresVersionNum: runtime.Target.PostgresVersionNum, + IsAurora: runtime.Target.IsAurora, + IsInRecovery: runtime.Target.IsInRecovery); } catch (OperationCanceledException) { @@ -208,6 +382,51 @@ public static async Task ProbeAsync(MonitoredServer confi } } + /// + /// The probed facts for a REACHABLE target, as one clause — shared by the --test-connection + /// PASS line (DarlingCliCommands.FormatProbeLine) and the add_servers MCP tool's detail + /// text, which previously each formatted their own and could drift. + /// The engine decides what is worth saying. A SQL Server target reports version, edition and + /// msdb access, because msdb access gates three collectors. A PostgreSQL target has none of those, + /// so it reports version, writer-vs-reader, Aurora-vs-not — and then the number that actually + /// answers "will this target give me what I expect", which is how many of the PostgreSQL collectors + /// clear the gate. A stock-PostgreSQL reader clears three of seven, and finding that out at + /// pre-flight is the point of the verb. + /// + public static string DescribeProbeFacts(ConnectionProbeResult probe) + { + ArgumentNullException.ThrowIfNull(probe); + + if (probe.Engine != CollectorTargetEngine.PostgreSql) + { + var edition = string.IsNullOrEmpty(probe.EngineEditionDescription) + ? DescribeEngineEdition(probe.EngineEdition) + : probe.EngineEditionDescription; + var msdb = probe.HasMsdbAccess ? "msdb access: yes" : "msdb access: NO (failed-job alerts unavailable)"; + return $"SQL major version {probe.MajorVersion}, {edition}, {msdb}"; + } + + var target = probe.ToTargetInfo(); + var postgresDefinitions = CollectorCatalog.All + .Where(d => d.TargetEngine == CollectorTargetEngine.PostgreSql) + .ToList(); + var skipped = postgresDefinitions + .Where(d => !CollectorCatalog.AppliesTo(d, target)) + .Select(d => d.Name) + .OrderBy(n => n, StringComparer.Ordinal) + .ToList(); + + var role = probe.IsInRecovery ? "reader (in recovery)" : "writer"; + var flavour = probe.IsAurora ? "Aurora" : "not Aurora"; + var applies = skipped.Count == 0 + ? $"all {postgresDefinitions.Count} PostgreSQL collectors apply" + : $"{postgresDefinitions.Count - skipped.Count} of {postgresDefinitions.Count} PostgreSQL collectors apply " + + $"(skipped: {string.Join(", ", skipped)})"; + + return $"PostgreSQL {probe.PostgresMajorVersion} (server_version_num {probe.PostgresVersionNum}), " + + $"{role}, {flavour} — {applies}"; + } + /// Human-readable SERVERPROPERTY('EngineEdition') description for the probe result. public static string DescribeEngineEdition(int engineEdition) => engineEdition switch { @@ -228,6 +447,11 @@ public static async Task ProbeAsync(MonitoredServer confi /// The outcome of a connect-and-probe attempt (): the /// success flag plus the probed target facts, or the error message on failure. Deliberately carries NO /// credentials so it is safe to serialize into config_command.result_json and print from the CLI. +/// The SQL Server facts come first because they came first; the PostgreSQL ones are trailing +/// optional parameters so every existing construction site — including the tests — keeps compiling and +/// keeps meaning "a SQL Server target". is what a reader should branch on: on a +/// PostgreSQL target and are 0 and +/// is meaningless, so reporting them would be worse than silence. /// public sealed record ConnectionProbeResult( bool Success, @@ -238,4 +462,33 @@ public sealed record ConnectionProbeResult( bool IsAzureManagedInstance, bool IsAwsRds, bool HasMsdbAccess, - string? Error); + string? Error, + CollectorTargetEngine Engine = CollectorTargetEngine.SqlServer, + int PostgresMajorVersion = 0, + int PostgresVersionNum = 0, + bool IsAurora = false, + bool IsInRecovery = false, + /* #2280: the database the connection ACTUALLY reached, so a registration-time collision check can compare + what the SERVER says against what other registrations claim, rather than comparing two claims. Trailing + and defaulted, so every existing construction of this record still compiles unchanged. */ + string? ConnectedDatabase = null) +{ + /// + /// Rebuilds the gate's-eye view of this target, so a caller can ask which collectors would actually + /// run against it. These are the same fields + /// reads, which is why a count derived from this is a real answer and not an estimate. + /// + public CollectorTargetInfo ToTargetInfo() => new() + { + Engine = Engine, + IsAzureSqlDb = IsAzureSqlDb, + IsAzureManagedInstance = IsAzureManagedInstance, + IsAwsRds = IsAwsRds, + SqlMajorVersion = MajorVersion, + HasMsdbAccess = HasMsdbAccess, + PostgresMajorVersion = PostgresMajorVersion, + PostgresVersionNum = PostgresVersionNum, + IsAurora = IsAurora, + IsInRecovery = IsInRecovery, + }; +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingStoreBootstrapEvidence.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreBootstrapEvidence.cs new file mode 100644 index 000000000..6762c08c1 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreBootstrapEvidence.cs @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// Whether a missing managed-store credential means "nobody has started the service yet" or "a bootstrap +/// already ran and FAILED" (#2197) — the question every missing-credential CLI message answered the same +/// way, and the wrong way in the case that actually produces it in the field. +/// +/// The bug this exists for: when a managed bootstrap dies, the operator's LAST message is +/// rarely the bootstrap error — it is whatever verb they run next, and every one of those said "Start the +/// PerformanceMonitor Darling service once so its first run initializes the store". In #2185 that is the +/// message the reporter led with, and it sent them to darling.json, which was never the fault. Starting a +/// service whose bootstrap has already failed just fails it again; the two situations want opposite +/// advice, so a message that cannot tell them apart has to be wrong in one of them. +/// +/// Why the evidence is only ever the store's own files. The tempting signal — the service's +/// log directory exists, therefore the service has run — is machine-global while the question is about ONE +/// store: a box that has monitored happily for a year has that directory, so a genuinely new second store +/// configured on it would be told its bootstrap had failed. Everything read here sits under or beside +/// postgres.dataDirectory and so can only describe the store being asked about. Absent evidence is +/// therefore reported as "no evidence", never as "the service has never run" — which is why the first-run +/// branch still carries one sentence for the operator who HAS already started it. +/// +/* Windows-only for the same reason DarlingManagedPostgres is: every path it reads belongs to a store the + product only builds on Windows (DPAPI credentials, the bundled runtime), and every caller is already + platform-guarded. */ +[SupportedOSPlatform("windows")] +internal static class DarlingStoreBootstrapEvidence +{ + /// + /// The service log the reason is in, named rather than referred to (the issue's ask), and named as a + /// SHAPE rather than today's file: the run that failed may have been days ago, and one file per day is + /// itself something the operator needs to know before they go looking. + /// + internal static string ServiceLogPath => + Path.Combine(DarlingFileLoggerProvider.DefaultLogDirectory(), "darling-service_yyyyMMdd.log"); + + /// + /// What on disk proves a bootstrap was ATTEMPTED against this store, as the phrase the message quotes — + /// null when nothing does. Quoting it keeps the verdict from being a bare assertion the operator has to + /// take on faith, which is the failure mode the old message had. + /// Ordered by how much each one settles. A cluster with a PG_VERSION is past initdb; a + /// pg.log means pg_ctl ran; the store's own credential is written IMMEDIATELY BEFORE initdb + /// ('s InitializeClusterAsync), so it survives the exact field + /// failure — an initdb that Windows killed in the loader — and is what makes this branch reachable at + /// all for the role-credential verbs. + /// + internal static string? FindBootstrapEvidence(string dataDirectory) + { + if (string.IsNullOrWhiteSpace(dataDirectory)) + { + /* No store to look at — never let an empty path become a relative one probed against the + working directory, which would answer about whatever the operator happened to cd into. */ + return null; + } + + try + { + if (File.Exists(Path.Combine(dataDirectory, "PG_VERSION"))) + { + return $"the cluster in {dataDirectory} is already initialized"; + } + + var storeFolder = Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(dataDirectory))); + if (!string.IsNullOrEmpty(storeFolder)) + { + var serverLog = Path.Combine(storeFolder, DarlingManagedPostgres.ServerLogFileName); + if (File.Exists(serverLog)) + { + return $"the store's server log {serverLog} is already there"; + } + + var storeCredential = Path.Combine(storeFolder, DarlingManagedPostgres.CredentialFileName); + if (File.Exists(storeCredential)) + { + return $"{storeCredential} is already there, and the service writes that one immediately before it runs initdb"; + } + + foreach (var roleCredential in s_roleCredentialFileNames) + { + var path = Path.Combine(storeFolder, roleCredential); + if (File.Exists(path)) + { + return $"{path} is already there"; + } + } + } + + if (Directory.Exists(dataDirectory) && Directory.EnumerateFileSystemEntries(dataDirectory).Any()) + { + return $"{dataDirectory} already holds a partly-built cluster"; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException) + { + /* A diagnostic must never become a second exception: an unreadable or malformed data directory + costs the operator the sharper branch, not the message. */ + } + + return null; + } + + /// The role credentials, provisioned AFTER the cluster is up — weaker evidence than the store's + /// own credential, but any of them proves the bootstrap got past initdb. + private static readonly string[] s_roleCredentialFileNames = + [ + DarlingManagedPostgres.AdminCredentialFileName, + DarlingManagedPostgres.ViewerCredentialFileName, + DarlingManagedPostgres.McpCredentialFileName, + ]; + + /// + /// The shared missing-credential message, in whichever of the two voices the evidence earns. The lead + /// clause is deliberately the same in both ("<subject> does not exist"), because the old wording is + /// already searchable in field reports and the issue tracker — the branch changes what follows it, not + /// what an operator pastes into a search box. + /// + /// The missing thing, naming its exact path. + /// What a genuine first run does, e.g. "initializes the store". + /// The store this verb is asking about. + internal static string MissingCredentialMessage(string subject, string firstRunAction, string dataDirectory) + { + var evidence = FindBootstrapEvidence(dataDirectory); + if (evidence is null) + { + return + $"{subject} does not exist yet. Start the PerformanceMonitor Darling service once so its first run " + + $"{firstRunAction}, then re-run this command. If you have ALREADY started it, its first run never got " + + $"this far and starting it again will not either — the reason is in the service log ({ServiceLogPath}), " + + "not in darling.json."; + } + + return + $"{subject} does not exist, and this is NOT a first run: {evidence}. The service has already run against " + + "this store and its bootstrap stopped before it got this far, so starting it again is not the fix. Read " + + $"the newest service log ({ServiceLogPath}) and work the FIRST error in it — a bundled Postgres tool that " + + "Windows killed is decoded there in words rather than left as a bare exit code. Nothing in darling.json " + + "produces this."; + } + + /// + /// The message for the store's OWN credential — the one four managed-store verbs refuse on, and the one + /// they never named a path for. Resolves the store's paths itself, and degrades (to the first-run branch, + /// and to a subject without a path) rather than throwing when the configured dataDirectory is + /// unresolvable: a message builder that can throw turns a diagnosable failure into an unhandled one. + /// + internal static string MissingStoreCredentialMessage(PostgresConfig postgres) + { + string dataDirectory; + string subject; + try + { + dataDirectory = DarlingManagedPostgres.ResolveDataDirectory(postgres); + subject = $"The managed store credential ({DarlingManagedPostgres.CredentialPathFor(dataDirectory)})"; + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or InvalidOperationException or IOException or UnauthorizedAccessException) + { + dataDirectory = string.Empty; + subject = "The managed store credential"; + } + + return MissingCredentialMessage(subject, "initializes the store", dataDirectory); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs index 9402580aa..fc27940bc 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs @@ -1539,14 +1539,18 @@ can be read rather than assumed ---- */ File.WriteAllText(passwordFile, context.Password + "\n"); DarlingFileSecurity.HardenFile(passwordFile, allowInteractiveRead: false); + var newInitDb = Path.Combine(context.NewBinDirectory, "initdb.exe"); var (initExit, initOutput) = await DarlingManagedPostgres.RunToolAsync( - Path.Combine(context.NewBinDirectory, "initdb.exe"), + newInitDb, BuildInitDbArguments(newDataDirectory, context.UserName, passwordFile, identity, context.NewMajor), s_initDbTimeout, cancellationToken); if (initExit != 0) { - throw new InvalidOperationException($"initdb of the new cluster failed (exit {initExit}):\n{initOutput}"); + throw new InvalidOperationException( + $"initdb of the new cluster failed (exit {DarlingToolExitCode.Describe(initExit)}):" + + DarlingToolExitCode.Diagnose(initExit, newInitDb) + + $"\n{DarlingToolExitCode.FormatOutput(initOutput, initExit)}"); } step = "conf-new-cluster"; @@ -1560,8 +1564,9 @@ can be read rather than assumed ---- */ step = "pg_upgrade-check"; var environment = BuildLibpqCredentialEnvironment(context.Password); + var pgUpgrade = Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"); var checkExit = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"), + pgUpgrade, BuildPgUpgradeArguments( context.OldBinDirectory, context.NewBinDirectory, context.DataDirectory, newDataDirectory, context.UserName, mode, checkOnly: true, jobs: 1, QuiesceTimescaleServerOptions), @@ -1571,8 +1576,16 @@ can be read rather than assumed ---- */ parent); if (checkExit != 0) { + /* "The clusters are not compatible" is pg_upgrade's verdict, and only pg_upgrade can reach it. + A Windows status means pg_upgrade never ran, so the compatibility claim would be invented + (#2186) — the same wrong-blame the pg_ctl status message carried. */ + var checkDiagnosis = DarlingToolExitCode.Diagnose(checkExit, pgUpgrade); throw new InvalidOperationException( - $"pg_upgrade --check failed (exit {checkExit}) — the clusters are not compatible and NOTHING has been changed.\n{ReadPgUpgradeLogTail(newDataDirectory)}"); + $"pg_upgrade --check failed (exit {DarlingToolExitCode.Describe(checkExit)})" + + (checkDiagnosis.Length == 0 + ? " — the clusters are not compatible and NOTHING has been changed." + : ". NOTHING has been changed." + checkDiagnosis) + + $"\n{ReadPgUpgradeLogTail(newDataDirectory)}"); } step = "pg_upgrade"; @@ -1585,7 +1598,7 @@ can be read rather than assumed ---- */ becomes the complaint. */ const int jobs = 1; var upgradeExit = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"), + pgUpgrade, BuildPgUpgradeArguments( context.OldBinDirectory, context.NewBinDirectory, context.DataDirectory, newDataDirectory, context.UserName, mode, checkOnly: false, jobs, QuiesceTimescaleServerOptions), @@ -1596,7 +1609,9 @@ becomes the complaint. */ if (upgradeExit != 0) { throw new InvalidOperationException( - $"pg_upgrade failed (exit {upgradeExit}).\n{ReadPgUpgradeLogTail(newDataDirectory)}"); + $"pg_upgrade failed (exit {DarlingToolExitCode.Describe(upgradeExit)})." + + DarlingToolExitCode.Diagnose(upgradeExit, pgUpgrade) + + $"\n{ReadPgUpgradeLogTail(newDataDirectory)}"); } /* ---- 7. swap the directories so the configured path holds the upgraded cluster. The @@ -1772,8 +1787,9 @@ private async Task StartClusterAsync(string binDirectory, string dataDirectory, Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(dataDirectory)))!, DarlingManagedPostgres.ServerLogFileName); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var exitCode = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"-D \"{dataDirectory}\" -o \"-p {port} -c listen_addresses=127.0.0.1\" -l \"{serverLog}\" -w -t 120 start", TimeSpan.FromMinutes(5), cancellationToken); @@ -1781,21 +1797,25 @@ private async Task StartClusterAsync(string binDirectory, string dataDirectory, if (exitCode != 0) { throw new InvalidOperationException( - $"could not start the cluster in {dataDirectory} with the runtime at {binDirectory} (pg_ctl exit {exitCode})"); + $"could not start the cluster in {dataDirectory} with the runtime at {binDirectory} (pg_ctl exit {DarlingToolExitCode.Describe(exitCode)})" + + DarlingToolExitCode.Diagnose(exitCode, pgCtl)); } } private async Task StopClusterAsync(string binDirectory, string dataDirectory, CancellationToken cancellationToken) { + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await DarlingManagedPostgres.RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"stop -D \"{dataDirectory}\" -m fast -w -t 120", s_toolTimeout, cancellationToken); if (exitCode != 0) { - throw new InvalidOperationException($"could not cleanly stop the cluster in {dataDirectory} (pg_ctl exit {exitCode}): {output}"); + throw new InvalidOperationException( + $"could not cleanly stop the cluster in {dataDirectory} (pg_ctl exit {DarlingToolExitCode.Describe(exitCode)}): {DarlingToolExitCode.FormatOutput(output, exitCode)}" + + DarlingToolExitCode.Diagnose(exitCode, pgCtl)); } } @@ -2229,8 +2249,10 @@ private async Task RunAnalyzeInStagesAsync( else { _logger.LogWarning( - "Post-upgrade analyze reported exit {ExitCode} ({Output}). The store is fully usable; autovacuum will build the remaining statistics.", - exitCode, output); + "Post-upgrade analyze reported exit {ExitCode} ({ExitCodeMeaning}): {Output}. The store is fully usable; autovacuum will build the remaining statistics.", + exitCode, + DarlingToolExitCode.Describe(exitCode), + DarlingToolExitCode.FormatOutput(output, exitCode)); } } catch (OperationCanceledException) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs new file mode 100644 index 000000000..daad24a8a --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// Turns a bundled-Postgres tool's exit code into words (#2186) — the shared vocabulary every +/// managed-store process failure is reported in. +/// +/// The bug this exists for, twice from the field: a managed bootstrap failed with +/// initdb failed (exit code -1073741515) for C:\ProgramData\PerformanceMonitorDarling\pg. Output: +/// and nothing more. -1073741515 is 0xC0000135, STATUS_DLL_NOT_FOUND: Windows killed +/// the process in the LOADER, before a line of its own code ran. Which is also why Output: was +/// empty and always will be for this class of failure — the one field an operator reads is guaranteed +/// blank exactly when the failure is a load failure, so it reads as "no information available" rather +/// than "this is a loader failure". The operator's attention then went to the follow-on +/// missing-credential message and darling.json, neither of which was the fault. +/// +/// Why a decoder and not one more format string: the sibling case already converts "the +/// binaries could not run" into words (MustRefuseUnidentifiableRuntime, #1738) and every other +/// bundled-tool failure printed the raw number. One shared decoder is what keeps the next one from +/// being the third report of the same defect. +/// +internal static class DarlingToolExitCode +{ + /// + /// NTSTATUS severity ERROR — the top two bits set. A process whose exit code lands in this range did + /// not choose it: Windows did, either in the loader or by ending a crashed process. Real programs exit + /// with small numbers (initdb 1, pg_ctl 3/4), so the range is a clean discriminator rather than a + /// heuristic, and treating the WHOLE range as "Windows killed it" is what keeps an unlisted status + /// from being as opaque as -1073741515 was. + /// + private const uint NtStatusErrorFloor = 0xC0000000; + + /// + /// The statuses worth naming. Loader statuses (the ones that mean the image or a dependency never + /// loaded) carry Loader = true and earn the full two-causes-two-checks diagnosis; the rest are + /// named so the number stops being a mystery, and get the shorter paragraph their kind deserves. + /// + private static readonly Dictionary s_knownStatuses = new() + { + [0xC0000135] = ("STATUS_DLL_NOT_FOUND", StatusKind.Loader), + [0xC0000139] = ("STATUS_ENTRYPOINT_NOT_FOUND", StatusKind.Loader), + [0xC000007B] = ("STATUS_INVALID_IMAGE_FORMAT", StatusKind.Loader), + [0xC0000142] = ("STATUS_DLL_INIT_FAILED", StatusKind.Loader), + [0xC0000022] = ("STATUS_ACCESS_DENIED", StatusKind.Loader), + [0xC0000005] = ("STATUS_ACCESS_VIOLATION", StatusKind.Crash), + [0xC0000374] = ("STATUS_HEAP_CORRUPTION", StatusKind.Crash), + [0xC0000409] = ("STATUS_STACK_BUFFER_OVERRUN", StatusKind.Crash), + [0xC000013A] = ("STATUS_CONTROL_C_EXIT", StatusKind.Terminated), + }; + + private enum StatusKind + { + Loader, + Crash, + Terminated, + } + + /// + /// The exit code as an operator should read it: the bare number for a tool's own status (initdb's 1, + /// pg_ctl status's 3), and the number PLUS its hex and NTSTATUS name when Windows set it. + /// + internal static string Describe(int exitCode) + { + var status = unchecked((uint)exitCode); + if (status < NtStatusErrorFloor) + { + return exitCode.ToString(CultureInfo.InvariantCulture); + } + + /* No parentheses of its own: every call site already sits inside one ("exit code {...}"), and + nesting them produced "(exit code -1073741515 (0xC0000135 STATUS_DLL_NOT_FOUND))". The "=" form + is also how the field report itself explained the number. */ + var hex = "0x" + status.ToString("X8", CultureInfo.InvariantCulture); + return s_knownStatuses.TryGetValue(status, out var known) + ? $"{exitCode.ToString(CultureInfo.InvariantCulture)} = {hex} {known.Name}" + : $"{exitCode.ToString(CultureInfo.InvariantCulture)} = {hex}, an unnamed Windows status rather than the program's own exit code"; + } + + /// + /// True when Windows killed the process in the LOADER — a dependency problem, not a tool error. The + /// caller uses this to decide whether gathering more evidence is worth a process launch (#2185). + /// + internal static bool IsLoaderStatus(int exitCode) + { + var status = unchecked((uint)exitCode); + return status >= NtStatusErrorFloor + && s_knownStatuses.TryGetValue(status, out var known) + && known.Kind == StatusKind.Loader; + } + + /// + /// Names WHICH binary could not load, from a --version probe of each (#2185). + /// + /// Why this exists. The field report that produced it took four exchanges and still is not + /// diagnosed, and the decisive clue was buried in the operator's shell rather than in any log: they ran + /// initdb --version by hand and it printed initdb (PostgreSQL) 18.4, while the service's + /// real initdb run died in the loader. A process that dies loading cannot print its own version, + /// so those two facts together rule out the two causes the loader diagnosis suggests — and neither the + /// operator nor the log could see that. + /// + /// The asymmetry that makes it diagnostic. initdb --version prints and exits, but a + /// real initdb run spawns postgres.exe in bootstrap mode to build the template database + /// and propagates its status. So a dependency missing only for postgres.exe produces exactly the + /// reported shape: the version probe succeeds and the bootstrap dies. Probing both separates that from + /// "the whole runtime cannot load", which is a different fix. + /// + internal static string DescribeRuntimeProbe(int initDbExitCode, int postgresExitCode) + { + var initDbLoaded = !IsLoaderStatus(initDbExitCode); + var postgresLoaded = !IsLoaderStatus(postgresExitCode); + + if (initDbLoaded && !postgresLoaded) + { + return "\nRuntime probe: initdb.exe loaded and reported its version, but postgres.exe did NOT — " + + $"it exited {Describe(postgresExitCode)}. That is the specific cause: a real initdb run spawns " + + "postgres.exe in bootstrap mode to build the template database and passes its status back, so a " + + "dependency missing only for postgres.exe fails the bootstrap while leaving `initdb --version` " + + "working. Compare postgres.exe's imports against the DLLs beside it; the bundle is the suspect, " + + "not the install location or the service account."; + } + + if (!initDbLoaded && !postgresLoaded) + { + return "\nRuntime probe: NEITHER initdb.exe nor postgres.exe could load, so this is not specific to " + + "one binary — the whole bundled runtime is failing to start. That points at the shared MSVC " + + "runtime beside the binaries or the machine's Universal CRT, rather than at any one tool."; + } + + if (initDbLoaded && postgresLoaded) + { + return "\nRuntime probe: BOTH initdb.exe and postgres.exe loaded and reported their versions when " + + "probed just now. The loader failure is therefore not a permanently missing dependency — it is " + + "specific to the failing invocation, so capture the Event Viewer > Windows Logs > Application " + + "entry at the failure time, which names the module."; + } + + return "\nRuntime probe: postgres.exe loaded but initdb.exe did not — unusual, since they share a " + + $"dependency set; initdb.exe exited {Describe(initDbExitCode)}. Treat initdb.exe itself as the " + + "damaged file and re-extract the package."; + } + + /// + /// The paragraph that follows the failure line: what Windows did, why the captured output is empty, + /// and the checks that separate the two causes. Empty string for a tool's own exit code — initdb + /// exiting 1 with a real error on stderr needs no help from here, and burying that message under + /// boilerplate would make the common failure worse to read. + /// + internal static string Diagnose(int exitCode, string exePath) + { + var status = unchecked((uint)exitCode); + if (status < NtStatusErrorFloor) + { + return string.Empty; + } + + var toolName = SafeFileName(exePath); + var kind = s_knownStatuses.TryGetValue(status, out var known) ? known.Kind : StatusKind.Terminated; + + return kind switch + { + StatusKind.Loader => LoaderDiagnosis(toolName, exePath), + StatusKind.Crash => + $"\n{toolName} started and then crashed — Windows ended it, so anything it had to say may be truncated or missing entirely. " + + "A crash is not a configuration problem and restarting is unlikely to clear it: capture the matching entry from Event Viewer > Windows Logs > Application " + + "(Application Error / Windows Error Reporting) together with the store's pg.log, and report it.", + _ => + $"\nWindows ended {toolName} rather than the tool exiting on its own, so its output may be empty or truncated and this is not a PostgreSQL error. " + + "Event Viewer > Windows Logs > Application, at the time of the failure, usually carries the matching entry.", + }; + } + + /// + /// The two causes and the two checks, in the order an operator should work them. Both causes are + /// specific to how this product ships PostgreSQL: the MSVC runtime is BUNDLED beside the binaries + /// (so a missing one means a partial extract, not a missing prerequisite), and the service runs as an + /// unprivileged virtual account that a user-profile install tree does not grant. + /// + private static string LoaderDiagnosis(string toolName, string exePath) + { + var binDirectory = SafeDirectoryName(exePath); + var builder = new StringBuilder(); + + builder.Append('\n').Append("Windows killed ").Append(toolName) + .Append(" in the LOADER: it never ran a line of its own code. This is a Windows failure, not a PostgreSQL one, and it is why the captured output is empty — for a loader failure an empty Output is EXPECTED, not missing information.\n"); + + builder.Append("Two causes account for nearly all of these:\n"); + builder.Append(" (1) The Microsoft Visual C++ runtime is missing from ").Append(binDirectory) + .Append(". Packaging bundles vcruntime140.dll, vcruntime140_1.dll and msvcp140.dll there so the box needs no prerequisite — ") + .Append("if any of the three is absent the install tree is a partial or damaged extract, so redeploy the package, or install the Microsoft Visual C++ 2015-2022 x64 redistributable.\n"); + builder.Append(" (2) The service account cannot read the install tree. The service runs as the virtual account NT SERVICE\\PerformanceMonitor Darling, which is neither you nor Administrators, ") + .Append("so an install under a user profile (Desktop, Downloads, anywhere below C:\\Users) is unreadable to it — reinstall to a machine-scoped path such as C:\\PerformanceMonitorDarling.\n"); + + builder.Append("Two checks that tell them apart:\n"); + builder.Append(" (a) Run \"").Append(exePath).Append("\" --version from an elevated prompt. Failing there too means (1). Succeeding points at (2) — and note that these tools re-execute themselves ") + .Append("under a restricted token that drops the Administrators group, so a tree readable only VIA Administrators still fails the real run even when it runs by hand.\n"); + builder.Append(" (b) Event Viewer > Windows Logs > Application, at the time of the failure: an Application Error or SideBySide entry usually names the exact module that could not be loaded."); + + return builder.ToString(); + } + + /// + /// The Output: field itself. Real output passes through untouched; a blank capture renders as + /// something readable instead of nothing at all, and says so explicitly when Windows is the reason + /// there was nothing to capture. + /// + internal static string FormatOutput(string? output, int exitCode) + { + if (!string.IsNullOrWhiteSpace(output)) + { + return output; + } + + return unchecked((uint)exitCode) >= NtStatusErrorFloor + ? "(none — Windows ended the process before it could write anything, which is expected for this failure rather than missing information)" + : "(none)"; + } + + /// Never let a malformed path turn a diagnostic into a second exception. + private static string SafeFileName(string exePath) + { + try + { + var name = Path.GetFileName(exePath); + return string.IsNullOrEmpty(name) ? "the tool" : name; + } + catch (ArgumentException) + { + return "the tool"; + } + } + + private static string SafeDirectoryName(string exePath) + { + try + { + var directory = Path.GetDirectoryName(exePath); + return string.IsNullOrEmpty(directory) ? "the pg-runtime\\pgsql\\bin directory" : directory; + } + catch (ArgumentException) + { + return "the pg-runtime\\pgsql\\bin directory"; + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs index 3d8dd2415..96c4ca53f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs @@ -1098,6 +1098,7 @@ private static CatalogRead R(string category, string description, params Catalog ["get_trace_flags"] = R(CatConfig, "Active trace flags for a server.", PServer()), ["get_database_config_changes"] = R(CatConfig, "Database-configuration changes over time.", PServer(), PHours(168)), ["get_database_scoped_config"] = R(CatConfig, "Database-scoped configuration for a database.", PServer(), PText("database_name")), + ["get_query_store_health"] = R(CatConfig, "Per-database Query Store health: actual vs desired state, readonly_reason, storage vs cap.", PServer(), PText("database_name")), ["get_server_config_changes"] = R(CatConfig, "Server-configuration changes over time.", PServer(), PHours(168)), ["get_trace_flag_changes"] = R(CatConfig, "Trace-flag changes over time.", PServer(), PHours(168)), @@ -1114,6 +1115,14 @@ private static CatalogRead R(string category, string description, params Catalog ["get_tempdb_trend"] = R(CatData, "tempdb space usage over time.", PServer(), PHours(24)), ["get_top_procedures_by_cpu"] = R(CatData, "Top stored procedures by CPU.", PServer(), PHours(24), PTop(20), PText("database_name")), ["get_top_queries_by_cpu"] = R(CatData, "Top queries by CPU, optionally parallel-only / min-DOP.", PServer(), PHours(24), PTop(20), PText("database_name"), PBool("parallel_only", false), PInt("min_dop", 0)), + ["get_pg_top_queries"] = R(CatData, "Top PostgreSQL query shapes by total execution time (Aurora targets).", PServer(), PHours(24), PLimit(20)), + ["get_pg_wraparound_risk"] = R(CatData, "PostgreSQL XID/MultiXact freeze headroom per database.", PServer(), PHours(24)), + ["get_pg_xmin_horizon"] = R(CatData, "What is holding back the PostgreSQL xmin horizon, by cause.", PServer(), PHours(24)), + ["get_pg_replication_slots"] = R(CatData, "PostgreSQL replication slot health, including whether retained WAL is still growing.", PServer(), PHours(24)), + ["get_pg_autovacuum_health"] = R(CatData, "PostgreSQL tables behind on vacuum or analyze, ranked by how far past each table's own threshold.", PServer(), PHours(24), PLimit(20)), + ["get_pg_io_stats"] = R(CatData, "PostgreSQL I/O by backend type, object and context, differenced across the window.", PServer(), PHours(24), PLimit(20)), + ["get_pg_wait_stats"] = R(CatData, "Top PostgreSQL wait events in the window (Aurora targets).", PServer(), PHours(24), PLimit(20)), + ["get_pg_blocking"] = R(CatData, "PostgreSQL blocking chains that were sampled, with the root blocker attributed. A sample, not an event log.", PServer(), PHours(24), PLimit(50)), ["get_wait_stats"] = R(CatData, "Top wait statistics in the window.", PServer(), PHours(24), PLimit(20)), ["get_wait_trend"] = R(CatData, "One wait type's totals over time (requires wait_type).", PReqText("wait_type"), PServer(), PHours(24)), ["get_wait_types"] = R(CatData, "The wait types observed in the window.", PServer(), PHours(24)), @@ -1525,6 +1534,7 @@ internal static IReadOnlyDictionary BuildReadDispatch() ["get_trace_flags"] = (c, pg, an) => DarlingMcpConfigTools.GetTraceFlags(pg, Server(c)), ["get_database_config_changes"] = (c, pg, an) => DarlingMcpConfigHistoryTools.GetDatabaseConfigChanges(pg, Server(c), Hours(c, 168)), ["get_database_scoped_config"] = (c, pg, an) => DarlingMcpConfigHistoryTools.GetDatabaseScopedConfig(pg, Server(c), Str(c, "database_name")), + ["get_query_store_health"] = (c, pg, an) => DarlingMcpConfigHistoryTools.GetQueryStoreHealth(pg, Server(c), Str(c, "database_name")), ["get_server_config_changes"] = (c, pg, an) => DarlingMcpConfigHistoryTools.GetServerConfigChanges(pg, Server(c), Hours(c, 168)), ["get_trace_flag_changes"] = (c, pg, an) => DarlingMcpConfigHistoryTools.GetTraceFlagChanges(pg, Server(c), Hours(c, 168)), @@ -1541,6 +1551,14 @@ internal static IReadOnlyDictionary BuildReadDispatch() ["get_tempdb_trend"] = (c, pg, an) => DarlingMcpDataTools.GetTempDbTrend(pg, Server(c), Hours(c, 24)), ["get_top_procedures_by_cpu"] = (c, pg, an) => DarlingMcpDataTools.GetTopProceduresByCpu(pg, Server(c), Hours(c, 24), Rows(c, "top", 20), Str(c, "database_name")), ["get_top_queries_by_cpu"] = (c, pg, an) => DarlingMcpDataTools.GetTopQueriesByCpu(pg, Server(c), Hours(c, 24), Rows(c, "top", 20), Str(c, "database_name"), QueryBool(c, "parallel_only", false), QueryInt(c, "min_dop", null, 0)), + ["get_pg_top_queries"] = (c, pg, an) => DarlingMcpPgStatementTools.GetPgTopQueries(pg, Server(c), Hours(c, 24), Rows(c, "limit", 20)), + ["get_pg_wraparound_risk"] = (c, pg, an) => DarlingMcpPgWraparoundTools.GetPgWraparoundRisk(pg, Server(c), Hours(c, 24)), + ["get_pg_xmin_horizon"] = (c, pg, an) => DarlingMcpPgXminTools.GetPgXminHorizon(pg, Server(c), Hours(c, 24)), + ["get_pg_replication_slots"] = (c, pg, an) => DarlingMcpPgSlotTools.GetPgReplicationSlots(pg, Server(c), Hours(c, 24)), + ["get_pg_autovacuum_health"] = (c, pg, an) => DarlingMcpPgAutovacuumTools.GetPgAutovacuumHealth(pg, Server(c), Hours(c, 24), Rows(c, "limit", 20)), + ["get_pg_io_stats"] = (c, pg, an) => DarlingMcpPgIoTools.GetPgIoStats(pg, Server(c), Hours(c, 24), Rows(c, "limit", 20)), + ["get_pg_wait_stats"] = (c, pg, an) => DarlingMcpPgWaitTools.GetPgWaitStats(pg, Server(c), Hours(c, 24), Rows(c, "limit", 20)), + ["get_pg_blocking"] = (c, pg, an) => DarlingMcpPgBlockingTools.GetPgBlocking(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50)), ["get_wait_stats"] = (c, pg, an) => DarlingMcpDataTools.GetWaitStats(pg, Server(c), Hours(c, 24), Rows(c, "limit", 20)), ["get_wait_trend"] = (c, pg, an) => RequireText(c, "wait_type", out var waitType) ? DarlingMcpDataTools.GetWaitTrend(pg, waitType, Server(c), Hours(c, 24)) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs index a8d3d56b9..2dc65b021 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs @@ -21,6 +21,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Npgsql; +using PerformanceMonitor.Darling.Service.Targets; using PerformanceMonitor.Alerting; using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; @@ -105,16 +106,54 @@ cancellation token and the guarded server snapshot. One slice per server per tic private const int MaxAnalysisIntervalMinutes = 360; /// - /// The bounded per-server collection concurrency (the fire-and-track sweep, #1553): at most this many - /// servers' collection bodies run at once, each opening at most ONE SQL connection (collectors stay - /// sequential within a body — Lite's RemoteCollectorService shape). Hardcoded, no control-plane knob - /// (defaults-over-config): 4 clears a 24-server worst case in ~6 waves while the 120s analysis budget stays - /// de-clustered by the cadence jitter, so one slow/hung server can never head-of-line-block the fleet the - /// way the old strictly-sequential foreach did (the 24-server field incident). Internal so a unit test pins - /// the value against this rationale (a cheap drift tripwire — see the plan). + /// The DEFAULT bounded per-server collection concurrency (the fire-and-track sweep, #1553): at most this + /// many servers' collection bodies run at once, each opening at most ONE SQL connection (collectors stay + /// sequential within a body — Lite's RemoteCollectorService shape). 4 clears a 24-server worst case in ~6 + /// waves while the 120s analysis budget stays de-clustered by the cadence jitter, so one slow/hung server + /// can never head-of-line-block the fleet the way the old strictly-sequential foreach did (the 24-server + /// field incident). + /// + /// #2170: no longer the hard ceiling — an operator knob (config_service.max_concurrent_sweeps, V59) + /// overrides it, because on a host with headroom watching a large fleet, 4-wide serialization is itself + /// what makes sweeps queue and the Fleet Health screen report staleness while every collector is healthy + /// (the reporter's 56-server case). This stays the DEFAULT and the seeded value. /// internal const int MaxConcurrentServerSweeps = 4; + /// + /// The gate is constructed at this ceiling and immediately narrowed to the configured width (#2170) — + /// a cannot be resized, so unused permits are drained rather than the + /// semaphore rebuilt (rebuilding would strand in-flight bodies releasing the old instance). Matches + /// , the store-read clamp ceiling. + /// + internal const int SweepGateCeiling = 16; + + /* Sweep-gate width state (#2170), all under _gateLock: the gate is built at SweepGateCeiling and its + effective width is (ceiling - _gateAbsorbed). _gateDesiredAbsorb is where the knob wants that to + land; a single absorber task closes the gap as in-flight bodies release permits. Holding the counts + (rather than per-call deltas) is what makes a widen landing mid-narrow safe — see ReconcileSweepGate. */ + private readonly object _gateLock = new(); + private int _gateAbsorbed; + private int _gateDesiredAbsorb; + private bool _gateAbsorberRunning; + + /// + /// The sweep gate's width right now (#2170) — the ceiling minus what has been absorbed. Reported by the + /// queued-behind-the-gate diagnostic, which an operator reads while deciding whether to raise the knob, + /// so it must never print the compile-time default once the knob has moved. Mid-narrow this reads the + /// TARGET rather than the momentarily-larger real count; that is the honest number to act on. + /// + internal int EffectiveSweepWidth + { + get + { + lock (_gateLock) + { + return SweepGateCeiling - _gateDesiredAbsorb; + } + } + } + /// /// Seconds an in-flight collection body may go unresolved before the sweep watchdog surfaces it. One /// threshold serves both channels below — what differs is WHICH clock it is measured against. @@ -211,6 +250,16 @@ MUST respect that ceiling or shutdown starts being force-killed mid-drain. */ /// Test hook: the hardcoded per-run analysis budget, pinned against Lite's default. internal static TimeSpan AnalysisTimeout => s_analysisTimeout; + /* #2299: how long a stopping sweep holds its analysis pass open so the pass can unwind BEFORE + the loop's data source is disposed at RunCollectionLoopAsync scope exit. The pass observes + the same stopping token (via AnalysisContext), so this is normally milliseconds; the bound + exists for a pass stuck inside a store read. Sized WELL INSIDE the 15s s_shutdownDrainBudget + (this await runs inside a drained sweep body) and the host's 30s ShutdownTimeout. */ + private static readonly TimeSpan s_analysisShutdownGrace = TimeSpan.FromSeconds(5); + + /// Test hook: the shutdown grace granted to an in-flight analysis pass (#2299). + internal static TimeSpan AnalysisShutdownGrace => s_analysisShutdownGrace; + /// /// The Stage 2 pause gate: whether the collection sweep does work this tick. FALSE while the service is /// paused (config_service.paused, mirrored into _paused on reload) — the loop then skips all @@ -335,6 +384,21 @@ branches the retention purge onto drop_chunks. */ self-alerts inherit its delivery/cooldown/restart-replay. Held as a field because the connection edge fires from TryConnectAsync and the reconcile drops per-server state through it. */ private DarlingSelfAlertEvaluator? _selfAlerts; + /* Concrete rather than IAlertDeliverer: there is exactly one implementation here and it is constructed + a few lines from where this is assigned, so the interface bought an indirection per delivered alert + and no seam (CA1859). */ + private DarlingAlertDeliverer? _alertDeliverer; + + /* The mute check and the cooldown stamps for the PostgreSQL predictors. These ride alongside the shared + AlertEngine rather than inside it, which is deliberate — but "alongside" was taken to mean "without", + and the PG path shipped with Muted hardcoded false and no cooldown at all. Every AlertEngine family + gates on both; a 30-second sweep without them writes ~2,880 history rows a day per breaching subject + and emails through a mute rule that says not to. */ + private Func? _isAlertMuted; + + private readonly ConcurrentDictionary _lastPostgresAlert = new(StringComparer.Ordinal); + + private int _alertCooldownMinutes = 15; /* #1560: the live MCP enable/port seam — published to the MCP host's supervisor at startup and on every control-plane reload, so the viewer's Settings toggle takes effect without a restart. */ @@ -345,12 +409,18 @@ supervisor at startup and on every control-plane reload so the viewer's Settings without a restart. */ private readonly WebRuntimeState _webState; - public DarlingWorker(ILogger logger, ILoggerFactory loggerFactory, McpRuntimeState mcpState, WebRuntimeState webState) + /* #2298: the live monitored-server registry seam — published beside the two above, read by the MCP + host's plan-fetch resolver so it never re-reads config_monitored_servers as the mcp role (whose + encrypted_password SELECT-carve fails that whole read). */ + private readonly MonitoredServerRegistryState _registryState; + + public DarlingWorker(ILogger logger, ILoggerFactory loggerFactory, McpRuntimeState mcpState, WebRuntimeState webState, MonitoredServerRegistryState registryState) { _logger = logger; _loggerFactory = loggerFactory; _mcpState = mcpState; _webState = webState; + _registryState = registryState; } private sealed class ServerLoopState @@ -360,6 +430,10 @@ private sealed class ServerLoopState public required MonitoredServer Config { get; set; } public ServerRuntime? Runtime { get; set; } + /* Set once per process after the PostgreSQL analysis-state row is written, so the explanation is + recorded without rewriting the same row every analysis interval forever. */ + public bool PostgresAnalysisStateWritten { get; set; } + /* ConcurrentDictionary (#1553 D1): with the fire-and-track sweep the per-server body runs on a pool thread, so a reload's RecomputeNextDueAsync on the OUTER thread can touch this map concurrently with the body's RunDueCollectorsAsync read-and-advance. It is only ever INDEXED by the static collector-catalog @@ -368,6 +442,21 @@ private sealed class ServerLoopState public ConcurrentDictionary NextDue { get; } = new(StringComparer.OrdinalIgnoreCase); public DateTime NextConnectAttempt { get; set; } = DateTime.MinValue; + /* #2255: the last connect-failure message logged in FULL, so an unchanged cause repeats as one terse + line instead of its whole explanation every 60 seconds forever. The field report is a DPAPI decrypt + failure — permanent by construction, since the blob can never become decryptable on this host — and + at Warning-with-full-text it buried the log while never once telling the operator anything new. + Compared on the message rather than the exception type so a changed cause (credential fixed, server + now genuinely unreachable) prints in full again. */ + public string? LastConnectFailureLogged { get; set; } + + /* #2228: the database-mismatch state last reported for this server, so the tripwire fires on the + TRANSITION rather than once per connect. A mismatch is a standing misconfiguration — it persists + until an operator edits the registration — so logging it every reconnect would bury the one line + that matters, which is how a tripwire gets trained past and stops working. Null = last seen + correct; the message itself = last seen wrong, compared so a change of mismatch re-reports. */ + public string? LastDatabaseMismatchLogged { get; set; } + /* MinValue = the first loop pass after connect evaluates alerts immediately. */ public DateTime NextAlertSweep { get; set; } = DateTime.MinValue; @@ -930,12 +1019,20 @@ read is empty (a partially-seeded store), so the service never starts up monitor holds — including on a store-unreachable boot. Re-published on every reload below. */ _mcpState.Publish(config.Mcp.Enabled, config.Mcp.Port); _webState.Publish(config.Web.Enabled, config.Web.Port); + /* #2298: publish the effective server set the same way — store-authoritative when the view loaded, + else the darling.json servers, which is what this run will actually collect from either way. The + MCP host's plan-fetch resolver reads this instead of re-reading the store as the mcp role. */ + _registryState.Publish(initialServers); /* Capture-plans is read live (() => config.CapturePlans) so a store reload of config_service.capture_plans is honored on the next collector cycle without rebuilding. CollectSchemaChangeEvents is a file-only knob (darling.json), read the same way for symmetry — default true keeps every SKU collecting Object DDL; set false to silence a benchmark box's flood. */ - var runner = new DarlingCollectorRunner(postgres, deltas, _logger, () => config.CapturePlans, () => config.CollectSchemaChangeEvents); + var runner = new DarlingCollectorRunner(postgres, deltas, _logger, () => config.CapturePlans, () => config.CollectSchemaChangeEvents, + () => StoreConfigProvider.ClampTextBudgetMb(config.QueryStoreTextBudgetMb), + /* #2171: live provider like its siblings — a store reload flipping plan_xml_compression + takes effect on the next write batch, no restart. */ + compressPlanContent: () => !string.Equals(config.PlanXmlCompression, "none", StringComparison.OrdinalIgnoreCase)); var servers = new List(); /* #1581 cold-start stagger: capture ONE startup instant so every initial server's first-sweep offset is measured from the same base — the deterministic per-server ColdStartFirstSweepDue then spreads the @@ -949,7 +1046,7 @@ measured from the same base — the deterministic per-server ColdStartFirstSweep { Config = server, FirstSweepDueUtc = ColdStartFirstSweepDue( - coldStartInstant, ServerIdHelper.GetDeterministicHashCode(server.StorageName)), + coldStartInstant, server.ServerId), }); } @@ -986,12 +1083,22 @@ are hoisted here because the AN3 analysis-notification path below shares them. T lock (_serversLock) { return servers - .Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == id) + .Find(s => s.Config.ServerId == id) ?.Config.AlertDeliveryModeOverride; } }); var engine = BuildAlertEngine(config, servers, alertSettings, historyStore, muteRuleService, deliverer); + /* Held for the PostgreSQL predictors, which deliver alongside the shared engine rather than + through it (see EvaluatePostgresAlertsAsync). Same deliverer instance, so a PostgreSQL alert + lands in the same history and obeys the same mute rules as an engine-emitted one — the point + of reusing it rather than building a second delivery path. */ + _alertDeliverer = deliverer; + /* Same instance the engine binds, so a mute-rule reload mutes the PostgreSQL predictors on the next + sweep exactly as it mutes every SQL Server family. */ + _isAlertMuted = muteRuleService.IsAlertMuted; + _alertCooldownMinutes = alertSettings.CooldownMinutes; + /* Stage 4: the service self-alerts, over the SAME deliverer + history + mute check the engine uses. collection-stopped / capture-down are polled from collection_log on the alert cadence below; connection lost/restored fire on the connect edges in TryConnectAsync. */ @@ -1008,7 +1115,9 @@ without a restart (and the clamps live on the settings properties, not here). */ notifyAgHealth: () => alertSettings.NotifyAgHealth, agLagAlertSeconds: () => alertSettings.AgLagAlertSeconds, agRedoQueueAlertKb: () => alertSettings.AgRedoQueueAlertKb, - agDisconnectRefireMinutes: () => alertSettings.AgDisconnectRefireMinutes); + agDisconnectRefireMinutes: () => alertSettings.AgDisconnectRefireMinutes, + /* #2136: the cadence warning threshold, read live like the AG seams (clamped on the property). */ + storeJobCadenceWarnPercent: () => alertSettings.StoreJobCadenceWarnPercent); /* #1706: report this start's store runtime upgrade, now that there IS an alert engine to report it through. Fired once, here, and never re-evaluated — the store is down while an upgrade runs, so @@ -1060,8 +1169,9 @@ collection loop stops so both drain cleanly on shutdown. */ Fills the two windows the live path discards by design — the 60-minute first-contact tail and 24h-clamped outage holes — newest-first, byte-budgeted, strictly BELOW the live path's floor, and never past the raw tier's horizon. Plan capture reads the same live provider the runner does. */ - var queryStoreBackfill = new QueryStoreBackfill(postgres, runner, deltas, _logger, () => config.CapturePlans); - var backfillLoop = RunQueryStoreBackfillLoopAsync(queryStoreBackfill, servers, stoppingToken); + var queryStoreBackfill = new QueryStoreBackfill(postgres, runner, deltas, _logger, () => config.CapturePlans, + () => StoreConfigProvider.ClampTextBudgetMb(config.QueryStoreTextBudgetMb)); + var backfillLoop = RunQueryStoreBackfillLoopAsync(queryStoreBackfill, servers, () => config.QueryStoreBackfillEnabled, stoppingToken); /* The fleet concurrency gate (#1553 D2): at most N=4 per-server collection bodies open a SQL connection at once, so one slow or hung server cannot head-of-line-block the fleet the way the old strictly @@ -1070,9 +1180,28 @@ never past the raw tier's horizon. Plan capture reads the same live provider the body detached from the drain list — would otherwise reach its finally { gate.Release() } on a disposed SemaphoreSlim and throw ObjectDisposedException, faulting an unobserved Task. A SemaphoreSlim needs no deterministic disposal unless its AvailableWaitHandle is used, which it never is here. */ + /* #2170: the width is now an operator knob, and a SemaphoreSlim cannot be resized — so the gate's + MAX is the clamp ceiling while its INITIAL count is the configured width. Later changes move + between the two: widening Releases permits, narrowing absorbs them as in-flight bodies finish + (see ReconcileSweepGate), which converges without ever blocking this loop or interrupting a + running collection. + + Starting AT the configured width rather than at the ceiling matters (review catch): reconciling + down only STARTS the absorber, so a gate born wide would offer ceiling-many permits for the + window before it retires them — and a restart with many servers simultaneously due (before the + #1581 cold-start stagger spreads them) is exactly when that window would be spent. Born narrow, + the window does not exist. */ + var initialSweepWidth = StoreConfigProvider.ClampConcurrentSweeps(config.MaxConcurrentSweeps); #pragma warning disable CA2000 - var serverSweepGate = new SemaphoreSlim(MaxConcurrentServerSweeps, MaxConcurrentServerSweeps); + var serverSweepGate = new SemaphoreSlim(initialSweepWidth, SweepGateCeiling); #pragma warning restore CA2000 + lock (_gateLock) + { + /* The permits the gate was never given ARE the absorbed ones — seed both counts so the first + reconcile computes its delta from reality instead of re-absorbing what was never issued. */ + _gateAbsorbed = SweepGateCeiling - initialSweepWidth; + _gateDesiredAbsorb = _gateAbsorbed; + } _logger.LogInformation("PerformanceMonitor Darling collection loop started"); @@ -1088,6 +1217,10 @@ never past the raw tier's horizon. Plan capture reads the same live provider the { _lastConfigVersion = configVersion.Value; await ReloadFromStoreAsync(configProvider, config, servers, muteRuleService, stoppingToken); + + /* #2170: the reload swapped the knob into the live config; move the gate to match. Safe here + by construction — top of the sweep, and narrowing never preempts a running body. */ + ReconcileSweepGate(serverSweepGate, StoreConfigProvider.ClampConcurrentSweeps(config.MaxConcurrentSweeps), stoppingToken); } /* Stage 2 pause gate (Lite's IsPaused): while paused, skip ALL collection/alert/analysis/purge @@ -1219,12 +1352,16 @@ This is the channel that must stay quiet on a healthy fleet so a real stall is s break; /* CAPACITY — still QUEUED behind the gate, so nothing is wrong with this server: the - fleet is simply wider than MaxConcurrentServerSweeps at this moment. Info, once. */ + fleet is simply wider than the configured sweep width at this moment. Info, once. + Reports the EFFECTIVE width, not the compile-time default (#2170 review catch): + this line is what an operator reads while deciding whether to raise the knob, so + printing 4 after they raised it to 12 would send them chasing a limit that is no + longer in force. */ case SweepEpisodeSignal.Queued: server.QueuedInfoThisEpisode = true; _logger.LogInformation( "[{Server}] collection body has waited {Elapsed:F0}s for a free slot (fleet concurrency limit {Limit}) — queued, not stalled; it has not started yet", - server.Config.DisplayName, episodeSeconds, MaxConcurrentServerSweeps); + server.Config.DisplayName, episodeSeconds, EffectiveSweepWidth); break; } @@ -1267,7 +1404,8 @@ Empty overrides (Stage 1 seeds none) resolve to the defaults — identical behav var overrides = _scheduleOverrides; await DarlingRetention.PurgeAsync( postgres, _timescaleAvailable, _logger, stoppingToken, - name => StoreConfigProvider.ResolveFleetRetentionDays(name, overrides)); + name => StoreConfigProvider.ResolveFleetRetentionDays(name, overrides), + config.PlanContentRetentionDays); /* AN3: findings retention. Both apps' finding stores declare a 30-day cleanup but neither app schedules it (Lite's DuckDB archive-reset bounds it @@ -1459,7 +1597,7 @@ on the connect edges in TryConnectAsync. (Uses the _postgres field — the loop- server.NextSelfAlertSweep = DateTime.UtcNow.Add(s_alertSweepInterval); await _selfAlerts!.EvaluateStoreAlertsAsync( _postgres!, - ServerIdHelper.GetDeterministicHashCode(server.Config.StorageName), + server.Config.ServerId, server.Config.DisplayName, connected: server.Runtime is not null, stoppingToken); @@ -1499,8 +1637,38 @@ on the connect edges in TryConnectAsync. (Uses the _postgres field — the loop- { var intervalMinutes = Math.Clamp(config.Analysis.IntervalMinutes, MinAnalysisIntervalMinutes, MaxAnalysisIntervalMinutes); server.NextAnalysisDue = DateTime.UtcNow.AddMinutes(intervalMinutes); - await RunScheduledAnalysisAsync( - server, planFetcher, notificationService, config.Analysis.NotificationsEnabled, stoppingToken); + + /* The analysis pipeline is SQL-Server-shaped: its facts come from wait_stats, query_stats, + cpu_utilization_stats and friends, none of which a PostgreSQL target ever writes. Running it + anyway is not harmless. RunAnalysisPassAsync takes a serverId and a storage name — not the + target — so it cannot gate itself, and it would read those tables, find nothing, hit the + 24-hour data-span gate and persist insufficient_data = true. FOREVER: those tables will + never have rows for a PostgreSQL server_id, so the Recommendations tab would say "still + collecting" for the life of the deployment, which is the one thing analysis_state exists to + distinguish from a genuine all-clear. Plus a fresh DarlingAnalysisService and up to a + 120-second pass per target per interval, producing nothing. + + So: skip the pass, and say why ONCE rather than leaving the tab silent. The message is the + honest state — not "still collecting", which is a lie about a young deployment. */ + if (server.Runtime?.Target.Engine == CollectorTargetEngine.PostgreSql) + { + if (!server.PostgresAnalysisStateWritten) + { + server.PostgresAnalysisStateWritten = true; + await DarlingObservability.WriteAnalysisStateAsync( + _postgres!, + server.Runtime.ServerId, + insufficientData: true, + message: PostgresAnalysisNotApplicable, + _logger, + stoppingToken); + } + } + else + { + await RunScheduledAnalysisAsync( + server, planFetcher, notificationService, config.Analysis.NotificationsEnabled, stoppingToken); + } } } catch (OperationCanceledException) @@ -1535,7 +1703,18 @@ private async Task ReconcileLongQueryTraceAsync(ServerLoopState server, DarlingC return; } - var serverId = ServerIdHelper.GetDeterministicHashCode(server.Config.StorageName); + /* XE is a SQL Server concept: there is nothing to create or drop on a PostgreSQL target, and the + un-gated form was the round-2 live catch — ReconcileLongQueryCompletionsAsync builds a + SqlConnection, the ctor throws "Keyword not supported: 'host'", the catch below skips the latch + assignment, and because LongQueryTraceApplied resets to null on every connect the failure retried + EVERY sweep forever (~1,440 warnings/day/server, the same order as the defect this PR fixed). + Same gate as EnsureAllAsync and FetchFailedJobsAsync, the two doors this class already closed. */ + if (server.Runtime.Target.Engine != CollectorTargetEngine.SqlServer) + { + return; + } + + var serverId = server.Config.ServerId; var enabled = StoreConfigProvider.ResolveSchedule("long_query_completions", serverId, _scheduleOverrides).Enabled; if (server.LongQueryTraceApplied == enabled) @@ -1587,6 +1766,113 @@ private async Task RunBaselineBackfillAsync(NpgsqlDataSource postgres, Cancellat } } + /// + /// Moves the sweep gate to concurrent servers (#2170). The gate is built at + /// and its width is expressed as how many permits are held OUT of + /// circulation, so narrowing "absorbs" permits and widening gives them back. + /// + /// State is the absorbed COUNT plus a desired count, both under , rather + /// than a per-call absorb loop: the first cut had a permit-stealing race (review catch) where a + /// still-running narrowing task would immediately re-absorb the permit a later widening had just + /// released, pinning the gate below the configured width. At most one absorber runs, and it re-reads + /// the desired count under the lock before AND after every wait — so a widening mid-absorb makes the + /// absorber hand its permit straight back and retire. + /// + /// Never blocks the caller and never preempts a running collection: narrowing only takes permits + /// as in-flight bodies release them, so the effective width converges within about one sweep. + /// + private void ReconcileSweepGate(SemaphoreSlim gate, int target, CancellationToken stoppingToken) + { + int toRelease; + bool startAbsorber; + lock (_gateLock) + { + var desired = SweepGateCeiling - target; + if (desired == _gateDesiredAbsorb && _gateAbsorbed == desired) + { + return; + } + + _gateDesiredAbsorb = desired; + toRelease = _gateAbsorbed > desired ? _gateAbsorbed - desired : 0; + _gateAbsorbed -= toRelease; + startAbsorber = _gateAbsorbed < desired && !_gateAbsorberRunning; + if (startAbsorber) + { + _gateAbsorberRunning = true; + } + } + + if (toRelease > 0) + { + gate.Release(toRelease); + _logger.LogInformation("Fleet sweep width widened to {Target} concurrent servers (#2170 knob)", target); + } + + if (startAbsorber) + { + _logger.LogInformation( + "Fleet sweep width narrowing to {Target} concurrent servers (#2170 knob) — permits retire as in-flight collections finish", + target); + _ = Task.Run(() => AbsorbSweepPermitsAsync(gate, stoppingToken), stoppingToken); + } + } + + /// + /// The single sweep-gate absorber (#2170): takes permits out of circulation until the absorbed count + /// reaches the desired count. Re-checks that target around every wait, so a widening that lands while + /// it is parked on is honored — the permit it + /// was granted goes straight back rather than being stolen from the wider gate. + /// + private async Task AbsorbSweepPermitsAsync(SemaphoreSlim gate, CancellationToken stoppingToken) + { + try + { + while (true) + { + lock (_gateLock) + { + if (_gateAbsorbed >= _gateDesiredAbsorb) + { + _gateAbsorberRunning = false; + return; + } + } + + await gate.WaitAsync(stoppingToken).ConfigureAwait(false); + + var giveBack = false; + lock (_gateLock) + { + if (_gateAbsorbed >= _gateDesiredAbsorb) + { + /* Widened while we waited — this permit is no longer surplus. */ + _gateAbsorberRunning = false; + giveBack = true; + } + else + { + _gateAbsorbed++; + } + } + + if (giveBack) + { + gate.Release(); + return; + } + } + } + catch (OperationCanceledException) + { + /* Shutdown — the gate goes away with the process. */ + lock (_gateLock) + { + _gateAbsorberRunning = false; + } + } + } + /// /// The #2022 backfill tick: at most one Query Store backfill slice per CONNECTED server per /// interval, sequentially — sequence IS the fleet-wide concurrency bound, so a fleet of slow @@ -1598,8 +1884,11 @@ private async Task RunBaselineBackfillAsync(NpgsqlDataSource postgres, Cancellat /// constraint — a backfill slice is read-only against the monitored server and writes on its /// own store connection, so running beside a live sweep is safe. /// - private async Task RunQueryStoreBackfillLoopAsync(QueryStoreBackfill backfill, List servers, CancellationToken stoppingToken) + private async Task RunQueryStoreBackfillLoopAsync(QueryStoreBackfill backfill, List servers, Func backfillEnabled, CancellationToken stoppingToken) { + /* #2167: transition-logged so a store-config flip is visible in the log exactly once per state + change, not once per idle cycle. */ + var lastEnabled = true; while (!stoppingToken.IsCancellationRequested) { try @@ -1611,6 +1900,25 @@ private async Task RunQueryStoreBackfillLoopAsync(QueryStoreBackfill backfill, L return; } + /* #2167: the off switch (config_service.query_store_backfill_enabled, V58) — read live each + cycle via the store-reload seam, so an operator can stop a runaway drain (a freshly restored + catalog on a cross-region server) without a restart and without touching plan capture. The + loop keeps ticking while disabled: a re-enable takes effect on the next cycle. */ + var enabled = backfillEnabled(); + if (enabled != lastEnabled) + { + _logger.LogInformation( + enabled + ? "query_store backfill re-enabled via config — resuming on the next cycle" + : "query_store backfill DISABLED via config (config_service.query_store_backfill_enabled) — loop idling, in-flight slices finish and no new ones start"); + lastEnabled = enabled; + } + + if (!enabled) + { + continue; + } + List runtimes; lock (_serversLock) { @@ -1627,25 +1935,209 @@ private async Task RunQueryStoreBackfillLoopAsync(QueryStoreBackfill backfill, L return; } - try + /* #2148 parity (review catch on the Lite fix): a slice that WEDGES — not throws — used + to hold this foreach forever, stalling backfill for the entire fleet with the + exception armor below intact. Per-SERVER abandonable steps, so one wedged server is + abandoned (loudly) and quarantined until its task actually dies, while every other + server's backfill continues. The deadline is a generous multiple of a healthy slice + (statement timeout 60s + store writes), so an abandonment is a defect signal. */ + /* #2165: the other half of the gate. Held for the WHOLE slice, and taken outside the + AbandonableStep so an abandoned-but-still-wedged slice keeps the gate closed — the tick must + keep yielding while that statement is genuinely still running on the server, which is exactly + the case the abandonment leaves behind. Zero-wait, so a tick already collecting simply defers + this server's slice to the next five-minute cycle. */ + var gate = _queryStoreGates.GetOrAdd(runtime.ServerId, static _ => new QueryStoreServerGate()).TryAcquire(); + if (gate is null) { - await backfill.RunServerSliceAsync(runtime, stoppingToken); + _logger.LogInformation( + "query_store backfill slice on '{Server}' deferred — the tick's Query Store collection is running (#2165)", + runtime.Config.DisplayName); + continue; } - catch (OperationCanceledException) + + using var backfillGate = gate; + + var step = _backfillSliceSteps.GetOrAdd(runtime.ServerId, static _ => new AbandonableStep()); + var result = await step.RunAsync( + () => backfill.RunServerSliceAsync(runtime, stoppingToken), + BackfillSliceDeadline, + onLateFault: ex => _logger.LogError(ex, + "query_store backfill slice on '{Server}' faulted AFTER being abandoned — this is the wedge's own exception (#2148)", + runtime.Config.DisplayName), + cancellationToken: stoppingToken); + + switch (result.Outcome) { - return; + case AbandonableStepOutcome.Cancelled: + return; + case AbandonableStepOutcome.Faulted when result.Exception is OperationCanceledException: + return; + case AbandonableStepOutcome.Faulted: + /* One server's slice failing (unreachable, permissions, a mid-tick disconnect) is + that server's problem for this tick; the loop and the rest of the fleet continue. */ + _logger.LogWarning("query_store backfill slice on '{Server}' failed: {Message}", + runtime.Config.DisplayName, result.Exception!.Message); + break; + case AbandonableStepOutcome.Abandoned: + _logger.LogError( + "query_store backfill slice on '{Server}' exceeded {Deadline}s and was ABANDONED — " + + "the fleet's backfill continues; this server's backfill is quarantined until the " + + "wedged task ends. Defect signal: report with this log (#2148).", + runtime.Config.DisplayName, (int)BackfillSliceDeadline.TotalSeconds); + break; + case AbandonableStepOutcome.SkippedStillRunning: + _logger.LogError( + "query_store backfill slice on '{Server}' skipped — a previously-abandoned slice is still wedged (#2148).", + runtime.Config.DisplayName); + break; } - catch (Exception ex) + } + } + } + + /// + /// #2165: per-server gates shared by the tick's Query Store pass and the backfill slice, so the two never + /// run heavy QS text extraction against one server at the same time. Keyed by ServerId and never pruned, + /// like its sibling — one small object per server ever monitored. + /// + /// Both loops must resolve the SAME gate instance for a server, which is what makes this one + /// dictionary rather than one per loop. Pinned by a test for that reason. + /// + private readonly ConcurrentDictionary _queryStoreGates = new(); + + /// + /// #2219: whether this is the PostgreSQL statement-stats collector, whose success is what triggers a text + /// refresh. Compared against the collector's OWN declared name rather than a literal, so renaming it cannot + /// silently unhook the text path — the same reasoning as . + /// + internal static bool IsPgStatementStatsCollector(string collectorName) => + string.Equals(collectorName, PgStatementStatsCollector.Instance.Name, StringComparison.OrdinalIgnoreCase); + + /// + /// #2165: whether a dispatched collector name is the Query Store collector the gate covers. Compared + /// against the collector's OWN declared name rather than a literal, so renaming the collector cannot + /// silently unhook the gate and let the two loops overlap again. + /// + internal static bool IsQueryStoreCollector(string collectorName) => + string.Equals(collectorName, QueryStoreCollector.Instance.Name, StringComparison.OrdinalIgnoreCase); + + /// + /// #2219: refreshes this PostgreSQL server's statement text if it is due, and swallows everything if not. + /// + /// Best-effort by construction. It runs after the statistics have already been collected and + /// logged, so nothing here can cost a collection: unreadable text is a degraded read, a lost collection is + /// lost data, and those are not the same severity. Every fault mode — the target refusing + /// aurora_stat_statements, a store write failing, the cadence query erroring — logs once and leaves + /// the statistics intact. + /// + /// Due-ness is asked of the STORE (), not remembered here. + /// A restart therefore cannot re-fetch the fleet, and two hosts writing one store cannot disagree about when + /// text was last written. The same now is used for the decision and the rows it stamps, so the cadence + /// cannot drift against its own timestamps. + /// + /// Only for PostgreSQL targets: aurora_stat_statements does not exist elsewhere, and the + /// statement-stats collector is already engine-gated, so this mirrors that gate rather than trusting it. + /// + private async Task TryRefreshPgStatementTextAsync(ServerRuntime runtime, CancellationToken cancellationToken) + { + if (runtime.Target.Engine != CollectorTargetEngine.PostgreSql) + { + return; + } + + try + { + var now = PgStatementText.Naive(DateTime.UtcNow); + var due = now - PgStatementText.RefreshInterval; + + await using (var isDue = _postgres!.CreateCommand(PgStatementText.IsDueSql)) + { + isDue.Parameters.AddWithValue(runtime.ServerId); + isDue.Parameters.AddWithValue(PgStatementText.Naive(due)); + if (await isDue.ExecuteScalarAsync(cancellationToken) is not true) { - /* One server's slice failing (unreachable, permissions, a mid-tick disconnect) is - that server's problem for this tick; the loop and the rest of the fleet continue. */ - _logger.LogWarning("query_store backfill slice on '{Server}' failed: {Message}", - runtime.Config.DisplayName, ex.Message); + return; } } + + var (queryIds, texts) = await ReadPgStatementTextAsync(runtime, cancellationToken); + if (queryIds.Count == 0) + { + return; + } + + var stamps = new DateTime[queryIds.Count]; + Array.Fill(stamps, now); + + await using var upsert = _postgres!.CreateCommand(PgStatementText.UpsertSql); + upsert.Parameters.AddWithValue(Enumerable.Repeat(runtime.ServerId, queryIds.Count).ToArray()); + upsert.Parameters.AddWithValue(queryIds.ToArray()); + upsert.Parameters.AddWithValue(texts.ToArray()); + upsert.Parameters.AddWithValue(stamps); + await upsert.ExecuteNonQueryAsync(cancellationToken); + + _logger.LogInformation( + " [{Server}] pg_statement_text => {Count} statement text(s) refreshed (#2219)", + runtime.Config.DisplayName, queryIds.Count); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + /* Deliberately broad — see the summary. The statistics for this cycle are already stored and logged; + losing their text is not worth failing the sweep over. */ + _logger.LogWarning( + " [{Server}] pg_statement_text refresh failed, statistics are unaffected: {Message} (#2219)", + runtime.Config.DisplayName, ex.Message); + } + } + + /// + /// Reads (queryid, query) from the monitored PostgreSQL server with showtext = true (#2219). + /// Capped, and ordered by total execution time so a catalog larger than the cap keeps the text for the + /// queries anyone would actually look at rather than an arbitrary slice. + /// + private static async Task<(List QueryIds, List Texts)> ReadPgStatementTextAsync( + ServerRuntime runtime, CancellationToken cancellationToken) + { + var queryIds = new List(); + var texts = new List(); + + await using var connection = new Npgsql.NpgsqlConnection(runtime.ConnectionString); + await connection.OpenAsync(cancellationToken); + await using var command = new Npgsql.NpgsqlCommand(PgStatementText.FetchSql, connection) { CommandTimeout = 60 }; + command.Parameters.AddWithValue(PgStatementTextRowCap); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + if (reader.IsDBNull(0) || reader.IsDBNull(1)) + { + continue; + } + + queryIds.Add(reader.GetInt64(0)); + texts.Add(reader.GetString(1)); } + + return (queryIds, texts); } + /// #2219: the row cap for one text fetch — comfortably above PostgreSQL's default + /// pg_stat_statements.max of 5,000, so a normally-configured instance is never truncated, while a + /// pathologically raised setting cannot turn one fetch into an unbounded transfer. + private const int PgStatementTextRowCap = 10_000; + + /// #2148: per-server abandonment guards for the backfill loop — keyed by ServerId so a + /// removed-and-re-added server reuses its guard (harmless), and a wedged server never blocks its + /// neighbors. Never pruned: one small object per server ever monitored, bounded by fleet size. + private readonly ConcurrentDictionary _backfillSliceSteps = new(); + + /// #2148: the hard ceiling one server's backfill slice may hold the fleet loop — a healthy + /// slice is one 60s-capped statement plus store writes. + private static readonly TimeSpan BackfillSliceDeadline = TimeSpan.FromSeconds(300); + /// /// The command plane's poll loop (Stage 2), run concurrently with the collection sweep on its own /// ~5-second tick. Each tick DRAINS every currently-pending command (claim one at a time until the @@ -1720,6 +2212,9 @@ private async Task ReloadFromStoreAsync( Settings toggles round-trip to a live start/stop/rebind with no service restart. */ _mcpState.Publish(config.Mcp.Enabled, config.Mcp.Port); _webState.Publish(config.Web.Enabled, config.Web.Port); + /* #2298: re-publish the server set on every reload, so a server added through add_servers or the + Viewer reaches the MCP host's plan-fetch resolver on its next resolution — no MCP restart. */ + _registryState.Publish(view.EnabledServers); _scheduleOverrides = view.ScheduleOverrides; /* Stage 2: honor a pause/resume issued through the store (config_service.paused) — the collection loop reads this on its next tick. Single writer (this reload), so no interlock needed. */ @@ -1763,13 +2258,13 @@ private void ReconcileServers(List servers, IReadOnlyList(); foreach (var d in desired) { - desiredById[ServerIdHelper.GetDeterministicHashCode(d.StorageName)] = d; + desiredById[d.ServerId] = d; } for (int i = servers.Count - 1; i >= 0; i--) { var state = servers[i]; - var id = ServerIdHelper.GetDeterministicHashCode(state.Config.StorageName); + var id = state.Config.ServerId; if (!desiredById.TryGetValue(id, out var desiredServer)) { _logger.LogInformation( @@ -1899,10 +2394,14 @@ internal static bool ServerDefinitionEquals(MonitoredServer a, MonitoredServer b /// /// A deterministic, restart-stable per-server phase offset within a cadence period (#1553 cadence jitter), /// used to break the fleet-wide lockstep at cadence boundaries — the field incident re-herded every server - /// at once, so at each boundary all collectors fired together. The is ALREADY an - /// FNV-1a hash (), so a plain modulo spreads it across + /// at once, so at each boundary all collectors fired together. The is + /// , which today is an FNV-1a hash + /// () — so a plain modulo spreads it across /// [0, period) without any further mixing (an extra multiply was reviewed out as unnecessary — the - /// input is already avalanched). Restart-stable because it is a pure function of the id — no . + /// input is already avalanched). This is the ONE consumer that wants the value only as a spreading + /// function rather than as an identity, so if #2218 ever makes ids sequential the extra mixing that was + /// reviewed out has to come back here: consecutive integers modulo a period do not spread, they line up. + /// Restart-stable because it is a pure function of the id — no . /// A non-positive period yields no offset (guards the callers where a period could in principle be zero, and /// keeps the result well-defined for tests). Applied ONLY at initial cadence stamps, never the steady-state /// advance: directly for the on-connect analysis stamp, and — capped at min(interval, 150s) via @@ -2145,6 +2644,16 @@ private async Task EvaluateAlertsAsync(AlertEngine engine, ServerLoopState serve Suppressed: false); await engine.EvaluateServerAsync(snapshot, cancellationToken); + + /* PostgreSQL predictors ride alongside rather than inside the shared engine — see + IPostgresAlertReadAdapter for why the read contract is separate. Gated on the probed engine, + so a SQL Server target does not pay for a read it can never satisfy, and Lite never sees any + of it. Awaited AFTER the shared sweep so an existing SQL Server alert is never delayed by a + PostgreSQL read. */ + if (runtime.Target.Engine == CollectorTargetEngine.PostgreSql) + { + await EvaluatePostgresAlertsAsync(runtime, snapshot, cancellationToken); + } } catch (OperationCanceledException) { @@ -2156,6 +2665,100 @@ private async Task EvaluateAlertsAsync(AlertEngine engine, ServerLoopState serve } } + /// + /// Evaluates the three PostgreSQL Tier 0 outage predictors and delivers whatever fired. + /// Failure-isolated from the shared sweep on purpose: these are additive signals, and a broken + /// PostgreSQL read must not cost a server its CPU or blocking alerts. Recording and mute handling stay + /// with the deliverer, exactly as for an engine-emitted alert, so a PostgreSQL alert lands in the same + /// history and obeys the same mute rules as every other one. + /// + private async Task EvaluatePostgresAlertsAsync( + ServerRuntime runtime, AlertServerSnapshot snapshot, CancellationToken cancellationToken) + { + if (_postgres is null || _alertDeliverer is null) + { + return; + } + + try + { + var adapter = new DarlingPostgresAlertReadAdapter(_postgres); + + var findings = PostgresAlertEvaluator.Evaluate( + await adapter.GetWraparoundRiskAsync(runtime.ServerId, cancellationToken), + await adapter.GetXminHorizonAsync(runtime.ServerId, cancellationToken), + await adapter.GetReplicationSlotRiskAsync(runtime.ServerId, cancellationToken)); + + var now = DateTime.UtcNow; + var cooldown = TimeSpan.FromMinutes(Math.Max(1, _alertCooldownMinutes)); + + foreach (var finding in findings) + { + /* Cooldown keyed per SUBJECT, not per metric. Two databases past the wraparound line are two + incidents; a metric-level key would have let the first one's stamp suppress the second. */ + var cooldownKey = string.Create( + CultureInfo.InvariantCulture, + $"{snapshot.ServerKey}|{finding.MetricName}|{finding.Subject}"); + + if (_lastPostgresAlert.TryGetValue(cooldownKey, out var last) && now - last < cooldown) + { + continue; + } + + /* Stamped even when muted, mirroring AlertEngine: a muted alert still consumes its cooldown, + so unmuting does not produce a backlog. */ + _lastPostgresAlert[cooldownKey] = now; + + var muted = _isAlertMuted?.Invoke(new AlertMuteContext + { + ServerName = snapshot.ServerName, + MetricName = finding.MetricName, + /* The subject is the database for wraparound and the slot/holder for the others, which is + what a DatabaseName mute rule is written against. */ + DatabaseName = finding.Subject, + }) ?? false; + + await _alertDeliverer.DeliverAsync( + new AlertOutcome( + snapshot.ServerKey, + snapshot.ServerName, + finding.MetricName, + finding.CurrentValue, + finding.ThresholdValue, + /* The subject reaches the deliverer as a #1140 incident fingerprint. It was computed + by the evaluator and then thrown away (Context: null), so the send-side + IncidentCooldown fell back to its metric-level key: two databases past the + wraparound line, or two bad slots, collapsed into one incident and the second was + silently suppressed for the whole cooldown window. The DedupKey is identity only — + no ages or byte counts — so a recurrence of the SAME subject collapses while a + different subject does not. */ + Context: new AlertContext + { + Incidents = new List + { + new(finding.Subject, new[] { finding.Subject }), + }, + }, + DetailText: null, + finding.NumericCurrentValue, + finding.NumericThresholdValue, + Muted: muted, + finding.Severity, + finding.ShortMessage), + cancellationToken); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogError("[{Server}] PostgreSQL alert evaluation failed: {Message}", + runtime.Config.DisplayName, ex.Message); + } + } + /// /// The latest collected CPU sample for the snapshot — Lite's overview read /// (LocalDataService.Overview.cs:37-51) against the raw PG table, and the @@ -2284,6 +2887,15 @@ await TimescaleSupport.ReadCompressionActivityAsync(connection, _logger, cancell stuckJobs, jobId => TimescaleSupport.TryRearmJobAsync(connection, jobId, _logger, cancellationToken), cancellationToken); + + /* #2136: the Store Job Over Cadence check rides the same connection and hourly cadence — a + background job whose last successful run reached the warning share of its own schedule + interval is the store outgrowing its job schedule, the number an onboarding wave moves + first. Same isolation posture: the evaluator wraps itself, and this whole method's catch + is the backstop. */ + var cadenceReadings = await TimescaleSupport.ReadJobCadenceReadingsAsync( + connection, _logger, cancellationToken); + await _selfAlerts!.EvaluateStoreJobCadenceAsync(cadenceReadings, cancellationToken); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -2306,10 +2918,20 @@ await TimescaleSupport.ReadCompressionActivityAsync(connection, _logger, cancell /// private async Task SweepStoreSelfMetricsAsync(CancellationToken cancellationToken) { + /* #2327 review catch: this sweep is AWAITED on the main loop, unlike the fire-and-track + per-server sweeps — so its worst case stalls per-server dispatch and the disk-pressure and + compression checks with it. The budget is therefore ONE SweepTimeoutSeconds for the WHOLE + sweep (a linked CTS), not per statement: worst-case loop block stays ~5 minutes, comparable + to the old default's 5 x 30s, instead of the 25 minutes five sequential 300s statements + could take against a genuinely wedged store. The per-statement CommandTimeout inside + StoreSelfMetrics stays as the belt for callers that pass no token. */ + using var budget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + budget.CancelAfter(TimeSpan.FromSeconds(StoreSelfMetrics.SweepTimeoutSeconds)); + try { - await using var connection = await _postgres!.OpenConnectionAsync(cancellationToken); - await StoreSelfMetrics.SweepAsync(connection, _timescaleAvailable, DateTime.UtcNow, _logger, cancellationToken); + await using var connection = await _postgres!.OpenConnectionAsync(budget.Token); + await StoreSelfMetrics.SweepAsync(connection, _timescaleAvailable, DateTime.UtcNow, _logger, budget.Token); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -2317,7 +2939,22 @@ private async Task SweepStoreSelfMetricsAsync(CancellationToken cancellationToke } catch (Exception ex) { - _logger.LogError("Store self-metrics sweep failed: {Message}", ex.Message); + /* #2317: a command TIMEOUT and a genuine fault are the same Npgsql message here ("Exception + while reading from stream" — the #2294 lesson), and on the dogfood box that costume produced + ~5 fake network-fault ERRORs a day. Name each cause; both are one-hour series gaps that + self-heal on the next tick. The budget CTS surfaces as OperationCanceledException — with + the SERVICE token untripped that can only be the sweep budget, so it takes the timeout + arm too. */ + if (PgBaselineProvider.IsCommandTimeout(ex) || (ex is OperationCanceledException && budget.IsCancellationRequested)) + { + _logger.LogError( + "Store self-metrics sweep did not finish within its {Timeout}s command timeout — this tick's metrics are skipped and the series gains a one-hour gap (the store side logs this as 'canceling statement due to user request'). If it repeats, the store's sizing queries have outgrown the timeout: {Message}", + StoreSelfMetrics.SweepTimeoutSeconds, ex.Message); + } + else + { + _logger.LogError("Store self-metrics sweep failed: {Message}", ex.Message); + } } } @@ -2388,7 +3025,7 @@ private async Task RunAnalysisPassAsync( try { var analysisService = new DarlingAnalysisService(_postgres!, planFetcher, _logger); - var analyzeTask = analysisService.AnalyzeAsync(serverId, storageName, hoursBack: 4); + var analyzeTask = analysisService.AnalyzeAsync(serverId, storageName, hoursBack: 4, stoppingToken); /* Clear the in-flight marker only when the task truly finishes — not when the timeout below moves us on — so a hung server is not relaunched. */ @@ -2400,6 +3037,29 @@ when the timeout below moves us on — so a hung server is not relaunched. */ if (stoppingToken.IsCancellationRequested) { + /* #2299: the pass observes the same token (AnalysisContext.CancellationToken), so + hold this sweep open for a bounded grace and let it unwind — the loop's data + source is disposed when the sweeps drain, and before this hold it was disposed + UNDERNEATH the still-running pass, which cost a clean stop seven ERRORs. A pass + that outlives the grace keeps running into the disposal; its residue is then + classified as shutdown (Information) by the pass itself, and the in-flight + marker keeps it from being relaunched either way. */ + try + { + /* CancellationToken.None on purpose: stoppingToken has already FIRED — passing + it would cancel this wait instantly and defeat the grace. */ + await analyzeTask.WaitAsync(s_analysisShutdownGrace, CancellationToken.None); + } + catch (OperationCanceledException) + { + /* Shutdown — quiet and expected. */ + } + catch (TimeoutException) + { + _logger.LogDebug( + "[{Server}] Analysis pass did not unwind within {Grace}s of shutdown — its residue is classified as shutdown, not fault", + displayName, (int)s_analysisShutdownGrace.TotalSeconds); + } return new AnalysisPassResult(AnalysisPassStatus.Skipped, 0, "service is stopping"); } @@ -2469,7 +3129,7 @@ private async Task RunAnalyzeNowAsync( ServerLoopState? server; lock (_serversLock) { - server = servers.Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == serverId); + server = servers.Find(s => s.Config.ServerId == serverId); } if (server is null) @@ -2477,6 +3137,34 @@ private async Task RunAnalyzeNowAsync( return new CommandOutcome(false, "server not monitored", JsonError($"no monitored server with server_id {serverId}")); } + /* The operator door the scheduled-path gate (see the PostgreSql arm in the analysis tick) did not + cover: "Generate now" against a PostgreSQL target ran the full SQL-Server-shaped pass, which + found nothing, persisted the GENERIC insufficient_data message, and thereby OVERWROTE the honest + engine tombstone the scheduled arm wrote — the Recommendations tab regressed from "does not + apply, use the PG reads" back to "still collecting" the moment an operator clicked the button. + Same decision, same honest answer, re-written here so the tombstone survives the click. */ + if (server.Runtime?.Target.Engine == CollectorTargetEngine.PostgreSql) + { + /* Mirror the scheduled arm's once-latch so the tick does not re-write what this just wrote. */ + server.PostgresAnalysisStateWritten = true; + await DarlingObservability.WriteAnalysisStateAsync( + _postgres!, + server.Runtime.ServerId, + insufficientData: true, + message: PostgresAnalysisNotApplicable, + _logger, + cancellationToken); + + return new CommandOutcome(true, "analysis not applicable", + JsonSerializer.Serialize(new + { + success = true, + server = server.Config.DisplayName, + message = "Analysis is SQL-Server-shaped and does not apply to a PostgreSQL target; " + + "use the get_pg_* MCP reads and the outage-predictor alerts instead.", + })); + } + var result = await RunAnalysisPassAsync( serverId, server.Config.StorageName, server.Config.DisplayName, planFetcher, notificationService, config.Analysis.NotificationsEnabled, cancellationToken); @@ -2507,7 +3195,7 @@ private async Task RunAnalyzeNowAsync( /// NO collection gate: unlike snapshot_now a purge writes no collector state and races no delta baseline, /// and PurgeAsync is idempotent + failure-isolated per table, so it may safely overlap the daily sweep. /// - private async Task RunPurgeNowAsync(int? customRetentionDays, CancellationToken cancellationToken) + private async Task RunPurgeNowAsync(DarlingConfig config, int? customRetentionDays, CancellationToken cancellationToken) { /* Reference read of the live overrides, matching the daily purge caller (never held under a lock — the reload swaps the whole list atomically). */ @@ -2517,7 +3205,8 @@ private async Task RunPurgeNowAsync(int? customRetentionDays, Ca : name => StoreConfigProvider.ResolveFleetRetentionDays(name, overrides); var summary = await DarlingRetention.PurgeAsync( - _postgres!, _timescaleAvailable, _logger, cancellationToken, resolver); + _postgres!, _timescaleAvailable, _logger, cancellationToken, resolver, + config.PlanContentRetentionDays); _logger.LogInformation( "purge_now purged {Tables} table(s), {Rows} row(s)/chunk(s){Custom}", @@ -2571,7 +3260,7 @@ public Task AnalyzeNowAsync(int serverId, CancellationToken canc => _worker.RunAnalyzeNowAsync(_servers, _planFetcher, _notificationService, _config, serverId, cancellationToken); public Task PurgeNowAsync(int? customRetentionDays, CancellationToken cancellationToken) - => _worker.RunPurgeNowAsync(customRetentionDays, cancellationToken); + => _worker.RunPurgeNowAsync(_config, customRetentionDays, cancellationToken); public Task FetchPlanAsync(int serverId, PlanFetchRequest request, CancellationToken cancellationToken) => _worker.RunFetchPlanAsync(_servers, _planFetcher, serverId, request, cancellationToken); @@ -2605,7 +3294,13 @@ then the connection I/O runs outside it. */ && string.Equals(r.ServerId.ToString(CultureInfo.InvariantCulture), serverKey, StringComparison.Ordinal)); } - if (runtime is null || runtime.Target.IsAzureSqlDb) + /* Engine first: msdb, SQL Agent and the whole FailedJobsQuery are SQL Server concepts, and this + opens a SqlConnection below. On a PostgreSQL target it threw "Keyword not supported: 'host'" once + per alert cycle. The IsAzureSqlDb arm stays for the same reason it always did — Azure SQL DB has + no msdb either. */ + if (runtime is null + || runtime.Target.Engine != CollectorTargetEngine.SqlServer + || runtime.Target.IsAzureSqlDb) { return new List(); } @@ -2654,6 +3349,42 @@ private async Task TryConnectAsync(ServerLoopState server, DarlingCollectorRunne { var runtime = await DarlingServerConnector.ConnectAsync(server.Config, _logger, cancellationToken); server.Runtime = runtime; + + /* #2255: cleared on success so a LATER failure prints in full even when it carries the same + message as one from before this connect. Without this, a fixed-then-broken-again cause would be + suppressed as a repeat of something the operator had already scrolled past. */ + server.LastConnectFailureLogged = null; + + /* #2228: the tripwire. The connection just told us which database it actually landed in, so this + is the one moment the registration's claim can be checked against the server's own answer. + Identity is registration-derived and never verified against the connection, so without this a + registration pointing somewhere else collects that other database's rows under its own id, + indefinitely and silently — and if a sibling registration names that database too, both collect + it and the history is duplicated under two identities (#2220's byte-identical graphs). + + ERROR, not Warning: nothing clears this on its own and every sweep in the meantime stores + mis-attributed rows. Logged on the TRANSITION so a standing misconfiguration does not bury + itself. */ + var mismatch = DarlingServerConnector.DescribeDatabaseMismatch( + server.Config.Database, runtime.ConnectedDatabase, server.Config.DisplayName); + + if (!string.Equals(server.LastDatabaseMismatchLogged, mismatch, StringComparison.Ordinal)) + { + server.LastDatabaseMismatchLogged = mismatch; + if (mismatch is not null) + { + _logger.LogError("[{Server}] {Mismatch}", server.Config.DisplayName, mismatch); + } + else + { + /* The transition BACK is worth one line too: it is the confirmation that an operator's + edit actually took, which otherwise requires trusting silence. */ + _logger.LogInformation( + "[{Server}] now connected to the database it is registered for ('{Database}') — the " + + "earlier mismatch (#2228) is resolved.", + server.Config.DisplayName, runtime.ConnectedDatabase); + } + } /* Force the long-query trace (#1496) to re-reconcile on the next sweep after every (re)connect: an Azure database-scoped session can stop on reconnect, so a still-"applied" flag would otherwise skip restarting it. Cheap — the reconcile no-ops unless the desired state differs @@ -2700,7 +3431,15 @@ unreachable. An Azure SQL DB firewall rejection or failover is reported with the await DarlingObservability.UpsertServerAsync(_postgres!, runtime, _logger, cancellationToken); - await DarlingXeSessions.EnsureAllAsync(runtime, runner, _logger, cancellationToken); + /* Extended Events are a SQL Server feature. Ungated, this ran SqlClient against a PostgreSQL + target on every connect and logged "Failed to ensure XE sessions: Keyword not supported: + 'host'. - deadlock/blocked-process collection will read zero rows until resolved" — a warning + that is both alarming and meaningless on an engine that has no XE, on a target whose deadlock + collectors are engine-gated off anyway. Confirmed on a live PostgreSQL target. */ + if (runtime.Target.Engine == CollectorTargetEngine.SqlServer) + { + await DarlingXeSessions.EnsureAllAsync(runtime, runner, _logger, cancellationToken); + } /* On-load config snapshots (effective FrequencyMinutes 0) run once per connect, then every scheduled collector becomes immediately due — mirrors Lite's server-open behavior. The @@ -2723,6 +3462,23 @@ watermark so a restart RESUMES the real cadence instead of re-phasing it up to a var watermarks = await ReadCollectorWatermarksAsync(_postgres!, serverId, _logger, cancellationToken); foreach (var name in CollectorScheduleDefaults.All.Keys) { + /* The SAME pre-dispatch engine gate the scheduled sweep applies (see RunDueCollectorsAsync), + which this loop never got. Without it, the on-load pass dispatches every foreign-engine + collector once per connect: a PostgreSQL target ran server_config, database_config, + database_scoped_config, trace_flags and server_properties as T-SQL and logged five fake + SUCCESS rows with zero rows collected — confirmed on a live PostgreSQL target. Those rows + feed the health bands and analysis, which key on status, so a fake success is worse than an + error. Re-read from server.Runtime because a preceding RunOneAsync in this loop can have + nulled it on a connection-level failure. */ + if (server.Runtime is null + || !CollectorCatalog.EngineMatches(name, server.Runtime.Target) + /* And the within-engine gate, PostgreSQL only — same reasoning as the scheduled sweep. */ + || (server.Runtime.Target.Engine == CollectorTargetEngine.PostgreSql + && !CollectorCatalog.AppliesTo(name, server.Runtime.Target))) + { + continue; + } + /* Captured serverId, not server.Runtime.ServerId: an earlier on-load RunOneAsync in this loop can null server.Runtime on a connection-level failure, which would otherwise NRE here. */ var effective = StoreConfigProvider.ResolveSchedule(name, serverId, _scheduleOverrides); @@ -2759,13 +3515,39 @@ still runs immediately (the sweep gates on config.Analysis.Enabled). */ { server.Runtime = null; server.NextConnectAttempt = DateTime.UtcNow.AddSeconds(60); - _logger.LogWarning("[{Server}] Connect failed, retrying in 60s: {Message}", server.Config.DisplayName, ex.Message); + /* #2255: full text on a NEW cause, one line while it persists. A credential that cannot be + decrypted on this host is not a transient connect failure, so its explanation is worth Error + once and worth almost nothing on the 1,440th repeat. */ + var failure = ex.Message; + if (!string.Equals(server.LastConnectFailureLogged, failure, StringComparison.Ordinal)) + { + server.LastConnectFailureLogged = failure; + if (ex is InvalidOperationException && failure.Contains("DPAPI-decrypt", StringComparison.Ordinal)) + { + /* Error, not Warning: nothing about this clears on its own, so it needs an operator. */ + _logger.LogError("[{Server}] Connect failed and will keep failing until fixed: {Message}", + server.Config.DisplayName, failure); + } + else + { + _logger.LogWarning("[{Server}] Connect failed, retrying in 60s: {Message}", + server.Config.DisplayName, failure); + } + } + else + { + _logger.LogWarning("[{Server}] Connect still failing, retrying in 60s (same cause as logged above)", + server.Config.DisplayName); + } /* Stage 4: the online->offline connection edge (Server Unreachable) — fires once when a previously-connected server can no longer be reached; a repeated failed reconnect does NOT - re-fire (the state machine dedups). server_id is derived from the config since Runtime is null. */ + re-fire (the state machine dedups). server_id comes from the CONFIG rather than the runtime, + because Runtime is null here by definition -- and post-#2218 the config carries the STORED id, + so an alert on a server that has never once connected keys on the same identity its collected + history does. */ await _selfAlerts!.ApplyConnectionOutcomeAsync( - ServerIdHelper.GetDeterministicHashCode(server.Config.StorageName), + server.Config.ServerId, server.Config.DisplayName, online: false, error: ex.Message, cancellationToken); } } @@ -2796,6 +3578,40 @@ long snapshot cannot starve collection of the OTHER servers. */ return; } + /* Wrong-engine collectors are dropped BEFORE dispatch, not gated inside the runner: a + definition whose dialect the target does not speak must leave no trace at all. The + runner's own CollectorCatalog.AppliesTo check returns 0 rows, which RunOneAsync would + record as SUCCESS — fine for the handful of Azure-gated collectors, but with a second + engine in the catalog every target would log a fake success per foreign collector per + cycle (most are 1-minute), flooding collection_log and feeding phantom successes to the + health bands and analysis, which key on status. This is Darling's equivalent of Lite's + pre-dispatch SKIPPED path: no dispatch, no log row, no NextDue churn. */ + if (!CollectorCatalog.EngineMatches(name, runtime.Target)) + { + continue; + } + + /* WITHIN-engine gates get the same treatment, on PostgreSQL targets only. + EngineMatches above drops the wrong DIALECT; it says nothing about a collector that is + right-dialect but inapplicable to this particular target — pg_wait_stats and + pg_statement_stats read Aurora-only functions, so on stock PostgreSQL they dispatched, came + back with 0 rows, and RunOneAsync recorded SUCCESS. Two collectors at a 1-minute cadence is + ~2,880 fake successes a day per server, and the PR promised "a graceful skip with an + explanation" instead. Confirmed on the review's live stock-PostgreSQL run. + + Scoped to PostgreSQL deliberately rather than applied to the composed gate for everyone: on + SQL Server the same zero-row-SUCCESS path covers a long-established handful of Azure-gated + collectors, and silencing those is a change to a shipping SKU's log semantics that deserves + its own decision rather than riding along here. + + No log row is the honest outcome, and it is not silent: --test-connection names exactly + which collectors do not apply to a target, and why, before the service ever runs. */ + if (runtime.Target.Engine == CollectorTargetEngine.PostgreSql + && !CollectorCatalog.AppliesTo(name, runtime.Target)) + { + continue; + } + /* Effective schedule = config_collector_schedules override layered on the code default. A disabled or on-load-only (freq 0) collector is skipped; the frequency the NextDue stamp advances by is the EFFECTIVE one, so an override takes effect immediately. */ @@ -2831,7 +3647,7 @@ private async Task RunSnapshotAsync( ServerLoopState? server; lock (_serversLock) { - server = servers.Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == serverId); + server = servers.Find(s => s.Config.ServerId == serverId); } if (server is null) @@ -2871,6 +3687,18 @@ enabled one NOW regardless of frequency or NextDue — that is what "snapshot" m continue; } + /* The THIRD dispatch loop, and it got neither engine gate in the first round: an operator + snapshot against a PostgreSQL target dispatched every SQL Server collector, whose + AppliesTo early-return yields zero rows and lands a burst of fake SUCCESS in + collection_log — the phantom-success class the other two loops (on-load :3157, scheduled + sweep) were gated against. Same predicate, same PostgreSQL-only scoping. */ + if (!CollectorCatalog.EngineMatches(name, runtime.Target) + || (runtime.Target.Engine == CollectorTargetEngine.PostgreSql + && !CollectorCatalog.AppliesTo(name, runtime.Target))) + { + continue; + } + totalRows += await RunOneAsync(server, runner, name, cancellationToken); collectorsRun++; } @@ -2918,7 +3746,7 @@ private async Task RunFetchPlanAsync( string displayName; lock (_serversLock) { - server = servers.Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == serverId); + server = servers.Find(s => s.Config.ServerId == serverId); connected = server?.Runtime is not null; displayName = server?.Config.DisplayName ?? serverId.ToString(CultureInfo.InvariantCulture); } @@ -2984,7 +3812,7 @@ private async Task RunFetchActiveQueriesLiveAsync( string displayName; lock (_serversLock) { - server = servers.Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == serverId); + server = servers.Find(s => s.Config.ServerId == serverId); runtime = server?.Runtime; displayName = server?.Config.DisplayName ?? serverId.ToString(CultureInfo.InvariantCulture); } @@ -3060,14 +3888,41 @@ ORDER BY collection_time DESC /// query_text is the same query either way, so this is strictly more robust — the sibling stored-plan /// readers' semantics. /// + /* #2150: the text now comes from collect.query_store_text and only FALLS BACK to the fact row's own + column, which is where it lived before the cutover — pre-cutover rows keep working unchanged, and + post-cutover rows (NULL inline) resolve from the side table. The lookup is keyed on exactly the + identifier this resolver already has, and all three keys are the statement's own parameters, so it + is an uncorrelated scalar subquery: resolved once, then named once by the derived table so the + IS NOT NULL filter can test the RESOLVED text rather than the raw column. Testing the raw column + is the trap — it would exclude every post-cutover row, the whole set this change exists to serve. */ public const string ResolveStoredQueryStoreForActualPlanSql = @" -SELECT query_text, query_plan_text, NULL::text AS transaction_isolation_level, NULL::bytea AS query_plan_gz -FROM query_store_stats -WHERE server_id = $1 -AND database_name = $2 -AND query_id = $3 -AND query_text IS NOT NULL -ORDER BY (query_plan_text IS NOT NULL) DESC, collection_time DESC +SELECT r.query_text, + r.query_plan_text, + NULL::text AS transaction_isolation_level, + NULL::bytea AS query_plan_gz +FROM +( + SELECT + COALESCE + ( + ( + SELECT x.query_sql_text + FROM query_store_text AS x + WHERE x.server_id = $1 + AND x.database_name = $2 + AND x.query_id = $3 + ), + s.query_text + ) AS query_text, + s.query_plan_text, + s.collection_time + FROM query_store_stats AS s + WHERE s.server_id = $1 + AND s.database_name = $2 + AND s.query_id = $3 +) AS r +WHERE r.query_text IS NOT NULL +ORDER BY (r.query_plan_text IS NOT NULL) DESC, r.collection_time DESC LIMIT 1"; /// The query_snapshots resolver — the Wait drill-down surface's identifier (server_id + @@ -3111,7 +3966,7 @@ immutable connection string + Azure flag as locals — never hold the runtime ac string displayName; lock (_serversLock) { - var server = servers.Find(s => ServerIdHelper.GetDeterministicHashCode(s.Config.StorageName) == serverId); + var server = servers.Find(s => s.Config.ServerId == serverId); serverExists = server is not null; connectionString = server?.Runtime?.ConnectionString; isAzureSqlDb = server?.Runtime?.Target.IsAzureSqlDb ?? false; @@ -3269,6 +4124,53 @@ private static void BindActualPlanResolveParameters(NpgsqlCommand command, int s _ => "row", }; + /// + /// Maps a PostgreSQL fault to a collection_log status plus the sentence an operator needs. + /// The store has five statuses and none of them is "this feature is not installed", so the + /// non-fatal-degradation bucket (PERMISSIONS) carries those cases and the MESSAGE distinguishes them — + /// the same division the Azure service-objective hint already uses. Returning "ERROR" means "let the + /// general handler have it", which keeps the genuinely unexpected loud. + /// + internal static (string Status, string Explanation) PostgresFaultOutcome( + PostgresException ex, string collectorName) + { + var fault = PostgresTargetProvider.Instance.Classify( + ex, CollectorCatalog.YieldsOnLockTimeout(collectorName)); + + return fault switch + { + CollectorTargetFault.Permissions => ("PERMISSIONS", + $"{ex.MessageText} (SQLSTATE {ex.SqlState}) — the monitoring login lacks a grant this " + + "source needs. pg_monitor covers every collector here; check that it is granted."), + + /* 42P01 / 42883: the relation or function is not there. Overwhelmingly an extension that was + never created in the connected database rather than anything to do with privileges. */ + CollectorTargetFault.ObjectMissing => ("PERMISSIONS", + $"{ex.MessageText} (SQLSTATE {ex.SqlState}) — the source object does not exist on this " + + "target. This is NOT a missing grant: it is normally an extension that was never " + + "created in the connected database (CREATE EXTENSION pg_stat_statements), so the " + + "collector will keep degrading until it is. Recorded as a non-fatal skip rather than an " + + "error so it does not fill the log every cycle."), + + /* 0A000 / 55000 / 55006: the server will not do this, permanently or by configuration — + pg_stat_wal on Aurora, or an optimized-reads cache that is switched off. */ + CollectorTargetFault.FeatureDisabled => ("PERMISSIONS", + $"{ex.MessageText} (SQLSTATE {ex.SqlState}) — this source is unsupported or disabled on " + + "this server. NOT a missing grant. Aurora does not implement some community sources at " + + "all, and others are gated by a parameter group. Recorded as a non-fatal skip because it " + + "will not change until the platform or the parameter group does."), + + CollectorTargetFault.LockTimeoutYield => ("YIELDED", + $"Lock-timeout yield (SQLSTATE {ex.SqlState}): the collector's lock-timeout guard fired " + + "rather than waiting in a blocking chain. One sweep skipped; evidence of lock contention " + + "on the monitored server, not a monitoring failure."), + + /* Everything else — including a command timeout and a fatal connection error — belongs to the + general handler, which logs ERROR and (for ConnectionFatal) forces the reprobe. */ + _ => ("ERROR", ex.Message), + }; + } + /// /// True when a SqlException is a permission denial — the expected failure when the least-privilege monitoring /// login (VIEW SERVER STATE only) re-executes a query that reads/writes user objects. Detected by the known @@ -3307,6 +4209,26 @@ private async Task RunOneAsync(ServerLoopState server, DarlingCollectorRunn return 0; } + /* #2165: the tick's Query Store pass and the backfill slice both do heavy QS text extraction, and + they used to be free to run against the SAME server at once — measured as ~128 MB in flight on a + 4-core box, because a big catalog arriving triggers BOTH loops. Gated HERE because this is the one + funnel that has the runtime and the collector name together. Never waits: see QueryStoreServerGate + for why blocking a shared fleet loop would recreate the #2148 wedge through a lock. Skipping is safe + for this collector because its window is watermark-driven (#1960) — the next pass resumes from the + same boundary, so a skipped pass defers rows rather than dropping them. */ + using var queryStoreGate = IsQueryStoreCollector(collectorName) + ? _queryStoreGates.GetOrAdd(runtime.ServerId, static _ => new QueryStoreServerGate()).TryAcquire() + : QueryStoreServerGate.NotGated; + + if (queryStoreGate is null) + { + _logger.LogInformation( + " [{Server}] query_store skipped this tick — its Query Store backfill slice is mid-flight (#2165). " + + "Resumes next tick from the same watermark; no rows are lost.", + server.Config.DisplayName); + return 0; + } + try { var result = await run(runner, runtime, cancellationToken); @@ -3319,6 +4241,16 @@ private async Task RunOneAsync(ServerLoopState server, DarlingCollectorRunn than on error_message, so the note is inert outside the Collection Log detail grid. */ await DarlingObservability.LogCollectionAsync( _postgres!, runtime, collectorName, "SUCCESS", result.Rows, result.SqlMs, result.StorageMs, result.Note, _logger, cancellationToken); + + /* #2219: statement TEXT rides alongside the statement stats, on its own hourly cadence. Hung off the + stats collector's success rather than given its own loop because it is meaningless without those + rows and must never run against a server whose stats collection is failing — one less loop that + can be independently wrong. Best-effort: a text fetch that fails leaves the statistics collected + and logs, because unreadable text is a degraded read while a failed collection is lost data. */ + if (IsPgStatementStatsCollector(collectorName)) + { + await TryRefreshPgStatementTextAsync(runtime, cancellationToken); + } return result.Rows; } catch (OperationCanceledException) @@ -3381,13 +4313,61 @@ await DarlingObservability.LogCollectionAsync( _postgres!, runtime, collectorName, "PERMISSIONS", 0, 0, 0, message, _logger, cancellationToken); return 0; } + catch (PostgresException ex) when ( + PostgresFaultOutcome(ex, collectorName) is { Status: not "ERROR" } outcome) + { + /* PostgreSQL faults classified by SQLSTATE through the same ITargetProvider.Classify the + engine seam already exposes, so the runner and the provider cannot disagree about what an + error means. + + Without this the general catch below claimed every one of them, and a PERSISTENT condition + would log ERROR every single cycle forever: pg_statement_stats against a database where the + extension was never created (42P01), a source Aurora does not implement at all (0A000), a + feature switched off in the parameter group (55006). Those are the exact PostgreSQL analogue + of the 8189 sys.traces denial above, which degrades to PERMISSIONS for the same reason — + it is a real, operator-actionable state, not a monitoring fault, and burying it in a + once-a-minute error is how it gets ignored. + + The message says WHICH kind it is rather than leaving "PERMISSIONS" to imply a missing + GRANT, following the AzureDmvPermissionHint precedent: the status is the store's + non-fatal-degradation bucket, the text is where the truth goes. */ + var (status, explanation) = (outcome.Status, outcome.Explanation); + + if (status == "YIELDED") + { + _logger.LogInformation(" [{Server}] {Collector} => YIELDED - {Explanation}", + server.Config.DisplayName, collectorName, explanation); + } + else + { + _logger.LogWarning(" [{Server}] {Collector} => {Status} ({SqlState}): {Message}", + server.Config.DisplayName, collectorName, status, ex.SqlState, explanation); + } + + await DarlingObservability.LogCollectionAsync( + _postgres!, runtime, collectorName, status, 0, 0, 0, explanation, _logger, cancellationToken); + return 0; + } catch (Exception ex) { _logger.LogError(" [{Server}] {Collector} => ERROR: {Message}", server.Config.DisplayName, collectorName, ex.Message); - /* A dead connection poisons every collector — force a reconnect + reprobe. */ - if (ex is SqlException sqlEx && (sqlEx.Class >= 20 || sqlEx.Number == -2)) + /* A dead connection poisons every collector — force a reconnect + reprobe. The Postgres arm + matters as much as the SQL Server one and is deliberately NARROWER than "any + PostgresException": a statement_timeout (57014) is a slow query, not a dead socket, and + dropping the connection over one would turn a tuning problem into a reconnect storm. Only + the 08 class and the shutdown/unavailability codes qualify, which is exactly what the + provider's ConnectionFatal means. */ + if ((ex is SqlException sqlEx && (sqlEx.Class >= 20 || sqlEx.Number == -2)) + /* ANY exception on a PostgreSQL target, not just a PostgresException. The pre-filter was the + bug: a dead socket surfaces as a plain NpgsqlException with no SQLSTATE — the provider + already classifies that as ConnectionFatal, and the call site could not reach it. So the + runtime stayed "connected", Server Unreachable never fired, and every collector errored + forever. Asymmetric with the SqlClient arm, which does reach its own classifier. */ + || (server.Runtime?.Target.Engine == CollectorTargetEngine.PostgreSql + && PostgresTargetProvider.Instance.Classify(ex, yieldsOnLockTimeout: false) + == CollectorTargetFault.ConnectionFatal)) { server.Runtime = null; server.NextConnectAttempt = DateTime.UtcNow.AddSeconds(60); @@ -3445,6 +4425,7 @@ await DarlingObservability.LogCollectionAsync( ["database_states"] = (r, s, ct) => r.RunAsync(DatabaseStateCollector.Instance, s, ct), ["trace_flags"] = RunTraceFlagsTolerantAsync, ["database_scoped_config"] = (r, s, ct) => r.RunAsync(DatabaseScopedConfigCollector.Instance, s, ct), + ["query_store_health"] = (r, s, ct) => r.RunAsync(QueryStoreHealthCollector.Instance, s, ct), ["session_stats"] = (r, s, ct) => r.RunAsync(SessionStatsCollector.Instance, s, ct), ["session_summary_stats"] = (r, s, ct) => r.RunAsync(SessionSummaryStatsCollector.Instance, s, ct), ["waiting_tasks"] = (r, s, ct) => r.RunAsync(WaitingTasksCollector.Instance, s, ct), @@ -3468,8 +4449,33 @@ await DarlingObservability.LogCollectionAsync( ["ag_database_replica_states"] = (r, s, ct) => r.RunAsync(AgDatabaseReplicaStatesCollector.Instance, s, ct), ["plan_correction"] = (r, s, ct) => r.RunAsync(PlanCorrectionCollector.Instance, s, ct), ["pvs_stats"] = (r, s, ct) => r.RunAsync(PvsStatsCollector.Instance, s, ct), + /* PostgreSQL. Dispatch is by name and engine-agnostic; the engine gate upstream in + RunDueCollectorsAsync means this lambda is only ever reached for a Postgres target. */ + ["pg_wait_stats"] = (r, s, ct) => r.RunAsync(PgWaitStatsCollector.Instance, s, ct), + ["pg_statement_stats"] = (r, s, ct) => r.RunAsync(PgStatementStatsCollector.Instance, s, ct), + ["pg_wraparound_stats"] = (r, s, ct) => r.RunAsync(PgWraparoundStatsCollector.Instance, s, ct), + ["pg_xmin_horizon"] = (r, s, ct) => r.RunAsync(PgXminHorizonCollector.Instance, s, ct), + ["pg_replication_slots"] = (r, s, ct) => r.RunAsync(PgReplicationSlotsCollector.Instance, s, ct), + ["pg_autovacuum_stats"] = (r, s, ct) => r.RunAsync(PgAutovacuumStatsCollector.Instance, s, ct), + ["pg_io_stats"] = (r, s, ct) => r.RunAsync(PgIoStatsCollector.Instance, s, ct), + ["pg_blocking"] = (r, s, ct) => r.RunAsync(PgBlockingCollector.Instance, s, ct), }; + /// + /// The analysis_state message for a PostgreSQL target, shared by the SCHEDULED pass and the manual + /// "Generate now" path. + /// One constant because there were two hand-maintained copies and they had already drifted: adding + /// get_pg_blocking to the scheduled one left the manual one listing seven tools, so an operator + /// clicking Generate now got different guidance from the same product depending on which door they came + /// through. The list grows with every PostgreSQL read, which guarantees the drift recurs. + /// + internal const string PostgresAnalysisNotApplicable = + "Scheduled analysis does not apply to a PostgreSQL target: its findings are derived from SQL Server " + + "collectors (waits, query stats, CPU) that this engine does not populate. This is not " + + "\"still collecting\" — use the PostgreSQL MCP reads (get_pg_wait_stats, get_pg_top_queries, " + + "get_pg_autovacuum_health, get_pg_wraparound_risk, get_pg_xmin_horizon, get_pg_replication_slots, " + + "get_pg_io_stats, get_pg_blocking) and the three outage-predictor alerts instead."; + /// /// Signals that a blocking/deadlock XE session is missing or inaccessible so the reader returned no /// events. throws it, catches it and logs a diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingXeSessions.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingXeSessions.cs index df748caec..2c8f859c9 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingXeSessions.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingXeSessions.cs @@ -35,6 +35,15 @@ public static class DarlingXeSessions { public static async Task EnsureAllAsync(ServerRuntime server, DarlingCollectorRunner runner, ILogger? logger, CancellationToken cancellationToken) { + /* Same self-gate as ReconcileLongQueryCompletionsAsync, same reason: this method constructs a + SqlConnection from the engine-ambiguous connection string, so it enforces its own precondition + rather than trusting the caller's gate (DarlingWorker's connect path) to be the only entry + forever. XE does not exist on PostgreSQL. */ + if (server.Target.Engine != PerformanceMonitor.Collectors.CollectorTargetEngine.SqlServer) + { + return; + } + if (server.Target.IsAzureSqlDb) { await EnsureDatabaseScopedAsync(server, runner, logger, cancellationToken); @@ -533,6 +542,15 @@ ADD TARGET package0.ring_buffer /// public static async Task ReconcileLongQueryCompletionsAsync(ServerRuntime server, DarlingCollectorRunner runner, bool enabled, ILogger? logger, CancellationToken cancellationToken) { + /* Belt to the worker's braces: the caller gates on engine (a PostgreSQL target has no XE to + reconcile), but this method constructs a SqlConnection from the engine-ambiguous connection + string below, so it enforces its own precondition rather than trusting every present and future + caller — the exact trust that put "Keyword not supported: 'host'" in the sweep log once a minute. */ + if (server.Target.Engine != PerformanceMonitor.Collectors.CollectorTargetEngine.SqlServer) + { + return; + } + if (server.Target.IsAzureSqlDb) { await ReconcileLongQueryCompletionsAzureAsync(server, runner, enabled, logger, cancellationToken); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs index e86b4080b..acf784f47 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs @@ -137,7 +137,14 @@ public sealed record AlertSettingsReadRow( bool PvsEnabled, int PvsThresholdPercent, int PvsFloorGb, - bool DatabaseStateEnabled); + bool DatabaseStateEnabled, + int SelfDiskFreeWarnPercent, + int CollectionStaleMinutes, + int CollectionFailureThreshold, + int DiskCriticalFreePercent, + int DiskCriticalFreeGb, + int AnalysisNotifyCooldownMinutes, + int StoreJobCadenceWarnPercent); /// The single global alert-settings row (id=1) — the viewer's AlertSettingsSelectSql. The /// 47 columns are read in the SAME order the service reads them (StoreConfigProvider). This had @@ -157,7 +164,10 @@ public sealed record AlertSettingsReadRow( notify_connection_down_at_startup, connection_refire_minutes, notify_ag_health, ag_lag_alert_seconds, ag_redo_queue_alert_kb, ag_disconnect_refire_minutes, blocking_wait_seconds_threshold, pvs_enabled, pvs_threshold_percent, - pvs_floor_gb, database_state_enabled + pvs_floor_gb, database_state_enabled, + self_disk_free_warn_percent, collection_stale_minutes, collection_failure_threshold, + disk_critical_free_percent, disk_critical_free_gb, analysis_notify_cooldown_minutes, + store_job_cadence_warn_percent FROM config_alert_settings WHERE id = 1"; @@ -191,6 +201,10 @@ FROM config_alert_settings reader.GetInt32(41), reader.GetInt32(42), reader.GetBoolean(43), reader.GetInt32(44), reader.GetInt32(45), - reader.GetBoolean(46)); + reader.GetBoolean(46), + /* #2107 threshold knobs (V55) at 47–52; #2136 cadence-warn knob (V57) at 53. */ + reader.GetInt32(47), reader.GetInt32(48), reader.GetInt32(49), + reader.GetInt32(50), reader.GetInt32(51), reader.GetInt32(52), + reader.GetInt32(53)); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs index fd3e08a90..055280369 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs @@ -43,6 +43,15 @@ internal static class DarlingConfigHistoryReader public sealed record DatabaseScopedConfigReadRow( string DatabaseName, string ConfigurationName, string? Value, string? ValueForSecondary); + /* ─────────────────────────── query store health snapshot row (not part of the change diff) ─────────────────────────── */ + + /* String fields coalesce DBNull to "" — the same defaults as both viewers' QueryStoreHealthRow — + so the two SKUs' MCP tools serialize identical JSON even in the never-observed null case. */ + public sealed record QueryStoreHealthReadRow( + string DatabaseName, string ActualState, string DesiredState, int ReadonlyReason, + long CurrentStorageMb, long MaxStorageMb, string SizeBasedCleanupMode, + long StaleQueryThresholdDays, long MaxPlansPerQuery, long IntervalLengthMinutes); + /* ─────────────────────────── server config snapshots ─────────────────────────── */ /// Every sys.configurations snapshot for a server, oldest-first — the change tool diffs @@ -215,4 +224,43 @@ public static async Task> GetLatestDatabaseSco return rows; } + + /* ─────────────────────────── query store health (latest snapshot) ─────────────────────────── */ + + /// The latest sys.database_query_store_options snapshot per database — the viewer's + /// QueryStoreHealthSql minus its grid database filter (the tool filters in memory, like the + /// scoped-config read above). Unlike the config-family reads this table is HOURLY, not on-connect, + /// so "latest" here is at most an hour old on a healthy schedule. $1 server_id. + public const string QueryStoreHealthSql = """ + SELECT database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes + FROM v_query_store_health + WHERE server_id = $1 + AND capture_time = (SELECT MAX(capture_time) FROM v_query_store_health WHERE server_id = $1) + ORDER BY database_name + """; + + public static async Task> GetLatestQueryStoreHealthAsync( + NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(QueryStoreHealthSql); + DarlingMcpReadParameters.AddInt(command, serverId); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new QueryStoreHealthReadRow( + reader.IsDBNull(0) ? "" : reader.GetString(0), + reader.IsDBNull(1) ? "" : reader.GetString(1), + reader.IsDBNull(2) ? "" : reader.GetString(2), + reader.IsDBNull(3) ? 0 : reader.GetInt32(3), + reader.IsDBNull(4) ? 0L : reader.GetInt64(4), + reader.IsDBNull(5) ? 0L : reader.GetInt64(5), + reader.IsDBNull(6) ? "" : reader.GetString(6), + reader.IsDBNull(7) ? 0L : reader.GetInt64(7), + reader.IsDBNull(8) ? 0L : reader.GetInt64(8), + reader.IsDBNull(9) ? 0L : reader.GetInt64(9))); + } + + return rows; + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs index 718e52978..fd265bee7 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs @@ -94,7 +94,12 @@ public sealed record TopQueryRow( long MinCpuUs, long MaxCpuUs, long MinElapsedUs, long MaxElapsedUs, string QueryText, /* #2012: distinct statement texts merged into this group; with stage 2's host-object split this flags the remaining ad-hoc literal blends (proc-hosted groups converge to 1). */ - long DistinctTexts); + long DistinctTexts, + /* #2235: how many DISTINCT query_hash values this row rolled up. Always 1 in the default + per-hash grouping — it is only interesting under host-object rollup, where it IS the finding: + a proc whose dynamic SQL fragments across 21 hashes reports 21 here, which is the number that + explains why top-N-by-hash could never surface it. */ + long DistinctQueryHashes = 1); /// One (database, schema, object) group's summed procedure-stats deltas over the window. public sealed record TopProcedureRow( @@ -169,6 +174,46 @@ public static async Task> GetCpuUtilizationAsync( return samples; } + public sealed record CpuWindowAggregate(int SampleCount, DateTime? FirstSample, DateTime? LastSample, double? AvgSqlCpuPercent); + + /// + /// The attributed-CPU denominator's pieces (#2320): sample count, coverage bounds, and average SQL + /// CPU% over the window. Windowed on collection_time — the SAME bounds the top-queries/procedures + /// rankings use — so numerator and denominator share collection gaps; sample_time skew is irrelevant + /// to an average. $1 server_id, $2/$3 window (naive UTC). + /// + public const string CpuWindowAggregateSql = """ + SELECT + COUNT(*), + MIN(collection_time), + MAX(collection_time), + AVG(sqlserver_cpu_utilization)::double precision + FROM cpu_utilization_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + """; + + public static async Task GetCpuWindowAggregateAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + { + await using var command = postgres.CreateCommand(CpuWindowAggregateSql); + AddInt(command, serverId); + AddTimestamp(command, startUtc); + AddTimestamp(command, endUtc); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return new CpuWindowAggregate(0, null, null, null); + } + + return new CpuWindowAggregate( + reader.IsDBNull(0) ? 0 : Convert.ToInt32(reader.GetValue(0), System.Globalization.CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? null : reader.GetDateTime(1), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? null : reader.GetDouble(3)); + } + /* ─────────────────────────── wait stats ─────────────────────────── */ /// @@ -605,11 +650,125 @@ ORDER BY r.total_elapsed_us DESC LIMIT $4 """; + /// + /// The same top-queries read, rolled up so proc-hosted dynamic SQL ranks as its PARENT (#2235). + /// + /// The defect this answers. query_hash is a shape hash, so dynamic SQL built with + /// per-value literals fragments one logical statement across as many hashes as there are literal sets — + /// measured at 21 for one API.GetInventoryWithLabsV5 statement. Ranking by hash therefore + /// STRUCTURALLY cannot surface it: two of its fragments together were 58-65% of the instance's + /// worker_time in every window sampled, while the hash itself never entered the 168-hour top 20. The + /// ranking looked healthy and explained roughly a tenth of the box. + /// + /// Why this is a sibling const rather than a parameter. Postgres cannot parameterize + /// GROUP BY, and every read here is a public const precisely so the suite can pin its dialect and + /// columns without a live store. Building the clause by string concatenation would trade both of those + /// for one saved copy. + /// + /// Ad-hoc rows keep their per-hash grouping, and that is load-bearing. A bare + /// GROUP BY host_object_name would pool EVERY unrelated ad-hoc statement in a database into one + /// meaningless row, because ad-hoc rows carry host_object_name = NULL — turning the fix into a + /// worse attribution bug than the one it fixes. The CASE in the grouping key keys ad-hoc rows on + /// their own query_hash (identical to the default read) and collapses only rows that actually name + /// a host object. + /// + /// The per-hash grouping (#2012 stage 2) stays the DEFAULT. Two procedures sharing a hash genuinely + /// are different work, which is why that split exists; this is an additional lens, not a replacement. + /// query_hash in a rolled-up row is one member of the group, exactly as query_text already + /// is when distinct_texts > 1distinct_query_hashes is what says so. + /// + public const string TopQueriesByHostObjectSql = """ + WITH ranked AS ( + SELECT + database_name, + MAX(query_hash) AS query_hash, + host_object_name, + CAST(SUM(delta_execution_count) AS bigint) AS total_executions, + CAST(SUM(delta_worker_time) AS bigint) AS total_cpu_us, + CAST(SUM(delta_elapsed_time) AS bigint) AS total_elapsed_us, + CAST(SUM(delta_logical_reads) AS bigint) AS total_reads, + CAST(SUM(delta_logical_writes) AS bigint) AS total_writes, + CAST(SUM(delta_physical_reads) AS bigint) AS total_physical_reads, + CAST(SUM(delta_rows) AS bigint) AS total_rows, + CAST(SUM(delta_spills) AS bigint) AS total_spills, + MIN(min_dop) AS min_dop, + MAX(max_dop) AS max_dop, + MIN(min_worker_time) AS min_worker_time, + MAX(max_worker_time) AS max_worker_time, + MIN(min_elapsed_time) AS min_elapsed_time, + MAX(max_elapsed_time) AS max_elapsed_time, + MAX(query_plan_hash) AS query_plan_hash, + MAX(sql_handle) AS sql_handle, + MAX(plan_handle) AS plan_handle, + COUNT(DISTINCT query_text_digest) AS distinct_texts, + /* #2235: the fragment count IS the finding — 21 here is why a per-hash ranking missed it. */ + COUNT(DISTINCT query_hash) AS distinct_query_hashes + FROM query_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + AND ($5::text IS NULL OR database_name = $5) + /* #2235: proc-hosted rows collapse to one row per (database, host object) — every literal + fragment of one statement lands together. Ad-hoc rows (host_object_name NULL) fall to the + CASE and stay keyed on their OWN query_hash, so they group exactly as the default read does; + without that arm every unrelated ad-hoc statement in a database would pool into one row. */ + GROUP BY database_name, host_object_name, + CASE WHEN host_object_name IS NULL THEN query_hash END + HAVING SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0 + ORDER BY SUM(delta_elapsed_time) DESC + LIMIT $4 + 5 + ) + SELECT + r.database_name, + r.query_hash, + r.host_object_name, + r.query_plan_hash, + r.sql_handle, + r.plan_handle, + r.total_executions, + r.total_cpu_us, + r.total_elapsed_us, + r.total_reads, + r.total_writes, + r.total_physical_reads, + r.total_rows, + r.total_spills, + r.min_dop, + r.max_dop, + r.min_worker_time, + r.max_worker_time, + r.min_elapsed_time, + r.max_elapsed_time, + t.query_text, + r.distinct_texts, + r.distinct_query_hashes + FROM ranked AS r + LEFT JOIN LATERAL ( + SELECT query_text + FROM v_query_stats + WHERE server_id = $1 + AND database_name = r.database_name + /* Mirrors the grouping: for a rolled-up proc any of its fragments' texts is a valid + representative, but an ad-hoc row must still match its own hash or the text could come from + an unrelated statement. */ + AND host_object_name IS NOT DISTINCT FROM r.host_object_name + AND (r.host_object_name IS NOT NULL OR query_hash = r.query_hash) + AND query_text IS NOT NULL + ORDER BY collection_time DESC + LIMIT 1 + ) AS t ON TRUE + WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' + ORDER BY r.total_elapsed_us DESC + LIMIT $4 + """; + public static async Task> GetTopQueriesByCpuAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int top, string? databaseName, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int top, string? databaseName, + bool rollUpByHostObject = false, CancellationToken cancellationToken = default) { var rows = new List(); - await using var command = postgres.CreateCommand(TopQueriesSql); + /* #2235: same parameters, same columns, different GROUP BY — see TopQueriesByHostObjectSql. */ + await using var command = postgres.CreateCommand(rollUpByHostObject ? TopQueriesByHostObjectSql : TopQueriesSql); AddWindow(command, serverId, startUtc, endUtc); AddInt(command, top); AddNullableText(command, databaseName); @@ -638,7 +797,8 @@ public static async Task> GetTopQueriesByCpuAsync( reader.IsDBNull(18) ? 0 : reader.GetInt64(18), reader.IsDBNull(19) ? 0 : reader.GetInt64(19), reader.IsDBNull(20) ? "" : reader.GetString(20), - reader.IsDBNull(21) ? 0 : reader.GetInt64(21))); + reader.IsDBNull(21) ? 0 : reader.GetInt64(21), + reader.FieldCount > 22 && !reader.IsDBNull(22) ? reader.GetInt64(22) : 1)); } return rows; @@ -798,15 +958,31 @@ ORDER BY SUM(execution_count) * AVG(CAST(avg_duration_us AS double precision)) D t.query_text, r.replica_role FROM ranked AS r + /* #2150: resolve the text inside the lateral so the projection and the WAITFOR self-exclusion below + both keep reading one t.query_text. First arm is collect.query_store_text (one row per + query_id, where the collector lands text once the separate fetch is on); second arm is the + newest inline query_text, which is where text lived before the cutover and is what keeps + existing history readable. */ LEFT JOIN LATERAL ( - SELECT query_text - FROM query_store_stats - WHERE server_id = $1 - AND query_id = r.query_id - AND database_name = r.database_name - AND query_text IS NOT NULL - ORDER BY collection_time DESC - LIMIT 1 + SELECT COALESCE( + ( + SELECT x.query_sql_text + FROM query_store_text AS x + WHERE x.server_id = $1 + AND x.database_name = r.database_name + AND x.query_id = r.query_id + ), + ( + SELECT s.query_text + FROM query_store_stats AS s + WHERE s.server_id = $1 + AND s.query_id = r.query_id + AND s.database_name = r.database_name + AND s.query_text IS NOT NULL + ORDER BY s.collection_time DESC + LIMIT 1 + ) + ) AS query_text ) AS t ON TRUE WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' ORDER BY r.total_executions * r.avg_duration_ms DESC @@ -1149,8 +1325,10 @@ internal sealed class CollectorHealth /// The collector's default cadence from the shared /// (0 for an on-load or unknown collector — both fall to the floor thresholds). The banding uses the - /// shipped default, not the resolved per-server override, so all three surfaces stay in parity. - private int FrequencyMinutes => + /// shipped default, not the resolved per-server override, so all three surfaces stay in parity. + /// Internal since #2296: the tool's sweep-pressure roll-up amortizes each collector's average + /// duration by this same cadence, so both readers of it share one resolution. + internal int FrequencyMinutes => CollectorScheduleDefaults.All.TryGetValue(CollectorName, out var schedule) ? schedule.FrequencyMinutes : 0; public string HealthStatus => CollectorHealthClassifier.Classify( diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingIncidentFingerprint.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingIncidentFingerprint.cs new file mode 100644 index 000000000..32c412991 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingIncidentFingerprint.cs @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Recomputes the #1140 alert fingerprint (the operator-visible Dedup Key) for STORED incident rows, so +/// the incident readers can jump straight from "this alert" to "this incident" (#2159). +/// +/// What this is for. The fingerprint already travels end to end — it is a fact on the alert and the +/// reporter sets it on their Azure DevOps tickets — but no reader accepted it, so triaging from an alert meant +/// pulling a server+time window and eyeballing rows for the one whose objects matched. That is slow and it is +/// easy to analyze the WRONG deadlock. +/// +/// Why it recomputes instead of reading a column. Nothing persists the key on the incident row, and +/// adding it would be a migration that back-fills nothing — historical rows would stay unsearchable, which is +/// exactly the history an operator triages. The key is a pure function of stored data, so deriving it on read +/// covers all retained history the moment this ships. +/// +/// It calls the SAME groupers the alert path calls, and that is the whole design. The fingerprint is +/// a SHA-256 over normalized identity members; any divergence in how those members are derived produces a +/// different hash and the filter silently matches NOTHING — the worst failure mode available here, because an +/// empty result is indistinguishable from "that incident is outside the window". So this does not reimplement +/// the derivation: it builds the same / +/// the alert builders build and hands them to the same +/// grouper. Re-deriving keys from stored rows is not novel either — the analysis drill-down +/// (AnalysisNotificationService.BuildIncidents) already does it from its own stored rows. +/// +/// THE SCOPE TRAP, and the reason this class takes a "fingerprint name" rather than a server name. +/// hashes the server name INTO the key, and the alert path passes +/// runtime.Config.DisplayName — that is Name if set, else Host. The MCP resolver returns the +/// STORAGE name (servers.server_name, i.e. host[:database][:RO]), which is a different string +/// whenever a server carries a custom display name, or whenever the registration names a database or read-only +/// intent. Fingerprinting with the storage name would therefore work on plain hosts and silently return nothing +/// on exactly the servers most likely to be carefully named. Callers must pass +/// 's value, which reproduces the alert path's choice. +/// +internal static class DarlingIncidentFingerprint +{ + /// + /// Normalizes a user-pasted key for comparison. The key is lowercase SHA-256 hex, but it arrives by copy + /// and paste out of a ticket or an alert card, so it can carry surrounding whitespace or have been + /// upper-cased by whatever rendered it. + /// + public static string NormalizeKey(string? dedupKey) => + (dedupKey ?? string.Empty).Trim().ToLowerInvariant(); + + /// + /// True when is absent, meaning "no fingerprint filter requested". Keeps the + /// filter's absence a single decision rather than a repeated null-or-empty test per tool. + /// + public static bool NoFilter(string? dedupKey) => NormalizeKey(dedupKey).Length == 0; + + /// + /// The dedup key for every deadlock row, in input order, using the same extraction and grouper the alert + /// path uses. A row whose graph yields no usable object has no incident and so gets null — it can + /// never match a filter, which is correct rather than a gap: an incident with no identity members is one + /// the alerting layer never emitted a key for either. + /// + public static List DeadlockKeys(string fingerprintName, IEnumerable graphXmls) + { + var keys = new List(); + foreach (var xml in graphXmls ?? Enumerable.Empty()) + { + var objects = string.IsNullOrEmpty(xml) + ? Array.Empty() + : (IReadOnlyList)DeadlockObjectExtractor.FromGraphXml(xml); + + /* Grouped ONE ROW AT A TIME on purpose. A deadlock's key hashes only its own object set + (AlertFingerprint.ForObjects over e.Objects), so per-row grouping yields the identical key while + keeping this a straight positional map from row to key — which is what the callers need in order + to filter their own row list without re-deriving the association. */ + var group = DeadlockIncidentGrouper.Group( + fingerprintName, + new[] { new DeadlockIncidentGrouper.DeadlockEvent(objects) }); + + keys.Add(group.Count > 0 ? group[0].Incident.DedupKey : null); + } + + return keys; + } + + /// + /// The dedup key for every blocked-process row, in input order. + /// + /// Unlike the deadlock case this CANNOT be done row at a time, and the difference is load-bearing. A + /// blocking key comes from the identity bucket's REPRESENTATIVE — the first event with that identity — and + /// falls back from the contentious object to a normalized query-pair key when no object resolved. So the + /// whole set is grouped once, exactly as the alert path groups it, and each row is then matched back to its + /// group by that same identity. Grouping per row would silently make every row its own representative. + /// + /// The grouper also normalizes the contentious-object label internally (#1876), which is another + /// reason to route through it rather than hash the stored column: the stored value can be a raw lock + /// resource that the label normalizer resolves, and the alert's key is over the NORMALIZED form. + /// + public static List BlockingKeys( + string fingerprintName, + IReadOnlyList events) + { + if (events is not { Count: > 0 }) + { + return new List(); + } + + var groups = BlockingIncidentGrouper.Group(fingerprintName, events); + + /* Match rows back to groups on the group's own representative fields. The grouper does not return the + per-event association, and reproducing IdentityKey here would be exactly the duplicated-derivation + this class exists to avoid — so rows are matched on the tuple the group exposes, which is what the + identity is built from. */ + var keys = new List(events.Count); + foreach (var e in events) + { + var normalizedObject = ContentiousObjectLabel.Normalize(e.ContentiousObject, e.Database); + var match = groups.FirstOrDefault(g => + string.Equals(g.ContentiousObject ?? string.Empty, normalizedObject ?? string.Empty, StringComparison.Ordinal) + && string.Equals(g.Database ?? string.Empty, e.Database ?? string.Empty, StringComparison.Ordinal) + && string.Equals(g.BlockedQuery ?? string.Empty, e.BlockedQuery ?? string.Empty, StringComparison.Ordinal) + && string.Equals(g.BlockingQuery ?? string.Empty, e.BlockingQuery ?? string.Empty, StringComparison.Ordinal)); + + keys.Add(match?.Incident.DedupKey); + } + + return keys; + } + + /// + /// The message a fingerprint filter returns when it matched nothing. Deliberately NOT the bare "empty" + /// status the unfiltered readers use. + /// + /// An empty result here has three quite different causes and the operator cannot tell them apart from + /// silence: the incident is outside the window (the common one, and the fixable one), the key belongs to a + /// DIFFERENT server than the one resolved, or the server has been renamed since the alert fired — which + /// re-keys every fingerprint it has ever produced, because the display name is hashed into the key. Saying + /// how many rows were examined separates "nothing to match against" from "matched against plenty and found + /// none", which is the first thing worth knowing. + /// + public static string NoMatchMessage(string kind, string dedupKey, string fingerprintName, int examined) => + $"No {kind} in the specified time range matches dedup_key {NormalizeKey(dedupKey)}. " + + $"Examined {examined} {kind} for server '{fingerprintName}'. " + + "The fingerprint is scoped to the server's DISPLAY name and to the incident's involved objects, so check " + + "that hours_back reaches back to when the alert fired, that the key came from this server, and that the " + + "server has not been renamed since — a rename changes the key for every incident on it. Re-run without " + + "dedup_key to see what the window does contain."; +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index 4d4ec2a14..e5f5406d8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -171,7 +171,25 @@ public static async Task GetAlertSettings( exclude_cdc = s.LongRunningQueryExcludeCdc }, tempdb_space = new { enabled = s.TempDbSpaceEnabled, threshold_percent = s.TempDbSpaceThresholdPercent }, - low_disk = new { enabled = s.LowDiskEnabled, threshold_percent = s.LowDiskThresholdPercent, threshold_gb = s.LowDiskThresholdGb }, + low_disk = new + { + enabled = s.LowDiskEnabled, + threshold_percent = s.LowDiskThresholdPercent, + threshold_gb = s.LowDiskThresholdGb, + /* #2107: the CRITICAL severity tier's floors (#1136) — previously compile-time. */ + critical_free_percent = s.DiskCriticalFreePercent, + critical_free_gb = s.DiskCriticalFreeGb + }, + /* #2107: the monitor's own self-alerts (store volume, collection health) — previously + compile-time constants. */ + self_alerts = new + { + disk_free_warn_percent = s.SelfDiskFreeWarnPercent, + collection_stale_minutes = s.CollectionStaleMinutes, + collection_failure_threshold = s.CollectionFailureThreshold, + /* #2136: the Store Job Over Cadence warning percent (Critical is fixed at 100). */ + store_job_cadence_warn_percent = s.StoreJobCadenceWarnPercent + }, pvs = new { enabled = s.PvsEnabled, threshold_percent = s.PvsThresholdPercent, floor_gb = s.PvsFloorGb }, long_running_job = new { enabled = s.LongRunningJobEnabled, multiplier = s.LongRunningJobMultiplier }, failed_job = new { enabled = s.FailedJobEnabled, lookback_minutes = s.FailedJobLookbackMinutes }, @@ -184,7 +202,9 @@ public static async Task GetAlertSettings( enabled = s.AnalysisEnabled, interval_minutes = s.AnalysisIntervalMinutes, notifications_enabled = s.AnalysisNotificationsEnabled, - notify_severity = s.AnalysisNotifySeverity + notify_severity = s.AnalysisNotifySeverity, + /* #2107: was a hardcoded 360 in Darling while Lite passed a configured value through. */ + notify_cooldown_minutes = s.AnalysisNotifyCooldownMinutes } }; @@ -617,11 +637,31 @@ void Group(JsonNode? node, string group, Action handleKey) case "enabled": AddBool("low_disk_enabled", n, "low_disk.enabled"); break; case "threshold_percent": AddInt("low_disk_threshold_percent", n, "low_disk.threshold_percent", 0, 100); break; case "threshold_gb": AddInt("low_disk_threshold_gb", n, "low_disk.threshold_gb", 0, int.MaxValue); break; + /* #2107: the CRITICAL tier floors, clamped like the warning thresholds. */ + case "critical_free_percent": AddInt("disk_critical_free_percent", n, "low_disk.critical_free_percent", 0, 100); break; + case "critical_free_gb": AddInt("disk_critical_free_gb", n, "low_disk.critical_free_gb", 0, int.MaxValue); break; default: error = $"Unknown field 'low_disk.{k}'."; break; } }); break; + case "self_alerts": + /* #2107: the monitor's own store-volume and collection-health thresholds. The + clamps match DarlingAlertSettings' read-side clamps, so a value stored here is + the value the sweep uses. */ + Group(prop.Value, "self_alerts", (k, n) => + { + switch (k) + { + case "disk_free_warn_percent": AddInt("self_disk_free_warn_percent", n, "self_alerts.disk_free_warn_percent", 0, 100); break; + case "collection_stale_minutes": AddInt("collection_stale_minutes", n, "self_alerts.collection_stale_minutes", 5, 1440); break; + case "collection_failure_threshold": AddInt("collection_failure_threshold", n, "self_alerts.collection_failure_threshold", 1, 1000); break; + case "store_job_cadence_warn_percent": AddInt("store_job_cadence_warn_percent", n, "self_alerts.store_job_cadence_warn_percent", 5, 100); break; + default: error = $"Unknown field 'self_alerts.{k}'."; break; + } + }); + break; + case "pvs": Group(prop.Value, "pvs", (k, n) => { @@ -691,6 +731,8 @@ void Group(JsonNode? node, string group, Action handleKey) case "interval_minutes": AddInt("analysis_interval_minutes", n, "analysis.interval_minutes", 5, 360); break; case "notifications_enabled": AddBool("analysis_notifications_enabled", n, "analysis.notifications_enabled"); break; case "notify_severity": AddDouble("analysis_notify_severity", n, "analysis.notify_severity", 0.0, 2.0); break; + /* #2107: the clamp matches the shared engine's documented [30, 10080]. */ + case "notify_cooldown_minutes": AddInt("analysis_notify_cooldown_minutes", n, "analysis.notify_cooldown_minutes", 30, 10080); break; default: error = $"Unknown field 'analysis.{k}'."; break; } }); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs index d11bb3da2..360a33d51 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs @@ -14,6 +14,7 @@ using ModelContextProtocol.Server; using Npgsql; using PerformanceMonitor.Common; +using PerformanceMonitor.Notifications; #pragma warning disable CA1707 // MCP tools use snake_case naming convention @@ -47,9 +48,10 @@ public static async Task GetBlocking( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 30.")] int limit = 30) + [Description("Maximum rows. Default 30.")] int limit = 30, + [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null) { - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + var (resolved, error) = await DarlingServerResolver.ResolveWithFingerprintNameAsync(postgres, server_name); if (error != null) return error; var validation = McpHelpers.ValidateHoursBack(hours_back); @@ -65,7 +67,29 @@ public static async Task GetBlocking( if (rows.Count == 0) return McpHelpers.Status("empty", "No blocking events found in the specified time range."); - var result = rows.Take(limit).Select(r => new + /* #2159: fingerprint the WHOLE window, then filter, then cap. Capping first would let `limit` + discard the very incident the key names — the caller asked for one specific incident, not for + the newest `limit` rows that happen to include it. */ + var examined = rows.Count; + var keys = DarlingIncidentFingerprint.BlockingKeys( + resolved.FingerprintName, + rows.Select(r => new BlockingIncidentGrouper.BlockedEvent( + r.DatabaseName, r.ContentiousObject, r.BlockedSqlText, r.BlockingSqlText, + r.WaitTimeMs, r.LockMode)).ToList()); + + if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + { + var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); + var kept = rows.Where((_, i) => keys[i] == wanted).ToList(); + if (kept.Count == 0) + return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( + "blocking events", dedup_key!, resolved.FingerprintName, examined)); + + keys = kept.Select(r => wanted).Cast().ToList(); + rows = kept; + } + + var result = rows.Take(limit).Select((r, i) => new { event_time = r.EventTime?.ToString("o"), source = r.Source, @@ -102,13 +126,15 @@ public static async Task GetBlocking( blocking_last_batch_completed = r.BlockingLastBatchCompleted?.ToString("o"), blocked_priority = r.BlockedPriority, blocking_priority = r.BlockingPriority, - has_report_xml = r.HasReportXml + has_report_xml = r.HasReportXml, + dedup_key = keys[i] }); return JsonSerializer.Serialize(new { server = resolved.ServerName, hours_back, + dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), total_events = rows.Count, events = result }, McpHelpers.JsonOptions); @@ -124,9 +150,10 @@ public static async Task GetDeadlocks( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 20.")] int limit = 20) + [Description("Maximum rows. Default 20.")] int limit = 20, + [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null) { - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + var (resolved, error) = await DarlingServerResolver.ResolveWithFingerprintNameAsync(postgres, server_name); if (error != null) return error; var validation = McpHelpers.ValidateHoursBack(hours_back); @@ -142,20 +169,39 @@ public static async Task GetDeadlocks( if (rows.Count == 0) return McpHelpers.Status("empty", "No deadlocks found in the specified time range."); - var result = rows.Take(limit).Select(r => new + /* #2159: see get_blocking — fingerprint the window, filter, then cap. */ + var examined = rows.Count; + var keys = DarlingIncidentFingerprint.DeadlockKeys( + resolved.FingerprintName, rows.Select(r => r.DeadlockGraphXml)); + + if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + { + var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); + var kept = rows.Where((_, i) => keys[i] == wanted).ToList(); + if (kept.Count == 0) + return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( + "deadlocks", dedup_key!, resolved.FingerprintName, examined)); + + keys = kept.Select(r => wanted).Cast().ToList(); + rows = kept; + } + + var result = rows.Take(limit).Select((r, i) => new { collection_time = r.CollectionTime.ToString("o"), deadlock_time = r.DeadlockTime?.ToString("o"), victim_process_id = r.VictimProcessId, victim_sql_text = McpHelpers.Truncate(r.VictimSqlText, 2000), process_summary = r.ProcessSummary, - has_deadlock_xml = r.HasDeadlockXml + has_deadlock_xml = r.HasDeadlockXml, + dedup_key = keys[i] }); return JsonSerializer.Serialize(new { server = resolved.ServerName, hours_back, + dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), total_deadlocks = rows.Count, deadlocks = result }, McpHelpers.JsonOptions); @@ -171,9 +217,10 @@ public static async Task GetDeadlockDetail( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum deadlocks to return. Default 5.")] int limit = 5) + [Description("Maximum deadlocks to return. Default 5.")] int limit = 5, + [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null) { - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + var (resolved, error) = await DarlingServerResolver.ResolveWithFingerprintNameAsync(postgres, server_name); if (error != null) return error; var validation = McpHelpers.ValidateHoursBack(hours_back); @@ -186,15 +233,38 @@ public static async Task GetDeadlockDetail( var now = DateTime.UtcNow; var rows = await DarlingBlockingReader.GetRecentDeadlocksAsync( postgres, resolved.ServerId, now.AddHours(-hours_back), now); - var withXml = rows.Where(r => r.HasDeadlockXml).Take(limit).ToList(); - if (withXml.Count == 0) + + /* #2159: the XML filter runs BEFORE the cap and before the fingerprint, because a row without a + graph has no objects to fingerprint — it could never match a key, and including it would only + consume one of the `limit` slots the caller wanted spent on real graphs. */ + var candidates = rows.Where(r => r.HasDeadlockXml).ToList(); + if (candidates.Count == 0) return McpHelpers.Status("empty", "No deadlock XML available in the specified time range."); - var result = withXml.Select(r => new + var examined = candidates.Count; + var keys = DarlingIncidentFingerprint.DeadlockKeys( + resolved.FingerprintName, candidates.Select(r => r.DeadlockGraphXml)); + + if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + { + var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); + var kept = candidates.Where((_, i) => keys[i] == wanted).ToList(); + if (kept.Count == 0) + return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( + "deadlocks with a graph", dedup_key!, resolved.FingerprintName, examined)); + + keys = kept.Select(r => wanted).Cast().ToList(); + candidates = kept; + } + + var withXml = candidates.Take(limit).ToList(); + + var result = withXml.Select((r, i) => new { collection_time = r.CollectionTime.ToString("o"), deadlock_time = r.DeadlockTime?.ToString("o"), victim_process_id = r.VictimProcessId, + dedup_key = keys[i], deadlock_graph_xml = r.DeadlockGraphXml }); @@ -202,6 +272,7 @@ public static async Task GetDeadlockDetail( { server = resolved.ServerName, hours_back, + dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), deadlocks = result }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs index 8b94a8838..f9a620cdb 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs @@ -22,12 +22,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// The config / trace-flag diagnostic-depth MCP tools — get_server_config_changes, -/// get_database_config_changes, get_trace_flag_changes (the Dashboard's change-history names) and -/// get_database_scoped_config (the Lite latest-snapshot name) — served over Darling's Postgres store. The -/// three change tools diff the store's append-only config snapshots (the Dashboard reads pre-materialized -/// report.*_changes tables that Darling does not have, so Darling computes the diff from the raw -/// snapshot history via ); get_database_scoped_config ports Lite's -/// tool over the viewer's latest-snapshot read. All are STORED reads (no live monitored-server hit). +/// get_database_config_changes, get_trace_flag_changes (the Dashboard's change-history names) and the two +/// Lite latest-snapshot names, get_database_scoped_config and get_query_store_health — served over Darling's +/// Postgres store. The three change tools diff the store's append-only config snapshots (the Dashboard reads +/// pre-materialized report.*_changes tables that Darling does not have, so Darling computes the diff +/// from the raw snapshot history via ); the latest-snapshot tools +/// port Lite's over the viewer's reads. All are STORED reads (no live monitored-server hit). /// /// /// Each change tool's Description states the two honest caveats plainly (they are NOT silently dropped): @@ -224,6 +224,58 @@ public static async Task GetDatabaseScopedConfig( } } + [McpServerTool(Name = "get_query_store_health"), Description("Gets per-database Query Store health (sys.database_query_store_options): actual vs desired state, readonly_reason (decoded), storage used vs cap, cleanup mode and thresholds, and the runtime-stats interval length. The classic silent failure is desired READ_WRITE with actual READ_ONLY after the storage cap hit — check this when Query Store data looks stale or missing. Collected hourly; OFF is recorded as OFF (an absent database means not collected, never off).")] + public static async Task GetQueryStoreHealth( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Filter to a specific database. Omit for all databases.")] string? database_name = null) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + try + { + var rows = await DarlingConfigHistoryReader.GetLatestQueryStoreHealthAsync(postgres, resolved.ServerId); + if (rows.Count == 0) + return McpHelpers.Status( + "unavailable", + "No Query Store health data available. The query_store_health collector runs hourly (SQL Server 2016+); a server with no rows either predates Query Store or has not completed a cycle yet."); + + IEnumerable filtered = rows; + if (!string.IsNullOrEmpty(database_name)) + filtered = filtered.Where(r => r.DatabaseName.Equals(database_name, StringComparison.OrdinalIgnoreCase)); + + var result = filtered.Select(r => new + { + database_name = r.DatabaseName, + actual_state = r.ActualState, + desired_state = r.DesiredState, + /* The condition this collector exists to surface, pre-folded so a client cannot miss it. */ + state_matches_desired = string.Equals(r.ActualState, r.DesiredState, StringComparison.OrdinalIgnoreCase), + readonly_reason = r.ReadonlyReason, + readonly_reason_decoded = r.ReadonlyReason == 0 ? null : QueryStoreReadonlyReason.Decode(r.ReadonlyReason), + current_storage_size_mb = r.CurrentStorageMb, + max_storage_size_mb = r.MaxStorageMb, + pct_of_cap = r.MaxStorageMb > 0 ? Math.Round(100.0 * r.CurrentStorageMb / r.MaxStorageMb, 1) : (double?)null, + size_based_cleanup_mode = string.IsNullOrEmpty(r.SizeBasedCleanupMode) ? null : r.SizeBasedCleanupMode, + stale_query_threshold_days = r.StaleQueryThresholdDays, + max_plans_per_query = r.MaxPlansPerQuery, + interval_length_minutes = r.IntervalLengthMinutes, + }).ToList(); + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + database_count = result.Count, + databases = result + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.FormatError("get_query_store_health", ex); + } + } + /// The number of distinct config CAPTURES (capture_time values) among the snapshot rows — the /// reads return one row per setting/database/flag per capture, so a raw row count would never hit the /// "fewer than two snapshots" branch on a freshly-connected server. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs index 68b4fa02a..8cb1fdcff 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs @@ -430,7 +430,7 @@ public static async Task GetPerfmonStats( /* ═══════════════════════════ query performance ═══════════════════════════ */ - [McpServerTool(Name = "get_top_queries_by_cpu"), Description("Gets expensive queries from sys.dm_exec_query_stats (plan cache). Best for: currently cached queries with detailed per-execution stats, DOP, spills, and query_hash for trending. Returns query_hash, query_plan_hash, sql_handle, plan_handle, and host_object (the hosting procedure/function for proc-hosted statements, null for ad-hoc) — groups key on (database, query_hash, host_object), so INSERT...EXEC callers in different procedures report separately with their own text. distinct_texts counts statement texts merged into a group (>1 = ad-hoc literal variants or pre-upgrade history; query_text is one representative, 0 means only rows predating the text dimension). Supports database and parallelism filtering.")] + [McpServerTool(Name = "get_top_queries_by_cpu"), Description("Gets expensive queries from sys.dm_exec_query_stats (plan cache). Best for: currently cached queries with detailed per-execution stats, DOP, spills, and query_hash for trending. Returns query_hash, query_plan_hash, sql_handle, plan_handle, and host_object (the hosting procedure/function for proc-hosted statements, null for ad-hoc) — groups key on (database, query_hash, host_object), so INSERT...EXEC callers in different procedures report separately with their own text. distinct_texts counts statement texts merged into a group (>1 = ad-hoc literal variants or pre-upgrade history; query_text is one representative, 0 means only rows predating the text dimension). Set group_by='host_object' to roll all of a procedure's statements into one row — necessary when dynamic SQL with per-value literals fragments one statement across many hashes, which no top-N-by-hash ranking can surface. Supports database and parallelism filtering. min/max_cpu_ms and min/max_elapsed_ms are LIFETIME extremes for the plan's time in cache (same semantics as max_dop), not windowed — totals and avgs are windowed deltas; rows where an extreme provably predates the window carry extremes_note. Also returns cpu_attribution: the returned rows' summed CPU-seconds against the SQL process's measured CPU-seconds for the window (avg cpu_utilization % x core count x window) - attributed_cpu_ratio says how much of the box the ranking explains; when the CPU series or core count is missing, or covers too little of the window, the ratio is omitted rather than invented.")] public static async Task GetTopQueriesByCpu( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -438,11 +438,22 @@ public static async Task GetTopQueriesByCpu( [Description("Number of top queries. Default 20.")] int top = 20, [Description("Filter to a specific database.")] string? database_name = null, [Description("If true, only return queries whose cached plan has EVER run at DOP > 1. Note: max_dop comes from sys.dm_exec_query_stats and is a lifetime-max for the plan's time in cache, so a plan compiled before MAXDOP was lowered keeps reporting the old higher value until it is evicted or recompiled. Confirm current parallelism with analyze_query_plan, which reads the actual plan.")] bool parallel_only = false, - [Description("Minimum DOP to filter on. Implies parallel filtering. Filters the same lifetime-max value as parallel_only, not current parallelism.")] int min_dop = 0) + [Description("Minimum DOP to filter on. Implies parallel filtering. Filters the same lifetime-max value as parallel_only, not current parallelism.")] int min_dop = 0, + [Description("Grouping. 'query_hash' (default) is one row per (database, query_hash, host_object). 'host_object' rolls every statement of a hosting procedure/function into ONE row — use it when dynamic SQL built with per-value literals fragments one logical statement across many query_hash values, which makes top-N-by-hash structurally unable to surface it (measured at 21 fragments for one statement, whose combined CPU was the largest on the instance while no single fragment ranked). Ad-hoc statements have no host object and stay grouped per hash in both modes. distinct_query_hashes reports how many hashes a row rolled up.")] string group_by = "query_hash") { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); if (error != null) return error; + /* #2235: an unrecognised value must not silently fall back to the default grouping — a caller who + asked for a rollup and got a per-hash ranking would read it as "this proc is not hot", which is + the exact wrong conclusion this option exists to prevent. */ + var rollUp = string.Equals(group_by, "host_object", StringComparison.OrdinalIgnoreCase); + if (!rollUp && !string.Equals(group_by, "query_hash", StringComparison.OrdinalIgnoreCase)) + { + return McpHelpers.Status("invalid", + $"group_by must be 'query_hash' or 'host_object' (got '{group_by}')."); + } + var validation = McpHelpers.ValidateHoursBack(hours_back); if (validation != null) return validation; validation = McpHelpers.ValidateTop(top, "top"); @@ -451,13 +462,29 @@ public static async Task GetTopQueriesByCpu( try { var now = DateTime.UtcNow; - var rows = await DarlingDataReader.GetTopQueriesByCpuAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name); + var rows = await DarlingDataReader.GetTopQueriesByCpuAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name, rollUpByHostObject: rollUp); if (rows.Count == 0) return McpHelpers.Status("unavailable", "No query stats available for the specified time range."); - IEnumerable filtered = rows; - if (parallel_only || min_dop > 1) - filtered = filtered.Where(r => r.MaxDop > 1 && r.MaxDop >= (min_dop > 1 ? min_dop : 2)); + var filtered = rows + .Where(r => !(parallel_only || min_dop > 1) || (r.MaxDop > 1 && r.MaxDop >= (min_dop > 1 ? min_dop : 2))) + .ToList(); + + /* #2320: what fraction of the box's measured CPU the RETURNED rows explain — numerator is + the caller-visible ranking (post top-N, post filters), denominator is measured, and the + ratio is omitted rather than invented when a denominator piece is missing. The two reads + are independent, so they run concurrently (review catch). */ + var cpuAggregateTask = DarlingDataReader.GetCpuWindowAggregateAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var propertiesTask = DarlingDataReader.GetLatestServerPropertiesAsync(postgres, resolved.ServerId); + await Task.WhenAll(cpuAggregateTask, propertiesTask); + var cpuAggregate = await cpuAggregateTask; + var properties = await propertiesTask; + var attribution = CpuAttribution.Compute( + filtered.Sum(r => r.TotalCpuUs) / 1_000_000.0, + now.AddHours(-hours_back), now, + cpuAggregate.SampleCount, cpuAggregate.FirstSample, cpuAggregate.LastSample, cpuAggregate.AvgSqlCpuPercent, + properties?.CpuCount ?? 0); var result = filtered.Select(r => new { @@ -475,6 +502,10 @@ public static async Task GetTopQueriesByCpu( max_cpu_ms = r.MaxCpuUs / 1000.0, min_elapsed_ms = r.MinElapsedUs / 1000.0, max_elapsed_ms = r.MaxElapsedUs / 1000.0, + /* #2235: min/max are lifetime extremes (see QueryStatExtremes) — flagged only on + the provable case, an extreme exceeding the whole window's total. */ + extremes_note = QueryStatExtremes.LifetimeExtremeNote( + r.TotalCpuUs, r.MaxCpuUs, r.TotalElapsedUs, r.MaxElapsedUs), min_dop = r.MinDop, max_dop = r.MaxDop, is_parallel = r.MaxDop > 1, @@ -496,6 +527,13 @@ public static async Task GetTopQueriesByCpu( distinct_texts = r.DistinctTexts, text_note = r.DistinctTexts > 1 ? $"this group blends {r.DistinctTexts} distinct statement texts (ad-hoc literal variants; or history predating the host-object split for INSERT...EXEC callers); query_text is one representative" + : null, + // #2235: under host_object rollup this is the finding, not a decoration — it is the number + // that explains why a per-hash ranking could not surface this statement. query_hash is one + // member of the group when it is > 1, exactly as query_text already is for distinct_texts. + distinct_query_hashes = r.DistinctQueryHashes, + rollup_note = r.DistinctQueryHashes > 1 + ? $"rolled up {r.DistinctQueryHashes} query_hash values belonging to {r.HostObjectName} — dynamic SQL with per-value literals fragments one statement across hashes, so none of these would rank individually; query_hash and query_text are one representative fragment" : null }); @@ -503,6 +541,16 @@ public static async Task GetTopQueriesByCpu( { server = resolved.ServerName, hours_back, + /* #2235: echoed so a stored or pasted payload cannot be misread as the other grouping — + the two answer different questions and the rows look alike. */ + group_by = rollUp ? "host_object" : "query_hash", + cpu_attribution = new + { + ranked_cpu_seconds = attribution.RankedCpuSeconds, + sql_cpu_seconds_in_window = attribution.SqlCpuSecondsInWindow, + attributed_cpu_ratio = attribution.AttributedCpuRatio, + note = attribution.Note + }, queries = result }, McpHelpers.JsonOptions); } @@ -512,7 +560,7 @@ public static async Task GetTopQueriesByCpu( } } - [McpServerTool(Name = "get_top_procedures_by_cpu"), Description("Gets the most expensive stored procedures ranked by total CPU time. Shows execution counts, CPU/elapsed times, and I/O metrics. Delta-based: requires ~30 minutes after adding a new server before data appears.")] + [McpServerTool(Name = "get_top_procedures_by_cpu"), Description("Gets the most expensive stored procedures ranked by total CPU time. Shows execution counts, CPU/elapsed times, and I/O metrics. Delta-based: requires ~30 minutes after adding a new server before data appears. min/max_cpu_ms and min/max_elapsed_ms are LIFETIME extremes for the plan's time in cache (same semantics as max_dop), not windowed — totals and avgs are windowed deltas; rows where an extreme provably predates the window carry extremes_note. Also returns cpu_attribution: the returned rows' summed CPU-seconds against the SQL process's measured CPU-seconds for the window (avg cpu_utilization % x core count x window) - attributed_cpu_ratio says how much of the box the ranking explains; when the CPU series or core count is missing, or covers too little of the window, the ratio is omitted rather than invented.")] public static async Task GetTopProceduresByCpu( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -537,6 +585,19 @@ public static async Task GetTopProceduresByCpu( "unavailable", "No procedure stats available. Delta-based collection requires at least two collection cycles (~30 minutes) to produce non-zero values."); + /* #2320: same attributed-CPU disclosure as the queries tool — one shared computation, + same concurrent independent reads. */ + var cpuAggregateTask = DarlingDataReader.GetCpuWindowAggregateAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var propertiesTask = DarlingDataReader.GetLatestServerPropertiesAsync(postgres, resolved.ServerId); + await Task.WhenAll(cpuAggregateTask, propertiesTask); + var cpuAggregate = await cpuAggregateTask; + var properties = await propertiesTask; + var attribution = CpuAttribution.Compute( + rows.Sum(r => r.TotalCpuUs) / 1_000_000.0, + now.AddHours(-hours_back), now, + cpuAggregate.SampleCount, cpuAggregate.FirstSample, cpuAggregate.LastSample, cpuAggregate.AvgSqlCpuPercent, + properties?.CpuCount ?? 0); + var result = rows.Select(r => new { database_name = r.DatabaseName, @@ -553,6 +614,9 @@ public static async Task GetTopProceduresByCpu( max_cpu_ms = r.MaxCpuUs / 1000.0, min_elapsed_ms = r.MinElapsedUs / 1000.0, max_elapsed_ms = r.MaxElapsedUs / 1000.0, + /* #2235: same lifetime-extremes flag as the queries tool. */ + extremes_note = QueryStatExtremes.LifetimeExtremeNote( + r.TotalCpuUs, r.MaxCpuUs, r.TotalElapsedUs, r.MaxElapsedUs), avg_reads = r.TotalExecutions > 0 ? (double)r.TotalLogicalReads / r.TotalExecutions : 0, total_logical_reads = r.TotalLogicalReads, total_logical_writes = r.TotalLogicalWrites, @@ -564,6 +628,13 @@ public static async Task GetTopProceduresByCpu( { server = resolved.ServerName, hours_back, + cpu_attribution = new + { + ranked_cpu_seconds = attribution.RankedCpuSeconds, + sql_cpu_seconds_in_window = attribution.SqlCpuSecondsInWindow, + attributed_cpu_ratio = attribution.AttributedCpuRatio, + note = attribution.Note + }, procedures = result }, McpHelpers.JsonOptions); } @@ -675,7 +746,7 @@ public static async Task ListServers( } } - [McpServerTool(Name = "get_collection_health"), Description("Shows the health status of all data collectors for a server — whether they're running successfully, failing, or stale. Check this before investigating data to ensure collectors are working properly. Each row also carries last_note/note_count: what a NON-failing run reported, e.g. an enumeration that came back with 0 items. note_count equal to total_runs means the collector has been collecting nothing all window — not a fault (the target may be legitimately empty), but the reason a HEALTHY collector can still have no data. target_has_user_databases tells those two apart: true means the target DID have user databases in the same window, so an all-window empty enumeration is worth investigating (a login that cannot enter them, an exclusion filter that matched everything); false means either no user databases or no inventory to go on.")] + [McpServerTool(Name = "get_collection_health"), Description("Shows the health status of all data collectors for a server — whether they're running successfully, failing, or stale. Check this before investigating data to ensure collectors are working properly. Each row also carries last_note/note_count: what a NON-failing run reported, e.g. an enumeration that came back with 0 items. note_count equal to total_runs means the collector has been collecting nothing all window — not a fault (the target may be legitimately empty), but the reason a HEALTHY collector can still have no data. target_has_user_databases tells those two apart: true means the target DID have user databases in the same window, so an all-window empty enumeration is worth investigating (a login that cannot enter them, an exclusion filter that matched everything); false means either no user databases or no inventory to go on. The sweep_pressure block is the server-level roll-up: it compares the collectors' combined execution demand (average duration amortized by cadence) against the minute the fastest cadence holds. SATURATED means the collection body cannot fit inside its cadence, so relaunches are skipped and the server collects at a multiple of its configured interval while every collector still reads healthy — heaviest_collectors names where that budget goes.")] public static async Task GetCollectionHealth( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -719,9 +790,43 @@ any other consumer cannot re-derive it differently. */ r.LastNote, r.NoteCount, r.TotalRuns, r.CollectorName, r.TargetHasUserDatabases) }); + /* #2296: the roll-up that makes half-rate collection visible. Every collector on a saturated + server reads HEALTHY — from each one's own seat nothing is wrong — so the condition only + existed as a service-log warning ("collection body has not completed … skipping relaunch"). + The verdict compares the collectors' combined execution demand (average duration amortized + by cadence) against the minute the fastest cadence holds; heaviest_collectors names where + the budget goes, which is the actionable half of the answer. */ + var pressure = SweepPressureClassifier.Compute( + rows.Select(r => (r.CollectorName, r.AvgDurationMs, r.FrequencyMinutes))); + var heaviest = rows + .Where(r => r.FrequencyMinutes > 0 && r.AvgDurationMs > 0) + .OrderByDescending(r => r.AvgDurationMs / r.FrequencyMinutes) + .Take(3) + .Select(r => new + { + collector = r.CollectorName, + avg_duration_ms = Math.Round(r.AvgDurationMs, 0), + frequency_minutes = r.FrequencyMinutes + }); + return JsonSerializer.Serialize(new { server = resolved.ServerName, + sweep_pressure = new + { + busy_ms_per_minute = Math.Round(pressure.BusyMsPerMinute, 0), + busy_percent = Math.Round(pressure.BusyPercent, 1), + verdict = pressure.Verdict, + heaviest_collectors = heaviest, + note = pressure.Verdict switch + { + SweepPressureClassifier.Saturated => + "The collection body cannot finish inside its cadence: relaunches are skipped every cycle and this server collects at a multiple of its configured interval, while each collector above correctly reads healthy from its own seat. The lever is capacity or placement (lighter or fewer scheduled collectors, a longer cadence, or a collector closer to the target), not collector repair.", + SweepPressureClassifier.AtRisk => + "The collection body's average demand is close to its cadence; variance will intermittently push it over, skipping relaunches and stretching the delivered interval.", + _ => null + } + }, collectors = result }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs index 1f92ddc7c..5a49a5e9e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs @@ -67,15 +67,22 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// pivot or the carved secret columns. It also gets its own . Store /// migration + role provisioning are the WORKER's job; the mcp-role credential is written AFTER /// migration (later than the owner's), so the first-boot poll budget tolerates the delay. The plan fetcher -/// resolves a finding's serverId to a live connection string built from darling.json (DPAPI resolution -/// lazy per fetch; any resolution/connection failure degrades the fetch to null inside -/// ). On a brand-new store, tool calls before the first migration/connect +/// resolves a finding's serverId to a live connection string from the worker-published registry +/// (, #2298 — darling.json only before the worker's first +/// publish; DPAPI resolution lazy per fetch; any resolution/connection failure degrades the fetch to null +/// inside ). On a brand-new store, tool calls before the first migration/connect /// simply return their error/miss envelopes. /// public sealed class DarlingMcpHostService : BackgroundService { private readonly ILogger _logger; private readonly McpRuntimeState _state; + + /* #2298: the worker-published monitored-server registry the plan-fetch resolver reads per fetch — + this host never re-reads config_monitored_servers itself (the mcp role's encrypted_password + SELECT-carve fails that whole read by design). */ + private readonly MonitoredServerRegistryState _registryState; + private WebApplication? _app; private NpgsqlDataSource? _appDataSource; private int _runningPort; @@ -87,10 +94,11 @@ public sealed class DarlingMcpHostService : BackgroundService /// failure logs on a calm cadence instead of every poll tick. internal static readonly TimeSpan FailedStartBackoff = TimeSpan.FromSeconds(30); - public DarlingMcpHostService(ILogger logger, McpRuntimeState state) + public DarlingMcpHostService(ILogger logger, McpRuntimeState state, MonitoredServerRegistryState registryState) { _logger = logger; _state = state; + _registryState = registryState; } /// The supervisor's per-tick verdict — pure over (running, runningPort, enabled, desiredPort) @@ -358,19 +366,52 @@ await CheckMcpFirewallAsync( var postgres = NpgsqlDataSource.Create(storeConnectionString); _appDataSource = postgres; - /* serverId → connection string from config (first entry wins on a duplicate storage - name, mirroring the worker's FirstOrDefault over runtimes). Resolution is lazy so - DPAPI decrypt runs only when a plan fetch actually needs the connection. */ - var serversById = new Dictionary(); + /* serverId → connection string, keyed by the STORE's identity (review catch on #2218). + Resolution is lazy so DPAPI decrypt runs only when a plan fetch actually needs the + connection; first entry wins on a duplicate storage name, mirroring the worker's + FirstOrDefault over runtimes. + + The server set comes from the WORKER's published registry (#2298), not a read of our own. + This host used to re-read config_monitored_servers over its mcp-role connection, and that + read selects encrypted_password — a column the section-6 secret ACL deliberately + SELECT-carves from mcp (DarlingManagedRoles: mcp can WRITE a credential blob but never READ + one back). The 42501 failed the whole config view read, so live plan fetch silently fell + back to darling.json — on a seeded box, exactly the set of servers the file does not know + about (#2254/#2256). The worker already loads the same rows over its privileged connection + (it must, or it could not collect), so the process already holds everything this host was + failing to re-read; a second, deliberately-restricted read of it was the defect. The mcp + DATABASE role keeps its carve untouched — this state feeds only the in-process resolver, + and no MCP tool exposes it, so a token-holder still cannot obtain a stored credential. + + Resolution reads the live snapshot PER FETCH rather than copying it once at host start: + before the worker's first publish it falls back to darling.json (this host's documented + store-down posture), and it heals on the next resolve after the publish — which also means + a server added later through add_servers or the Viewer reaches this resolver on the + worker's next reload, with no MCP restart. */ + var fileFallbackById = new Dictionary(); foreach (var server in config.Servers) { - serversById.TryAdd(ServerIdHelper.GetDeterministicHashCode(server.StorageName), server); + fileFallbackById.TryAdd(server.ServerId, server); + } + + /* Review note on #2298: the old permanent-failure WARN is gone with the failing read, but the + transient pre-publish window deserves a breadcrumb — once per inner-server (re)start (the + supervisor's Start and port-rebind Restart both come through here), at Debug, because it is + self-healing by design and a per-fetch log would just be noise. */ + if (_registryState.Read() is null) + { + _logger.LogDebug( + "MCP starting before the worker's first registry publish — live plan fetch resolves from darling.json until it arrives (self-healing; store-registered servers reach the resolver on the worker's next reload)."); } var planFetcher = new PgPlanFetcher( - serverId => serversById.TryGetValue(serverId, out var server) - ? DarlingServerConnector.ResolveConnectionString(server, _logger) - : null, + serverId => + { + var byId = _registryState.Read()?.ById ?? fileFallbackById; + return byId.TryGetValue(serverId, out var server) + ? DarlingServerConnector.ResolveConnectionString(server, _logger) + : null; + }, _logger); var builder = WebApplication.CreateBuilder(); @@ -472,6 +513,41 @@ These are the tools the analysis findings' next_tools recommendations point at. is served; Darling's delta collectors store no sample_interval_seconds, so per-second rates are derived from the LAG interval. */ .WithGeminiCompatibleTools() + /* get_pg_wait_stats — PostgreSQL wait events for an Aurora target, paired with the + pg_wait_stats collector. A separate tool from get_wait_stats rather than a widened + one: PostgreSQL's waits are a two-level type/event taxonomy with no signal-wait + concept, reported in microseconds, so the two engines cannot share a result shape + without lying about a unit or emitting mostly-null columns. */ + .WithGeminiCompatibleTools() + /* get_pg_top_queries — PostgreSQL query shapes by total time, paired with the + pg_statement_stats collector. Carries Aurora's I/O source split and per-statement + peak memory, neither of which the SQL Server tools have an equivalent for. */ + .WithGeminiCompatibleTools() + /* get_pg_wraparound_risk — XID/MultiXact freeze headroom, the highest-consequence + PostgreSQL signal and one with no SQL Server counterpart. Not Aurora-gated. */ + .WithGeminiCompatibleTools() + /* get_pg_xmin_horizon — why vacuum reclaims nothing, attributed to one of four causes + that are indistinguishable by symptom and need different fixes. */ + .WithGeminiCompatibleTools() + /* get_pg_replication_slots — the other half of the abandoned-slot story. The xmin tool + reports a slot pinning the horizon; this one reports the WAL it is retaining, which is + unbounded by default and fills the volume regardless of what vacuum is doing. */ + .WithGeminiCompatibleTools() + /* get_pg_autovacuum_health — which tables autovacuum is not keeping up with, ranked by + how far past each table's OWN threshold it is. The ratio is the whole tool: a + dead-tuple count is not comparable between a 50-million-row table and a 10,000-row + one, and the threshold is what makes it so. */ + .WithGeminiCompatibleTools() + /* get_pg_io_stats — I/O attributed to who/what/why rather than to a file. The context + dimension has no SQL Server counterpart and is what separates a buffer-pool miss that + more memory would fix from a ring-buffered sequential scan that it would not. */ + .WithGeminiCompatibleTools() + /* get_pg_blocking — who is blocked by whom, assembled from the stored edge list into chains + with the ROOT attributed. The one PostgreSQL read whose caveat has to travel WITH the + answer: SQL Server's blocked-process report is engine-recorded, this is periodically + sampled, so "no blocking" here means "none was sampled" and the tool reports its own + capture count so that distinction cannot be lost. */ + .WithGeminiCompatibleTools() .WithGeminiCompatibleTools() .WithGeminiCompatibleTools() .WithGeminiCompatibleTools() diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs index b4e09a6b2..085d4f4a8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs @@ -41,17 +41,17 @@ internal static class DarlingMcpInstructions ## Tool Reference - This server exposes ninety tools. Seventy-three are the same names Performance Monitor Lite and the Dashboard expose: six diagnostic-analysis tools, five plan-analysis tools, fifteen core data-read tools, twenty-one diagnostic-depth data-read tools, eight resource-contention + jobs data-read tools, five trend data-read tools, eight system-health parse-on-read tools, five alert + health-overview tools, and one Default Trace tool. The remaining seventeen are unique to Darling: eight are the Custom Views tools (seven manage the saved views — the one view-authoring write surface — and `describe_custom_view_catalog` returns the read-only compose vocabulary those authoring tools draw from), three are alert-tuning write tools (`update_alert_settings` tunes the alert engine's thresholds; `create_mute_rule` / `delete_mute_rule` manage the mute rules) that write only the shared alert configuration in the monitoring store, two are server-onboarding write tools (`add_servers` bulk-adds monitored servers; `remove_server` removes one) that add or remove rows in the monitoring store's monitored-server registry, `get_fleet_overview` and `get_ag_health` are the two cross-server reads only a central store can answer, `get_store_metrics` reads the monitoring store's OWN hourly size/compression/growth series for capacity forecasting, and `get_blocking` is Darling's name for the blocked-process-report read that Lite exposes as `get_blocked_process_reports` — a naming difference, not a capability gap. Every data-read tool reads the data the collectors already captured into the store — a stored read, never a live query against the monitored server. + This server exposes 101 tools. 76 are the same names Performance Monitor Lite exposes, spanning diagnostic analysis, plan analysis, data reads at core and diagnostic depth, resource contention + jobs, trends, system-health parse-on-read, alerts + health overview, and the Default Trace. The remaining 25 are unique to Darling: eight are the PostgreSQL reads (Aurora/PostgreSQL targets only Darling's central store can hold), eight are the Custom Views tools (seven manage the saved views — the one view-authoring write surface — and `describe_custom_view_catalog` returns the read-only compose vocabulary those authoring tools draw from), three are alert-tuning write tools (`update_alert_settings` tunes the alert engine's thresholds; `create_mute_rule` / `delete_mute_rule` manage the mute rules) that write only the shared alert configuration in the monitoring store, two are server-onboarding write tools (`add_servers` bulk-adds monitored servers; `remove_server` removes one) that add or remove rows in the monitoring store's monitored-server registry, `get_fleet_overview` and `get_ag_health` are the two cross-server reads only a central store can answer, `get_store_metrics` reads the monitoring store's OWN hourly size/compression/growth series for capacity forecasting, and `get_blocking` is Darling's name for the blocked-process-report read that Lite exposes as `get_blocked_process_reports` — a naming difference, not a capability gap. Every data-read tool reads the data the collectors already captured into the store — a stored read, never a live query against the monitored server. ### Diagnostic-analysis tools | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `analyze_server` | Runs the inference engine: scores facts, traverses relationship graph, returns evidence-backed findings with severity and recommended next tools. A remediable finding also carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), with a two-sided risk-disclosure header on destructive changes; advisory only, never executed | `server_name`, `hours_back` (default 4) | + | `analyze_server` | Runs the inference engine: scores facts, traverses relationship graph, returns evidence-backed findings with severity and recommended next tools. A remediable finding also carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), with a two-sided risk-disclosure header on destructive changes; advisory only, never executed. Force-plan findings additionally carry `structured_remediation`: the verdict as machine-readable fields (eligible + named blockers), evidence, and split force/unforce/verify SQL | `server_name`, `hours_back` (default 4) | | `get_analysis_facts` | Exposes raw scored facts from the collect+score pipeline — every observation the engine sees with base severity, amplifiers, and metadata | `server_name`, `hours_back` (default 4), `source` (filter), `min_severity` | | `compare_analysis` | Compares two time periods (e.g., peak vs off-peak, before vs after a change) showing severity deltas for each fact | `server_name`, `hours_back` (default 4), `baseline_hours_back` (default 28) | | `audit_config` | Edition-aware configuration audit: evaluates CTFP, MAXDOP, max memory, and max worker threads against best practices | `server_name` | - | `get_analysis_findings` | Retrieves persisted findings from previous analysis runs (the service also analyzes on its own schedule, every 30 minutes per server), deduplicated to one entry per diagnostic chain (`story_path_hash` + `incident_id`): the latest occurrence plus `occurrences`/`first_seen`/`last_seen`/`peak_severity` spanning the window; each remediable finding carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the persisted action, advisory only and never executed | `server_name`, `hours_back` (default 24) | + | `get_analysis_findings` | Retrieves persisted findings from previous analysis runs (the service also analyzes on its own schedule, every 30 minutes per server), deduplicated to one entry per diagnostic chain (`story_path_hash` + `incident_id`): the latest occurrence plus `occurrences`/`first_seen`/`last_seen`/`peak_severity` spanning the window; each remediable finding carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the persisted action, advisory only and never executed; force-plan findings additionally carry `structured_remediation` (verdict + evidence + split artifacts, machine-readable) | `server_name`, `hours_back` (default 24) | | `mute_analysis_finding` | Mutes a finding pattern by story_path_hash so it won't appear in future runs | `story_path_hash` (required), `server_name`, `reason` | ### Plan-analysis tools @@ -81,11 +81,11 @@ internal static class DarlingMcpInstructions | `get_file_io_stats` | Latest per-file I/O: reads/writes/bytes/stall and computed read/write latency | `server_name` | | `get_tempdb_trend` | TempDB space over time (user / internal / version store / unallocated) + top consumer | `server_name`, `hours_back` (default 24) | | `get_perfmon_stats` | Latest perfmon counters (value + delta); filter by counter / instance | `server_name`, `counter_name`, `instance_name` | - | `get_top_queries_by_cpu` | Expensive queries from query stats (plan cache) with query_hash / sql_handle | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `parallel_only`, `min_dop` | - | `get_top_procedures_by_cpu` | Most expensive stored procedures by total CPU | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name` | + | `get_top_queries_by_cpu` | Expensive queries from query stats (plan cache) with query_hash / sql_handle; `cpu_attribution.attributed_cpu_ratio` says how much of the box's measured CPU the returned rows explain | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `parallel_only`, `min_dop` | + | `get_top_procedures_by_cpu` | Most expensive stored procedures by total CPU, with the same `cpu_attribution` disclosure | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name` | | `get_query_store_top` | Expensive queries from Query Store with query_id / plan_id (survives restarts) | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name` | | `list_servers` | All monitored servers with collection-freshness status and last collection time | none | - | `get_collection_health` | Per-collector health (running / failing / stale) over the last 7 days | `server_name` | + | `get_collection_health` | Per-collector health (running / failing / stale) over the last 7 days, plus the server's sweep_pressure verdict (a SATURATED body collects at a multiple of its configured cadence with every collector healthy) | `server_name` | | `get_server_properties` | Instance properties: edition, version, CPU count, memory, socket/core topology, HADR | `server_name` | ### Diagnostic-depth data-read tools @@ -94,9 +94,9 @@ internal static class DarlingMcpInstructions | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `get_blocking` | Recent blocked/blocking pairs from the blocked-process-report XE + the always-on DMV fallback | `server_name`, `hours_back` (default 24), `limit` (default 30) | - | `get_deadlocks` | Recent deadlocks: victim process/SQL + a process summary | `server_name`, `hours_back` (default 24), `limit` (default 20) | - | `get_deadlock_detail` | The raw deadlock graph XML for the recent deadlocks | `server_name`, `hours_back` (default 24), `limit` (default 5) | + | `get_blocking` | Recent blocked/blocking pairs from the blocked-process-report XE + the always-on DMV fallback | `server_name`, `hours_back` (default 24), `limit` (default 30), `dedup_key` (optional) | + | `get_deadlocks` | Recent deadlocks: victim process/SQL + a process summary | `server_name`, `hours_back` (default 24), `limit` (default 20), `dedup_key` (optional) | + | `get_deadlock_detail` | The raw deadlock graph XML for the recent deadlocks | `server_name`, `hours_back` (default 24), `limit` (default 5), `dedup_key` (optional) | | `get_blocked_process_xml` | The raw blocked-process-report XML | `server_name`, `hours_back` (default 24), `limit` (default 5) | | `get_long_query_completions` | Longest completed queries (rpc/batch over the trace threshold) + attentions/cancels from the opt-in long-query trace, duration DESC (empty until the collector is enabled) | `server_name`, `hours_back` (default 24), `limit` (default 30) | | `get_blocking_trend` | Per-minute blocking-incident counts over time (XE, DMV-snapshot fallback) | `server_name`, `hours_back` (default 24) | @@ -108,6 +108,7 @@ internal static class DarlingMcpInstructions | `get_database_config_changes` | sys.databases setting changes, diffed from config snapshots | `server_name`, `hours_back` (default 168) | | `get_trace_flag_changes` | Trace flags enabled/disabled/modified, diffed from config snapshots | `server_name`, `hours_back` (default 168) | | `get_database_scoped_config` | Latest database-scoped configuration (MAXDOP, legacy CE, ...) | `server_name`, `database_name` | + | `get_query_store_health` | Per-database Query Store health (latest hourly snapshot) — actual vs desired state, readonly_reason decoded, storage vs cap, cleanup thresholds | `server_name`, `database_name` | | `get_server_config` | CURRENT sys.configurations (latest snapshot) — what CTFP / MAXDOP / max memory are set to now | `server_name` | | `get_database_config` | CURRENT per-database settings (latest snapshot) — recovery model, RCSI, Query Store, ... | `server_name`, `database_name` | | `get_trace_flags` | CURRENT active trace flags (latest snapshot) — flag number, enabled, global/session | `server_name` | @@ -237,6 +238,16 @@ A SQL password is encrypted at rest (DPAPI, the service identity) and is NEVER r The three config-change tools diff the store's config snapshots. This edition captures configuration WHEN THE SERVICE CONNECTS to a server (not on a fixed schedule), so a change is detected between two connect snapshots and at least two are needed — a stable, always-connected deployment may show no changes until the next connect. They emit only the values the collectors capture; the Dashboard's `requires_restart` / setting `description` / `setting_type` / generated change-narrative enrichment is not collected here and is omitted. `get_blocking_deadlock_stats` (the Dashboard's blocking/deadlock aggregate) is NOT hosted: this edition has no blocking/deadlock rollup table — use `get_blocking` / `get_deadlocks` for the raw events. + Jumping from an alert to its incident: `get_blocking`, `get_deadlocks` and `get_deadlock_detail` accept an + optional `dedup_key` — the #1140 alert fingerprint, shown to operators as the alert's **Dedup Key** fact and + carried onto downstream tickets. Pass it to get exactly that incident instead of pulling a server+time window + and guessing which row the alert meant. Those three tools also RETURN a `dedup_key` on every row, so an + incident found by browsing can be correlated back to its alerts, or handed to another agent as a stable + identifier. Two things to know when it matches nothing: the fingerprint is scoped to the server's DISPLAY name + and to the incident's involved objects, so a server renamed since the alert fired has different keys now; and + `hours_back` still bounds the search, so widen it before concluding the incident is gone. The no-match response + says how many rows it examined, which distinguishes those cases. + Note on `next_tools`: analyze_server findings include `next_tools` recommendations. Most are hosted on this server — the plan-analysis tools (`analyze_query_plan`, `analyze_query_store_plan`) and the data-read tools listed above (`get_wait_stats`, `get_top_queries_by_cpu`, `get_cpu_utilization`, `get_memory_stats`, `get_file_io_stats`, `get_tempdb_trend`, `get_blocking`, `get_deadlocks`, `get_waiting_tasks`, `get_active_queries`, ...) — so follow those here. `get_top_queries_by_cpu` / `get_top_procedures_by_cpu` / `get_query_store_top` are where the `query_hash` / `sql_handle` / `query_id` + `plan_id` keys for the plan-analysis tools come from. The resource-contention + jobs tools (`get_latch_stats`, `get_spinlock_stats`, `get_resource_semaphore`, `get_memory_grants`, `get_plan_cache_bloat`, `get_cpu_scheduler_pressure`, `get_running_jobs`), the trend siblings (`get_memory_trend`, `get_perfmon_trend`, `get_file_io_trend`, `get_query_trend`, `get_query_duration_trend`), the `get_health_parser_*` system-health family, and the blocking/deadlock trend + memory-pressure reads (`get_blocking_trend`, `get_deadlock_trend`, `get_memory_pressure_events`) are all hosted here too — follow those `next_tools` on this server. Two `next_tools` names differ from what this edition hosts: `get_blocked_process_reports` (a Lite name) is served here as `get_blocked_process_xml` (with `get_blocking` for a quick overview), and `get_blocking_deadlock_stats` (the Dashboard's blocking/deadlock rollup) is not hosted at all — use `get_blocking` / `get_deadlocks` instead. ## Recommended Workflow diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs new file mode 100644 index 000000000..ea6d85e86 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for autovacuum health, paired with the pg_autovacuum_stats collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgAutovacuumTools +{ + /// + /// Classifies a table by how far past its OWN threshold it is, not by its dead-tuple count. The same + /// count is routine on a large table and urgent on a small one, so the ratio is the only comparable + /// figure — and whether the pile is growing separates autovacuum losing a race from autovacuum not + /// running at all. + /// + internal static string Classify(bool autovacuumDisabled, double? thresholdRatio, bool deadTuplesGrowing) + { + /* A configuration finding, and the one case where the count is beside the point: this table will + never be vacuumed by autovacuum no matter how bad it gets, and it holds back the whole + database's freeze horizon while it sits there. */ + if (autovacuumDisabled) + { + return "critical_autovacuum_disabled_on_table"; + } + + if (thresholdRatio is null) + { + return "unknown_no_threshold"; + } + + return thresholdRatio switch + { + /* Ten times past the line is not a busy table, it is a stuck one — most often autovacuum + being cancelled repeatedly by conflicting locks, or starved of workers. */ + >= 10 => "critical_far_past_threshold", + >= 2 when deadTuplesGrowing => "warning_past_threshold_and_growing", + >= 2 => "warning_past_threshold", + >= 1 => "info_at_threshold", + _ => "ok", + }; + } + + [McpServerTool(Name = "get_pg_autovacuum_health"), Description("Gets PostgreSQL per-table autovacuum health: which tables are behind on vacuum or analyze, ranked by how far past each table's OWN trigger threshold it is. This ratio is the point of the tool - a dead-tuple count alone is not actionable, because autovacuum fires at autovacuum_vacuum_threshold + scale_factor * reltuples, so 500,000 dead tuples is routine on a 50-million-row table and urgent on a 10,000-row one. Thresholds honour per-table reloptions overrides, not just the server settings, since ALTER TABLE SET (autovacuum_*) is common on exactly the big hot tables where the global default is wrong. Also reports tables with autovacuum switched off entirely, whether dead tuples are still growing (autovacuum losing a race) or flat (autovacuum blocked or not running), and the analyze backlog that drives bad row estimates. Works on any PostgreSQL target; collected per database.")] + public static async Task GetPgAutovacuumHealth( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze, used for the dead-tuple growth comparison. Default 24.")] int hours_back = 24, + [Description("Maximum tables to return, worst first. Default 20.")] int limit = 20) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + validation = McpHelpers.ValidateTop(limit); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgAutovacuumReader.GetPgAutovacuumAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit); + + if (rows.Count == 0) + { + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "no_pending_maintenance", + finding = "No table on this server has dead tuples, pending analyze work, inserts since " + + "its last vacuum, or autovacuum disabled. The collector records only tables with " + + "pending work, so an empty result is the healthy case rather than missing data.", + }, McpHelpers.JsonOptions); + } + + var tables = rows.Select(r => + { + /* -1 is the collector's not-applicable sentinel, and a 0 threshold happens on a table that + has never been analyzed. Neither can produce a ratio, and inventing one would rank a + table we know nothing about above tables we have measured. */ + double? ratio = r.VacuumThreshold > 0 + ? Math.Round((double)r.DeadTuples / r.VacuumThreshold, 2) + : null; + double? analyzeRatio = r.AnalyzeThreshold > 0 + ? Math.Round((double)r.ModsSinceAnalyze / r.AnalyzeThreshold, 2) + : null; + var growing = r.DeadTuples > r.FirstDeadTuples; + + return new + { + database_name = r.DatabaseName, + table_name = $"{r.SchemaName}.{r.TableName}", + severity = Classify(r.AutovacuumDisabled, ratio, growing), + dead_tuples = r.DeadTuples, + vacuum_threshold = r.VacuumThreshold >= 0 ? r.VacuumThreshold : (long?)null, + /* The headline number: 1.0 means autovacuum should be triggering right now. */ + threshold_ratio = ratio, + dead_tuples_growing = growing, + dead_tuple_change = r.DeadTuples - r.FirstDeadTuples, + live_tuples = r.LiveTuples, + /* The analyze half. Stale statistics produce bad row estimates and bad plans, which is + a different symptom from bloat and gets missed because both come from one process. */ + mods_since_analyze = r.ModsSinceAnalyze, + analyze_threshold = r.AnalyzeThreshold >= 0 ? r.AnalyzeThreshold : (long?)null, + analyze_threshold_ratio = analyzeRatio, + /* The append-only path, PG13+: a table with no dead tuples is invisible to the vacuum + rule, and a table never vacuumed is never frozen either. */ + inserts_since_vacuum = r.InsertsSinceVacuum >= 0 ? r.InsertsSinceVacuum : (long?)null, + insert_vacuum_threshold = r.InsertVacuumThreshold >= 0 ? r.InsertVacuumThreshold : (long?)null, + autovacuum_disabled = r.AutovacuumDisabled, + total_bytes = r.TotalBytes >= 0 ? r.TotalBytes : (long?)null, + total_gb = r.TotalBytes >= 0 ? Math.Round(r.TotalBytes / 1024.0 / 1024.0 / 1024.0, 2) : (double?)null, + last_autovacuum = r.LastAutovacuum, + last_vacuum = r.LastVacuum, + last_autoanalyze = r.LastAutoanalyze, + last_analyze = r.LastAnalyze, + autovacuum_count = r.AutovacuumCount, + /* Never autovacuumed AND past threshold is the strongest single signal that something + is preventing it, rather than that it has not got round to this table yet. */ + never_autovacuumed = r.LastAutovacuum is null && r.AutovacuumCount == 0, + measured_at = r.MeasuredAt, + }; + }) + .ToList(); + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "tables_with_pending_maintenance", + table_count = tables.Count, + autovacuum_disabled_count = tables.Count(t => t.autovacuum_disabled), + past_threshold_count = tables.Count(t => t.threshold_ratio >= 1), + growing_count = tables.Count(t => t.dead_tuples_growing), + worst_table = tables[0].table_name, + worst_severity = tables[0].severity, + tables, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL autovacuum health failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgBlockingTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgBlockingTools.cs new file mode 100644 index 000000000..fc22c91e9 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgBlockingTools.cs @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for PostgreSQL blocking chains, paired with the pg_blocking collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgBlockingTools +{ + /// + /// What to do about a root blocker, keyed on the root's own state — which is the reason the collector + /// captures the blocker's state and not just its pid. Every branch here is a different action, and + /// picking the wrong one is worse than doing nothing: killing a backend that is mid-query loses work, + /// and tuning a query that is not running fixes nothing. + /// + internal static string RemedyFor(string? rootState, bool idleInTransaction, long xactDurationMs) + { + if (idleInTransaction || string.Equals(rootState, "idle in transaction", StringComparison.Ordinal)) + { + return "The root is IDLE IN TRANSACTION: it holds locks and is running nothing, so there is no " + + "query to tune and no work in progress to protect. This is an application defect — a " + + "transaction opened and then left open across think time, a missing commit on an error " + + "path, or a connection pool handing back a dirty connection. Find the code path from " + + "application_name; bound the class of failure with " + + "idle_in_transaction_session_timeout so it cannot recur unbounded."; + } + + if (string.Equals(rootState, "idle in transaction (aborted)", StringComparison.Ordinal)) + { + return "The root is IDLE IN TRANSACTION (ABORTED): its transaction already failed and every " + + "further statement will error, yet it still holds its locks until the client issues " + + "ROLLBACK. Nothing it is doing can succeed, so this is the cheapest root to clear. The " + + "client is not handling an error it already received — that is the bug."; + } + + if (string.Equals(rootState, "active", StringComparison.Ordinal)) + { + return "The root is ACTIVE — a real query holding locks while it runs. Tune the query or shorten " + + "the transaction; killing it discards work and the next execution will block the same way. " + + "If the statement is fast but the TRANSACTION is long, the lock is being held across " + + "statements, and moving the write later in the transaction is usually the fix." + + (xactDurationMs > 0 + ? $" Its transaction had been open {xactDurationMs} ms when sampled." + : string.Empty); + } + + if (rootState is null) + { + return "The root's own state was not captured — it left pg_stat_activity between the blocked " + + "backends being read and its own row being looked up, which means the chain resolved on " + + "its own. Nothing to act on unless it recurs."; + } + + return $"The root is in state '{rootState}' while holding locks. Identify it in pg_stat_activity by " + + "pid and establish what transaction it has open; a root that is neither active nor idle in " + + "transaction usually means it is waiting on something else — a lock outside this chain, or a " + + "client that stopped reading."; + } + + [McpServerTool(Name = "get_pg_blocking"), Description("Gets PostgreSQL blocking chains that were captured for a server, assembled from the stored edge list into one entry per chain with its ROOT blocker identified and attributed. Use this when sessions are timing out, waiting, or piling up on a PostgreSQL target, or to check whether a past slowdown involved lock contention. Reports for each captured chain: the root blocker's pid, state, application, username and query text, how many sessions were behind it in total and directly, how deep the chain went, the longest-waiting victim, and how many separate captures that same backend has been the root of - which distinguishes one stuck session from a recurring pattern. Also returns a specific remedy per root state, because an 'idle in transaction' root is an application defect while an 'active' root is a query-tuning problem and the two need opposite responses. IMPORTANT: this is a periodic SAMPLE, not an event log. Unlike SQL Server's blocked-process report, PostgreSQL records nothing on its own, so blocking shorter than the collection interval is never seen and an empty result means 'none was sampled', not 'none happened' - the capture counts in the response say how many samples the window actually contains. Works on any PostgreSQL target including standbys.")] + public static async Task GetPgBlocking( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, + [Description("Maximum chains to return, worst-first by victim count. Default 50.")] int limit = 50) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + + var limitValidation = McpHelpers.ValidateTop(limit); + if (limitValidation != null) return limitValidation; + + try + { + var now = DateTime.UtcNow; + var startUtc = now.AddHours(-hours_back); + + var chains = await DarlingPgBlockingReader.GetPgBlockingChainsAsync( + postgres, resolved.ServerId, startUtc, now, limit); + + /* The denominator comes first because it is what makes an empty answer honest. */ + var captures = await DarlingPgBlockingReader.GetPgBlockingCaptureCountsAsync( + postgres, resolved.ServerId, startUtc, now); + + /* Cycles are read separately and MUST be, because the chain query structurally cannot see them: + it finds a root by absence, and in a cycle every participant is blocked. Without this the + tool would report "no blocking" from a capture that recorded a deadlock. */ + var cycles = await DarlingPgBlockingReader.GetPgBlockingCyclesAsync( + postgres, resolved.ServerId, startUtc, now, limit); + + var cycleEntries = cycles.Select(c => new + { + captured_at = c.CapturedAt, + participant_count = c.ParticipantCount, + pids = c.Pids, + database = c.DatabaseName, + application = c.ApplicationName, + /* Sessions queued behind the deadlock without being part of it. Previously invisible to + BOTH reads — chains cannot see them (no cycle member qualifies as a root) and the cycle + walk cannot either (their walks never close). This count is usually what decides + urgency: a two-way deadlock is a bug, a two-way deadlock with forty sessions behind it + is an outage. */ + blocked_behind_count = c.BlockedBehindCount, + blocked_behind_pids = c.BlockedBehindPids, + finding = + "These backends were each waiting on a lock held by another member of the same set — a " + + "genuine cycle, which is a deadlock. PostgreSQL's deadlock detector resolves it after " + + "deadlock_timeout by killing one participant, so this capture landed inside that " + + "window and is likely the only record that will ever exist. The fix is ordering: make " + + "every code path acquire these objects in the same sequence.", + }).ToList(); + + if (chains.Count == 0 && cycleEntries.Count > 0) + { + /* Cycles but no chains. Reporting "no blocking sampled" here would be a flat lie. */ + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "cycles_only", + captures_total = captures.CapturesTotal, + captures_with_blocking = captures.CapturesWithBlocking, + finding = + "Blocking was captured, but every participant was itself blocked — a lock cycle " + + "(deadlock) rather than a chain with a root. There is no root blocker to name, " + + "which is why these are reported separately.", + cycles = cycleEntries, + }, McpHelpers.JsonOptions); + } + + if (chains.Count == 0) + { + /* Two very different empty answers, and conflating them would be the whole failure mode of + a sampled signal. No captures at all means the collector never ran — nothing is known + about this window either way. Captures with no blocking is a real all-clear, bounded by + the sampling interval. */ + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = captures.CapturesTotal == 0 ? "not_sampled" : "no_blocking_sampled", + captures_total = captures.CapturesTotal, + captures_with_blocking = 0, + first_capture_at = captures.FirstCaptureAt, + last_capture_at = captures.LastCaptureAt, + finding = captures.CapturesTotal == 0 + ? "No pg_blocking captures exist for this server in this window, so nothing is known " + + "about whether blocking occurred. Check that the collector is enabled and that " + + "collection is succeeding (get_collection_health) before concluding anything." + : $"No blocking was present in any of the {captures.CapturesTotal} captures in this " + + "window. Note the limit of that statement: captures are periodic, so blocking that " + + "started and cleared between two of them left no trace. PostgreSQL has no " + + "engine-side blocked-process recorder to fall back on.", + }, McpHelpers.JsonOptions); + } + + var entries = chains.Select(c => new + { + captured_at = c.CapturedAt, + root_pid = c.RootPid, + /* Surfaced because it is what samples_as_root counts, and a reader comparing pids across + captures without it can be fooled by pid reuse. */ + root_backend_id = c.RootBackendId, + databases = c.Databases, + root_username = c.RootUsername, + root_application = c.RootApplicationName, + root_state = c.RootState, + root_is_idle_in_transaction = c.RootIsIdleInTransaction, + root_query = c.RootQuery, + root_xact_duration_ms = c.RootXactDurationMs, + root_query_duration_ms = c.RootQueryDurationMs, + total_victims = c.TotalVictims, + direct_victims = c.DirectVictims, + max_chain_depth = c.MaxDepth, + worst_victim_wait_ms = c.WorstVictimWaitMs, + worst_victim_query = c.WorstVictimQuery, + /* The one-off vs. pattern discriminator, keyed on the stable backend id. NULL when the + root's own identity did not resolve — reported as unknown rather than as 1, because a + fabricated "seen once" reads as a real finding. */ + samples_as_root = c.SamplesAsRoot, + samples_as_root_note = c.SamplesAsRoot is null + ? "Unknown: this root had already left pg_stat_activity when the edge was captured, so " + + "it has no stable backend identity to count appearances of. Not a sign it is new." + : null, + /* Chain-wide, not root-only: the stored flag is an OR across both sides of an edge, so it + answers "some text in this chain may be clipped". */ + query_text_may_be_truncated = c.QueryTextMayBeTruncated, + chain_may_be_truncated = c.ChainMayBeTruncated, + chain_truncation_note = c.ChainMayBeTruncated + ? "This chain hit the read's 32-level walk cap, so total_victims, max_depth and the " + + "worst victim are computed over a truncated walk and are FLOORS, not totals." + : null, + recommended_action = RemedyFor(c.RootState, c.RootIsIdleInTransaction, c.RootXactDurationMs), + }).ToList(); + + var worst = entries[0]; + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "blocking_sampled", + captures_total = captures.CapturesTotal, + captures_with_blocking = captures.CapturesWithBlocking, + pct_of_captures_with_blocking = captures.CapturesTotal > 0 + ? Math.Round((double)captures.CapturesWithBlocking / captures.CapturesTotal * 100, 1) + : 0, + /* Lead with the worst chain and what to do about it, the same shape as the xmin tool: + the cause, then the action for that cause. */ + worst_chain_victims = worst.total_victims, + worst_chain_root_state = worst.root_state, + worst_chain_root_application = worst.root_application, + recommended_action = worst.recommended_action, + sampling_caveat = + "These are periodic samples of pg_stat_activity, not an event log. PostgreSQL records " + + "no blocking on its own, so any episode shorter than the collection interval is " + + "invisible here and the counts below are a floor, not a total.", + chains = entries, + /* Always present, even when empty, so its absence is never mistaken for "not checked". */ + cycles_sampled = cycleEntries.Count, + cycles = cycleEntries, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL blocking chains failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs new file mode 100644 index 000000000..ff9e13f60 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for I/O attribution, paired with the pg_io_stats collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgIoTools +{ + /// + /// Explains what a context value means, because it is the dimension with no SQL Server + /// counterpart and the one that changes what you do about a number. + /// + internal static string ContextMeaning(string? context) => context switch + { + "normal" => "Ordinary buffer-pool traffic. Reads here are cache misses that shared_buffers could " + + "have absorbed, so a high read share with a low hit share is the classic case for more " + + "memory or a better index.", + "bulkread" => "A sequential scan deliberately using a small ring buffer so it cannot evict the " + + "buffer pool. High volume here is a scan-heavy workload, NOT memory pressure — adding " + + "shared_buffers will not reduce it, because these reads bypass the pool by design.", + "bulkwrite" => "A bulk write (COPY, CREATE TABLE AS, some ALTER TABLE) using its own ring buffer.", + "vacuum" => "Vacuum's ring buffer. Volume here is autovacuum doing its job; pair it with " + + "get_pg_autovacuum_health to see whether it is keeping up.", + "index" => "Index-specific I/O, reported separately from the relation's own.", + "walreplay" => "A standby applying WAL. This is replica catch-up work, not query I/O, and it is the " + + "first thing to check when a reader lags.", + _ => "Unrecognized context — treat the raw counters as authoritative and check the PostgreSQL " + + "documentation for this server's major version.", + }; + + [McpServerTool(Name = "get_pg_io_stats"), Description("Gets PostgreSQL I/O attributed to WHO did it, to WHAT, and WHY - the (backend_type, object, context) breakdown from pg_stat_io, differenced across the requested window. Richer than SQL Server's file-level dm_io_virtual_file_stats: instead of 'this file is busy' you get 'autovacuum workers are reading relations in the vacuum context', which names the cause. The context dimension is the one with no SQL Server equivalent and the one that changes the remedy - it separates ordinary buffer-pool misses (where more shared_buffers or a better index helps) from sequential scans that deliberately bypass the pool via a ring buffer (where it will not help at all), from vacuum's ring buffer, from a standby applying WAL. Reports whether write counters are TRACKED at all, because on Amazon Aurora they are always null - backends there do not write data files, the storage layer does - and a zero would otherwise read as 'no writes happened'. Requires PostgreSQL 16 or later; valid on a standby.")] + public static async Task GetPgIoStats( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, + [Description("Maximum (backend_type, object, context) combinations to return, busiest first. Default 20.")] int limit = 20) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + validation = McpHelpers.ValidateTop(limit); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgIoReader.GetPgIoAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit); + + if (rows.Count == 0) + { + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "no_io_activity", + finding = "No (backend_type, object, context) combination recorded read, write, extend or " + + "hit activity in this window. On a busy server that more likely means the " + + "collector has not run yet than that the server is idle — pg_stat_io needs " + + "PostgreSQL 16 or later, so check the target's major version.", + }, McpHelpers.JsonOptions); + } + + var totalReads = rows.Sum(r => r.Reads); + var totalReadTime = rows.Sum(r => r.ReadTimeMs); + + var combinations = rows.Select(r => + { + var accesses = r.Reads + r.Hits; + return new + { + backend_type = r.BackendType, + object_type = r.ObjectType, + context = r.Context, + context_meaning = ContextMeaning(r.Context), + reads = r.Reads, + read_time_ms = Math.Round(r.ReadTimeMs, 1), + /* Per-read latency is the figure that separates "a lot of I/O" from "slow I/O", and + they have completely different remedies. */ + avg_read_ms = r.Reads > 0 ? Math.Round(r.ReadTimeMs / r.Reads, 3) : (double?)null, + hits = r.Hits, + /* A hit ratio scoped to this combination, which is the only scope where it means + anything: a server-wide ratio averages bulkread's deliberate misses together with + normal-context misses and understates both. */ + hit_pct = accesses > 0 ? Math.Round((double)r.Hits / accesses * 100, 1) : (double?)null, + pct_of_total_reads = totalReads > 0 ? Math.Round((double)r.Reads / totalReads * 100, 1) : 0, + pct_of_total_read_time = totalReadTime > 0 ? Math.Round(r.ReadTimeMs / totalReadTime * 100, 1) : 0, + extends = r.Extends, + extend_time_ms = Math.Round(r.ExtendTimeMs, 1), + evictions = r.Evictions, + /* Ring-buffer reuse, NOT eviction pressure. Conflating the two is the standard + misreading of this view: reuses are a bulk operation recycling its OWN buffers. */ + reuses = r.Reuses, + writes = r.WriteCountersTracked ? r.Writes : (long?)null, + write_time_ms = r.WriteCountersTracked ? Math.Round(r.WriteTimeMs, 1) : (double?)null, + write_counters_tracked = r.WriteCountersTracked, + block_bytes = r.OpBytes > 0 ? r.OpBytes : (long?)null, + read_bytes = r.OpBytes > 0 ? r.Reads * r.OpBytes : (long?)null, + stats_reset = r.StatsReset, + }; + }) + .ToList(); + + var anyWritesTracked = rows.Any(r => r.WriteCountersTracked); + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "io_activity", + combination_count = combinations.Count, + total_reads = totalReads, + total_read_time_ms = Math.Round(totalReadTime, 1), + busiest_by_read_time = $"{rows[0].BackendType}/{rows[0].ObjectType}/{rows[0].Context}", + /* Said once at the top rather than repeated per row: on Aurora this is false everywhere, + and a caller needs to know the write side is unmeasured before it concludes anything + from the absence of writes. */ + write_counters_tracked_anywhere = anyWritesTracked, + note = anyWritesTracked + ? "All counters are windowed differences, clamped per interval so a stats reset cannot " + + "produce a negative figure." + : "All counters are windowed differences. This server tracks NO write counters — the " + + "signature of Amazon Aurora, where backends do not write data files and the storage " + + "layer does. Absent writes here mean unmeasured, not zero.", + combinations, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL I/O stats failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs new file mode 100644 index 000000000..25b5b509d --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for replication slot health, paired with the pg_replication_slots collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgSlotTools +{ + /// + /// Severity from slot state, not from the retained figure alone. The size of the hole matters far + /// less than whether it is still being dug: a slot holding 45 GB steadily is a consumer keeping pace, + /// while one that grew from 2 GB to 45 GB in an hour is a volume filling in front of you. + /// + internal static string Classify(string? walStatus, bool isActive, bool retainedWalGrowing) => + walStatus switch + { + /* The slot is already unusable — its consumer cannot resume and needs recreating. */ + "lost" => "critical_slot_lost", + /* Required WAL has been removed; the consumer is about to find that out. */ + "unreserved" => "critical_wal_already_removed", + /* WAL is being retained BECAUSE of this slot. Inactive and still growing is the disk bomb. */ + "extended" when !isActive && retainedWalGrowing => "critical_orphan_filling_disk", + "extended" => "warning_retaining_wal", + _ when !isActive && retainedWalGrowing => "warning_inactive_and_growing", + _ when !isActive => "info_inactive", + _ => "ok", + }; + + [McpServerTool(Name = "get_pg_replication_slots"), Description("Gets PostgreSQL replication slot health, including whether any slot is retaining WAL without bound. An abandoned slot is one of the few PostgreSQL conditions that can take a server down by itself, and it does so two independent ways: it retains every WAL segment its consumer has not confirmed - unbounded by default, so it will fill the volume and stop the server - and it simultaneously pins the vacuum horizon so nothing gets reclaimed cluster-wide. Reports whether retained WAL is still GROWING across the window, which is the difference between a consumer that is merely behind and a volume filling in front of you. Common orphan sources are a removed CDC task, a finished blue/green deployment, a decommissioned Debezium consumer, or a failed major-version upgrade. Works on any PostgreSQL target.")] + public static async Task GetPgReplicationSlots( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze, used for the WAL growth comparison. Default 24.")] int hours_back = 24) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgSlotReader.GetPgSlotsAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); + + /* No slots is the common, healthy case on most servers — say so rather than returning an + "unavailable" envelope that reads like a collection problem. + The scope caveat is not hedging. Replication slots live on the WRITER, so an empty result + read from a replica says nothing about its cluster, and a caller that treats "no slots" as + a cluster-wide all-clear would be drawing the one conclusion this result cannot support. */ + if (rows.Count == 0) + { + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "no_slots", + finding = "This instance has no replication slots in the window, so it is not itself " + + "retaining WAL or pinning vacuum through one.", + scope = "Per-instance. Slots live on the writer, so if this target is a replica, check " + + "the cluster's writer before concluding the cluster has no abandoned slot.", + }, McpHelpers.JsonOptions); + } + + var slots = rows.Select(r => + { + var growthBytes = r.RetainedWalBytes >= 0 && r.FirstRetainedWalBytes >= 0 + ? r.RetainedWalBytes - r.FirstRetainedWalBytes + : 0; + var hours = Math.Max((r.MeasuredAt - r.FirstSeenAt).TotalHours, 0); + var growing = growthBytes > 0; + + return new + { + slot_name = r.SlotName, + severity = Classify(r.WalStatus, r.IsActive, growing), + slot_type = r.SlotType, + plugin = r.Plugin, + database_name = r.DatabaseName, + is_active = r.IsActive, + wal_status = r.WalStatus, + retained_wal_bytes = r.RetainedWalBytes, + retained_wal_gb = r.RetainedWalBytes >= 0 + ? Math.Round(r.RetainedWalBytes / 1024.0 / 1024.0 / 1024.0, 2) + : (double?)null, + /* Growth is the actionable half. Rate is only reported when the window actually spans + time, so a single-sample window cannot produce a fabricated per-hour figure. */ + retained_wal_growth_bytes = growthBytes, + retained_wal_growth_gb_per_hour = hours >= 0.05 + ? Math.Round(growthBytes / 1024.0 / 1024.0 / 1024.0 / hours, 3) + : (double?)null, + /* -1 is the collector's not-applicable sentinel: safe_wal_size is NULL whenever + max_slot_wal_keep_size is -1, which is the default, so on a stock server there is + no configured ceiling at all. */ + has_configured_wal_ceiling = r.SafeWalSizeBytes >= 0, + safe_wal_size_bytes = r.SafeWalSizeBytes >= 0 ? r.SafeWalSizeBytes : (long?)null, + /* The second failure mode, from the same slot. */ + xmin_age = r.XminAge >= 0 ? r.XminAge : (long?)null, + catalog_xmin_age = r.CatalogXminAge >= 0 ? r.CatalogXminAge : (long?)null, + inactive_since = r.InactiveSince, + invalidation_reason = r.InvalidationReason, + conflicting = r.Conflicting, + }; + }) + .OrderByDescending(s => s.retained_wal_bytes) + .ToList(); + + var worst = slots[0]; + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "slots_present", + slot_count = slots.Count, + inactive_count = slots.Count(s => !s.is_active), + worst_slot = worst.slot_name, + worst_severity = worst.severity, + total_retained_wal_gb = Math.Round( + slots.Where(s => s.retained_wal_bytes > 0).Sum(s => s.retained_wal_bytes) / 1024.0 / 1024.0 / 1024.0, 2), + slots, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL replication slots failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs new file mode 100644 index 000000000..e5367f298 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for PostgreSQL query statistics, paired with the pg_statement_stats +/// collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgStatementTools +{ + [McpServerTool(Name = "get_pg_top_queries"), Description("Gets the top PostgreSQL query shapes by total execution time over a time period, for Amazon Aurora PostgreSQL targets. Includes Aurora's I/O source breakdown, which stock PostgreSQL cannot provide: a block 'read' may have come from the distributed storage volume or from the local NVMe Optimized Reads cache, and the two have very different costs. Also reports peak memory per statement, the closest PostgreSQL equivalent of a memory grant, and WAL bytes generated, which has no SQL Server DMV counterpart. Returns query_text for each statement, captured hourly and keyed on queryid, or null when none has been captured yet (a statement first seen minutes ago, or a queryid minted by a major-version upgrade). queryid itself is stable within a major version but changes across a major upgrade — which is exactly why the text is STORED rather than fetched live: after an upgrade the live view no longer holds the old ids, so their text would otherwise be unrecoverable and the history would read as a list of integers. This is a separate tool from get_top_queries_by_cpu, which covers SQL Server.")] + public static async Task GetPgTopQueries( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, + [Description("Maximum rows to return. Default 20.")] int limit = 20) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + validation = McpHelpers.ValidateTop(limit); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgStatementReader.GetPgTopQueriesAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); + + if (rows.Count == 0) + { + return McpHelpers.Status( + "unavailable", + "No PostgreSQL query statistics for this server and window. If this server is SQL " + + "Server, use get_top_queries_by_cpu instead. If it is Aurora PostgreSQL, check that " + + "pg_stat_statements is installed in the database the collector connects to — on some " + + "clusters it exists only in the application database, not in postgres."); + } + + var totalTimeMs = rows.Sum(r => r.TotalExecTimeMs); + + var result = rows.Take(limit).Select(r => + { + var storageAndCache = r.StorageBlocksRead + r.OrcacheBlocksHit; + return new + { + queryid = r.QueryId, + database_id = r.DatabaseId, + calls = r.Calls, + total_exec_time_ms = r.TotalExecTimeMs, + avg_exec_time_ms = r.Calls > 0 ? Math.Round((double)r.TotalExecTimeMs / r.Calls, 3) : 0, + max_exec_time_ms = Math.Round(r.MaxExecTimeMs, 3), + rows_returned = r.RowsReturned, + pct_of_total_time = totalTimeMs > 0 ? Math.Round((double)r.TotalExecTimeMs / totalTimeMs * 100, 1) : 0, + /* Aurora's I/O split, which is the point of using aurora_stat_statements over the + vanilla view. A high orcache share means the reads were cheap local NVMe hits; a + high storage share means network round trips to the cluster volume. The community + cache-hit ratio cannot distinguish these and so overstates the cost of one and + understates the other. */ + shared_blks_hit = r.SharedBlocksHit, + shared_blks_read = r.SharedBlocksRead, + storage_blks_read = r.StorageBlocksRead, + orcache_blks_hit = r.OrcacheBlocksHit, + orcache_hit_pct_of_reads = storageAndCache > 0 + ? Math.Round((double)r.OrcacheBlocksHit / storageAndCache * 100, 1) + : (double?)null, + /* Spills. temp blocks are sort/hash spill to disk, NOT temporary tables - those are + the local_blks_* family and a different problem. */ + temp_blks_read = r.TempBlocksRead, + temp_blks_written = r.TempBlocksWritten, + wal_bytes = r.WalBytes, + max_exec_peakmem_bytes = r.MaxPeakMemBytes, + // #2219: the statement text, or null when none has been captured for this queryid yet. + // Null is honest rather than a placeholder — text refreshes hourly, so a statement first + // seen minutes ago has none, and after a major-version upgrade re-keys queryid the new ids + // have none until the next refresh. An empty string would read as "the query is blank". + query_text = r.QueryText, + }; + }); + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + total_exec_time_ms = totalTimeMs, + /* Every counter here covers the window, so a caller can safely divide one by another. + Only the two high-water marks are not counters, and saying which is cheaper than + letting someone assume max_exec_time_ms is a windowed total. */ + note = "All counters cover the requested window: calls, total_exec_time_ms and " + + "rows_returned from stored per-interval deltas, and the block and WAL figures " + + "differenced across the window's snapshots. max_exec_time_ms and " + + "max_exec_peakmem_bytes are high-water marks, not windowed totals.", + queries = result, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL query stats failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs new file mode 100644 index 000000000..e0d9ea017 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for PostgreSQL wait statistics, paired with the pg_wait_stats collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgWaitTools +{ + [McpServerTool(Name = "get_pg_wait_stats"), Description("Gets the top PostgreSQL wait events aggregated over a time period, for Amazon Aurora PostgreSQL targets. Waits reveal what the database spends time waiting on: IO events point at storage or cache misses, Lock events at blocking between sessions, LWLock at internal contention, and LSN is Aurora's storage-durability wait. Background-worker and client-idle waits are already excluded by the collector, so every row here is real work. Note this is a separate tool from get_wait_stats, which covers SQL Server: PostgreSQL has a two-level type/event taxonomy, no signal-wait concept, and reports in microseconds, so the two cannot share one result shape.")] + public static async Task GetPgWaitStats( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, + [Description("Maximum rows to return. Default 20.")] int limit = 20) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + validation = McpHelpers.ValidateTop(limit); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgWaitReader.GetPgWaitStatsAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit); + + /* An empty result is genuinely ambiguous here in a way it is not for SQL Server: it means + either no data in the window, or that this server is not a PostgreSQL target at all. Say + so, rather than letting a caller read "no waits" as "no waiting". */ + if (rows.Count == 0) + { + return McpHelpers.Status( + "unavailable", + "No PostgreSQL wait data for this server and window. If this server is SQL Server, " + + "use get_wait_stats instead; if it is PostgreSQL but not Aurora, cumulative wait " + + "counters are not available (core PostgreSQL does not provide them)."); + } + + var totalWaitMs = rows.Sum(r => r.TotalWaitTimeMs); + + /* No Take(limit) — the SQL now applies the cap, so the rows returned ARE the rows asked for. */ + var result = rows.Select(r => new + { + wait_type = r.WaitType, + wait_event = r.WaitEvent, + total_wait_time_ms = Math.Round(r.TotalWaitTimeMs, 1), + waits = r.TotalWaits, + avg_wait_time_ms = Math.Round(r.AvgWaitTimeMs, 3), + /* Share of the window's total wait time. The absolute figure alone does not say whether + an event is the story or a rounding error. */ + pct_of_total_wait = totalWaitMs > 0 ? Math.Round(r.TotalWaitTimeMs / totalWaitMs * 100, 1) : 0, + }); + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + total_wait_time_ms = Math.Round(totalWaitMs, 1), + waits = result, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL wait stats failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWraparoundTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWraparoundTools.cs new file mode 100644 index 000000000..1c22cae69 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWraparoundTools.cs @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for PostgreSQL freeze headroom, paired with the pg_wraparound_stats collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgWraparoundTools +{ + /// + /// Severity from the documented escalation ladder rather than from a round number. Each boundary is + /// a real PostgreSQL behaviour change, which is what makes the label actionable instead of decorative. + /// + internal static string Classify( + double pctTowardWraparound, + double pctTowardEmergencyVacuum, + double pctTowardMultixactWraparound = 0, + double pctTowardMultixactEmergency = 0) + { + /* BOTH counters, graded on the same ladder and the worse label winning. This used to take the XID + percentage only, so a database at 80% on MultiXacts and 3% on XIDs was labelled "ok" — while the + tool's own description promises "a server can look fine on transaction IDs and be in trouble on + MultiXacts". MultiXact exhaustion stops writes exactly as XID exhaustion does, and is burned much + faster by SELECT FOR UPDATE and foreign-key-heavy workloads, which is precisely the workload that + gets there first. */ + var xid = Grade(pctTowardWraparound, pctTowardEmergencyVacuum); + var multi = Grade(pctTowardMultixactWraparound, pctTowardMultixactEmergency); + + /* Rank by how bad the label is, not by which counter it came from. */ + var worst = Rank(multi) > Rank(xid) ? multi : xid; + + /* Name the counter when MultiXacts are the reason, because the remedy differs and the reader would + otherwise go looking at transaction IDs. */ + return worst != "ok" && Rank(multi) > Rank(xid) ? worst + "_multixact" : worst; + } + + private static string Grade(double pctTowardWraparound, double pctTowardEmergencyVacuum) => + pctTowardWraparound switch + { + /* PostgreSQL starts its own "must be vacuumed within N transactions" warnings at + wrapLimit - 100M for BOTH counters (varsup.c SetTransactionIdLimit, multixact.c + SetMultiXactIdLimit) = ~95.3% of 2^31 — an earlier draft said 98% ("~40M left"), + which put this rung ~57M ids AFTER the server began warning. The tool should never + grade calmer than the engine. */ + >= 95.3 => "critical_wraparound_imminent", + /* vacuum_failsafe_age (1.6B) — cost limits and index cleanup are abandoned to catch up. */ + >= 74.5 => "critical_failsafe_range", + >= 50.0 => "warning", + _ => pctTowardEmergencyVacuum >= 100.0 ? "info_anti_wraparound_vacuum_expected" : "ok", + }; + + private static int Rank(string label) => label switch + { + "critical_wraparound_imminent" => 4, + "critical_failsafe_range" => 3, + "warning" => 2, + "info_anti_wraparound_vacuum_expected" => 1, + _ => 0, + }; + + [McpServerTool(Name = "get_pg_wraparound_risk"), Description("Gets PostgreSQL transaction ID and MultiXact ID freeze headroom per database - how close the server is to a write outage. This is the highest-consequence PostgreSQL signal and has no SQL Server equivalent. PostgreSQL transaction IDs are 32-bit and wrap; if the oldest unfrozen ID gets too old the server stops accepting new write transactions entirely, and no failover helps because every replica shares the condition. Reports two independent counters, because MultiXact IDs exhaust separately and are burned much faster by SELECT FOR UPDATE and foreign-key-heavy workloads - a server can look fine on transaction IDs and be in trouble on MultiXacts. Also reports whether autovacuum is winning: compare the current age against the window peak. Works on any PostgreSQL target, not only Aurora.")] + public static async Task GetPgWraparoundRisk( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze, used for the peak comparison. Default 24.")] int hours_back = 24) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgWraparoundReader.GetPgWraparoundAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); + + if (rows.Count == 0) + { + return McpHelpers.Status( + "unavailable", + "No PostgreSQL freeze-headroom data for this server and window. This collector runs on " + + "any PostgreSQL target, so an empty result means the server is SQL Server, or " + + "pg_wraparound_stats has not collected yet."); + } + + var databases = rows + .Select(r => new + { + database_name = r.DatabaseName, + severity = Classify( + r.PctTowardWraparound, + r.PctTowardEmergencyVacuum, + r.PctTowardMultixactWraparound, + r.PctTowardMultixactEmergency), + measured_at = r.MeasuredAt, + /* Transaction IDs. */ + frozen_xid_age = r.FrozenXidAge, + xids_remaining = r.XidsRemaining, + pct_toward_wraparound = Math.Round(r.PctTowardWraparound, 2), + pct_toward_emergency_vacuum = Math.Round(r.PctTowardEmergencyVacuum, 1), + autovacuum_freeze_max_age = r.AutovacuumFreezeMaxAge, + /* MultiXacts - the independently fatal second counter. */ + min_multixid_age = r.MinMultiXidAge, + multixids_remaining = r.MultiXidsRemaining, + pct_toward_multixact_wraparound = Math.Round(r.PctTowardMultixactWraparound, 2), + pct_toward_multixact_emergency = Math.Round(r.PctTowardMultixactEmergency, 1), + /* Is autovacuum winning? A current age below the window peak means freezing has + clawed age back at least once — the healthy sawtooth. Equal to the peak means it + has only ever climbed within this window, which is the shape that ends badly. */ + window_peak_frozen_xid_age = r.WindowPeakFrozenXidAge, + freezing_is_keeping_up = r.FrozenXidAge < r.WindowPeakFrozenXidAge, + allows_connections = r.AllowsConnections, + }) + .OrderByDescending(d => Math.Max(d.pct_toward_wraparound, d.pct_toward_multixact_wraparound)) + .ToList(); + + var worst = databases[0]; + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + /* Fleet-triage summary first: the worst database is the server's real state, since one + database hitting the wall stops writes for the whole instance. */ + worst_database = worst.database_name, + worst_severity = worst.severity, + worst_pct_toward_wraparound = worst.pct_toward_wraparound, + thresholds = new + { + anti_wraparound_vacuum_forced_at_pct_of_freeze_max_age = 100, + failsafe_engages_around_pct = 74.5, + server_warnings_begin_around_pct = 95.3, + writes_stop_at_pct = 99.86, + }, + databases, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading PostgreSQL freeze headroom failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs new file mode 100644 index 000000000..f33499358 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The MCP surface for xmin-horizon attribution, paired with the pg_xmin_horizon collector. +/// +[McpServerToolType] +public sealed class DarlingMcpPgXminTools +{ + /// + /// The remedy for a source, which is the whole reason attribution matters: the four causes look + /// identical from the symptom side and each needs a different action. + /// + internal static string RemedyFor(string source) => source switch + { + "session" => + "A backend is holding an old snapshot. Find it in pg_stat_activity by the holder pid. " + + "'idle in transaction' means the application opened a transaction and stopped using it — fix " + + "the client, or bound it with idle_in_transaction_session_timeout. A genuinely long-running " + + "query is a different problem: make it faster rather than killing it blindly.", + "replication_slot" => + "A replication slot is retaining an old xmin. If it is inactive and nothing consumes it, it is " + + "abandoned and should be dropped — an inactive slot also retains WAL without limit by default " + + "and can fill the volume. Common orphan sources: a removed CDC task, a finished blue/green " + + "deployment, or a failed major-version upgrade.", + "replication_slot_catalog" => + "A logical decoding slot is holding catalog_xmin, which blocks catalog cleanup specifically. " + + "Check whether its consumer is still running and keeping up; a stalled logical subscriber " + + "produces exactly this.", + "standby_feedback" => + "A standby with hot_standby_feedback=on is reporting its xmin back, so a long-running query on " + + "the REPLICA is preventing cleanup on the primary. That trade is deliberate — it stops the " + + "replica's queries being cancelled — so the fix is usually the replica query, not the setting. " + + "Note this is expected to be absent on Aurora, whose replicas share the storage volume.", + "prepared_transaction" => + "An orphaned prepared (two-phase) transaction. These survive disconnects and restarts and hold " + + "their snapshot until explicitly resolved: COMMIT PREPARED or ROLLBACK PREPARED by gid. Almost " + + "always a distributed transaction coordinator that failed mid-protocol.", + _ => "Unrecognized holder source.", + }; + + [McpServerTool(Name = "get_pg_xmin_horizon"), Description("Gets what is holding back the PostgreSQL xmin horizon, attributed by cause. Use this whenever dead tuples or table bloat are growing while autovacuum appears to be running normally - that symptom has four unrelated causes which look identical from the outside, and each needs a completely different fix: a long-running or idle-in-transaction session, an abandoned replication slot, a logical slot holding catalog_xmin, a standby feeding back its xmin, or an orphaned prepared transaction. Reports the oldest holder for each source, which one is currently winning, and how persistent each has been across the window, so a chronic holder can be told apart from a query that merely ran long. Also relevant to wraparound risk: a pinned horizon blocks freezing, so an unattended holder here is an upstream cause of the risk get_pg_wraparound_risk measures. Works on any PostgreSQL target.")] + public static async Task GetPgXminHorizon( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to analyze, used for the persistence figures. Default 24.")] int hours_back = 24) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + var validation = McpHelpers.ValidateHoursBack(hours_back); + if (validation != null) return validation; + + try + { + var now = DateTime.UtcNow; + var rows = await DarlingPgXminReader.GetPgXminHorizonAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); + + /* Nothing holding the horizon is the HEALTHY answer, and saying so plainly matters more here + than for most tools: an operator arrives at this tool BECAUSE bloat is growing, so "no + holder" is a real finding that redirects the investigation rather than a dead end. */ + if (rows.Count == 0) + { + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "no_holder", + finding = "Nothing is holding back the xmin horizon in this window. Vacuum is free to " + + "reclaim dead rows, so bloat growth has a different cause — look at whether " + + "autovacuum is being triggered at all (per-table thresholds and dead-tuple " + + "counts) rather than at whether it is being blocked.", + }, McpHelpers.JsonOptions); + } + + var holders = rows.Select(r => new + { + source = r.Source, + is_currently_winning = r.IsWinner, + xmin_age = r.XminAge, + holder = r.Holder, + detail = r.Detail, + measured_at = r.MeasuredAt, + peak_xmin_age = r.PeakXminAge, + /* Persistence, not just presence. A source that won nearly every sample is a standing + problem someone must own; one that won twice was a query that ran long and finished. */ + samples_as_winner = r.SamplesAsWinner, + samples = r.Samples, + pct_of_window_winning = r.Samples > 0 + ? Math.Round((double)r.SamplesAsWinner / r.Samples * 100, 1) + : 0, + remedy = RemedyFor(r.Source), + }).ToList(); + + var winner = holders.FirstOrDefault(h => h.is_currently_winning) ?? holders[0]; + + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "holder_present", + /* Lead with the actionable pair: which cause, and what to do about that cause. */ + winning_source = winner.source, + winning_holder = winner.holder, + winning_xmin_age = winner.xmin_age, + recommended_action = winner.remedy, + holders, + }, McpHelpers.JsonOptions); + } + catch (Exception ex) + { + return McpHelpers.Status("error", $"Reading the PostgreSQL xmin horizon failed: {ex.Message}"); + } + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs index a666cafac..abf07aca3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs @@ -74,14 +74,18 @@ private static Task DefaultProbeAsync(MonitoredServer ser DarlingServerConnector.ProbeAsync(server, null, cancellationToken); [McpServerTool(Name = "add_servers"), Description( - "Adds one or more SQL Servers to the fleet the Darling service monitors — BULK onboarding: pass a JSON " + + "Adds one or more database servers to the fleet the Darling service monitors — BULK onboarding: pass a JSON " + "ARRAY of server objects and each is validated, connection-tested, and (if new and reachable) saved to the " + "central monitoring store, which the running service picks up within one collection sweep (no restart). " + "Each object: host (REQUIRED); display_name (optional, defaults to host); database (optional — set it only " + - "to monitor a single database, e.g. one Azure SQL Database); auth (\"Windows\" for integrated security or " + - "\"SQL\" for a SQL login — default \"Windows\"); username + password (REQUIRED for \"SQL\" auth, ignored for " + - "\"Windows\"); encrypt_mode (\"Optional\"|\"Mandatory\"|\"Strict\", default \"Mandatory\"); " + - "trust_server_certificate (bool, default false — set true to accept a self-signed server cert); " + + "to monitor a single database, e.g. one Azure SQL Database); engine (\"sqlserver\" default, or \"postgres\" " + + "for PostgreSQL / Amazon Aurora PostgreSQL); auth (\"Windows\" for integrated security or " + + "\"SQL\" for a SQL login — default \"Windows\"; a PostgreSQL target REQUIRES \"SQL\"); username + password " + + "(REQUIRED for \"SQL\" auth, ignored for " + + "\"Windows\"); port (optional, PostgreSQL only — omit for 5432); " + + "encrypt_mode (\"Optional\"|\"Mandatory\"|\"Strict\", default \"Mandatory\"); " + + "trust_server_certificate (bool, default false — set true to accept a self-signed server cert, and " + + "typically REQUIRED for Aurora, which presents an RDS CA a stock trust store does not know); " + "read_only_intent (bool, default false); multi_subnet_failover (bool, default false). Servers are processed " + "IN ORDER, one at a time. A case-variant or exact duplicate of an already-monitored server (or an earlier " + "entry in the same array) is skipped as status \"duplicate\". A server that fails to connect is recorded as " + @@ -89,11 +93,13 @@ private static Task DefaultProbeAsync(MonitoredServer ser "Principal / Managed Identity auth is rejected (status \"invalid\") — the service connects with Windows or " + "SQL authentication only. A SQL password is encrypted at rest (DPAPI, the service identity) and is never " + "returned. Returns {added:N, skipped:N, failed:N, results:[{server, status:\"added\"|\"duplicate\"|" + - "\"connection_failed\"|\"invalid\", detail}]}. NOTE: the password travels to this endpoint in the request; " + + "\"connection_failed\"|\"invalid\", detail}]}, where an added server's detail reports what the probe found " + + "— for a PostgreSQL target that includes writer-vs-reader, Aurora-vs-not, and how many of the PostgreSQL " + + "collectors apply to it. NOTE: the password travels to this endpoint in the request; " + "on a LAN use the documented TLS reverse proxy.")] public static Task AddServers( NpgsqlDataSource postgres, - [Description("A JSON ARRAY of server objects to add (see the tool description for the per-object fields), e.g. [{\"host\":\"sql01\",\"auth\":\"SQL\",\"username\":\"monitor\",\"password\":\"...\",\"encrypt_mode\":\"Mandatory\",\"trust_server_certificate\":true},{\"host\":\"sql02\"}].")] string servers_json) => + [Description("A JSON ARRAY of server objects to add (see the tool description for the per-object fields), e.g. [{\"host\":\"sql01\",\"auth\":\"SQL\",\"username\":\"monitor\",\"password\":\"...\",\"encrypt_mode\":\"Mandatory\",\"trust_server_certificate\":true},{\"host\":\"aurora.cluster-abc.us-east-1.rds.amazonaws.com\",\"engine\":\"postgres\",\"auth\":\"SQL\",\"username\":\"darling_monitor\",\"password\":\"...\",\"trust_server_certificate\":true}].")] string servers_json) => AddServersAsync(postgres, servers_json, DefaultProbeAsync, CancellationToken.None); /// The testable core of add_servers: validates + dedupes + probes (through the injected @@ -127,6 +133,16 @@ a duplicate (of an existing server OR an earlier entry in this batch, first occu var (ready, duplicates) = PartitionDuplicates(entries, existingKeys); results.AddRange(duplicates); + /* #2280: the identities claimed so far — the store's, plus every entry this batch is about to add. + The gate above compares DECLARED identities; the check inside the loop compares each entry's ACTUAL + database (what the server just told the probe) against this set, which is what catches two + registrations resolving to one database while claiming different ones. */ + var claimed = new HashSet(existingKeys, StringComparer.OrdinalIgnoreCase); + foreach (var entry in ready) + { + claimed.Add(entry.StorageKey); + } + foreach (var entry in ready) { /* Validate the connection IN-PROCESS (the service holds the network path + credentials). A failure @@ -141,6 +157,22 @@ is recorded and the batch CONTINUES — one unreachable server never aborts the continue; } + /* #2280: the probe just asked the server which database it actually reached. If that is a + DIFFERENT database from the one this entry names, and some other registration already claims + that one, then adding this would give one real database two identities and two full copies of + every collected row — the #2220 field report, prevented at the point of creation instead of + reported at every connect by #2277's tripwire. + + Compared against the ACTUAL database and only when it differs from the declared one: an entry + that names what it reached is the normal case and is already covered by the declared gate + above, so re-checking it would just re-detect that gate's own decision. */ + var collision = ActualIdentityCollision(entry, probeResult.ConnectedDatabase, claimed); + if (collision is not null) + { + results.Add(new ServerResult(entry.Order, entry.DisplayName, "collides", collision)); + continue; + } + /* DPAPI-encrypt the SQL password for storage (the service identity encrypts here and decrypts it during collection, so it round-trips); Windows-auth servers store no secret. The plaintext never leaves this method — it is not logged, not echoed in a result. */ @@ -353,6 +385,29 @@ private static (ParsedServerEntry? Entry, ServerResult? Result) ParseEntry(int i return (null, Invalid(msError)); } + var (engine, engineError) = ResolveEngine(TryGetString(obj, "engine")); + if (engineError != null) + { + return (null, Invalid(engineError)); + } + + /* The same rule DarlingConfig.Validate enforces for a file entry, enforced here so the two onboarding + paths cannot disagree: PostgreSQL has no integrated-auth path, and defaulting auth to Windows means + an entry that just says {"host": ..., "engine": "postgres"} would otherwise be accepted and then + fail at every connect. */ + if (engine != EngineSqlServer && storeAuth != ServerStoreAuth.Sql) + { + return (null, Invalid( + "a PostgreSQL target requires auth \"SQL\" with a username and password " + + "(integrated/Kerberos auth is not supported for PostgreSQL targets).")); + } + + var (port, portError) = ResolvePort(obj); + if (portError != null) + { + return (null, Invalid(portError)); + } + var probeConfig = new MonitoredServer { Name = displayName, @@ -367,12 +422,70 @@ private static (ParsedServerEntry? Entry, ServerResult? Result) ParseEntry(int i TrustServerCertificate = trustCert, ReadOnlyIntent = readOnlyIntent, MultiSubnetFailover = multiSubnet, + Engine = engine, + Port = port, }; - var storageKey = ServerIdHelper.BuildStorageName(host, database, readOnlyIntent); + /* #2218: the FULL identity, matching what the store derives — engine and port included, so a PostgreSQL + entry does not collide with a SQL Server one on the same host. */ + var storageKey = ServerIdHelper.BuildStorageName(host, database, readOnlyIntent, engine, port); return (new ParsedServerEntry(index, displayName, storageKey, probeConfig, plaintextPassword), null); } + /// + /// The #2280 check: why this entry must not be added, or null when it may be. + /// + /// Compares the identity this entry would have if keyed on the database the server ACTUALLY reached + /// against the identities already claimed. Only fires when the actual database DIFFERS from the declared one + /// — an entry that reached what it named is the ordinary case and the declared gate has already ruled on it, + /// so re-checking would only re-detect that gate's decision under a more confusing name. + /// + /// Silent when the probe did not report a database (a stub probe, or a target that returned + /// none): unknown is not the same as colliding, and refusing on an absent value would block registrations + /// for a reason nobody could act on. + /// + /// Keyed on the FULL identity, not on (host, database). A read-only-intent registration + /// alongside a read-write one for the same database is legitimate and read_only_intent is part of the + /// identity, so comparing without it would refuse a valid pair. Same for engine and port after #2218. + /// + /// Note the asymmetry this cannot see: existing rows record only the database they DECLARE, so this + /// catches "the new one lands where an existing one lives" and not "both mis-resolve to a database neither + /// names". Closing that needs the actual database persisted per row; #2277's tripwire reports it at connect + /// for both in the meantime, which is why this is a guard and not the whole answer. + /// + internal static string? ActualIdentityCollision( + ParsedServerEntry entry, string? connectedDatabase, ISet claimedKeys) + { + if (entry is null || claimedKeys is null || string.IsNullOrWhiteSpace(connectedDatabase)) + { + return null; + } + + var declared = entry.ProbeConfig.Database; + if (!string.IsNullOrWhiteSpace(declared) + && string.Equals(declared.Trim(), connectedDatabase.Trim(), StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var actualKey = ServerIdHelper.BuildStorageName( + entry.ProbeConfig.Host, connectedDatabase.Trim(), entry.ProbeConfig.ReadOnlyIntent, + entry.ProbeConfig.Engine, entry.ProbeConfig.Port); + + if (string.Equals(actualKey, entry.StorageKey, StringComparison.OrdinalIgnoreCase) + || !claimedKeys.Contains(actualKey)) + { + return null; + } + + var declaredText = string.IsNullOrWhiteSpace(declared) ? "no database" : $"database '{declared}'"; + return $"Not added: this registration names {declaredText} but its connection lands in " + + $"'{connectedDatabase.Trim()}', which another monitored server already covers. Adding it would " + + "store that one database's history under two identities and alert twice for every incident. " + + "Point it at the database you meant (check Initial Catalog), or monitor the existing " + + "registration instead."; + } + /// /// PURE dedupe partition — the case-folded gate seeded with the /// existing store keys, first-occurrence-wins within the batch (the #1549 idiom). Returns the Ready @@ -405,8 +518,14 @@ internal static (List Ready, List Duplicates) P /* ─────────────────────────────── store I/O ─────────────────────────────── */ /// Reads the identity fields of every existing monitored server so the dedupe gate can be seeded from - /// the authoritative set (mirrors the bulk dialog's LoadExistingKeysAsync). Non-secret columns only. - public const string ExistingServersSql = "SELECT host, database, read_only_intent FROM config_monitored_servers"; + /// the authoritative set (mirrors the bulk dialog's LoadExistingKeysAsync). Non-secret columns only. + /// + /// #2218 added engine and port to the identity, so they have to be read here too. Without + /// them the gate keys on a NARROWER identity than the product does, and a PostgreSQL instance on a host that + /// already has a SQL Server registration reads as a duplicate and is refused — a valid pair rejected because + /// the gate could not see what distinguishes them. + public const string ExistingServersSql = + "SELECT host, database, read_only_intent, engine, port FROM config_monitored_servers"; /// The INSERT — column set + shape mirrored from StoreConfigProvider.SeedMonitoredServersAsync /// (the seed authority), so a tool-added row is byte-identical to a seeded one. capture_plans and @@ -418,8 +537,8 @@ internal static (List Ready, List Duplicates) P INSERT INTO config_monitored_servers ( server_id, name, host, database, auth, username, encrypted_password, encrypt_mode, trust_server_certificate, read_only_intent, multi_subnet_failover, excluded_databases, - monthly_cost_usd, capture_plans, alert_delivery_mode_override, is_enabled, created_at, modified_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL, NULL, TRUE, $14, $14) + monthly_cost_usd, capture_plans, alert_delivery_mode_override, engine, port, is_enabled, created_at, modified_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL, NULL, $15, $16, TRUE, $14, $14) ON CONFLICT (server_id) DO NOTHING"; private static async Task> LoadExistingStorageKeysAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken) @@ -432,7 +551,9 @@ private static async Task> LoadExistingStorageKeysAsync(NpgsqlDataS var host = reader.GetString(0); var database = reader.IsDBNull(1) ? null : reader.GetString(1); var readOnlyIntent = !reader.IsDBNull(2) && reader.GetBoolean(2); - keys.Add(ServerIdHelper.BuildStorageName(host, database, readOnlyIntent)); + var engine = reader.IsDBNull(3) ? null : reader.GetString(3); + var port = reader.IsDBNull(4) ? 0 : reader.GetInt32(4); + keys.Add(ServerIdHelper.BuildStorageName(host, database, readOnlyIntent, engine, port)); } return keys; @@ -459,6 +580,8 @@ private static async Task InsertServerAsync( command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Text, Value = Array.Empty() }); // $12 command.Parameters.Add(new NpgsqlParameter { TypedValue = 0m }); // $13 command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = now }); // $14 + command.Parameters.Add(new NpgsqlParameter { TypedValue = config.Engine }); // $15 + command.Parameters.Add(new NpgsqlParameter { TypedValue = config.Port }); // $16 await command.ExecuteNonQueryAsync(cancellationToken); } @@ -508,15 +631,90 @@ private static string Aggregate(List results) }, McpHelpers.JsonOptions); } - /// The probed facts for an added server — edition / major version / msdb access, mirroring the - /// --test-connection CLI line (DarlingCliCommands.FormatProbeLine). - private static string DescribeProbe(ConnectionProbeResult probe) + /// The probed facts for an added server, from the same describer the + /// --test-connection CLI line uses (), so the + /// two cannot drift and a PostgreSQL target reads as one here too. + private static string DescribeProbe(ConnectionProbeResult probe) => + $"Connected — {DarlingServerConnector.DescribeProbeFacts(probe)}."; + + /// The canonical engine values, as written to the store. + internal const string EngineSqlServer = "sqlserver"; + internal const string EnginePostgres = "postgres"; + + /// + /// Validates the optional engine; absent → , the + /// default. + /// Deliberately STRICTER than , which resolves anything + /// unrecognized to SQL Server so that one typo in darling.json cannot take the whole fleet down. That is + /// the right call for a file read at startup and the wrong one here: onboarding is a single deliberate act, + /// and silently turning "postgress" into a SQL Server target would hand back connection_failed + /// against a Postgres port with nothing pointing at the typo. Accepts the same aliases the parser does, so + /// the two never disagree about a value they both accept. + /// + internal static (string Engine, string? Error) ResolveEngine(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + { + return (EngineSqlServer, null); + } + + return raw.Trim().ToLowerInvariant() switch + { + "sqlserver" or "sql" or "mssql" => (EngineSqlServer, null), + "postgres" or "postgresql" or "pg" or "aurora-postgresql" or "aurora" => (EnginePostgres, null), + _ => (EngineSqlServer, + "engine must be \"sqlserver\" (default) or \"postgres\" — also accepted: \"postgresql\", " + + "\"pg\", \"aurora\", \"aurora-postgresql\"."), + }; + } + + /// + /// Validates the optional port; absent or 0 → the driver's default. Consumed only by the PostgreSQL + /// connection builder (a SQL Server target carries its port in the host string), and range-checked here to + /// match DarlingConfig.Validate rather than failing later inside Npgsql. + /// + internal static (int Port, string? Error) ResolvePort(JsonObject obj) { - var edition = string.IsNullOrEmpty(probe.EngineEditionDescription) - ? DarlingServerConnector.DescribeEngineEdition(probe.EngineEdition) - : probe.EngineEditionDescription; - var msdb = probe.HasMsdbAccess ? "msdb access: yes" : "msdb access: NO (SQL Agent job data unavailable)"; - return $"Connected — SQL major version {probe.MajorVersion}, {edition}, {msdb}."; + var node = obj["port"]; + if (node is null) + { + return (0, null); + } + + if (!TryGetInt(node, out var port)) + { + return (0, "port must be a number."); + } + + if (port is not 0 && port is < 1 or > 65535) + { + return (0, $"port must be between 1 and 65535 (got {port})."); + } + + return (port, null); + } + + private static bool TryGetInt(JsonNode node, out int value) + { + try + { + value = node.GetValue(); + return true; + } + catch (Exception ex) when (ex is FormatException or InvalidOperationException or OverflowException) + { + /* A JSON string ("5432") is a plausible thing for a caller to send; accept it rather than + refusing on a type technicality. */ + if (node.GetValueKind() == JsonValueKind.String + && int.TryParse(node.GetValue(), System.Globalization.NumberStyles.Integer, + System.Globalization.CultureInfo.InvariantCulture, out value)) + { + return true; + } + + value = 0; + return false; + } } /// Validates the optional encrypt_mode ("Optional"/"Mandatory"/"Strict"); absent → the diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs index 21fd3a8df..6b5e6f8c1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs @@ -35,7 +35,7 @@ public sealed class DarlingMcpStoreMetricsTools public const int MaxDaysBack = StoreSelfMetrics.RetentionDays; [McpServerTool(Name = "get_store_metrics"), Description( - "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot: per-hypertable total size, pre/post-compression bytes and chunk count; the query-text and query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; and the whole store's size with the enabled-server count. Returns the latest snapshot per object plus a daily series over the window, with the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth / enabled servers). Use for capacity forecasting: what is driving store growth, how fast, and what adding N servers would multiply.")] + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot: per-hypertable total size, pre/post-compression bytes and chunk count; the query-text and query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; the whole store's size with the enabled-server count; and one row per TimescaleDB background job (CAGG refresh, compression, retention) with its last run duration, schedule interval, duration-vs-cadence percent, and run/failure totals — the jobs whose runtimes scale with fleet size. Returns the latest snapshot per object plus a daily series over the window, with the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth / enabled servers). Use for capacity forecasting: what is driving store growth, how fast, what adding N servers would multiply, and which background job is closest to outgrowing its own cadence.")] public static async Task GetStoreMetrics( NpgsqlDataSource postgres, [Description("Days of daily-series history. Default 30; max 400 (the series' own retention).")] int days_back = 30) @@ -101,6 +101,16 @@ way the operator already talks about it (6-36x measured on the motivating store) : (double?)null, chunk_count = r.ChunkCount, row_count = r.RowCount, + /* #2136 background_job rows only (NULL elsewhere): last run duration, the job's own + cadence, and how much of that cadence the run consumed — the ceiling-proximity + number an onboarding wave moves first. */ + last_run_duration_ms = r.LastRunDurationMs, + schedule_interval_ms = r.ScheduleIntervalMs, + duration_vs_cadence_percent = r.LastRunDurationMs is > 0 && r.ScheduleIntervalMs is > 0 + ? Math.Round(100.0 * r.LastRunDurationMs.Value / r.ScheduleIntervalMs.Value, 1) + : (double?)null, + total_runs = r.TotalRuns, + total_failures = r.TotalFailures, }), daily = daily .Where(p => p.ObjectKind != "store") @@ -119,6 +129,10 @@ way the operator already talks about it (6-36x measured on the motivating store) compressed_after_bytes = p.CompressedAfterBytes, chunk_count = p.ChunkCount, row_count = p.RowCount, + last_run_duration_ms = p.LastRunDurationMs, + schedule_interval_ms = p.ScheduleIntervalMs, + total_runs = p.TotalRuns, + total_failures = p.TotalFailures, }), }), }, McpHelpers.JsonOptions); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs index f2bb51a7f..e4217f5bd 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs @@ -36,7 +36,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpTools { - [McpServerTool(Name = "analyze_server"), Description("Runs the diagnostic inference engine against a server's collected data. Scores wait stats, blocking, memory, config, and other facts, then traverses a relationship graph to build evidence-backed stories about what's wrong and why. Anomaly detection compares the analysis window against 30-day time-bucketed baselines (hour-of-day x day-of-week) to identify deviations that are unusual for this specific time slot, not just unusual overall. Returns structured findings with severity scores, evidence chains, baseline context for anomalies, and recommended next tools to call. A remediable finding also carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed.")] + [McpServerTool(Name = "analyze_server"), Description("Runs the diagnostic inference engine against a server's collected data. Scores wait stats, blocking, memory, config, and other facts, then traverses a relationship graph to build evidence-backed stories about what's wrong and why. Anomaly detection compares the analysis window against 30-day time-bucketed baselines (hour-of-day x day-of-week) to identify deviations that are unusual for this specific time slot, not just unusual overall. Returns structured findings with severity scores, evidence chains, baseline context for anomalies, and recommended next tools to call. A remediable finding also carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose.")] public static async Task AnalyzeServer( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -125,6 +125,11 @@ surfaced with the shared miss vocabulary so callers branch on it uniformly. */ // finding has no remediable action. PRODUCE ONLY — advisory text; the read-only // MCP never executes it. remediation_command = FactRemediation.RenderCopyPasteCommand(f.Remediation), + // #2138: the machine-first projection — verdict (eligible/blockers, the future + // bot's policy gate), evidence, and split force/unforce/verify artifacts as + // named fields, so an agent never regexes the comment prose above. Null for + // non-force-plan remediations. ADVISORY like everything else here. + structured_remediation = FactRemediation.BuildStructuredRemediation(f.Remediation), // B3 Phase 3 (§6): two-sided risk DISCLOSURE for a destructive // remediation, read-only (like Lite, Darling has no Apply path; its // RCSI fields are null/0 so the inaction side shows the weak-case baseline). @@ -525,7 +530,7 @@ public static async Task AuditConfig( } } - [McpServerTool(Name = "get_analysis_findings"), Description("Gets persisted findings from previous analysis runs without running a new analysis, deduplicated to one entry per diagnostic chain (story_path_hash + incident_id) - the engine re-persists the same stories every cycle, so each entry is the chain's LATEST occurrence plus occurrence stats (occurrences, first_seen, last_seen, peak_severity) spanning the window. Use this to review historical findings or check if anything has changed since the last analysis. A remediable finding carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the finding's persisted action and including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. Set include_drilldown to also return each chain's persisted evidence rows (the specific plans/queries behind the finding, capped at write time with an explicit _truncation_note; null on findings persisted before the column existed).")] + [McpServerTool(Name = "get_analysis_findings"), Description("Gets persisted findings from previous analysis runs without running a new analysis, deduplicated to one entry per diagnostic chain (story_path_hash + incident_id) - the engine re-persists the same stories every cycle, so each entry is the chain's LATEST occurrence plus occurrence stats (occurrences, first_seen, last_seen, peak_severity) spanning the window. Use this to review historical findings or check if anything has changed since the last analysis. A remediable finding carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the finding's persisted action and including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose. Set include_drilldown to also return each chain's persisted evidence rows (the specific plans/queries behind the finding, capped at write time with an explicit _truncation_note; null on findings persisted before the column existed).")] public static async Task GetAnalysisFindings( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -643,7 +648,9 @@ stats. The store keeps every row — this shapes the read only. */ // persisted action via the shared renderer (all seven shapes + the two-sided // risk-disclosure comment header on the destructive ones). Null when the finding // has no remediable action. PRODUCE ONLY — the read-only MCP never executes it. - remediation_command = FactRemediation.RenderCopyPasteCommand(f.Remediation) + remediation_command = FactRemediation.RenderCopyPasteCommand(f.Remediation), + // #2138: the machine-first projection — see analyze_server's twin field. + structured_remediation = FactRemediation.BuildStructuredRemediation(f.Remediation) }; }) }, McpHelpers.JsonOptions); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs index 03db5dbf0..07e04a765 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs @@ -137,11 +137,18 @@ the collected names so the caller can correct it. */ new { collected_counters = collected }); } + /* sample_interval_seconds is the delta's denominator, and the only way a caller can tell a + fabricated zero from an idle interval: 0 means no delta was knowable (first sighting, + counter reset, or a gap past the policy), so delta_value = 0 with an interval of 0 must + NOT be read as "no activity". Derive rates as delta_value / sample_interval_seconds + rather than assuming a fixed cadence — fleet gaps run p50 299 s, p99 830 s, so dividing + by the configured 60 s is wrong by whatever the jitter was (#2233, #2234). */ var result = points.Select(p => new { time = p.CollectionTime.ToString("o"), value = p.Value, - delta_value = p.DeltaValue + delta_value = p.DeltaValue, + sample_interval_seconds = p.SampleIntervalSeconds }); return JsonSerializer.Serialize(new diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgAutovacuumReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgAutovacuumReader.cs new file mode 100644 index 000000000..5a49b0b8d --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgAutovacuumReader.cs @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads per-table autovacuum state (pg_autovacuum_stats), ranked by how far past its own +/// threshold each table is. +/// +public static class DarlingPgAutovacuumReader +{ + public sealed record PgAutovacuumRow( + string? DatabaseName, + string? SchemaName, + string? TableName, + DateTime MeasuredAt, + long LiveTuples, + long DeadTuples, + long VacuumThreshold, + long ModsSinceAnalyze, + long AnalyzeThreshold, + long InsertsSinceVacuum, + long InsertVacuumThreshold, + bool AutovacuumDisabled, + long TotalBytes, + DateTime? LastVacuum, + DateTime? LastAutovacuum, + DateTime? LastAnalyze, + DateTime? LastAutoanalyze, + long AutovacuumCount, + long FirstDeadTuples, + DateTime FirstSeenAt); + + /// + /// Latest state per table, joined to that table's earliest reading in the window. + /// Ordered by the RATIO of dead tuples to that table's own threshold, not by the raw count. + /// Ordering by count would put the biggest tables on top permanently — they always have the most dead + /// tuples and are usually fine — and bury the small hot table that is fifty times past its line. The + /// threshold is what makes the two comparable. + /// The earliest reading answers the follow-up question: dead tuples that are climbing mean + /// autovacuum is losing, while a flat figure at ten times the threshold usually means it is blocked or + /// switched off. Those need different fixes. + /// $1 server_id, $2/$3 window (naive UTC), $4 row limit. + /// + public const string PgAutovacuumSql = """ + WITH latest AS ( + SELECT DISTINCT ON (database_name, schema_name, table_name) + database_name, schema_name, table_name, collection_time, + live_tuples, dead_tuples, vacuum_threshold, mods_since_analyze, analyze_threshold, + inserts_since_vacuum, insert_vacuum_threshold, autovacuum_disabled, total_bytes, + last_vacuum, last_autovacuum, last_analyze, last_autoanalyze, autovacuum_count + FROM pg_autovacuum_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY database_name, schema_name, table_name, collection_time DESC + ), + earliest AS ( + SELECT DISTINCT ON (database_name, schema_name, table_name) + database_name, schema_name, table_name, + dead_tuples AS first_dead_tuples, + collection_time AS first_seen_at + FROM pg_autovacuum_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY database_name, schema_name, table_name, collection_time ASC + ) + SELECT + l.database_name, + l.schema_name, + l.table_name, + l.collection_time, + l.live_tuples, + l.dead_tuples, + l.vacuum_threshold, + l.mods_since_analyze, + l.analyze_threshold, + l.inserts_since_vacuum, + l.insert_vacuum_threshold, + l.autovacuum_disabled, + l.total_bytes, + l.last_vacuum, + l.last_autovacuum, + l.last_analyze, + l.last_autoanalyze, + l.autovacuum_count, + e.first_dead_tuples, + e.first_seen_at + FROM latest AS l + JOIN earliest AS e + ON e.database_name IS NOT DISTINCT FROM l.database_name + AND e.schema_name IS NOT DISTINCT FROM l.schema_name + AND e.table_name IS NOT DISTINCT FROM l.table_name + /* Ratio, not raw count — and a table with autovacuum switched off sorts to the top regardless, + because that is a configuration finding rather than a workload one. NULLIF guards the + never-analyzed case, where the threshold can be 0. + + The ratio is the WORSE of the two, dead-tuple and insert-only. Ranking on dead tuples alone buried + append-only tables at ratio 0, below the LIMIT — and those are the classic wraparound route the + collector gathers inserts_since_vacuum for in the first place: never vacuumed means relfrozenxid + never advances. A table taking 10x its insert threshold with zero dead tuples is a finding, and it + used to be invisible here. + + insert_vacuum_threshold carries -1 as the not-applicable sentinel on a major that has no + autovacuum_vacuum_insert_threshold, so the CASE keeps that out of the arithmetic rather than + producing a negative ratio. GREATEST ignores NULLs (verified on live Aurora), so a NULL dead ratio + does not swallow a real insert ratio. */ + ORDER BY + l.autovacuum_disabled DESC, + GREATEST( + l.dead_tuples::numeric / NULLIF(l.vacuum_threshold, 0), + CASE + WHEN l.insert_vacuum_threshold > 0 + THEN l.inserts_since_vacuum::numeric / l.insert_vacuum_threshold + ELSE 0 + END + ) DESC NULLS LAST, + GREATEST(l.dead_tuples, l.inserts_since_vacuum) DESC + LIMIT $4 + """; + + public static async Task> GetPgAutovacuumAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgAutovacuumSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgAutovacuumRow( + reader.IsDBNull(0) ? null : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.GetDateTime(3), + reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + reader.IsDBNull(6) ? -1 : reader.GetInt64(6), + reader.IsDBNull(7) ? 0 : reader.GetInt64(7), + reader.IsDBNull(8) ? -1 : reader.GetInt64(8), + reader.IsDBNull(9) ? -1 : reader.GetInt64(9), + reader.IsDBNull(10) ? -1 : reader.GetInt64(10), + !reader.IsDBNull(11) && reader.GetBoolean(11), + reader.IsDBNull(12) ? -1 : reader.GetInt64(12), + reader.IsDBNull(13) ? null : reader.GetDateTime(13), + reader.IsDBNull(14) ? null : reader.GetDateTime(14), + reader.IsDBNull(15) ? null : reader.GetDateTime(15), + reader.IsDBNull(16) ? null : reader.GetDateTime(16), + reader.IsDBNull(17) ? 0 : reader.GetInt64(17), + reader.IsDBNull(18) ? 0 : reader.GetInt64(18), + reader.GetDateTime(19))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgBlockingReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgBlockingReader.cs new file mode 100644 index 000000000..0a35574c6 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgBlockingReader.cs @@ -0,0 +1,607 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads blocking chains from the stored edge list (pg_blocking_edges), assembled into one row per +/// captured chain with its root blocker attributed. +/// This is where storing edges rather than a rendered tree pays for itself: the collector wrote +/// (blocked, blocking) pairs and knew nothing about chains, and the questions that actually matter — who is +/// at the root, how deep does it go, how many sessions are behind it, has this same backend been the root +/// all afternoon — are all answered here in SQL over those pairs. +/// +public static class DarlingPgBlockingReader +{ + public sealed record PgBlockingChainRow( + DateTime CapturedAt, + long RootBackendId, + int RootPid, + string[] Databases, + string? RootUsername, + string? RootApplicationName, + string? RootState, + string? RootQuery, + bool RootIsIdleInTransaction, + long RootXactDurationMs, + long RootQueryDurationMs, + int TotalVictims, + int DirectVictims, + int MaxDepth, + long WorstVictimWaitMs, + string? WorstVictimQuery, + /* NULL, not 0 or 1, when the root's own backend id did not resolve (the collector's + vanished-blocker sentinel). Recurrence is genuinely UNKNOWN there, and "seen once" is a + different claim from "cannot tell". */ + long? SamplesAsRoot, + bool QueryTextMayBeTruncated, + bool ChainMayBeTruncated); + + /// + /// One row per (capture, root blocker), with the chain behind it measured and the root's own state + /// attached. + /// + /// Roots are found by absence. A backend is a root when it blocks something and is not + /// itself blocked in the same capture. That definition is why the collector had to store the whole edge + /// set per capture rather than only the pairs someone asked about — a root cannot be recognised from one + /// edge in isolation. + /// + /// The recursion is depth-capped at 32, and that guard is not decoration. A cycle in the + /// edge set would make an uncapped recursive CTE run until it exhausted memory. Cycles are rare but + /// genuinely possible: PostgreSQL's deadlock detector resolves them, but only after + /// deadlock_timeout (1s by default), so a capture can land inside that window and record a true + /// cycle. No real chain approaches 32, so the cap costs nothing and removes the failure mode. + /// + /// samples_as_root is keyed on the synthetic backend id, not the pid, which is the + /// whole reason that column exists. It answers "has this been the same stuck backend all along, or a + /// succession of different ones that happened to reuse a pid" — and those two have different remedies. + /// A pid-keyed count cannot tell them apart and would silently merge them on a busy instance. + /// + /// Ordered worst-first, not newest-first (widest chain, then deepest, then most recent). + /// The question this read serves is "what was the worst blocking in this window", and a newest-first + /// ordering under a row limit would answer a different one — it would return the most recent captures + /// and could omit the incident entirely. + /// + /// WITH RECURSIVE, and the keyword goes on the FIRST CTE. PostgreSQL scopes + /// RECURSIVE to the whole WITH clause, not to the one CTE that needs it, so writing + /// WITH edges AS ... chain AS (... UNION ALL ... FROM chain ...) fails outright with + /// relation "chain" does not exist — a forward reference it will not resolve. It is a runtime + /// error on the first call, not a compile-time one, which is why this was found by running the text + /// against a real instance rather than by reading it. + /// + /// $1 server_id, $2/$3 window (naive UTC), $4 row limit. + /// + public const string PgBlockingChainsSql = """ + WITH RECURSIVE edges AS ( + SELECT + collection_id, + collection_time, + blocked_pid, + blocking_pid, + blocking_backend_id, + blocked_query, + blocked_query_duration_ms, + blocking_username, + blocking_application_name, + blocking_state, + blocking_query, + blocking_is_idle_in_transaction, + blocking_xact_duration_ms, + blocking_query_duration_ms, + database_name, + query_text_may_be_truncated + FROM pg_blocking_edges + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ), + roots AS ( + SELECT DISTINCT + e.collection_id, + e.collection_time, + e.blocking_pid, + e.blocking_backend_id + FROM edges AS e + WHERE NOT EXISTS ( + SELECT 1 + FROM edges AS upstream + WHERE upstream.collection_id = e.collection_id + AND upstream.blocked_pid = e.blocking_pid + ) + ), + chain AS ( + SELECT + r.collection_id, + r.blocking_pid AS root_pid, + e.blocked_pid, + e.blocked_query, + e.blocked_query_duration_ms, + 1 AS depth, + ARRAY[r.blocking_pid, e.blocked_pid] AS visited + FROM roots AS r + JOIN edges AS e + ON e.collection_id = r.collection_id + AND e.blocking_pid = r.blocking_pid + + UNION ALL + + SELECT + c.collection_id, + c.root_pid, + e.blocked_pid, + e.blocked_query, + e.blocked_query_duration_ms, + c.depth + 1, + c.visited || e.blocked_pid + FROM chain AS c + JOIN edges AS e + ON e.collection_id = c.collection_id + AND e.blocking_pid = c.blocked_pid + WHERE c.depth < 32 + /* Never revisit a backend already on this walk. Without it a cycle hanging off an otherwise + legitimate root is walked until the depth cap: root A blocks B while B/C/D form a cycle among + themselves, B is correctly excluded from roots (it IS blocked) but A still qualifies, and the + walk goes B -> C -> D -> B -> ... to 32. The cap stops the runaway but chain_stats then reports + max_depth = 32 and a worst victim drawn from repeated revisits of the same three backends, + which is indistinguishable from a genuine 32-deep chain. With the guard the counts are the + DISTINCT set, and the cycle itself is reported by PgBlockingCyclesSql instead. */ + AND e.blocked_pid <> ALL(c.visited) + ), + chain_stats AS ( + SELECT + collection_id, + root_pid, + count(DISTINCT blocked_pid)::int AS total_victims, + count(DISTINCT blocked_pid) FILTER (WHERE depth = 1)::int AS direct_victims, + max(depth)::int AS max_depth, + max(coalesce(blocked_query_duration_ms, -1)) AS worst_victim_wait_ms + FROM chain + GROUP BY collection_id, root_pid + ), + worst_victim AS ( + SELECT DISTINCT ON (collection_id, root_pid) + collection_id, + root_pid, + blocked_query + FROM chain + ORDER BY collection_id, root_pid, coalesce(blocked_query_duration_ms, -1) DESC + ), + /* Split deliberately in two. DISTINCT ON picks ONE of the root's edges, which is correct only for + columns that are constant per BACKEND — username, application, state, query, the durations all + come from the same blocker row whichever edge is chosen. It is NOT correct for the two columns + the collector computes per EDGE from both sides: database_name is + coalesce(blocked.datname, blocker.datname) and query_text_may_be_truncated is an OR across both + queries. Taking those from an arbitrary edge attributes a victim's truncated text, or a victim's + database, to the root. They are aggregated over all the root's edges instead. */ + root_detail AS ( + SELECT DISTINCT ON (e.collection_id, e.blocking_pid) + e.collection_id, + e.collection_time, + e.blocking_pid, + e.blocking_backend_id, + e.blocking_username, + e.blocking_application_name, + e.blocking_state, + e.blocking_query, + e.blocking_is_idle_in_transaction, + e.blocking_xact_duration_ms, + e.blocking_query_duration_ms + FROM edges AS e + JOIN roots AS r + ON r.collection_id = e.collection_id + AND r.blocking_pid = e.blocking_pid + ORDER BY e.collection_id, e.blocking_pid, e.blocked_pid + ), + root_edge_agg AS ( + SELECT + e.collection_id, + e.blocking_pid, + array_agg(DISTINCT e.database_name) AS databases, + bool_or(e.query_text_may_be_truncated) AS query_text_may_be_truncated + FROM edges AS e + JOIN roots AS r + ON r.collection_id = e.collection_id + AND r.blocking_pid = e.blocking_pid + GROUP BY e.collection_id, e.blocking_pid + ), + recurrence AS ( + SELECT + blocking_backend_id, + count(DISTINCT collection_id) AS samples_as_root + FROM roots + /* Exclude the vanished-blocker sentinel. The collector stores + coalesce(blocker.backend_id, 0), so every root whose own row had already left + pg_stat_activity lands on id 0 — and grouping those together counts unrelated one-off + incidents in different captures as repeat appearances of one backend. That is precisely the + conflation the synthetic backend id exists to prevent, arriving through the fallback instead + of through pid reuse. Excluded rather than counted, so the final LEFT JOIN yields NULL and + the read reports recurrence as UNKNOWN rather than inventing a number. */ + WHERE blocking_backend_id <> 0 + GROUP BY blocking_backend_id + ) + /* Every output column is aliased, including the ones whose name looks obvious. The C# reader is + positional so it does not care — but an unaliased coalesce() comes back named "coalesce", and + three of them did, so a psql session debugging this query saw three identical column headings. + A query this intricate has to be readable in the tool people will actually reach for. */ + SELECT + d.collection_time AS captured_at, + d.blocking_backend_id AS root_backend_id, + d.blocking_pid AS root_pid, + a.databases AS databases, + d.blocking_username AS root_username, + d.blocking_application_name AS root_application_name, + d.blocking_state AS root_state, + d.blocking_query AS root_query, + d.blocking_is_idle_in_transaction AS root_is_idle_in_transaction, + coalesce(d.blocking_xact_duration_ms, -1) AS root_xact_duration_ms, + coalesce(d.blocking_query_duration_ms, -1) AS root_query_duration_ms, + s.total_victims AS total_victims, + s.direct_victims AS direct_victims, + s.max_depth AS max_depth, + s.worst_victim_wait_ms AS worst_victim_wait_ms, + v.blocked_query AS worst_victim_query, + c.samples_as_root AS samples_as_root, + a.query_text_may_be_truncated AS query_text_may_be_truncated, + /* #5: the depth cap must announce itself. With the revisit guard in place a max_depth of 32 is + no longer a masked cycle — it means a genuinely 32-level walk that the cap stopped, so + total_victims and the worst victim are computed over a TRUNCATED walk and read identically to + a complete one. Implausible in practice; reported anyway, because this collector's premise is + that a short answer must never pass for the whole picture. */ + (s.max_depth >= 32) AS chain_may_be_truncated + FROM root_detail AS d + JOIN chain_stats AS s + ON s.collection_id = d.collection_id + AND s.root_pid = d.blocking_pid + LEFT JOIN worst_victim AS v + ON v.collection_id = d.collection_id + AND v.root_pid = d.blocking_pid + JOIN root_edge_agg AS a + ON a.collection_id = d.collection_id + AND a.blocking_pid = d.blocking_pid + LEFT JOIN recurrence AS c + ON c.blocking_backend_id = d.blocking_backend_id + ORDER BY s.total_victims DESC, s.max_depth DESC, d.collection_time DESC + LIMIT $4 + """; + + public static async Task> GetPgBlockingChainsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgBlockingChainsSql); + command.Parameters.AddWithValue(serverId); + /* SpecifyKind(Unspecified), not the bare value. Npgsql does not reject Kind=Utc — it infers + timestamptz, and PostgreSQL then zone-shifts the window against the store's NAIVE timestamp + columns, so east of UTC the window silently slides off the data. Same convention as every + other PostgreSQL read (DarlingPgXminReader, and the alert adapter's NaiveUtcNow). */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(MapChainRow(reader)); + } + + return rows; + } + + /// + /// Maps one chain row by ORDINAL, extracted so it can be tested against a fake reader. + /// + /// Everything else about this read is pinned by asserting on the SQL TEXT, which cannot see the one + /// defect that matters here: the projection's column order and this method's ordinals are two lists that + /// must agree, and nothing makes them. Reordering root_username and root_application_name — + /// both string?, adjacent, and semantically confusable — would silently transpose them and every + /// text assertion would still pass. The same hazard on the probe/mapper pair got its own pin + /// (StoreSchemaProbe_ColumnCount_MatchesTheMapArity) and on the collector side too + /// (WritesEveryDeclaredPayloadColumn); the reader side had none, which review caught. + /// + /// The projection was edited three times while this PR was open — databases replaced a + /// scalar, samples_as_root became nullable, chain_may_be_truncated was appended — so the + /// risk was live rather than hypothetical. + /// + internal static PgBlockingChainRow MapChainRow(DbDataReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + + return new PgBlockingChainRow( + reader.GetDateTime(0), + reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + reader.IsDBNull(2) ? 0 : reader.GetInt32(2), + reader.IsDBNull(3) ? Array.Empty() : reader.GetFieldValue(3), + reader.IsDBNull(4) ? null : reader.GetString(4), + reader.IsDBNull(5) ? null : reader.GetString(5), + reader.IsDBNull(6) ? null : reader.GetString(6), + reader.IsDBNull(7) ? null : reader.GetString(7), + !reader.IsDBNull(8) && reader.GetBoolean(8), + reader.IsDBNull(9) ? -1 : reader.GetInt64(9), + reader.IsDBNull(10) ? -1 : reader.GetInt64(10), + reader.IsDBNull(11) ? 0 : reader.GetInt32(11), + reader.IsDBNull(12) ? 0 : reader.GetInt32(12), + reader.IsDBNull(13) ? 0 : reader.GetInt32(13), + reader.IsDBNull(14) ? -1 : reader.GetInt64(14), + reader.IsDBNull(15) ? null : reader.GetString(15), + reader.IsDBNull(16) ? null : reader.GetInt64(16), + !reader.IsDBNull(17) && reader.GetBoolean(17), + !reader.IsDBNull(18) && reader.GetBoolean(18)); + } + + public sealed record PgBlockingCycleRow( + DateTime CapturedAt, + int ParticipantCount, + int[] Pids, + string? DatabaseName, + string? ApplicationName, + int BlockedBehindCount, + int[] BlockedBehindPids); + + /// + /// Backends caught in a lock CYCLE — each one reachable from itself through the edge list. + /// + /// This exists because the chain read cannot report them, and finding that out was the point of + /// probing it. chains identifies a root by absence: a backend that blocks something and is not + /// itself blocked. In a cycle every participant is blocked, so there is no root, so the entire cyclic + /// component is silently dropped — 0 rows from a capture that recorded real blocking. For a collector + /// whose whole design is about never letting an empty answer mean "nothing happened", that was the one + /// place the read did exactly that. + /// + /// Rare but genuinely reachable: PostgreSQL's deadlock detector resolves cycles, but only after + /// deadlock_timeout (1s by default), and a capture can land inside that window. When it does, this + /// is the only evidence that will ever exist — the edges are stored, and the engine kills one of the + /// participants a moment later. + /// + /// Detected by reachability rather than by "the collection has no root", which would miss a cycle + /// sharing a capture with an ordinary chain. Recursion stops as soon as a walk returns to where it + /// started (at_pid <> start_pid), refuses to wander into a foreign cycle, and is + /// depth-capped besides. + /// + /// One row per CYCLE, not per capture, and the attributed names come from the cycle's own + /// edges. Both of those were wrong first time and both failed the same way — silently, with a + /// plausible number. Grouping on collection_id alone merged two independent deadlocks that landed + /// in one sample into a single bogus component; joining the edge rows on collection_id alone + /// aggregated database_name over every edge in the capture, so a cycle sharing a sample with an + /// unrelated chain reported whichever database sorted first. Each walk's members array is carried + /// specifically so the component can be canonicalised (sorted, then DISTINCT collapses the rotations one + /// per participant) and used to scope the join. + /// + /// $1 server_id, $2/$3 window (naive UTC), $4 row limit. + /// + public const string PgBlockingCyclesSql = """ + WITH RECURSIVE edges AS ( + SELECT + collection_id, + collection_time, + blocked_pid, + blocking_pid, + database_name, + blocked_application_name + FROM pg_blocking_edges + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ), + walk AS ( + SELECT + collection_id, + blocked_pid AS start_pid, + blocking_pid AS at_pid, + 1 AS depth, + ARRAY[blocked_pid] AS members + FROM edges + + UNION ALL + + SELECT + w.collection_id, + w.start_pid, + e.blocking_pid, + w.depth + 1, + w.members || e.blocked_pid + FROM walk AS w + JOIN edges AS e + ON e.collection_id = w.collection_id + AND e.blocked_pid = w.at_pid + WHERE w.depth < 32 + /* Stop the moment the walk closes on where it started — that IS the detection. */ + AND w.at_pid <> w.start_pid + /* And never wander into a FOREIGN cycle: without this, a walk that starts outside a cycle and + reaches one loops inside it to the depth cap, doing 32 rounds of work per starting edge. */ + AND (e.blocking_pid = w.start_pid OR e.blocking_pid <> ALL(w.members)) + ), + closed AS ( + /* A walk that returned to its own start. Its `members` array is exactly that cycle's + participants, which is why the array is carried at all. */ + SELECT collection_id, members + FROM walk + WHERE at_pid = start_pid + ), + components AS ( + /* Canonicalise: every participant of one cycle produces its own closed walk with the same + member SET in a rotated order, so sorting collapses them to one row per actual cycle. + DISTINCT then dedupes the rotations. + + Grouping by the component rather than by the capture is load-bearing: two independent + deadlocks landing in the same one-minute capture are two findings, and grouping on + collection_id alone merged their pids into one bogus connected component. */ + SELECT DISTINCT + collection_id, + (SELECT array_agg(m ORDER BY m) FROM unnest(members) AS m) AS members + FROM closed + ), + behind AS ( + /* Backends stuck BEHIND the cycle: blocked by a member, transitively, without being a member. + These were invisible to BOTH reads, which is the one outcome this collector's design forbids. + chains cannot see them — every cycle member is itself blocked, so no member qualifies as a + root and no root walk ever reaches their edges. And this query's own walk cannot see them + either: a walk starting at such an edge extends into the cycle but is barred from closing on + its own start, so it never lands in `closed`. A real, captured blocking relationship therefore + appeared nowhere at all. Reported here, attached to the cycle that is causing it, because + "this deadlock also has nine sessions queued behind it" is the part that decides urgency. */ + SELECT + c.collection_id, + c.members, + e.blocked_pid, + 1 AS depth, + ARRAY[e.blocked_pid] AS seen + FROM components AS c + JOIN edges AS e + ON e.collection_id = c.collection_id + AND e.blocking_pid = ANY(c.members) + WHERE e.blocked_pid <> ALL(c.members) + + UNION ALL + + SELECT + b.collection_id, + b.members, + e.blocked_pid, + b.depth + 1, + b.seen || e.blocked_pid + FROM behind AS b + JOIN edges AS e + ON e.collection_id = b.collection_id + AND e.blocking_pid = b.blocked_pid + WHERE b.depth < 32 + AND e.blocked_pid <> ALL(b.members) + AND e.blocked_pid <> ALL(b.seen) + ), + behind_stats AS ( + SELECT + collection_id, + members, + count(DISTINCT blocked_pid)::int AS blocked_behind_count, + array_agg(DISTINCT blocked_pid) AS blocked_behind_pids + FROM behind + GROUP BY collection_id, members + ) + SELECT + e.collection_time AS captured_at, + cardinality(c.members) AS participant_count, + c.members AS pids, + min(e.database_name) AS database_name, + min(e.blocked_application_name) AS application_name, + coalesce(max(b.blocked_behind_count), 0) AS blocked_behind_count, + coalesce(max(b.blocked_behind_pids), ARRAY[]::int[]) AS blocked_behind_pids + FROM components AS c + LEFT JOIN behind_stats AS b + ON b.collection_id = c.collection_id + AND b.members = c.members + JOIN edges AS e + ON e.collection_id = c.collection_id + /* Scoped to the cycle's OWN participants. Joining on collection_id alone aggregated + database_name and application_name over every edge in the capture, so a cycle in one database + sharing a sample with an ordinary chain in another reported whichever name sorted first — + pointing the reader at a database the deadlock never touched. */ + AND e.blocked_pid = ANY(c.members) + GROUP BY e.collection_id, e.collection_time, c.members + ORDER BY e.collection_time DESC + LIMIT $4 + """; + + public static async Task> GetPgBlockingCyclesAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgBlockingCyclesSql); + command.Parameters.AddWithValue(serverId); + /* SpecifyKind(Unspecified), not the bare value. Npgsql does not reject Kind=Utc — it infers + timestamptz, and PostgreSQL then zone-shifts the window against the store's NAIVE timestamp + columns, so east of UTC the window silently slides off the data. Same convention as every + other PostgreSQL read (DarlingPgXminReader, and the alert adapter's NaiveUtcNow). */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(MapCycleRow(reader)); + } + + return rows; + } + + /// Maps one cycle row by ORDINAL — same seam, same reason, see . + internal static PgBlockingCycleRow MapCycleRow(DbDataReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + + return new PgBlockingCycleRow( + reader.GetDateTime(0), + reader.IsDBNull(1) ? 0 : reader.GetInt32(1), + reader.IsDBNull(2) ? Array.Empty() : reader.GetFieldValue(2), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.IsDBNull(4) ? null : reader.GetString(4), + reader.IsDBNull(5) ? 0 : reader.GetInt32(5), + reader.IsDBNull(6) ? Array.Empty() : reader.GetFieldValue(6)); + } + + /// + /// How many captures in the window recorded any blocking at all, and how many recorded none. + /// Reported alongside the chains because the denominator is the honest part of a sampled signal. + /// "Three chains" means something different in a window of 60 captures than in a window of 4, and the + /// stored table cannot say which on its own — an absent capture and a capture that found nothing look + /// identical in a table that only holds edges. The blocking-free count comes from + /// collection_log, which records a SUCCESS with zero rows, so the two really are + /// distinguishable — but only by looking there. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgBlockingCaptureCountsSql = """ + SELECT + count(*) FILTER (WHERE l.rows_collected > 0), + count(*), + min(l.collection_time), + max(l.collection_time) + FROM collection_log AS l + WHERE l.server_id = $1 + AND l.collector_name = 'pg_blocking' + AND l.status = 'SUCCESS' + AND l.collection_time >= $2 + AND l.collection_time <= $3 + """; + + public sealed record PgBlockingCaptureCounts( + long CapturesWithBlocking, + long CapturesTotal, + DateTime? FirstCaptureAt, + DateTime? LastCaptureAt); + + public static async Task GetPgBlockingCaptureCountsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + await using var command = postgres.CreateCommand(PgBlockingCaptureCountsSql); + command.Parameters.AddWithValue(serverId); + /* SpecifyKind(Unspecified), not the bare value. Npgsql does not reject Kind=Utc — it infers + timestamptz, and PostgreSQL then zone-shifts the window against the store's NAIVE timestamp + columns, so east of UTC the window silently slides off the data. Same convention as every + other PostgreSQL read (DarlingPgXminReader, and the alert adapter's NaiveUtcNow). */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (await reader.ReadAsync(cancellationToken)) + { + return new PgBlockingCaptureCounts( + reader.IsDBNull(0) ? 0 : reader.GetInt64(0), + reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? null : reader.GetDateTime(3)); + } + + return new PgBlockingCaptureCounts(0, 0, null, null); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgIoReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgIoReader.cs new file mode 100644 index 000000000..7f375b8f4 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgIoReader.cs @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads I/O by (backend_type, object, context) from pg_io_stats, differenced across the window. +/// +public static class DarlingPgIoReader +{ + public sealed record PgIoRow( + string? BackendType, + string? ObjectType, + string? Context, + long Reads, + double ReadTimeMs, + long Hits, + long Extends, + double ExtendTimeMs, + long Evictions, + long Reuses, + long Writes, + double WriteTimeMs, + long OpBytes, + bool WriteCountersTracked, + DateTime? StatsReset); + + /// + /// Positive-difference-per-interval, summed over the window — the same rule the statement read uses, + /// for the same reason: these are cumulative counters, so a plain last-minus-first goes negative + /// whenever pg_stat_reset_shared('io') runs or the server restarts. + /// The numeric columns DO come back as 0 for an untracked counter, and + /// write_counters_tracked is what makes that safe. PostgreSQL uses NULL for "this counter does + /// not apply to this combination", and on Aurora the entire write side is NULL because backends there do + /// not write data files. Two things then flatten it: GREATEST(NULL, 0) returns 0 — GREATEST + /// ignores NULLs, verified against live Aurora 17.7 — and the outer coalesce(SUM(...), 0) would do + /// it anyway. So a caller MUST read write_counters_tracked to tell "no writes happened" from + /// "writes are not measured here"; the zero alone cannot distinguish them, and averaging latency over it + /// divides by a number that was never measured. + /// This comment previously claimed NULL survived the arithmetic. It does not, and the claim was + /// worse than useless: it would have licensed someone to drop the tracked flag believing the NULLs were + /// carrying the information. The flag is not belt-and-braces — it is the only discriminator. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgIoSql = """ + WITH differenced AS ( + SELECT + backend_type, + object_type, + context, + stats_reset, + GREATEST(reads - LAG(reads) OVER series, 0) AS d_reads, + GREATEST(read_time_ms - LAG(read_time_ms) OVER series, 0) AS d_read_time_ms, + GREATEST(hits - LAG(hits) OVER series, 0) AS d_hits, + GREATEST(extends - LAG(extends) OVER series, 0) AS d_extends, + GREATEST(extend_time_ms - LAG(extend_time_ms) OVER series, 0) AS d_extend_time_ms, + GREATEST(evictions - LAG(evictions) OVER series, 0) AS d_evictions, + GREATEST(reuses - LAG(reuses) OVER series, 0) AS d_reuses, + GREATEST(writes - LAG(writes) OVER series, 0) AS d_writes, + GREATEST(write_time_ms - LAG(write_time_ms) OVER series, 0) AS d_write_time_ms, + op_bytes, + /* Whether this combination tracks writes AT ALL, as opposed to having written nothing. + Aurora reports NULL here across the board; a self-managed target reports numbers. */ + (writes IS NOT NULL) AS writes_tracked + FROM pg_io_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + WINDOW series AS ( + PARTITION BY backend_type, object_type, context + ORDER BY collection_time + ) + ) + SELECT + backend_type, + object_type, + context, + CAST(coalesce(SUM(d_reads), 0) AS bigint) AS reads, + coalesce(SUM(d_read_time_ms), 0) AS read_time_ms, + CAST(coalesce(SUM(d_hits), 0) AS bigint) AS hits, + CAST(coalesce(SUM(d_extends), 0) AS bigint) AS extends, + coalesce(SUM(d_extend_time_ms), 0) AS extend_time_ms, + CAST(coalesce(SUM(d_evictions), 0) AS bigint) AS evictions, + CAST(coalesce(SUM(d_reuses), 0) AS bigint) AS reuses, + CAST(coalesce(SUM(d_writes), 0) AS bigint) AS writes, + coalesce(SUM(d_write_time_ms), 0) AS write_time_ms, + CAST(coalesce(MAX(op_bytes), 0) AS bigint) AS op_bytes, + bool_or(writes_tracked) AS write_counters_tracked, + MAX(stats_reset) AS stats_reset + FROM differenced + GROUP BY backend_type, object_type, context + /* Anything that moved, ordered by the work that actually costs time. A combination with no + activity in the window is not a finding and would crowd out the ones that are. */ + HAVING coalesce(SUM(d_reads), 0) + coalesce(SUM(d_writes), 0) + + coalesce(SUM(d_extends), 0) + coalesce(SUM(d_hits), 0) > 0 + ORDER BY coalesce(SUM(d_read_time_ms), 0) DESC, coalesce(SUM(d_reads), 0) DESC + LIMIT $4 + """; + + public static async Task> GetPgIoAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgIoSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgIoRow( + reader.IsDBNull(0) ? null : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.GetInt64(3), + reader.GetDouble(4), + reader.GetInt64(5), + reader.GetInt64(6), + reader.GetDouble(7), + reader.GetInt64(8), + reader.GetInt64(9), + reader.GetInt64(10), + reader.GetDouble(11), + reader.GetInt64(12), + !reader.IsDBNull(13) && reader.GetBoolean(13), + reader.IsDBNull(14) ? null : reader.GetDateTime(14))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgSlotReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgSlotReader.cs new file mode 100644 index 000000000..9c93b8be1 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgSlotReader.cs @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads replication slot state (pg_replication_slots) — current state plus whether retained WAL +/// is still growing. +/// +public static class DarlingPgSlotReader +{ + public sealed record PgSlotRow( + string SlotName, + DateTime MeasuredAt, + string? SlotType, + string? Plugin, + string? DatabaseName, + bool IsActive, + string? WalStatus, + long RetainedWalBytes, + long SafeWalSizeBytes, + long XminAge, + long CatalogXminAge, + DateTime? InactiveSince, + string? InvalidationReason, + bool Conflicting, + long FirstRetainedWalBytes, + DateTime FirstSeenAt); + + /// + /// Latest state per slot, joined to that slot's earliest reading in the window. + /// The earliest reading is what makes retained WAL actionable. A slot holding 45 GB that has + /// held 45 GB all window is a consumer that is behind but keeping pace; a slot that held 2 GB an hour + /// ago and holds 45 GB now is a volume filling in front of you, and only the second one is an + /// emergency. A single current figure cannot distinguish them. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgSlotsSql = """ + WITH latest AS ( + SELECT DISTINCT ON (slot_name) + slot_name, collection_time, slot_type, plugin, database_name, is_active, + wal_status, retained_wal_bytes, safe_wal_size_bytes, xmin_age, catalog_xmin_age, + inactive_since, invalidation_reason, conflicting + FROM collect.pg_replication_slot_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY slot_name, collection_time DESC + ), + earliest AS ( + SELECT DISTINCT ON (slot_name) + slot_name, + retained_wal_bytes AS first_retained_wal_bytes, + collection_time AS first_seen_at + FROM collect.pg_replication_slot_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY slot_name, collection_time ASC + ) + SELECT + l.slot_name, + l.collection_time, + l.slot_type, + l.plugin, + l.database_name, + l.is_active, + l.wal_status, + l.retained_wal_bytes, + l.safe_wal_size_bytes, + l.xmin_age, + l.catalog_xmin_age, + l.inactive_since, + l.invalidation_reason, + l.conflicting, + e.first_retained_wal_bytes, + e.first_seen_at + FROM latest AS l + JOIN earliest AS e ON e.slot_name = l.slot_name + ORDER BY l.retained_wal_bytes DESC + """; + + public static async Task> GetPgSlotsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgSlotsSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgSlotRow( + reader.GetString(0), + reader.GetDateTime(1), + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.IsDBNull(4) ? null : reader.GetString(4), + !reader.IsDBNull(5) && reader.GetBoolean(5), + reader.IsDBNull(6) ? null : reader.GetString(6), + reader.IsDBNull(7) ? -1 : reader.GetInt64(7), + reader.IsDBNull(8) ? -1 : reader.GetInt64(8), + reader.IsDBNull(9) ? -1 : reader.GetInt64(9), + reader.IsDBNull(10) ? -1 : reader.GetInt64(10), + reader.IsDBNull(11) ? null : reader.GetDateTime(11), + reader.IsDBNull(12) ? null : reader.GetString(12), + !reader.IsDBNull(13) && reader.GetBoolean(13), + reader.IsDBNull(14) ? -1 : reader.GetInt64(14), + reader.GetDateTime(15))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgStatementReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgStatementReader.cs new file mode 100644 index 000000000..1298a32db --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgStatementReader.cs @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads the PostgreSQL statement store (pg_statement_stats) — the top-queries surface for an +/// Aurora target. +/// +public static class DarlingPgStatementReader +{ + /// + /// One row per query shape over the window. + /// QueryId rather than text: the collector does not store statement text yet, and + /// queryid is the join key anyway. It is stable within a major version but NOT across one, so + /// a consumer must not treat it as a permanent identifier. + /// + public sealed record PgStatementRow( + long QueryId, + long DatabaseId, + long Calls, + long TotalExecTimeMs, + long RowsReturned, + double MaxExecTimeMs, + long SharedBlocksHit, + long SharedBlocksRead, + long StorageBlocksRead, + long OrcacheBlocksHit, + long TempBlocksRead, + long TempBlocksWritten, + long WalBytes, + long MaxPeakMemBytes, + /* #2219: the statement text, or null when none has been captured for this queryid yet. Null is the + HONEST answer rather than a placeholder: text is refreshed hourly, so a statement first seen minutes + ago genuinely has none, and after a major-version upgrade re-keys queryid the new ids have none until + the next refresh. Distinguishing "not captured yet" from "" is what stops a caller reading an empty + string as the query. */ + string? QueryText = null); + + /// + /// Every counter is reported for the WINDOW, never as a lifetime total. Summing the cumulative + /// counters directly would multiply each query's whole history by the number of snapshots in the + /// window, so the time/call/row figures come from the stored delta columns and the block/WAL figures + /// are differenced here. + /// The block and WAL columns keep no stored deltas, and reporting them as the window's MAX — + /// which is what this read used to do — put a lifetime cumulative figure in the same row as a + /// windowed one. A consumer has no way to see that: it reads total_exec_time_ms for the last + /// hour beside shared_blks_read since the last pg_stat_statements_reset(), possibly + /// weeks earlier, and any per-call ratio it derives is nonsense. So the difference is computed at + /// read time instead, per series, which needs no new stored state. + /// GREATEST(value - LAG(value), 0) is what makes that safe. A counter reset — an + /// explicit pg_stat_statements_reset(), an eviction and re-entry, or a major-version upgrade + /// (queryid is not stable across majors) — makes one difference negative, and a plain + /// last-minus-first would report that as a large negative or silently wrong figure. Clamping each + /// interval at zero drops exactly the reset interval and keeps the rest, which is the same rule the + /// stored delta machinery applies. + /// The LAG partition is the FULL series identity (queryid, database_id, user_id, toplevel), + /// matching how the stored deltas are keyed: the same normalized statement run by another user or + /// against another database is a separate pg_stat_statements entry with its own counters, so + /// differencing across those would interleave series. The outer aggregation then rolls up to + /// (queryid, database_id), which is the grain a "top queries" answer wants. + /// max_exec_peakmem_bytes and max_exec_time_ms stay MAX — they are high-water + /// marks, not counters, and differencing a high-water mark would be meaningless. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgTopQueriesSql = """ + WITH differenced AS ( + SELECT + queryid, + database_id, + delta_calls, + delta_total_exec_time_ms, + delta_rows, + max_exec_time_ms, + max_exec_peakmem_bytes, + GREATEST(shared_blks_hit - LAG(shared_blks_hit) OVER series, 0) AS d_shared_blks_hit, + GREATEST(shared_blks_read - LAG(shared_blks_read) OVER series, 0) AS d_shared_blks_read, + GREATEST(storage_blks_read - LAG(storage_blks_read) OVER series, 0) AS d_storage_blks_read, + GREATEST(orcache_blks_hit - LAG(orcache_blks_hit) OVER series, 0) AS d_orcache_blks_hit, + GREATEST(temp_blks_read - LAG(temp_blks_read) OVER series, 0) AS d_temp_blks_read, + GREATEST(temp_blks_written - LAG(temp_blks_written) OVER series, 0) AS d_temp_blks_written, + GREATEST(wal_bytes - LAG(wal_bytes) OVER series, 0) AS d_wal_bytes + FROM pg_statement_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + WINDOW series AS ( + PARTITION BY queryid, database_id, user_id, toplevel + ORDER BY collection_time + ) + ) + SELECT + queryid, + database_id, + CAST(SUM(delta_calls) AS bigint) AS calls, + CAST(SUM(delta_total_exec_time_ms) AS bigint) AS total_exec_time_ms, + CAST(SUM(delta_rows) AS bigint) AS rows_returned, + MAX(max_exec_time_ms) AS max_exec_time_ms, + /* coalesce to 0, not to the cumulative value: a series with a single sample in the window + has no measurable interval, and its increment happened before the window began. */ + CAST(coalesce(SUM(d_shared_blks_hit), 0) AS bigint) AS shared_blks_hit, + CAST(coalesce(SUM(d_shared_blks_read), 0) AS bigint) AS shared_blks_read, + CAST(coalesce(SUM(d_storage_blks_read), 0) AS bigint) AS storage_blks_read, + CAST(coalesce(SUM(d_orcache_blks_hit), 0) AS bigint) AS orcache_blks_hit, + CAST(coalesce(SUM(d_temp_blks_read), 0) AS bigint) AS temp_blks_read, + CAST(coalesce(SUM(d_temp_blks_written), 0) AS bigint) AS temp_blks_written, + CAST(coalesce(SUM(d_wal_bytes), 0) AS bigint) AS wal_bytes, + CAST(MAX(max_exec_peakmem_bytes) AS bigint) AS max_exec_peakmem_bytes, + /* #2219: the statement text, from the (server_id, queryid) store. MAX rather than a join column + because the grain here is (queryid, database_id) while text is keyed on queryid alone — one text + per group by construction, so MAX picks it without widening the GROUP BY. LEFT JOIN, so a + queryid whose text has not been captured yet still ranks; it simply reads as null. */ + MAX(t.query_text) AS query_text + FROM differenced + LEFT JOIN collect.pg_statement_text AS t + ON t.server_id = $1 + AND t.queryid = differenced.queryid + GROUP BY queryid, database_id + HAVING SUM(delta_total_exec_time_ms) > 0 + ORDER BY SUM(delta_total_exec_time_ms) DESC + LIMIT 50 + """; + + public static async Task> GetPgTopQueriesAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgTopQueriesSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgStatementRow( + reader.GetInt64(0), + reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + reader.IsDBNull(5) ? 0 : reader.GetDouble(5), + reader.IsDBNull(6) ? 0 : reader.GetInt64(6), + reader.IsDBNull(7) ? 0 : reader.GetInt64(7), + reader.IsDBNull(8) ? 0 : reader.GetInt64(8), + reader.IsDBNull(9) ? 0 : reader.GetInt64(9), + reader.IsDBNull(10) ? 0 : reader.GetInt64(10), + reader.IsDBNull(11) ? 0 : reader.GetInt64(11), + reader.IsDBNull(12) ? 0 : reader.GetInt64(12), + reader.IsDBNull(13) ? 0 : reader.GetInt64(13), + /* #2219: null stays null — see PgStatementRow.QueryText for why an empty string would be a lie. */ + reader.IsDBNull(14) ? null : reader.GetString(14))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWaitReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWaitReader.cs new file mode 100644 index 000000000..ffc9c5eee --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWaitReader.cs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads the PostgreSQL wait store (pg_wait_stats) — the counterpart of +/// for an Aurora target. +/// Deliberately a separate reader and a separate MCP tool rather than a widened +/// get_wait_stats: the two engines' wait models do not line up column for column. Postgres has +/// a two-level type/event taxonomy where SQL Server has one flat name, carries no signal-wait concept +/// at all, and reports microseconds where SQL Server reports milliseconds. Folding them into one +/// result would mean either lying about a unit or emitting mostly-null columns. +/// +public static class DarlingPgWaitReader +{ + /// + /// One row per wait event over the window, heaviest first. + /// WaitTimeMs is converted from the stored microseconds so the field means the same + /// thing it does everywhere else in this product. The store keeps microseconds because that is + /// what Aurora reports; the read layer converts once, here, rather than leaving every consumer to + /// remember. + /// + public sealed record PgWaitRow( + string WaitType, + string WaitEvent, + long TotalWaits, + double TotalWaitTimeMs, + double AvgWaitTimeMs); + + /// + /// Aggregated over the window from the delta columns, not the raw cumulative counters — summing + /// cumulative values across snapshots would multiply the whole history by the snapshot count. + /// Unnamed events are surfaced rather than filtered: wait_type and wait_event + /// are nullable because their lookups are LEFT JOINed, and an event Aurora reports but does not + /// name is exactly the new-wait-type case an operator should see. They get a synthetic label built + /// from the numeric ids so the row is still identifiable. + /// $1 server_id, $2/$3 window (naive UTC, matching every other read in this store), $4 row cap. + /// The cap is a PARAMETER, not a literal. It was LIMIT 50 while the tool advertised a + /// caller-supplied limit and then applied it with Take(limit) — so a caller asking for more than 50 + /// silently got 50, and every request below that fetched rows only to discard them. Same shape as every + /// other read in this store. + /// + public const string PgWaitStatsSql = """ + SELECT + COALESCE(wait_type, 'unknown_type_' || wait_type_id::text) AS wait_type, + COALESCE(wait_event, 'unknown_event_' || wait_event_id::text) AS wait_event, + CAST(SUM(delta_waits) AS bigint) AS total_waits, + SUM(delta_wait_time_us) / 1000.0 AS total_wait_time_ms, + CASE + WHEN SUM(delta_waits) > 0 + THEN (SUM(delta_wait_time_us) / 1000.0) / SUM(delta_waits) + ELSE 0 + END AS avg_wait_time_ms + FROM pg_wait_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + GROUP BY + COALESCE(wait_type, 'unknown_type_' || wait_type_id::text), + COALESCE(wait_event, 'unknown_event_' || wait_event_id::text) + HAVING SUM(delta_wait_time_us) > 0 + ORDER BY SUM(delta_wait_time_us) DESC + LIMIT $4 + """; + + public static async Task> GetPgWaitStatsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgWaitStatsSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgWaitRow( + reader.GetString(0), + reader.GetString(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : Convert.ToDouble(reader.GetValue(3)), + reader.IsDBNull(4) ? 0 : Convert.ToDouble(reader.GetValue(4)))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWraparoundReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWraparoundReader.cs new file mode 100644 index 000000000..2523f344b --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgWraparoundReader.cs @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads freeze headroom (pg_wraparound_stats) — the current distance to a write outage, per +/// database. +/// +public static class DarlingPgWraparoundReader +{ + public sealed record PgWraparoundRow( + string DatabaseName, + DateTime MeasuredAt, + long FrozenXidAge, + long MinMultiXidAge, + long AutovacuumFreezeMaxAge, + long AutovacuumMultixactFreezeMaxAge, + double PctTowardEmergencyVacuum, + double PctTowardWraparound, + double PctTowardMultixactEmergency, + double PctTowardMultixactWraparound, + long XidsRemaining, + long MultiXidsRemaining, + bool AllowsConnections, + long WindowPeakFrozenXidAge, + long WindowPeakMinMultiXidAge); + + /// + /// The LATEST reading per database, plus the window's peak for each counter. + /// Deliberately not an aggregate over the window the way the rate collectors are read. Freeze + /// age is a level, not accumulated work: averaging it would blur the only number that matters, and + /// summing it would be nonsense. The current value answers "how much time is left"; the window peak + /// answers "did autovacuum actually claw it back, or has it only ever climbed" — a pair that + /// distinguishes a healthy sawtooth from a monotonic march at a glance. + /// DISTINCT ON with the ordering below takes the newest row per database. The window + /// maxima are computed before DISTINCT is applied, so they see every row in the window, not just the + /// surviving one. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgWraparoundSql = """ + SELECT DISTINCT ON (database_name) + database_name, + collection_time, + frozen_xid_age, + min_multixid_age, + autovacuum_freeze_max_age, + autovacuum_multixact_freeze_max_age, + pct_toward_emergency_vacuum, + pct_toward_wraparound, + pct_toward_multixact_emergency, + pct_toward_multixact_wraparound, + xids_remaining, + multixids_remaining, + allows_connections, + MAX(frozen_xid_age) OVER (PARTITION BY database_name) AS window_peak_frozen_xid_age, + MAX(min_multixid_age) OVER (PARTITION BY database_name) AS window_peak_min_multixid_age + FROM pg_wraparound_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY database_name, collection_time DESC + """; + + public static async Task> GetPgWraparoundAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgWraparoundSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgWraparoundRow( + reader.GetString(0), + reader.GetDateTime(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + reader.IsDBNull(6) ? 0 : reader.GetDouble(6), + reader.IsDBNull(7) ? 0 : reader.GetDouble(7), + reader.IsDBNull(8) ? 0 : reader.GetDouble(8), + reader.IsDBNull(9) ? 0 : reader.GetDouble(9), + reader.IsDBNull(10) ? 0 : reader.GetInt64(10), + reader.IsDBNull(11) ? 0 : reader.GetInt64(11), + !reader.IsDBNull(12) && reader.GetBoolean(12), + reader.IsDBNull(13) ? 0 : reader.GetInt64(13), + reader.IsDBNull(14) ? 0 : reader.GetInt64(14))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgXminReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgXminReader.cs new file mode 100644 index 000000000..9106f1815 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgXminReader.cs @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// Reads what is holding back the xmin horizon (pg_xmin_horizon), by cause. +/// +public static class DarlingPgXminReader +{ + public sealed record PgXminRow( + string Source, + DateTime MeasuredAt, + long XminAge, + string? Holder, + string? Detail, + bool IsWinner, + long PeakXminAge, + long SamplesAsWinner, + long Samples); + + /// + /// The current holder per source, joined to that source's behaviour across the window. + /// The window columns are what separate a chronic holder from a transient one, and that + /// distinction changes the response. A slot that won 58 of 60 samples is a standing problem someone + /// needs to own; a session that won twice is a query that ran long and finished. Reporting only the + /// current state would make those two look the same, and reporting only the window would hide which + /// one is holding the horizon right now. + /// Both branches read the same window, so a source present in one is present in the other and + /// the join cannot drop a row. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string PgXminHorizonSql = """ + WITH latest AS ( + SELECT DISTINCT ON (source) + source, collection_time, xmin_age, holder, detail, is_winner + FROM pg_xmin_horizon + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ORDER BY source, collection_time DESC + ), + window_stats AS ( + SELECT + source, + MAX(xmin_age) AS peak_xmin_age, + COUNT(*) FILTER (WHERE is_winner) AS samples_as_winner, + COUNT(*) AS samples + FROM pg_xmin_horizon + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + GROUP BY source + ) + SELECT + l.source, + l.collection_time, + l.xmin_age, + l.holder, + l.detail, + l.is_winner, + w.peak_xmin_age, + w.samples_as_winner, + w.samples + FROM latest AS l + JOIN window_stats AS w ON w.source = l.source + ORDER BY l.xmin_age DESC + """; + + public static async Task> GetPgXminHorizonAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + var rows = new List(); + await using var command = postgres.CreateCommand(PgXminHorizonSql); + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the BIND, per the store's naive-UTC discipline: a Kind=Utc DateTime makes + Npgsql infer timestamptz, and PostgreSQL then resolves the comparison against these naive + timestamp columns by converting THEM at the store session's TimeZone - east of UTC every fresh + row falls out of the window and the read silently returns nothing. Hidden by UTC-hosted test + stores; found by the round-2 review. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgXminRow( + reader.GetString(0), + reader.GetDateTime(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.IsDBNull(4) ? null : reader.GetString(4), + !reader.IsDBNull(5) && reader.GetBoolean(5), + reader.IsDBNull(6) ? 0 : reader.GetInt64(6), + reader.IsDBNull(7) ? 0 : reader.GetInt64(7), + reader.IsDBNull(8) ? 0 : reader.GetInt64(8))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingServerResolver.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingServerResolver.cs index a1a1cf751..601983091 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingServerResolver.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingServerResolver.cs @@ -84,6 +84,60 @@ internal static ((int ServerId, string ServerName) resolved, string? error) Reso : (resolved.Value, null); } + /// + /// The server name the ALERT path hashes into a #1140 fingerprint, for a resolved server (#2159). + /// + /// This is not the resolved name, and the difference is the whole reason this exists. + /// returns servers.server_name — the STORAGE name, + /// host[:database][:RO], which is right for every read because it is what the collectors stamp on + /// each row. But AlertFingerprint hashes the server name into the dedup key, and the alerting path + /// passes DarlingConfig.DisplayName: Name if one is set, else Host. Those two strings + /// differ whenever a server has a custom display name, and also whenever the registration names a database + /// or read-only intent — server_name carries those suffixes and DisplayName does not. + /// + /// So a reader that recomputed a fingerprint from the resolved name would agree with the alert only on + /// plain, un-renamed hosts, and return NOTHING on the rest — silently, because no match is + /// indistinguishable from no incident. Hence one helper, next to the resolution it corrects. + /// + /// Falls back to the storage name when the registry's display_name is null or blank, matching + /// the convention the fleet reader already applies to the same column. DisplayName itself is never + /// blank at alert time (it falls back to Host), so this only covers a registry row written without + /// one. + /// + public static string FingerprintNameOf(RegisteredServer server) => + string.IsNullOrWhiteSpace(server.DisplayName) ? server.ServerName : server.DisplayName!; + + /// + /// Resolves a server AND the fingerprint name for it, in one registry read — the incident readers that + /// accept a dedup_key need both, and reading the registry twice could disagree with itself. + /// + public static async Task<((int ServerId, string ServerName, string FingerprintName) resolved, string? error)> + ResolveWithFingerprintNameAsync(NpgsqlDataSource postgres, string? serverName) + { + List servers; + try + { + servers = await LoadEnabledAsync(postgres); + } + catch (Exception ex) + { + return (default, $"Could not read the servers registry from the Postgres store: {ex.Message}"); + } + + var (resolved, error) = ResolveOrError(servers, serverName); + if (error != null) + { + return (default, error); + } + + /* Re-find the row by the id just resolved rather than re-running the name match: the match is + first-wins over a partial, so a second pass is a second chance to pick a different row. */ + var row = servers.FirstOrDefault(s => s.ServerId == resolved.ServerId); + var fingerprintName = row is null ? resolved.ServerName : FingerprintNameOf(row); + + return ((resolved.ServerId, resolved.ServerName, fingerprintName), null); + } + private static (int ServerId, string ServerName)? Resolve( IReadOnlyList servers, string? serverName) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs index 394c7320e..7f6a85c3d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs @@ -38,7 +38,11 @@ SELECT DISTINCT ON (object_kind, object_name) compressed_after_bytes, chunk_count, row_count, - enabled_server_count + enabled_server_count, + last_run_duration_ms, + schedule_interval_ms, + total_runs, + total_failures FROM collect.store_metrics ORDER BY object_kind, object_name, metric_time DESC"; @@ -55,12 +59,17 @@ SELECT DISTINCT ON (object_kind, object_name, date_trunc('day', metric_time)) compressed_after_bytes, chunk_count, row_count, - enabled_server_count + enabled_server_count, + last_run_duration_ms, + schedule_interval_ms, + total_runs, + total_failures FROM collect.store_metrics WHERE metric_time >= $1 ORDER BY object_kind, object_name, date_trunc('day', metric_time), metric_time DESC"; - /// One object's newest self-metrics row. + /// One object's newest self-metrics row. The four job fields (#2136, V56) are non-null only + /// on background_job rows — every other kind leaves them NULL, as the sweep writes them. public sealed record StoreMetricRow( string ObjectKind, string ObjectName, @@ -70,9 +79,14 @@ public sealed record StoreMetricRow( long? CompressedAfterBytes, int? ChunkCount, long? RowCount, - int? EnabledServerCount); - - /// One object's settled point for one day (the day's last sample). + int? EnabledServerCount, + long? LastRunDurationMs = null, + long? ScheduleIntervalMs = null, + long? TotalRuns = null, + long? TotalFailures = null); + + /// One object's settled point for one day (the day's last sample). Job fields as on + /// . public sealed record StoreMetricDailyPoint( string ObjectKind, string ObjectName, @@ -82,7 +96,11 @@ public sealed record StoreMetricDailyPoint( long? CompressedAfterBytes, int? ChunkCount, long? RowCount, - int? EnabledServerCount); + int? EnabledServerCount, + long? LastRunDurationMs = null, + long? ScheduleIntervalMs = null, + long? TotalRuns = null, + long? TotalFailures = null); /// One day's whole-store growth: the byte delta from the previous day's settled point, and /// that delta divided by the day's enabled-server count — the number onboarding N servers multiplies. @@ -108,7 +126,11 @@ public static async Task> GetLatestAsync( reader.IsDBNull(5) ? null : reader.GetInt64(5), reader.IsDBNull(6) ? null : reader.GetInt32(6), reader.IsDBNull(7) ? null : reader.GetInt64(7), - reader.IsDBNull(8) ? null : reader.GetInt32(8))); + reader.IsDBNull(8) ? null : reader.GetInt32(8), + reader.IsDBNull(9) ? null : reader.GetInt64(9), + reader.IsDBNull(10) ? null : reader.GetInt64(10), + reader.IsDBNull(11) ? null : reader.GetInt64(11), + reader.IsDBNull(12) ? null : reader.GetInt64(12))); } return rows; @@ -133,7 +155,11 @@ public static async Task> GetDailyAsync( reader.IsDBNull(5) ? null : reader.GetInt64(5), reader.IsDBNull(6) ? null : reader.GetInt32(6), reader.IsDBNull(7) ? null : reader.GetInt64(7), - reader.IsDBNull(8) ? null : reader.GetInt32(8))); + reader.IsDBNull(8) ? null : reader.GetInt32(8), + reader.IsDBNull(9) ? null : reader.GetInt64(9), + reader.IsDBNull(10) ? null : reader.GetInt64(10), + reader.IsDBNull(11) ? null : reader.GetInt64(11), + reader.IsDBNull(12) ? null : reader.GetInt64(12))); } return rows; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs index 28d4a1f11..2780bebe6 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs @@ -46,9 +46,16 @@ public sealed record MemoryTrendPoint( DateTime CollectionTime, double TotalServerMemoryMb, double TargetServerMemoryMb, double BufferPoolMb, double PlanCacheMb); - /// One perfmon-trend point for a single counter: the counter value and per-interval delta, - /// both summed across the counter's instances at that collection (Lite's PerfmonTrendPoint). - public sealed record PerfmonTrendPoint(DateTime CollectionTime, long Value, long DeltaValue); + /// One perfmon-trend point for a single counter: the counter value, the per-interval delta, + /// and the wall-clock seconds that delta covers, all summed across the counter's instances at that + /// collection (Lite's PerfmonTrendPoint, plus the interval Lite does not carry). + /// SampleIntervalSeconds is what makes a zero readable: the collector reports 0 in + /// exactly the cases where no delta was knowable (first sighting, counter reset, gap past the + /// policy), so (0, 0) is "unknown" while (0, n) is "genuinely idle". Without it the two are the same + /// number and a fabricated zero reads as quiet (#2234). + /// Rows written before that fix carry a hard-coded 60 regardless of the real gap, so a rate + /// derived over a window spanning the upgrade is only as good as its newest rows. + public sealed record PerfmonTrendPoint(DateTime CollectionTime, long Value, long DeltaValue, long SampleIntervalSeconds); /// One file I/O-latency-trend point: average read/write latency (stall-ms / op) per collection /// for one (database, file) — the tool surfaces database_name + latencies, mirroring Lite's @@ -116,12 +123,22 @@ public static async Task> GetMemoryTrendAsync( /// viewer's perfmon read: SUM the counter's instances per collection. Postgres SUM(bigint) is /// numeric, so both SUMs CAST back to bigint for the typed GetInt64 reader (the viewer's PerfmonTrendsSql /// makes the same cast). $1 server_id, $2 counter_name, $3/$4 window (naive UTC). + /// The interval is MAX, not SUM, and that distinction is load-bearing. cntr_value and + /// delta_cntr_value are additive across a counter's instance rows — summing Transactions/sec over + /// every database is a meaningful total — but the interval is the same measured sweep gap repeated + /// once per instance, so summing it multiplies the denominator by the instance count. Measured on + /// the fleet: Transactions/sec, Log Flushes/sec and Log Bytes Flushed/sec carry a median of 12 and + /// up to 17 rows per collection_time, so a summed denominator would report rates 12-17x too LOW — + /// the same silent corruption this read exists to expose, pointed the other way. MAX also ignores a + /// 0 from an instance seen for the first time, while still reporting 0 when every instance is + /// unknown. /// public const string PerfmonTrendSql = """ SELECT collection_time, CAST(SUM(cntr_value) AS bigint) AS cntr_value, - CAST(SUM(delta_cntr_value) AS bigint) AS delta_cntr_value + CAST(SUM(delta_cntr_value) AS bigint) AS delta_cntr_value, + CAST(MAX(sample_interval_seconds) AS bigint) AS sample_interval_seconds FROM v_perfmon_stats WHERE server_id = $1 AND counter_name = $2 @@ -146,7 +163,8 @@ public static async Task> GetPerfmonTrendAsync( items.Add(new PerfmonTrendPoint( reader.GetDateTime(0), reader.IsDBNull(1) ? 0 : reader.GetInt64(1), - reader.IsDBNull(2) ? 0 : reader.GetInt64(2))); + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3))); } return items; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/MonitoredServerRegistryState.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/MonitoredServerRegistryState.cs new file mode 100644 index 000000000..b57b6f6b7 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/MonitoredServerRegistryState.cs @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The live monitored-server registry, published by the WORKER after every privileged config load and +/// observed by 's plan-fetch resolver (#2298). This is the same +/// publish/observe seam as (#1560), carrying the server set instead of the +/// control-plane knobs. +/// +/// Why it exists: the MCP host used to re-read config_monitored_servers over its own +/// least-privilege mcp-role connection, and that read selects encrypted_password — a column +/// the section-6 secret ACL deliberately SELECT-carves from mcp. The 42501 failed the WHOLE config +/// view read, so live plan fetch silently fell back to darling.json — which, on a seeded box where the +/// store is authoritative (#2254), is exactly the set of servers the file does not know about. The worker +/// already loads the same rows over its privileged connection (it must, or it could not collect), so the +/// process already holds everything the MCP host was failing to re-read. Source it from here instead. +/// +/// The security property this PRESERVES: a token-holder talking to the MCP server still cannot obtain +/// a stored credential — no MCP tool exposes this state; it feeds only the in-process +/// resolver, and the mcp database +/// role's grants are untouched. The carve was never about keeping credentials out of this process (the +/// worker holds them); it is about keeping them off the MCP wire, which they remain. +/// +/// Thread-safety: one writer (the worker's startup/reload path), readers on the MCP host's per-fetch +/// resolution. State swaps as one immutable snapshot reference, so a reader always sees a coherent set — +/// never a torn mix of old and new. Null until the worker first publishes; the reader's documented posture +/// there is the darling.json fallback (the same one it had when the store could not answer), and it heals +/// on the next resolve after the first publish because resolution reads this state per call, not once at +/// host start. +/// +public sealed class MonitoredServerRegistryState +{ + /// A coherent published registry snapshot; null until the worker first publishes. + public sealed record Snapshot(IReadOnlyList Servers, IReadOnlyDictionary ById); + + private volatile Snapshot? _current; + + /// + /// Publishes the effective monitored-server set (worker only; called at startup and on every + /// control-plane reload). First entry wins on a duplicate server id, mirroring the worker's + /// FirstOrDefault over runtimes and the resolver map this replaces. + /// + public void Publish(IReadOnlyList servers) + { + var byId = new Dictionary(); + foreach (var server in servers) + { + byId.TryAdd(server.ServerId, server); + } + + _current = new Snapshot(servers, byId); + } + + /// The latest published snapshot, or null when the worker has not published yet. + public Snapshot? Read() => _current; +} diff --git a/Darling/PerformanceMonitor.Darling.Service/MonitoredServerConnection.cs b/Darling/PerformanceMonitor.Darling.Service/MonitoredServerConnection.cs index 05dc33009..a81c894dc 100644 --- a/Darling/PerformanceMonitor.Darling.Service/MonitoredServerConnection.cs +++ b/Darling/PerformanceMonitor.Darling.Service/MonitoredServerConnection.cs @@ -8,6 +8,7 @@ using System; using Microsoft.Data.SqlClient; +using Npgsql; namespace PerformanceMonitor.Darling.Service; @@ -26,6 +27,11 @@ public static string BuildConnectionString(MonitoredServer server, string? resol throw new ArgumentNullException(nameof(server)); } + if (server.IsPostgres) + { + return BuildPostgresConnectionString(server, resolvedPassword); + } + var builder = new SqlConnectionStringBuilder { DataSource = server.Host, @@ -60,4 +66,55 @@ public static string BuildConnectionString(MonitoredServer server, string? resol return builder.ConnectionString; } + + /// + /// The PostgreSQL equivalent, keeping the same posture the SQL Server path establishes: a + /// 15-second connect budget, a 60-second command budget, TLS required unless explicitly relaxed, + /// and an application name the DBA can see in pg_stat_activity. + /// Deliberate differences from the SQL Server builder, each because the concept does not + /// exist here: there is no MARS (Npgsql multiplexes differently), no ApplicationIntent (a + /// PostgreSQL read replica is a separate endpoint, not a routing hint — point the entry at the + /// reader's own host), and no MultiSubnetFailover. + /// Integrated auth is rejected rather than silently ignored. Npgsql can do Kerberos, but a + /// Windows service account authenticating to Aurora is not a path anyone has configured here, and + /// quietly producing a connection string that cannot authenticate would fail later and less + /// clearly than failing now. + /// + private static string BuildPostgresConnectionString(MonitoredServer server, string? resolvedPassword) + { + if (!server.UsesSqlAuth) + { + throw new InvalidOperationException( + $"Server '{server.DisplayName}' is a PostgreSQL target, which requires auth \"sql\" with a username " + + "and password (integrated/Kerberos auth is not supported for PostgreSQL targets)."); + } + + var builder = new NpgsqlConnectionStringBuilder + { + Host = server.Host, + /* "postgres" is the maintenance database every cluster has. Per-database collectors + override this; the connect probe and every instance-wide view work from here. */ + Database = string.IsNullOrWhiteSpace(server.Database) ? "postgres" : server.Database, + Username = server.Username, + Password = resolvedPassword + ?? throw new InvalidOperationException($"Server '{server.DisplayName}' uses sql auth but no password was resolved."), + ApplicationName = "PerformanceMonitorDarling", + Timeout = 15, + CommandTimeout = 60, + /* Same fail-closed intent as the SQL Server path: anything but an explicit opt-out gets + TLS. TrustServerCertificate maps to VerifyFull-vs-Require rather than to disabling TLS — + Aurora presents an RDS CA that a stock trust store does not know, which is the case + TrustServerCertificate exists to cover. */ + SslMode = server.EncryptMode?.Trim().ToUpperInvariant() == "OPTIONAL" + ? SslMode.Prefer + : server.TrustServerCertificate ? SslMode.Require : SslMode.VerifyFull, + }; + + if (server.Port > 0) + { + builder.Port = server.Port; + } + + return builder.ConnectionString; + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj b/Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj index f920909ef..8a1f4f119 100644 --- a/Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj +++ b/Darling/PerformanceMonitor.Darling.Service/PerformanceMonitor.Darling.Service.csproj @@ -5,9 +5,13 @@ disable PerformanceMonitor.Darling.Service PerformanceMonitor.Darling.Service - 3.4.0 - 3.3.0.0 - 3.3.0.0 + 3.5.0 + + false Darling Data, LLC Copyright © 2026 Darling Data, LLC true @@ -17,20 +21,20 @@ - - - - + + + + - + - - + + diff --git a/Darling/PerformanceMonitor.Darling.Service/PgAlertStateStore.cs b/Darling/PerformanceMonitor.Darling.Service/PgAlertStateStore.cs index 802ba7f96..a46c90b23 100644 --- a/Darling/PerformanceMonitor.Darling.Service/PgAlertStateStore.cs +++ b/Darling/PerformanceMonitor.Darling.Service/PgAlertStateStore.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -163,10 +164,235 @@ ON CONFLICT (server_id, metric_name) DO UPDATE SET } } + /// + /// #2166: stamps the alerted state onto the database's row in config.database_state_expected, + /// the table that already holds this alert's per-database config. + /// + /// UPDATE, never upsert. An INSERT here would have to supply expected_state (NOT NULL), and + /// the only value available is the state being alerted ON — so a database first observed SUSPECT would + /// get SUSPECT written as its accepted baseline. It would then stop deviating, drop out of the deviation + /// query, be read as RECOVERED (firing a false "resolved" on a still-corrupt database), and never alert + /// again. The seed logic refuses to baseline any of + /// for exactly this reason; this write must not do behind its back what it declines to do in front. + /// + /// Nothing is lost by skipping the no-row case: a database with no baseline row was first observed + /// in an integrity state, and those states are never edge-suppressed (RepeatsAreNoise is false for them), + /// so the memory this writes is never consulted for them. The parked-database case that NEEDS the memory + /// always has a row — it was baselined when the database was still healthy. + /// + public async Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState) + { + try + { + await using var connection = await _postgres.OpenConnectionAsync(); + using var command = new NpgsqlCommand(@" +UPDATE config.database_state_expected +SET last_alerted_state = $3, + last_alerted_at = (now() AT TIME ZONE 'UTC') +WHERE server_id = $1 +AND database_name = $2", connection); + command.Parameters.AddWithValue(ParseServerKey(serverKey)); + command.Parameters.AddWithValue(databaseName); + command.Parameters.AddWithValue(effectiveState); + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + /* Same posture as the watermark writes: a failed stamp costs a duplicate alert next cycle, + never a missed one, so it logs and continues rather than failing the sweep. */ + _logger?.LogWarning("Could not record the alerted database state for {Database}: {Message}", databaseName, ex.Message); + } + } + + /// + /// #2166: forgets the alerted state when a database returns to its expected one, so the NEXT episode is + /// a fresh transition. Without this the memory is permanent: park a database OFFLINE (alerts), bring it + /// back (resolves), park it OFFLINE again weeks later — the stale last_alerted_state still reads + /// OFFLINE, the repeat is judged already-announced, and the second parking never alerts at all. Edge + /// triggering has to reset on the falling edge or it only ever fires once per database, forever. + /// + public async Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName) + { + try + { + await using var connection = await _postgres.OpenConnectionAsync(); + using var command = new NpgsqlCommand(@" +UPDATE config.database_state_expected +SET last_alerted_state = NULL, + last_alerted_at = NULL +WHERE server_id = $1 +AND database_name = $2", connection); + command.Parameters.AddWithValue(ParseServerKey(serverKey)); + command.Parameters.AddWithValue(databaseName); + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + /* A failed clear costs a MISSED alert on the next episode rather than a duplicate, so it is the + more consequential of the two failures — logged at warning with the database named, same as + its sibling, because the sweep must still finish for every other database. */ + _logger?.LogWarning("Could not clear the alerted database state for {Database}: {Message}", databaseName, ex.Message); + } + } + + /// + /// #2216: loads the per-fingerprint occurrence accounting for one server/metric from the V61 + /// config.incident_occurrences table. + /// + /// Returns an EMPTY map on any failure rather than the rows it managed to read. A partial map is + /// the worst of the three outcomes: the fingerprints that made it keep accumulating while the ones that + /// did not silently restart their totals mid-incident, so the same alert reports some counters + /// continuing and others reset with a fresh start time. Empty is at least uniform — every fingerprint + /// reads as new, totals equal the window counts, which is the documented degradation. + /// + public async Task> LoadIncidentOccurrencesAsync( + string serverKey, string metricName) + { + var states = new Dictionary(StringComparer.Ordinal); + + try + { + await using var connection = await _postgres.OpenConnectionAsync(); + using var command = new NpgsqlCommand(@" +SELECT dedup_key, total_occurrences, observed_window_count, incident_started_at, last_observed_at +FROM config.incident_occurrences +WHERE server_id = $1 +AND metric_name = $2", connection); + command.Parameters.AddWithValue(ParseServerKey(serverKey)); + command.Parameters.AddWithValue(metricName); + + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + /* The two timestamps come back Kind=Unspecified from a `timestamp` column, and they are + naive UTC by the store-wide convention. Deliberately NOT coerced to Kind=Utc: the + accumulator only ever subtracts them from its own UTC clock, which is plain arithmetic + on both Kinds, and a ToUniversalTime() "fix" here would shift every value by the host's + offset and make live incidents look stale. */ + states[reader.GetString(0)] = new IncidentOccurrenceState( + reader.GetInt64(1), + reader.GetInt32(2), + reader.GetDateTime(3), + reader.GetDateTime(4)); + } + } + catch (Exception ex) + { + _logger?.LogError("Could not load incident occurrences ({Metric}): {Message}", metricName, ex.Message); + return new Dictionary(StringComparer.Ordinal); + } + + return states; + } + + /// + /// #2216: REPLACES the persisted occurrence set for one server/metric — upserts every row in + /// and deletes the (server, metric)'s rows that are not in it. An empty map + /// clears the metric, which is how the falling edge is recorded. + /// + /// ONE transaction, because the delete and the upsert are two halves of one replacement: a delete + /// that commits without its upsert would zero live counters, and an upsert without its delete strands + /// finished fingerprints for the accumulator's staleness horizon to clean up later. The horizon exists + /// for crashes, not as cover for a half-applied write. + /// + public async Task SaveIncidentOccurrencesAsync( + string serverKey, string metricName, IReadOnlyDictionary states) + { + if (states is null) + { + return; + } + + try + { + int serverId = ParseServerKey(serverKey); + + var dedupKeys = new string[states.Count]; + var totals = new long[states.Count]; + var observed = new int[states.Count]; + var started = new DateTime[states.Count]; + var lastObserved = new DateTime[states.Count]; + + int n = 0; + foreach (var entry in states) + { + dedupKeys[n] = entry.Key; + totals[n] = entry.Value.TotalOccurrences; + observed[n] = entry.Value.ObservedWindowCount; + started[n] = Naive(entry.Value.IncidentStartedUtc); + lastObserved[n] = Naive(entry.Value.LastObservedUtc); + n++; + } + + await using var connection = await _postgres.OpenConnectionAsync(); + await using var transaction = await connection.BeginTransactionAsync(); + + /* `<> ALL` over an EMPTY array is true for every row, so the no-states case degenerates to + "delete them all" with no special-casing — the clear path and the replace path are one + statement. It relies on the array being NULL-free, since `<> ALL` over an array containing + NULL yields NULL and deletes nothing: the keys come from the accumulator's state map, which + never admits a blank or null fingerprint (it passes those incidents through unkeyed). The + explicit cast is here because the bind is the only thing telling PostgreSQL the element type, + and an inference failure on this shape is a RUNTIME error, not a compile one. */ + using (var prune = new NpgsqlCommand(@" +DELETE FROM config.incident_occurrences +WHERE server_id = $1 +AND metric_name = $2 +AND dedup_key <> ALL($3::text[])", connection, transaction)) + { + prune.Parameters.AddWithValue(serverId); + prune.Parameters.AddWithValue(metricName); + prune.Parameters.AddWithValue(dedupKeys); + await prune.ExecuteNonQueryAsync(); + } + + if (states.Count > 0) + { + using var upsert = new NpgsqlCommand(@" +INSERT INTO config.incident_occurrences + (server_id, metric_name, dedup_key, total_occurrences, observed_window_count, incident_started_at, last_observed_at) +SELECT $1, $2, d.dedup_key, d.total_occurrences, d.observed_window_count, d.incident_started_at, d.last_observed_at +FROM unnest($3::text[], $4::bigint[], $5::integer[], $6::timestamp[], $7::timestamp[]) + AS d(dedup_key, total_occurrences, observed_window_count, incident_started_at, last_observed_at) +ON CONFLICT (server_id, metric_name, dedup_key) DO UPDATE SET + total_occurrences = EXCLUDED.total_occurrences, + observed_window_count = EXCLUDED.observed_window_count, + incident_started_at = EXCLUDED.incident_started_at, + last_observed_at = EXCLUDED.last_observed_at", connection, transaction); + upsert.Parameters.AddWithValue(serverId); + upsert.Parameters.AddWithValue(metricName); + upsert.Parameters.AddWithValue(dedupKeys); + upsert.Parameters.AddWithValue(totals); + upsert.Parameters.AddWithValue(observed); + upsert.Parameters.AddWithValue(started); + upsert.Parameters.AddWithValue(lastObserved); + await upsert.ExecuteNonQueryAsync(); + } + + await transaction.CommitAsync(); + } + catch (Exception ex) + { + /* Same posture as the watermark writes: a dropped write costs accuracy on the NEXT delivery's + total (the fingerprint reads as new and restarts, with a fresh start time saying so), never + a missed or duplicated alert. The alert itself has already been decided by the gate. */ + _logger?.LogError("Could not persist incident occurrences ({Metric}): {Message}", metricName, ex.Message); + } + } + /// Naive-UTC now, Kind-Unspecified — the product's PG timestamp discipline. private static DateTime NaiveUtcNow() => DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + /// + /// Kind-stripped for a `timestamp` bind. Npgsql does NOT reject Kind=Utc on this version — it infers + /// timestamptz and PostgreSQL casts into the SERVER's zone, silently storing a value offset from every + /// other naive-UTC timestamp in the store (see the failed-job write's remarks; that quiet zone-shift + /// misled two reviewers in one night). + /// + private static DateTime Naive(DateTime value) => + DateTime.SpecifyKind(value, DateTimeKind.Unspecified); + private static int ParseServerKey(string serverKey) => int.Parse(serverKey, CultureInfo.InvariantCulture); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Program.cs b/Darling/PerformanceMonitor.Darling.Service/Program.cs index d7f259672..1300bcaf2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Program.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Program.cs @@ -211,6 +211,21 @@ the running service only verifies. Reads darling.json only (no store, no credent return await DarlingCliCommands.DisableWebAsync(configPath, Console.Out, Console.Error, CancellationToken.None); } +/* CLI verb: --add-server / --add-servers (#2256) — register monitored server(s) in the store from a JSON + array on STDIN. The store is authoritative after the first seed, so darling.json edits are ignored, and the + web surface deliberately excludes the write tools; a headless host (the field report ran Windows Server 2012, + which cannot run the Viewer) had no supported path at all. Goes through the same AddServers path the MCP tool + uses, so validation, dedupe, the connection probe, password encryption and the server_id computation are + shared rather than reimplemented. NO Windows guard here on purpose: the verb needs Windows only for a MANAGED + store credential (DPAPI), which it checks itself, so a Linux host with bring-your-own Postgres can use it. + Reads stdin rather than argv so a password never lands in the process list or shell history. */ +if (args.Length > 0 && DarlingCliCommands.IsAddServerVerb(args[0])) +{ + var addServerConfigPath = args.Length > 1 ? args[1] : null; + return await DarlingCliCommands.AddServerAsync( + addServerConfigPath, Console.In, Console.Out, Console.Error, CancellationToken.None); +} + /* CLI verb: --backfill-rollups (#1759 Phase 2) — materialize the query-acceleration rollups back over pre-existing history so the #1680 arming gate can release the held raw retention policies by itself. An OPERATOR verb, deliberately not a startup step: the gate is all-or-nothing, so a store with a year of raw @@ -338,6 +353,11 @@ diagnostic surface (see DarlingFileLoggerProvider remarks). Registered unconditi builder.Services.AddSingleton(); builder.Services.AddSingleton(); +/* #2298: the worker-published monitored-server registry the MCP host's plan-fetch resolver reads, + replacing its own mcp-role re-read of rows whose encrypted_password column that role is + deliberately denied. */ +builder.Services.AddSingleton(); + builder.Services.AddHostedService(); /* AN4: the analysis MCP tools over Streamable HTTP — registered always, self-gating on diff --git a/Darling/PerformanceMonitor.Darling.Service/QueryStoreBackfill.cs b/Darling/PerformanceMonitor.Darling.Service/QueryStoreBackfill.cs index 74c73ec2c..c45380df1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/QueryStoreBackfill.cs +++ b/Darling/PerformanceMonitor.Darling.Service/QueryStoreBackfill.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Threading; @@ -23,8 +24,11 @@ namespace PerformanceMonitor.Darling.Service; /// #2022 — Query Store phase 2 (of #1960): the newest-first backfill worker for the history the /// live path never takes. Phase 1 made the LIVE path hole-free, but two bounded windows still /// discard history by design: first contact takes only the trailing 60 minutes of a ~30-day -/// catalog, and post-outage catch-up is clamped to 24h (the #1556 incident fix) as a bounded, -/// logged hole. One mechanism fills both: +/// catalog, and post-outage catch-up is clamped to (the +/// #1556 incident fix, tightened to 1h by #2102) as a bounded, logged hole. One mechanism fills +/// both, and every slice of it windows at most +/// at a time (#2102 — the query's cost grows with window width, so an unchunked wide range on a +/// big database re-times-out forever instead of draining): /// /// The tail (first contact). The backfill ceiling is DERIVED, exactly like the live /// watermark: MIN(last_execution_time) over the rows already stored for a database. Everything at @@ -83,18 +87,27 @@ public sealed class QueryStoreBackfill private readonly ILogger? _logger; private readonly Func _capturePlans; + /* #2164: the per-database text budget override in MB, read live like _capturePlans. Backfill slices + carry the SAME nvarchar(max) query-text/plan-XML payload over the same link as a live tick, so the + operator knob has to reach here too — a knob that only bounds the tick would leave the heavier of + the two paths at the compile-time 64 MB, which is precisely the drain the knob exists to shorten. */ + private readonly Func _textBudgetMb; + public QueryStoreBackfill( NpgsqlDataSource postgres, DarlingCollectorRunner runner, CollectorDeltaCalculator deltas, ILogger? logger, - Func? capturePlans = null) + Func? capturePlans = null, + Func? textBudgetMb = null) { _postgres = postgres ?? throw new ArgumentNullException(nameof(postgres)); _runner = runner ?? throw new ArgumentNullException(nameof(runner)); _deltas = deltas ?? throw new ArgumentNullException(nameof(deltas)); _logger = logger; _capturePlans = capturePlans ?? (() => true); + /* Null provider = keep the collector's compile-time budget (tests and any non-Darling host). */ + _textBudgetMb = textBudgetMb ?? (() => 0); } /// @@ -106,11 +119,31 @@ public QueryStoreBackfill( /// public async Task RunServerSliceAsync(ServerRuntime server, CancellationToken cancellationToken) { - if (!QueryStoreCollector.Instance.AppliesTo(server.Target)) + /* The COMPOSED gate, not the definition's own AppliesTo. Query Store is a SQL Server feature and this + method opens SqlConnections, but the raw override never checks the engine — it reads + SqlMajorVersion, and CollectorTargetInfo treats 0 as "assume newest" so a PostgreSQL target (which + has no SqlMajorVersion at all) sails straight through. Latent today only because the caller happens + to be reached from a SQL-Server-shaped path; one new call site and it becomes B1 again. */ + if (!CollectorCatalog.AppliesTo(QueryStoreCollector.Instance, server.Target)) { return false; } + /* #2111 yield-to-live: a backfill slice scans the same QS internal tables the live sweep + reads, on a replica that is often MAXDOP-1 — when the live path is failing on this + server, running a slice anyway is the contention that keeps it failing. Skip the server + this tick (false = the tick is free for another server); the hole waits, live recovers, + backfill resumes. Debug, not Warning: the live failure already logs loudly every cycle, + and this is the designed response to it. */ + if (QueryStoreBackfillState.ShouldYieldToLive( + _runner.LastQueryStoreItemFailureUtc(server.ServerId), DateTime.UtcNow)) + { + _logger?.LogDebug( + "query_store backfill on '{Server}': yielding to the live path (recent live query_store failure)", + server.Config.DisplayName); + return false; + } + var state = await _runner.GetCollectorStateAsync(server.ServerId, StateCollectorName, cancellationToken); var databases = await GetCandidateDatabasesAsync(server.ServerId, cancellationToken); @@ -134,7 +167,7 @@ public async Task RunServerSliceAsync(ServerRuntime server, CancellationTo } var holeFloor = holeFrom > floorLimit ? holeFrom : floorLimit; - await RunSliceAsync(server, databaseName, holeFloor, holeTo, isHole: true, cancellationToken); + await RunCountedSliceAsync(server, databaseName, holeFloor, holeTo, isHole: true, cancellationToken); return true; } @@ -160,13 +193,40 @@ without shipping a row so the steady state never re-probes it. */ continue; } - await RunSliceAsync(server, databaseName, floorLimit, storedFloor.Value, isHole: false, cancellationToken); + await RunCountedSliceAsync(server, databaseName, floorLimit, storedFloor.Value, isHole: false, cancellationToken); return true; } return false; } + /// + /// Consecutive failed slices per server — the adaptive-shrink signal's backfill half (#2111 + /// promoted): a server whose hour-wide slices keep dying at the command timeout digs in + /// progressively narrower chunks () until one + /// fits. Reset by any completed slice; in-memory on purpose, like the live counters — a restart + /// forgetting it costs one full-width slice. Concurrent for symmetry with the Lite twin — the + /// worker is single-threaded today, but nothing pins that. + /// + private readonly ConcurrentDictionary _consecutiveSliceFailures = new(); + + /// Runs one slice with the failure accounting wrapped around it — the worker's outer + /// catch still logs the throw exactly as before. + private async Task RunCountedSliceAsync( + ServerRuntime server, string databaseName, DateTime floorUtc, DateTime ceilingUtc, bool isHole, CancellationToken cancellationToken) + { + try + { + await RunSliceAsync(server, databaseName, floorUtc, ceilingUtc, isHole, cancellationToken); + _consecutiveSliceFailures.TryRemove(server.ServerId, out _); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _consecutiveSliceFailures.AddOrUpdate(server.ServerId, 1, static (_, count) => count + 1); + throw; + } + } + /// /// One byte-budgeted, newest-first slice for one database: probe PRODUCTVERSION (the same /// version gates as the live path, so the reader ordinals cannot differ), run the backfill @@ -178,6 +238,17 @@ without shipping a row so the steady state never re-probes it. */ private async Task RunSliceAsync( ServerRuntime server, string databaseName, DateTime floorUtc, DateTime ceilingUtc, bool isHole, CancellationToken cancellationToken) { + /* #2102: one slice queries at most the top MaxSliceSpan of the remaining range. The byte + budget bounds what SHIPS, not what the query aggregates and sorts — an unchunked wide + window on a big database times out at the command timeout every tick and the range never + drains, the same row-cap-is-not-a-cost-cap flaw that wedged the live path. */ + /* #2111 adaptive shrink: after consecutive failed slices this server digs in narrower + chunks until one fits its command timeout; a completed slice resets to full width. */ + var sliceSpan = QueryStoreBackfillState.AdaptiveSpan( + QueryStoreBackfillState.MaxSliceSpan, + _consecutiveSliceFailures.TryGetValue(server.ServerId, out var recentFailures) ? recentFailures : 0); + var sliceFloor = QueryStoreBackfillState.BoundSliceFloor(floorUtc, ceilingUtc, sliceSpan); + var definition = QueryStoreCollector.Instance; var context = new CollectorContext { @@ -188,6 +259,8 @@ private async Task RunSliceAsync( Target = server.Target, ExcludedDatabases = server.Config.ExcludedDatabases?.ToArray() ?? Array.Empty(), CapturePlanXml = _capturePlans(), + /* #2164: 0 from the default provider means "no override" — the collector keeps its constant. */ + TextByteBudgetOverride = _textBudgetMb() > 0 ? _textBudgetMb() * 1024 * 1024 : null, }; var timeout = definition.CommandTimeoutSecondsOverride ?? DarlingCollectorRunner.CommandTimeoutSeconds; @@ -201,7 +274,7 @@ private async Task RunSliceAsync( needed; CurrentDatabaseName feeds ReadAsync's database attribution exactly as on the live Azure path. */ context.CurrentDatabaseName = databaseName; - var azurePlan = definition.BuildBackfillQuery(context, floorUtc, ceilingUtc); + var azurePlan = definition.BuildBackfillQuery(context, sliceFloor, ceilingUtc); using var dbConnection = await _runner.OpenAzureDatabaseConnectionAsync(server, databaseName, cancellationToken); using var dbCommand = DarlingCollectorRunner.CreateCollectorCommand(azurePlan, dbConnection, timeout); using var dbReader = await dbCommand.ExecuteReaderAsync(cancellationToken); @@ -233,7 +306,7 @@ the live Azure path. */ } } - var plan = definition.BuildBackfillPerItemQuery(databaseName, context, floorUtc, ceilingUtc); + var plan = definition.BuildBackfillPerItemQuery(databaseName, context, sliceFloor, ceilingUtc); using var command = DarlingCollectorRunner.CreateCollectorCommand(plan, sqlConnection, timeout); using var reader = await command.ExecuteReaderAsync(cancellationToken); await definition.ReadItemAsync(databaseName, reader, rows, context, cancellationToken); @@ -241,6 +314,27 @@ the live Azure path. */ if (rows.Count == 0) { + if (sliceFloor > floorUtc) + { + /* Only this CHUNK is quiet — the range below it is unexplored, so this is an + advance, not a terminal verdict (#2102). The persisted hole ceiling shrinks past + the quiet chunk; a derived-boundary tail converts its remainder to a hole record, + because MIN over stored rows cannot walk through quiet space (an empty chunk + ships nothing, so the derived ceiling would re-ask the same chunk forever). The + tail marks done in the same breath — the hole owns the rest of the dig, and the + scan services holes first. */ + await SaveStateAsync(server.ServerId, QueryStoreBackfillState.HoleKeyPrefix + databaseName, QueryStoreBackfillState.EncodeHole(floorUtc, sliceFloor), cancellationToken); + if (!isHole) + { + await SaveStateAsync(server.ServerId, QueryStoreBackfillState.DoneKeyPrefix + databaseName, DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture), cancellationToken); + } + + _logger?.LogInformation( + "query_store backfill on '{Server}' [{Database}]: quiet chunk {Floor:o}..{Ceiling:o}, continuing below ({Range}).", + server.Config.DisplayName, databaseName, sliceFloor, ceilingUtc, isHole ? "hole" : "tail"); + return; + } + /* Query Store retains nothing inside the window — the monitored catalog is shorter than the horizon (or the hole's span was never persisted at the source). Terminal for this range, and cheaper to record than to re-ask every tick. */ @@ -266,7 +360,11 @@ than the horizon (or the hole's span was never persisted at the source). Termina var boundary = context.PerItemShippedBoundary; if (isHole) { - if (boundary is null || boundary <= floorUtc) + /* A chunked slice's rows all sit at or above its own chunk floor, so a missing shipped + boundary falls back to the chunk floor rather than deleting (#2102) — deletion under + a bounded window would orphan the unexplored range below it. */ + var shippedTo = boundary ?? sliceFloor; + if (shippedTo <= floorUtc) { await _runner.DeleteCollectorStateKeyAsync(server.ServerId, StateCollectorName, QueryStoreBackfillState.HoleKeyPrefix + databaseName, cancellationToken); } @@ -274,7 +372,7 @@ than the horizon (or the hole's span was never persisted at the source). Termina { /* Shrink the ceiling to the oldest shipped row; the from-side stays at the floor we actually used (anything below it is horizon-expired either way). */ - await SaveStateAsync(server.ServerId, QueryStoreBackfillState.HoleKeyPrefix + databaseName, QueryStoreBackfillState.EncodeHole(floorUtc, boundary.Value), cancellationToken); + await SaveStateAsync(server.ServerId, QueryStoreBackfillState.HoleKeyPrefix + databaseName, QueryStoreBackfillState.EncodeHole(floorUtc, shippedTo), cancellationToken); } } else if (boundary is not null && boundary <= floorUtc) diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 3e80a23bb..b29a03f9c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -109,6 +109,14 @@ public async Task SeedIfEmptyAsync(DarlingConfig config, CancellationToken cance { await SeedMonitoredServersAsync(connection, config, now, cancellationToken); } + else + { + /* #2254: the seed is skipped, so any server added to darling.json AFTER the first start is + silently ignored — and --test-connection reads the FILE, so it validates those servers + happily while the service never monitors them. Say so once per start instead of leaving the + operator to discover it. */ + await WarnAboutFileOnlyServersAsync(connection, config, cancellationToken); + } /* LAST — its presence marks the seed complete (the reload gate keys on config_version). */ if (await CountAsync(connection, "config_service", cancellationToken) == 0) @@ -124,6 +132,196 @@ public async Task SeedIfEmptyAsync(DarlingConfig config, CancellationToken cance } } + /// + /// #2254: names the servers present in darling.json that the store does not have, once per start. + /// + /// The seed runs only while config_monitored_servers is empty, so a server added to the file + /// after the first successful start is a permanent no-op. What made that expensive in the field is that + /// --test-connection reads the FILE and validated the new server as PASS, so the operator had two + /// outputs that were each correct about different things and no way to see the disagreement: config edit, + /// service restart, support round trip. + /// + /// Compared on server_id OR name (#2158). It used to be id alone, on the grounds that the id + /// is what the collectors key on — correct while every row's id equalled the hash of its own address, and + /// wrong the moment an edit began PRESERVING a row's identity so a re-addressed server keeps its history. + /// After such an edit the file's derived id matches nothing, and an id-only comparison would report a + /// server that IS monitored as absent, then advise re-adding it — wrong advice, on every start, about the + /// one server the operator had just fixed. The name arm covers that; a genuinely removed server is gone + /// from the store under both keys, so the Viewer-Remove case still reports exactly as before. + /// + private async Task WarnAboutFileOnlyServersAsync( + NpgsqlConnection connection, DarlingConfig config, CancellationToken ct) + { + var storeIds = new HashSet(); + var storeNames = new HashSet(StringComparer.OrdinalIgnoreCase); + using (var command = new NpgsqlCommand("SELECT server_id, name FROM config_monitored_servers", connection)) + await using (var reader = await command.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + storeIds.Add(reader.GetInt32(0)); + if (!reader.IsDBNull(1)) + { + storeNames.Add(reader.GetString(1)); + } + } + } + + var fileOnly = ServersOnlyInFile(config.Servers, storeIds, storeNames); + if (fileOnly.Count == 0) + { + return; + } + + /* #2258: the OBSERVED registry is the tombstone, and it already exists. collect.servers gets a row + upserted on every successful connect, and the Viewer's Remove deletes only from + config_monitored_servers (the DESIRED config) — so a row surviving there means "this server really + was monitored once", which is exactly the fact that separates the two causes. Nothing purges it + either: it is a registry, not a time series, so retention leaves it alone. */ + var everMonitored = new HashSet(StringComparer.OrdinalIgnoreCase); + using (var observed = new NpgsqlCommand("SELECT display_name, server_name FROM collect.servers", connection)) + await using (var reader = await observed.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + if (!reader.IsDBNull(0)) + { + everMonitored.Add(reader.GetString(0)); + } + + if (!reader.IsDBNull(1)) + { + everMonitored.Add(reader.GetString(1)); + } + } + } + + var (neverRegistered, deliberatelyRemoved) = SplitByEverMonitored(fileOnly, everMonitored); + + /* Cause 1 — in the file, never monitored. This is the field report (#2252): the operator edited the + file expecting it to be picked up, and it silently was not. A WARNING, because it is the case where + something the operator wants is not happening and only they can fix it. */ + if (neverRegistered.Count > 0) + { + _logger?.LogWarning( + "darling.json lists {Count} server(s) that are NOT monitored and never have been: {Servers}. " + + "The store is authoritative after the first seed, so adding a server to the file does not " + + "register it and a restart cannot change that — add them with the Viewer's Add Server dialog " + + "or the MCP add_servers tool. Note --test-connection reads darling.json, so it will keep " + + "reporting them as PASS while they collect nothing.", + neverRegistered.Count, + string.Join(", ", neverRegistered)); + } + + /* Cause 2 — monitored once, then removed, and the file was left alone. A CORRECT state, so this is + Information and says so plainly rather than advising anything. It is not silent because the file + still names them and --test-connection will still call them PASS, which is worth one line at + startup; but it no longer tells the operator to re-add a server they deliberately dropped. */ + if (deliberatelyRemoved.Count > 0) + { + _logger?.LogInformation( + "darling.json still lists {Count} server(s) that were monitored and have since been removed: " + + "{Servers}. That is expected — the Viewer's Remove deletes the registration and never edits " + + "the file. Delete them from darling.json to silence this line; their collected history is " + + "kept either way.", + deliberatelyRemoved.Count, + string.Join(", ", deliberatelyRemoved)); + } + } + + /// + /// Splits the file-only servers into "never monitored" and "monitored once, since removed" (#2258), using the + /// observed registry as the evidence. + /// + /// Why this needs no tombstone table. #2258 proposed one — a config_removed_servers table + /// or an is_removed flag, plus a rung. But the fact it wanted is already recorded: + /// collect.servers holds a row per server the service has successfully connected to, the Viewer's + /// Remove deletes only from config_monitored_servers, and nothing purges the observed registry. So + /// "was this ever really monitored" is answerable today, for free, without a schema change and without a + /// second piece of state that could disagree with the first. + /// + /// An is_removed flag was the option worth rejecting explicitly: is_enabled = FALSE + /// already means "registered but paused", so a second flag on the same row makes + /// (is_enabled, is_removed) a four-state space where two combinations are meaningless, and every + /// existing reader of that table would have to learn the new flag or silently start including removed + /// servers. That is the same seam failure that #2280 had to fix in the dedupe gate — a widened concept that + /// old call sites never learned about. + /// + /// The limits, stated because they bound what the log may claim. A server registered but never + /// successfully connected to has no observed row, so it reports as never-monitored — which is the right + /// answer to the operator's actual question ("is this being monitored?"), even though it is the wrong answer + /// to "was it ever registered?". And a store rebuilt from scratch has no observed rows at all, so everything + /// reads as never-monitored until it connects once; that degrades to a warning rather than to silence, which + /// is the safe direction for a fresh store where the file genuinely is the intent. + /// + /// Matched on either name for the reason gives: the observed registry + /// carries both the storage name and the display name, and identity drift predating #2158 means the id is + /// the less reliable key of the three. A miss here warns rather than going quiet, so the failure direction + /// is the harmless one. + /// + internal static (IReadOnlyList NeverRegistered, IReadOnlyList DeliberatelyRemoved) + SplitByEverMonitored(IEnumerable fileOnlyNames, ISet everMonitoredNames) + { + var never = new List(); + var removed = new List(); + + foreach (var name in fileOnlyNames ?? Enumerable.Empty()) + { + if (everMonitoredNames is not null && everMonitoredNames.Contains(name)) + { + removed.Add(name); + } + else + { + never.Add(name); + } + } + + return (never, removed); + } + + /// + /// The file entries whose server_id is absent from the store, by display name. Pure so the + /// comparison is testable without a store — the log line above is the only part that needs one. + /// + internal static IReadOnlyList ServersOnlyInFile( + IEnumerable fileServers, ISet storeServerIds, ISet? storeNames = null) + { + var missing = new List(); + if (fileServers is null) + { + return missing; + } + + foreach (var server in fileServers) + { + /* A file entry has no StoredServerId, so ServerId here IS the derivation: "would the id this file + entry describes be in the store". That was the whole test until #2158 made an edit preserve its + identity — a re-addressed server keeps its own id so its history stays attached, which means the + file's derived id no longer matches it and the id arm alone now reports a monitored server as + absent. The NAME arm answers the question the log actually asks, "is this file entry represented + in the store at all", and it is the operator-facing key: the display name is what they typed and + what the Viewer shows, and an edit does not change it. + + Deliberately either-or rather than name-only. Two different file entries can share a display + name (nothing enforces uniqueness), so name-only would hide a genuinely unmonitored server + behind a same-named sibling; and the id arm still resolves the common case exactly. */ + if (storeServerIds.Contains(server.ServerId)) + { + continue; + } + + if (storeNames is not null && storeNames.Contains(server.DisplayName)) + { + continue; + } + + missing.Add(server.DisplayName); + } + + return missing; + } + private static async Task CountAsync(NpgsqlConnection connection, string table, CancellationToken ct) { /* table is a compile-time constant name, never user input — interpolation is safe. */ @@ -136,8 +334,8 @@ private static async Task SeedServiceRowAsync(NpgsqlConnection connection, Darli /* config_version starts at 0; the four desired-state seed writes below bump it via the trigger, so the worker's post-seed baseline read reflects the seeded state and triggers no spurious reload. */ using var command = new NpgsqlCommand(@" -INSERT INTO config_service (id, paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port, config_version, updated_at, updated_by) -VALUES (1, FALSE, $1, $2, $3, $4, $5, 0, $6, 'seed') +INSERT INTO config_service (id, paused, capture_plans, query_store_backfill_enabled, query_store_text_budget_mb, max_concurrent_sweeps, plan_xml_compression, mcp_enabled, mcp_port, web_enabled, web_port, plan_content_retention_days, config_version, updated_at, updated_by) +VALUES (1, FALSE, $1, $7, $8, $9, $10, $2, $3, $4, $5, $11, 0, $6, 'seed') ON CONFLICT (id) DO NOTHING", connection); command.Parameters.AddWithValue(config.CapturePlans); command.Parameters.AddWithValue(config.Mcp.Enabled); @@ -145,6 +343,15 @@ INSERT INTO config_service (id, paused, capture_plans, mcp_enabled, mcp_port, we command.Parameters.AddWithValue(config.Web.Enabled); command.Parameters.AddWithValue(config.Web.Port); command.Parameters.AddWithValue(now); + command.Parameters.AddWithValue(config.QueryStoreBackfillEnabled); + command.Parameters.AddWithValue(config.QueryStoreTextBudgetMb); + command.Parameters.AddWithValue(config.MaxConcurrentSweeps); + /* Normalized at the WRITE too, not just the read: the V62 CHECK is case-sensitive by design + (it mirrors this normalizer's output), so seeding the raw file value would turn + "planXmlCompression": "GZIP" in darling.json into a CHECK violation during store bring-up — + the seed is the last step of first contact, and a cosmetic casing choice must not fail it. */ + command.Parameters.AddWithValue(NormalizePlanXmlCompression(config.PlanXmlCompression)); + command.Parameters.AddWithValue(ClampPlanContentRetentionDays(config.PlanContentRetentionDays)); await command.ExecuteNonQueryAsync(ct); } @@ -167,10 +374,13 @@ INSERT INTO config_alert_settings ( notify_connection_down_at_startup, connection_refire_minutes, notify_ag_health, ag_lag_alert_seconds, ag_redo_queue_alert_kb, ag_disconnect_refire_minutes, blocking_wait_seconds_threshold, pvs_enabled, pvs_threshold_percent, - pvs_floor_gb, modified_at, database_state_enabled) + pvs_floor_gb, modified_at, database_state_enabled, + self_disk_free_warn_percent, collection_stale_minutes, collection_failure_threshold, + disk_critical_free_percent, disk_critical_free_gb, analysis_notify_cooldown_minutes, + store_job_cadence_warn_percent) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, - $43, $44, $45, $46, $47, $48) + $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55) ON CONFLICT (id) DO NOTHING", connection); command.Parameters.AddWithValue(a.Enabled); command.Parameters.AddWithValue(a.CpuEnabled); @@ -229,6 +439,14 @@ INSERT INTO config_alert_settings ( command.Parameters.AddWithValue(now); /* V49 database-state alert master switch (appended last, matching the ALTER's physical order). */ command.Parameters.AddWithValue(a.DatabaseStateEnabled); + /* V55 #2107: the previously-hardcoded threshold knobs, appended in the ALTER's order. */ + command.Parameters.AddWithValue(a.SelfDiskFreeWarnPercent); + command.Parameters.AddWithValue(a.CollectionStaleMinutes); + command.Parameters.AddWithValue(a.CollectionFailureThreshold); + command.Parameters.AddWithValue(a.DiskCriticalFreePercent); + command.Parameters.AddWithValue(a.DiskCriticalFreeGb); + command.Parameters.AddWithValue(a.AnalysisNotifyCooldownMinutes); + command.Parameters.AddWithValue(a.StoreJobCadenceWarnPercent); await command.ExecuteNonQueryAsync(ct); } @@ -268,10 +486,13 @@ Viewer deletion (Stage 3) is never resurrected by a re-seed. */ INSERT INTO config_monitored_servers ( server_id, name, host, database, auth, username, encrypted_password, encrypt_mode, trust_server_certificate, read_only_intent, multi_subnet_failover, excluded_databases, - monthly_cost_usd, capture_plans, alert_delivery_mode_override, is_enabled, created_at, modified_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL, $14, TRUE, $15, $15) + monthly_cost_usd, capture_plans, alert_delivery_mode_override, engine, port, is_enabled, created_at, modified_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NULL, $14, $16, $17, TRUE, $15, $15) ON CONFLICT (server_id) DO NOTHING", connection); - command.Parameters.AddWithValue(ServerIdHelper.GetDeterministicHashCode(server.StorageName)); + /* THE ALLOCATION SITE. A darling.json entry has no StoredServerId, so this is the derivation — + and this is where it is minted and made permanent. When new rows stop being hash-keyed + (#2218), this is the write that changes; every READ already goes through the stored value. */ + command.Parameters.AddWithValue(server.ServerId); command.Parameters.AddWithValue(server.DisplayName); command.Parameters.AddWithValue(server.Host); AddNullableText(command, server.Database); @@ -289,6 +510,14 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ /* Per-server delivery override (#1236): the enum name or NULL = "inherit the global". */ AddNullableText(command, server.AlertDeliveryModeOverride?.ToString()); command.Parameters.AddWithValue(now); + /* V68: the engine, persisted as the raw darling.json string rather than the parsed enum, so the + store round-trips exactly what the operator wrote — including an alias like "aurora" — and the + single parse in MonitoredServer.TargetEngine stays the only place that interprets it. */ + command.Parameters.AddWithValue(server.Engine); + /* V68: the port, PostgreSQL-only (0 = the driver's default). Persisted for the same reason as the + engine — a non-default port dropped here would connect to 5432 and fail with an error naming + the right host. */ + command.Parameters.AddWithValue(server.Port); await command.ExecuteNonQueryAsync(ct); } } @@ -301,7 +530,8 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ /// SQL-auth servers whose store row carries no DPAPI blob (never persisted; matched by server_id). /// Returns null when the store is unreachable, so the caller keeps the current live config. /// - public async Task LoadViewAsync(DarlingConfig bootstrap, CancellationToken cancellationToken) + public async Task LoadViewAsync( + DarlingConfig bootstrap, CancellationToken cancellationToken) { if (bootstrap is null) { @@ -312,9 +542,22 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ { await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); - var (paused, capturePlans, mcpEnabled, mcpPort, webEnabled, webPort, configVersion) = await ReadServiceRowAsync(connection, cancellationToken); + var (paused, capturePlans, backfillEnabled, textBudgetMb, maxSweeps, planXmlCompression, mcpEnabled, mcpPort, webEnabled, webPort, planContentRetentionDays, configVersion) = await ReadServiceRowAsync(connection, cancellationToken); var (alerts, analysis) = await ReadAlertSettingsAsync(connection, cancellationToken); + + /* The notification row is the ONLY read here that touches secret columns — the SMTP password and + username, and the Teams/Slack/generic/PagerDuty bearer URLs. DarlingManagedRoles deliberately + revokes table-wide SELECT on config_notification from BOTH viewer and mcp and re-grants only + the non-secret columns, so a caller connecting as one of those roles and asking for the full + row gets 42501 for the whole table — and because every section here shares one try/catch, one + denied column would cost the WHOLE view (that is how #2293 lost the MCP host its registry). + That is precisely why no restricted-role caller reads this view any more: the MCP host used to + (skipping this row via an includeNotification parameter, #2293) until #2298 removed its config + read entirely — the worker publishes the server registry to it instead. Every remaining caller + is the worker or a test on the privileged connection, so the row is read unconditionally and + the skip parameter is gone with its last caller. */ var (smtp, webhooks) = await ReadNotificationAsync(connection, cancellationToken); + var servers = await ReadMonitoredServersAsync(connection, bootstrap, cancellationToken); var schedules = await ReadScheduleOverridesAsync(connection, cancellationToken); @@ -323,6 +566,11 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ ConfigVersion = configVersion, Paused = paused, CapturePlans = capturePlans, + QueryStoreBackfillEnabled = backfillEnabled, + QueryStoreTextBudgetMb = textBudgetMb, + PlanContentRetentionDays = planContentRetentionDays, + MaxConcurrentSweeps = maxSweeps, + PlanXmlCompression = planXmlCompression, McpEnabled = mcpEnabled, McpPort = mcpPort, WebEnabled = webEnabled, @@ -342,20 +590,53 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ } } - private static async Task<(bool Paused, bool CapturePlans, bool McpEnabled, int McpPort, bool WebEnabled, int WebPort, long ConfigVersion)> + /// The V59 collector-memory knob clamps (#2164/#2170) — a bad stored value degrades to a sane + /// one rather than failing the config load, matching the alert knobs' posture. + internal const int MinTextBudgetMb = 4; + internal const int MaxTextBudgetMb = 256; + internal const int MinConcurrentSweeps = 1; + internal const int MaxConcurrentSweepsLimit = 16; + + internal static int ClampTextBudgetMb(int value) => Math.Clamp(value, MinTextBudgetMb, MaxTextBudgetMb); + + internal static int ClampConcurrentSweeps(int value) => Math.Clamp(value, MinConcurrentSweeps, MaxConcurrentSweepsLimit); + + /// The V75 plan-content horizon clamps (#2316) — 0 (and any negative) means DISABLED + /// (the fact-coupled dimension horizon stands alone); an enabled value clamps to [7,365], because a + /// sub-week horizon would age plan XML out from under the viewer's default history windows. + internal const int MinPlanContentRetentionDays = 7; + internal const int MaxPlanContentRetentionDays = 365; + + internal static int ClampPlanContentRetentionDays(int value) => + value <= 0 ? 0 : Math.Clamp(value, MinPlanContentRetentionDays, MaxPlanContentRetentionDays); + + /// #2171: unknown values normalize to 'gzip' (fail to the shipped default) so a hand-edited + /// row cannot switch the writer into an undefined mode; the V62 CHECK constraint enforces the same + /// set DB-side, and this guard covers pre-constraint rows and direct writes with the constraint + /// dropped. + internal static string NormalizePlanXmlCompression(string? value) => + string.Equals(value?.Trim(), "none", StringComparison.OrdinalIgnoreCase) ? "none" : "gzip"; + + private static async Task<(bool Paused, bool CapturePlans, bool QueryStoreBackfillEnabled, int QueryStoreTextBudgetMb, int MaxConcurrentSweeps, string PlanXmlCompression, bool McpEnabled, int McpPort, bool WebEnabled, int WebPort, int PlanContentRetentionDays, long ConfigVersion)> ReadServiceRowAsync(NpgsqlConnection connection, CancellationToken ct) { using var command = new NpgsqlCommand( - "SELECT paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port, config_version FROM config_service WHERE id = 1", connection); + "SELECT paused, capture_plans, query_store_backfill_enabled, query_store_text_budget_mb, max_concurrent_sweeps, plan_xml_compression, mcp_enabled, mcp_port, web_enabled, web_port, plan_content_retention_days, config_version FROM config_service WHERE id = 1", connection); using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) { - /* Row missing (unseeded) — treat as defaults; capture stays on (Darling's SKU default). */ - return (false, true, false, 5152, false, 5153, 0); + /* Row missing (unseeded) — treat as defaults; capture and backfill stay on, the memory + knobs reproduce the pre-V59 compile-time constants (64 MB budget, 4-wide sweep), and + plan content keeps the V75 default 21-day horizon. */ + return (false, true, true, 64, 4, "gzip", false, 5152, false, 5153, 21, 0); } - return (reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetInt32(3), - reader.GetBoolean(4), reader.GetInt32(5), reader.GetInt64(6)); + return (reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), + ClampTextBudgetMb(reader.GetInt32(3)), ClampConcurrentSweeps(reader.GetInt32(4)), + NormalizePlanXmlCompression(reader.GetString(5)), + reader.GetBoolean(6), reader.GetInt32(7), + reader.GetBoolean(8), reader.GetInt32(9), + ClampPlanContentRetentionDays(reader.GetInt32(10)), reader.GetInt64(11)); } private static async Task<(AlertsConfig Alerts, AnalysisConfig Analysis)> ReadAlertSettingsAsync(NpgsqlConnection connection, CancellationToken ct) @@ -374,7 +655,10 @@ backfilled at read time (BuildServerFromRow's bootstrap merge). */ notify_connection_down_at_startup, connection_refire_minutes, notify_ag_health, ag_lag_alert_seconds, ag_redo_queue_alert_kb, ag_disconnect_refire_minutes, blocking_wait_seconds_threshold, pvs_enabled, pvs_threshold_percent, - pvs_floor_gb, database_state_enabled + pvs_floor_gb, database_state_enabled, + self_disk_free_warn_percent, collection_stale_minutes, collection_failure_threshold, + disk_critical_free_percent, disk_critical_free_gb, analysis_notify_cooldown_minutes, + store_job_cadence_warn_percent FROM config_alert_settings WHERE id = 1", connection); using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) @@ -447,6 +731,19 @@ reset the knob on every worker start. */ /* database-state alert master switch appended (V49) at ordinal 46; NOT NULL DEFAULT true so a pre-V49 row can't reach here without the column present. */ DatabaseStateEnabled = reader.GetBoolean(46), + /* #2107 threshold knobs appended (V55) at ordinals 47–52; NOT NULL DEFAULTs are the + constants they replace, so a pre-V55 row can't reach here without the columns present + and the wholesale ApplyToConfig replacement never resets a knob. */ + SelfDiskFreeWarnPercent = reader.GetInt32(47), + CollectionStaleMinutes = reader.GetInt32(48), + CollectionFailureThreshold = reader.GetInt32(49), + DiskCriticalFreePercent = reader.GetInt32(50), + DiskCriticalFreeGb = reader.GetInt32(51), + AnalysisNotifyCooldownMinutes = reader.GetInt32(52), + /* #2136 cadence-warn knob appended (V57) at ordinal 53; NOT NULL DEFAULT 25, and the same + reachability rule as every appended knob above: ApplyToConfig replaces config.Alerts + wholesale, so a column missing here would silently reset the knob on every worker start. */ + StoreJobCadenceWarnPercent = reader.GetInt32(53), }; var analysis = new AnalysisConfig { @@ -504,9 +801,14 @@ private static async Task> ReadMonitoredServersAs NpgsqlConnection connection, DarlingConfig bootstrap, CancellationToken ct) { var servers = new List(); + /* server_id is LAST rather than first (#2218): every ordinal in BuildServerFromRow is positional, so + appending is the only addition that cannot silently re-map an existing column onto the wrong + property. It was absent entirely before this — the registry's own PRIMARY KEY was read past, and + twelve downstream sites re-derived it from the mutable columns instead. */ using var command = new NpgsqlCommand(@" SELECT name, host, database, auth, username, encrypted_password, encrypt_mode, trust_server_certificate, - read_only_intent, multi_subnet_failover, excluded_databases, monthly_cost_usd, alert_delivery_mode_override + read_only_intent, multi_subnet_failover, excluded_databases, monthly_cost_usd, alert_delivery_mode_override, + engine, port, server_id FROM config_monitored_servers WHERE is_enabled = TRUE ORDER BY name", connection); using var reader = await command.ExecuteReaderAsync(ct); @@ -546,6 +848,16 @@ private static MonitoredServer BuildServerFromRow(NpgsqlDataReader reader, Darli MonthlyCostUsd = reader.GetDecimal(11), /* #1236: the per-server delivery override (null = inherit the global), available at delivery time. */ AlertDeliveryModeOverride = ParseDeliveryOverride(reader.IsDBNull(12) ? null : reader.GetString(12)), + /* V68. Without this the registry — which is authoritative once seeded — silently downgraded every + PostgreSQL target to the "sqlserver" property default, and the service opened a SqlConnection to + port 5432. NOT NULL DEFAULT in both columns means the DBNull guards are belt-and-braces for a + store mid-migration, not an expected path. */ + Engine = reader.IsDBNull(13) ? "sqlserver" : reader.GetString(13), + Port = reader.IsDBNull(14) ? 0 : reader.GetInt32(14), + /* #2218: the row's OWN primary key, which this read used to discard. NOT NULL in the table, so + the DBNull guard is for a store mid-migration rather than an expected path — and a null there + falls back to the derivation, which is exactly what it did before this column was read at all. */ + StoredServerId = reader.IsDBNull(15) ? null : reader.GetInt32(15), }; if (server.UsesSqlAuth && string.IsNullOrWhiteSpace(server.EncryptedPassword)) @@ -609,6 +921,11 @@ public static void ApplyToConfig(DarlingConfig config, StoreConfigView view) config.Smtp = view.Smtp; config.Webhooks = view.Webhooks; config.CapturePlans = view.CapturePlans; + config.QueryStoreBackfillEnabled = view.QueryStoreBackfillEnabled; + config.QueryStoreTextBudgetMb = view.QueryStoreTextBudgetMb; + config.PlanContentRetentionDays = view.PlanContentRetentionDays; + config.MaxConcurrentSweeps = view.MaxConcurrentSweeps; + config.PlanXmlCompression = view.PlanXmlCompression; config.Mcp.Enabled = view.McpEnabled; config.Mcp.Port = view.McpPort; config.Web.Enabled = view.WebEnabled; @@ -745,6 +1062,30 @@ public sealed class StoreConfigView public bool Paused { get; init; } public bool CapturePlans { get; init; } + + /// The #2167 backfill off switch (config_service, V58) — worker reads it live each backfill cycle. + public bool QueryStoreBackfillEnabled { get; init; } = true; + + /// The #2164 per-database query_store text budget in MB (config_service, V59), already clamped. + public int QueryStoreTextBudgetMb { get; init; } = 64; + + /// The V75 plan-content horizon (#2316): days a stored plan XML outlives its last sighting. + /// 0 = disabled (the fact-coupled dimension horizon stands alone). + public int PlanContentRetentionDays { get; init; } = 21; + + /// + /// The #2171 plan-XML storage codec (config_service, V62), already normalized to 'gzip' or 'none'. + /// 'gzip' (the default, unchanged behavior): the plan dim stores gzip bytes in query_plan_gz, + /// 14.0x measured, and only the apps/MCP can read plans back. 'none': plain text into + /// query_plan_xml - lz4 TOAST compresses at ~8.9x and any direct-SQL consumer (Grafana, report + /// tooling) reads the column bare, no extension, no UDF. Existing rows are untouched either way; + /// the readers' text-first-else-gz resolution covers every mix of eras and modes. + /// + public string PlanXmlCompression { get; init; } = "gzip"; + + /// The #2170 fleet sweep width (config_service, V59), already clamped. + public int MaxConcurrentSweeps { get; init; } = 4; + public bool McpEnabled { get; init; } public int McpPort { get; init; } public bool WebEnabled { get; init; } diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs b/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs new file mode 100644 index 000000000..6bab7fc1a --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Net; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// The store's TLS material, generated as a REAL two-cert chain (#2117): a throwaway local root CA +/// whose private key is discarded the moment it has signed the one server leaf, and the leaf +/// postgres serves. The old single self-signed end-entity cert (critical CA=false Basic +/// Constraints) was its own trust anchor. The field report (#2117) shows that shape failing +/// VerifyFull chain validation on a real Windows viewer while the same certificate imported +/// into the OS trust store validated fine — and the E2E pins in NpgsqlRootCertificateValidationTests +/// show STOCK Windows CI accepting it, so the refusal is environmental (hardening policy is the +/// likely class: "an end-entity certificate may not be its own anchor" is exactly what strict +/// chain-policy configurations enforce). A leaf under a real CA root is the textbook shape every +/// chain engine and policy regime accepts — Windows, macOS, Linux, and libpq for non-Npgsql +/// clients — which is why the chain is the durable fix even though stock platforms tolerate the +/// old shape. +/// +/// Discarding the CA key is load-bearing: nothing can ever mint another certificate under +/// the distributed root, so trusting root.crt pins exactly one server identity, the same +/// security property the single self-signed cert had. Rotation regenerates BOTH (delete the server +/// cert + key and restart, exactly the old delete-to-rotate contract). +/// +/// Pure — no file I/O, no logger — so the chain's validity under Npgsql's exact custom-root +/// trust semantics is pinned by tests on every OS CI runs. +/// +internal static class StoreTlsCertificates +{ + /// What postgres serves (ssl_cert_file takes the whole chain, leaf first), the + /// leaf's private key, and the root the operator distributes to viewers. + internal sealed record Generated(string ServerCertChainPem, string ServerKeyPem, string RootCertPem); + + internal static Generated Create(string hostName, IPAddress listenIp, int validityYears) + { + ArgumentException.ThrowIfNullOrEmpty(hostName); + ArgumentNullException.ThrowIfNull(listenIp); + + var notBefore = DateTimeOffset.UtcNow.AddDays(-1); + var notAfter = notBefore.AddYears(validityYears); + + using var caKey = RSA.Create(2048); + var caRequest = new CertificateRequest( + $"CN=PerformanceMonitor Darling store root ({hostName})", caKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + /* pathLenConstraint 0: this root may sign end-entity certs only — even with the key discarded, + the constraint documents the intent in the certificate itself. */ + caRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, true, 0, true)); + caRequest.CertificateExtensions.Add(new X509KeyUsageExtension(X509KeyUsageFlags.KeyCertSign, true)); + using var caCertificate = caRequest.CreateSelfSigned(notBefore, notAfter); + + using var leafKey = RSA.Create(2048); + var leafRequest = new CertificateRequest( + $"CN={hostName}", leafKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddIpAddress(listenIp); + sanBuilder.AddDnsName(hostName); + leafRequest.CertificateExtensions.Add(sanBuilder.Build()); + leafRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(false, false, 0, true)); + leafRequest.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, true)); + leafRequest.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension(new OidCollection { new Oid("1.3.6.1.5.5.7.3.1") /* serverAuth */ }, false)); + + var serialNumber = new byte[12]; + RandomNumberGenerator.Fill(serialNumber); + /* The leaf's window may not exceed the issuer's — same instants, which Create() accepts. */ + using var leafCertificate = leafRequest.Create(caCertificate, notBefore, notAfter, serialNumber); + + return new Generated( + ServerCertChainPem: leafCertificate.ExportCertificatePem() + "\n" + caCertificate.ExportCertificatePem() + "\n", + ServerKeyPem: leafKey.ExportPkcs8PrivateKeyPem(), + RootCertPem: caCertificate.ExportCertificatePem() + "\n"); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Targets/PostgresTargetProvider.cs b/Darling/PerformanceMonitor.Darling.Service/Targets/PostgresTargetProvider.cs new file mode 100644 index 000000000..c5be8f94b --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Targets/PostgresTargetProvider.cs @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using Npgsql; +using NpgsqlTypes; +using PerformanceMonitor.Collectors; + +namespace PerformanceMonitor.Darling.Service.Targets; + +/// +/// The PostgreSQL implementation of , covering Amazon Aurora PostgreSQL. +/// Npgsql is already a dependency of this project — it is the store driver — so monitoring a Postgres +/// target needs no new package, only this. +/// Classification is by SQLSTATE, not by message text. PostgreSQL's SQLSTATEs are stable across +/// versions and locales; its error messages are neither. +/// +public sealed class PostgresTargetProvider : ITargetProvider +{ + public static readonly PostgresTargetProvider Instance = new(); + + public CollectorTargetEngine Engine => CollectorTargetEngine.PostgreSql; + + public DbConnection CreateConnection(string connectionString) => new NpgsqlConnection(connectionString); + + public DbCommand CreateCommand(CollectorQuery query, DbConnection connection, int commandTimeoutSeconds) + { + if (connection is not NpgsqlConnection npgsqlConnection) + { + throw new ArgumentException( + $"PostgreSQL provider requires an NpgsqlConnection, got {connection?.GetType().Name ?? "null"}", + nameof(connection)); + } + + var command = new NpgsqlCommand(query.Text, npgsqlConnection) { CommandTimeout = commandTimeoutSeconds }; + + foreach (var parameter in query.Parameters) + { + command.Parameters.Add(ToNpgsqlParameter(parameter)); + } + + return command; + } + + /// + /// Maps a collector parameter to its PostgreSQL type. + /// Two deliberate choices. First, the NVarChar128/NVarChar260 lengths are + /// dropped: those exist to match SQL Server column widths, and PostgreSQL's text has no + /// length to declare — carrying a bogus length would imply a constraint the engine does not + /// have. Second, DateTime2 maps to timestamp WITHOUT time zone, matching the store's + /// own naive-UTC convention; mapping it to timestamptz would make Npgsql reject a + /// DateTimeKind.Unspecified value, and every timestamp in this product is Unspecified. + /// + private static NpgsqlParameter ToNpgsqlParameter(CollectorParameter parameter) => parameter.Type switch + { + CollectorParameterType.DateTime2 => new NpgsqlParameter(parameter.Name, NpgsqlDbType.Timestamp) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.NVarChar128 => new NpgsqlParameter(parameter.Name, NpgsqlDbType.Text) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.NVarChar260 => new NpgsqlParameter(parameter.Name, NpgsqlDbType.Text) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.Int32 => new NpgsqlParameter(parameter.Name, NpgsqlDbType.Integer) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.BigInt => new NpgsqlParameter(parameter.Name, NpgsqlDbType.Bigint) { Value = parameter.Value ?? DBNull.Value }, + _ => throw new ArgumentOutOfRangeException(nameof(parameter), parameter.Type, "Unmapped collector parameter type"), + }; + + /// + /// SQLSTATE-based classification. Every code here was observed on our own Aurora fleet while + /// probing which monitoring sources are readable, not taken from documentation: + /// + /// 42501 — pg_monitor lacks the grant (e.g. aurora_stat_logical_wal_cache(), + /// which needs rds_replication). + /// 42P01 / 42883 — the relation or function is not there, which is how a version-gated + /// feature or an uncreated extension presents (pg_stat_statements in a database where the + /// view was never created; apg_plan_mgmt.dba_plans without the extension). + /// 0A000 — feature_not_supported. Aurora returns this for pg_stat_wal. + /// 55000 / 55P03 — the object is not in the right state, or a lock could not be taken. + /// Aurora raises a 55-class error for aurora_stat_optimized_reads_cache() when the feature + /// is disabled, which a naive collector logs as a failure every single cycle. + /// 57014 — query_canceled, which is what statement_timeout produces. + /// 08* — connection exceptions; 57P01/57P02/57P03 — shutdown and unavailability. + /// + /// + public CollectorTargetFault Classify(Exception exception, bool yieldsOnLockTimeout) + { + /* Unwrapped OR wrapped. Npgsql surfaces a command timeout as an NpgsqlException whose INNER + exception is the TimeoutException — the bare shape is what a test constructs, not what the driver + throws. Checking only the outer type sent every real command timeout down the NpgsqlException arm + below and classified it ConnectionFatal, so a slow query forced a reconnect: precisely the + reconnect storm the SQLSTATE arm is careful to avoid. */ + if (exception is TimeoutException + || exception.InnerException is TimeoutException + || (exception is NpgsqlException && exception.GetBaseException() is TimeoutException)) + { + return CollectorTargetFault.CommandTimeout; + } + + if (exception is not PostgresException pg) + { + /* A connection-level Npgsql failure that never reached the server carries no SQLSTATE. */ + return exception is NpgsqlException ? CollectorTargetFault.ConnectionFatal : CollectorTargetFault.Unclassified; + } + + var state = pg.SqlState ?? string.Empty; + + if (state.StartsWith("08", StringComparison.Ordinal) + || state is "57P01" or "57P02" or "57P03") + { + return CollectorTargetFault.ConnectionFatal; + } + + return state switch + { + "42501" => CollectorTargetFault.Permissions, + "42P01" or "42883" => CollectorTargetFault.ObjectMissing, + "0A000" => CollectorTargetFault.FeatureDisabled, + "55000" or "55006" => CollectorTargetFault.FeatureDisabled, + "55P03" => yieldsOnLockTimeout ? CollectorTargetFault.LockTimeoutYield : CollectorTargetFault.Unclassified, + "57014" => CollectorTargetFault.CommandTimeout, + _ => CollectorTargetFault.Unclassified, + }; + } + + public string WithDatabase(string connectionString, string databaseName) + => new NpgsqlConnectionStringBuilder(connectionString) { Database = databaseName }.ConnectionString; + + /// + /// Enumerates from wherever the service is already connected — pg_database is a shared + /// catalog, so unlike SQL Server there is no equivalent of hopping to master first. + /// Both filters are load-bearing. datistemplate excludes template0 and + /// template1; template0 in particular is frozen and rejects connections outright, so + /// including it would guarantee one failed connection per collection cycle forever. datallowconn + /// excludes any database an administrator has deliberately closed — most often one mid-restore or + /// being retired, exactly the databases where an extra connection attempt is least welcome. + /// + public (string ConnectionString, CollectorQuery Query) BuildDatabaseListPlan( + string connectionString, IReadOnlyList? excludedDatabases) + { + var (exclusionClause, exclusionParameters) = DatabaseExclusionFilter.Build(excludedDatabases, "datname"); + + return (connectionString, new CollectorQuery( + $@" +SELECT datname +FROM pg_database +WHERE datallowconn +AND NOT datistemplate +{exclusionClause} +ORDER BY datname", + exclusionParameters)); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Targets/SqlServerTargetProvider.cs b/Darling/PerformanceMonitor.Darling.Service/Targets/SqlServerTargetProvider.cs new file mode 100644 index 000000000..9927685c4 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Targets/SqlServerTargetProvider.cs @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using Microsoft.Data.SqlClient; +using PerformanceMonitor.Collectors; + +namespace PerformanceMonitor.Darling.Service.Targets; + +/// +/// The SQL Server implementation of . This is a lift of the connection, +/// command, and parameter-mapping code that was inline in DarlingCollectorRunner — the +/// behaviour is deliberately unchanged, including the throw on an unmapped parameter type. +/// reproduces the error numbers the runner's catch filters already use, +/// so the two cannot disagree. It does not replace those filters in this change; it exists so the +/// same decisions are expressible for a non-SQL-Server target. +/// +public sealed class SqlServerTargetProvider : ITargetProvider +{ + public static readonly SqlServerTargetProvider Instance = new(); + + public CollectorTargetEngine Engine => CollectorTargetEngine.SqlServer; + + public DbConnection CreateConnection(string connectionString) => new SqlConnection(connectionString); + + public DbCommand CreateCommand(CollectorQuery query, DbConnection connection, int commandTimeoutSeconds) + { + if (connection is not SqlConnection sqlConnection) + { + throw new ArgumentException( + $"SQL Server provider requires a SqlConnection, got {connection?.GetType().Name ?? "null"}", + nameof(connection)); + } + + var command = new SqlCommand(query.Text, sqlConnection) { CommandTimeout = commandTimeoutSeconds }; + + foreach (var parameter in query.Parameters) + { + command.Parameters.Add(ToSqlParameter(parameter)); + } + + return command; + } + + /// + /// Maps a collector parameter to its SQL Server type. Throws rather than defaulting on an + /// unmapped type: a silently wrong parameter type yields a wrong result set, which is worse than + /// a failure. + /// + private static SqlParameter ToSqlParameter(CollectorParameter parameter) => parameter.Type switch + { + CollectorParameterType.DateTime2 => new SqlParameter(parameter.Name, SqlDbType.DateTime2) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.NVarChar128 => new SqlParameter(parameter.Name, SqlDbType.NVarChar, 128) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.NVarChar260 => new SqlParameter(parameter.Name, SqlDbType.NVarChar, 260) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.Int32 => new SqlParameter(parameter.Name, SqlDbType.Int) { Value = parameter.Value ?? DBNull.Value }, + CollectorParameterType.BigInt => new SqlParameter(parameter.Name, SqlDbType.BigInt) { Value = parameter.Value ?? DBNull.Value }, + _ => throw new ArgumentOutOfRangeException(nameof(parameter), parameter.Type, "Unmapped collector parameter type"), + }; + + /// + /// The error numbers below are the ones the runner and worker already branch on. + /// is deliberately NOT produced here. + /// Whether a 297 means "the XE session is gone" or "permission denied" depends on which collector + /// asked and how the worker wrapped the call — the worker raises its own exception type for the XE + /// case before any classification happens, and that type is private to it. Keeping that decision + /// in the worker is correct: it is collector context, not engine semantics. + /// + public CollectorTargetFault Classify(Exception exception, bool yieldsOnLockTimeout) + { + if (exception is not SqlException sql) + { + return CollectorTargetFault.Unclassified; + } + + /* Class 20+ is a fatal, connection-level error; -2 is a command timeout. Both force the + caller to drop and re-probe the connection rather than just failing one collector. */ + if (sql.Class >= 20) + { + return CollectorTargetFault.ConnectionFatal; + } + + if (sql.Number == -2) + { + return CollectorTargetFault.CommandTimeout; + } + + /* 1222 is a lock-request timeout. It is a YIELD only for a collector that deliberately set a + short LOCK_TIMEOUT; from any other collector it is a genuine error. */ + if (sql.Number == 1222) + { + return yieldsOnLockTimeout ? CollectorTargetFault.LockTimeoutYield : CollectorTargetFault.Unclassified; + } + + if (sql.Number is 229 or 297 or 300 or 8189 or 916) + { + return CollectorTargetFault.Permissions; + } + + if (sql.Number == 208) + { + return CollectorTargetFault.ObjectMissing; + } + + return CollectorTargetFault.Unclassified; + } + + public string WithDatabase(string connectionString, string databaseName) + => new SqlConnectionStringBuilder(connectionString) { InitialCatalog = databaseName }.ConnectionString; + + /// + /// Enumeration runs against master, which is why the plan carries its own connection string: + /// on an Azure SQL DB logical server the configured entry points at one user database, and + /// sys.databases there lists only itself. + /// database_id > 0 drops the resource database and state_desc = 'ONLINE' drops + /// anything a per-database connection would fail on anyway (restoring, offline, recovery pending). + /// This is the query Lite has always used, kept identical so both editions fan out over the same + /// set — a difference here would show up as one edition silently monitoring fewer databases. + /// + public (string ConnectionString, CollectorQuery Query) BuildDatabaseListPlan( + string connectionString, IReadOnlyList? excludedDatabases) + { + var (exclusionClause, exclusionParameters) = DatabaseExclusionFilter.Build(excludedDatabases, "name"); + + return ( + WithDatabase(connectionString, "master"), + new CollectorQuery( + $"SELECT name FROM sys.databases WHERE state_desc = N'ONLINE' AND database_id > 0 {exclusionClause} ORDER BY name;", + exclusionParameters)); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Targets/TargetProviders.cs b/Darling/PerformanceMonitor.Darling.Service/Targets/TargetProviders.cs new file mode 100644 index 000000000..2cfd2794b --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Targets/TargetProviders.cs @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Collectors; + +namespace PerformanceMonitor.Darling.Service.Targets; + +/// +/// Resolves the for an engine. A plain switch rather than a registry +/// dictionary: the set of engines is closed and known at compile time, and a switch makes an +/// unhandled engine a build-time concern instead of a runtime lookup miss. +/// +public static class TargetProviders +{ + /// + /// The provider for . Throws on an engine with no provider — that is a + /// programming error, not a runtime condition, and must not degrade into a silent skip. + /// + public static ITargetProvider For(CollectorTargetEngine engine) => engine switch + { + CollectorTargetEngine.SqlServer => SqlServerTargetProvider.Instance, + CollectorTargetEngine.PostgreSql => PostgresTargetProvider.Instance, + _ => throw new ArgumentOutOfRangeException(nameof(engine), engine, "No target provider for this engine"), + }; + + /// The provider for a monitored target, from the engine on its probed target info. + public static ITargetProvider For(CollectorTargetInfo target) => For(target.Engine); +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensionWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensionWriter.cs index 1b4aaa6d7..7e7b19e56 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensionWriter.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensionWriter.cs @@ -41,7 +41,8 @@ public static async Task FlushAsync( NpgsqlTransaction transaction, PayloadDimensionBatch batch, DateTime collectionTime, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool compressPlanContent = true) { if (connection is null) { @@ -71,10 +72,14 @@ public static async Task FlushAsync( /* #2069: the plan dim stores gzip bytes (measured 14.0x vs lz4-TOAST's 8.9x on live content). Compressed HERE — one seam — with the digest untouched: it was computed - over the uncompressed text upstream, so content identity is format-stable. */ - var compress = string.Equals(dimTable, PayloadDimensions.CompressedContentDimTable, StringComparison.Ordinal); + over the uncompressed text upstream, so content identity is format-stable. + #2171: plan_xml_compression = 'none' turns the seam off — the plan dim takes the same + text path as every other dim, and lz4 TOAST carries the compression so direct-SQL + consumers can read query_plan_xml bare. The digest is identical either way. */ + var compress = compressPlanContent + && string.Equals(dimTable, PayloadDimensions.CompressedContentDimTable, StringComparison.Ordinal); - await using var command = new NpgsqlCommand(PayloadDimensions.UpsertSql(dimTable), connection, transaction); + await using var command = new NpgsqlCommand(PayloadDimensions.UpsertSql(dimTable, compress), connection, transaction); command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bytea, diff --git a/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensions.cs b/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensions.cs index 70b4e9284..b40b378e6 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensions.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PayloadDimensions.cs @@ -328,9 +328,14 @@ gzip bytes only. Other dims keep the original NOT NULL text shape. */ /// #2069: the compressed-content dim upserts gzip BYTES into /// (text column left NULL on new rows); every other dim /// keeps the text shape. One method so the two shapes cannot drift on conflict semantics. - public static string UpsertSql(string dimTable) + /// #2171: false routes the plan dim through the TEXT + /// branch instead - query_plan_xml written, query_plan_gz left NULL - which is the + /// plan_xml_compression = 'none' store mode for direct-SQL consumers (Grafana and friends + /// read the column with no extension; lz4 TOAST does the compressing). Readers need no new + /// arm: the text-first-else-gz resolution below already covers every mix of eras and modes. + public static string UpsertSql(string dimTable, bool compressContent = true) { - if (string.Equals(dimTable, CompressedContentDimTable, StringComparison.Ordinal)) + if (compressContent && string.Equals(dimTable, CompressedContentDimTable, StringComparison.Ordinal)) { return $"INSERT INTO {dimTable} ({DigestColumn}, {CompressedContentColumn}, {LastSeenColumn})\n" + diff --git a/Darling/PerformanceMonitor.Darling.Storage/PerformanceMonitor.Darling.Storage.csproj b/Darling/PerformanceMonitor.Darling.Storage/PerformanceMonitor.Darling.Storage.csproj index 393201d15..f73174fca 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PerformanceMonitor.Darling.Storage.csproj +++ b/Darling/PerformanceMonitor.Darling.Storage/PerformanceMonitor.Darling.Storage.csproj @@ -14,7 +14,7 @@ - + diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index 33968e339..26ffdaac8 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -99,10 +99,40 @@ renumbered to 47 rather than shipped into that hole. Version numbers only have t new Migration(48, "pvs-pressure-alert", V48Sql), new Migration(49, "database-state-alert", V49Sql), new Migration(50, "server-tag-colour", V50Sql), - new Migration(51, "query-stats-host-object", V51Sql + "\n" + PgSchemaGenerator.GenerateQueryStatsResolvingView()), + /* #2119: V54Sql is PREPENDED ahead of the generated view. This rung's view SQL comes from the + LIVE generator, which since #2069 emits the V54 gz column — a ≤V50 store replaying this rung + on current code referenced a column three rungs before the ALTER that adds it (42703, the + ladder halts, every 3.3.0→3.4.0 upgrade failed). V54's ALTERs are idempotent, so pre-adding + here costs a fresh-through-this-rung store nothing and rung 54's own copy no-ops. This is + the standing hazard of generator-built rungs: any LATER column the generator learns must be + pre-added in EVERY earlier rung that re-emits generated SQL over existing tables — pinned by + MigrationLadderPins so the next collision fails in CI, not on an operator's store. */ + new Migration(51, "query-stats-host-object", V51Sql + "\n" + V54Sql + "\n" + PgSchemaGenerator.GenerateQueryStatsResolvingView()), new Migration(52, "finding-drilldown-json", V52Sql), new Migration(53, "store-self-metrics", V53Sql), new Migration(54, "plan-dim-gzip", V54Sql + "\n" + PgSchemaGenerator.GenerateQueryStatsResolvingView()), + new Migration(55, "self-alert-knobs", V55Sql), + new Migration(56, "store-metrics-background-jobs", V56Sql), + new Migration(57, "store-job-cadence-knob", V57Sql), + new Migration(58, "qs-backfill-switch", V58Sql), + new Migration(59, "collector-memory-knobs", V59Sql), + new Migration(60, "database-state-edge-memory", V60Sql), + new Migration(61, "incident-occurrence-counters", V61Sql), + new Migration(62, "plan-xml-compression-knob", V62Sql), + new Migration(63, "pg-wait-stats", V63Sql), + new Migration(64, "pg-statement-stats", V64Sql), + new Migration(65, "pg-wraparound-stats", V65Sql), + new Migration(66, "pg-xmin-horizon", V66Sql), + new Migration(67, "pg-replication-slots", V67Sql), + new Migration(68, "pg-autovacuum-stats", V68Sql), + new Migration(69, "pg-io-stats", V69Sql), + new Migration(70, "monitored-server-engine", V70Sql), + new Migration(71, "pg-blocking-edges", V71Sql), + new Migration(72, "query-store-plan-map", V72Sql), + new Migration(73, "pg-statement-text", V73Sql), + new Migration(74, "query-store-text", V74Sql), + new Migration(75, "plan-content-retention-knob", V75Sql), + new Migration(76, "query-store-health", V76Sql), }; /// @@ -1059,6 +1089,723 @@ ALTER TABLE query_plan_dim ALTER TABLE query_plan_dim ALTER COLUMN query_plan_xml DROP NOT NULL;"; + /// + /// V55 — the #2107 alert-threshold knobs, all previously compile-time constants: the store + /// volume's self-alert warning percent, the Collection Stopped staleness window and + /// consecutive-failure fast path, the low-disk CRITICAL severity tier's two floors (#1136 — + /// these grade the shared target-volume alert, not just the self-alert), and the analysis + /// notification cooldown Lite already passed through while Darling hardcoded 360. Defaults are + /// the constants they replace, NOT NULL so pre-V55 rows read cleanly at the appended ordinals. + /// + private const string V55Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS self_disk_free_warn_percent integer NOT NULL DEFAULT 10; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS collection_stale_minutes integer NOT NULL DEFAULT 30; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS collection_failure_threshold integer NOT NULL DEFAULT 10; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS disk_critical_free_percent integer NOT NULL DEFAULT 3; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS disk_critical_free_gb integer NOT NULL DEFAULT 2; +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS analysis_notify_cooldown_minutes integer NOT NULL DEFAULT 360;"; + + /// + /// V56 — background-job telemetry columns on the #2068 self-metrics series (#2136): the store's own + /// TimescaleDB background jobs (CAGG refreshes, compression, retention) are its heaviest recurring + /// work — measured on the production store, the four most expensive jobs are all the + /// query_store_stats family (compression 157s, interval_hourly refresh 96s) — and their runtimes + /// scale serially with raw volume, so an onboarding wave moves them first. Job rows ride the same + /// hourly sweep under object_kind = 'background_job'. All nullable, appended (the V55/#1984 + /// ordinal rule); non-job rows simply leave them NULL. + /// + private const string V56Sql = @" +ALTER TABLE collect.store_metrics + ADD COLUMN IF NOT EXISTS last_run_duration_ms bigint; +ALTER TABLE collect.store_metrics + ADD COLUMN IF NOT EXISTS schedule_interval_ms bigint; +ALTER TABLE collect.store_metrics + ADD COLUMN IF NOT EXISTS total_runs bigint; +ALTER TABLE collect.store_metrics + ADD COLUMN IF NOT EXISTS total_failures bigint;"; + + /// + /// V57 — the Store Job Over Cadence warning knob (#2136, the alert half of the V56 job telemetry): + /// a background job whose last run reaches this percent of its own schedule interval fires the + /// Warning tier of the new self-alert (the Critical tier is fixed at 100 — a job outrunning its + /// cadence is compounding refresh lag, which is the failure the telemetry exists to catch). + /// Store-backed like the V55 knobs (#2107 pattern): the column is the control plane, the C# default + /// remains only the shipped seed. Default 25: the production 52-server store's worst job runs at + /// ~7% of cadence, so 25 sits 3.5x above the observed ceiling but far ahead of real compounding. + /// + private const string V57Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS store_job_cadence_warn_percent integer NOT NULL DEFAULT 25;"; + + /// + /// V58 — the Query Store backfill off switch (#2167): a service-wide toggle the worker's backfill loop + /// reads live (store reload, no restart), because the #2058 backfill previously ran unconditionally — + /// during the 2026-08-10 consolidation a freshly restored catalog put it into sustained 64MB drains + /// against a cross-region production primary with no way to stop it short of gutting plan capture + /// fleet-wide. Default TRUE preserves today's behavior; the column rides config_service so the + /// existing config_version trigger makes a flip visible to the service's next reload poll. + /// + private const string V58Sql = @" +ALTER TABLE config.config_service + ADD COLUMN IF NOT EXISTS query_store_backfill_enabled boolean NOT NULL DEFAULT TRUE;"; + + /// + /// V59 — the two collector memory knobs that were compile-time constants (#2164 + #2170). They ride + /// ONE rung deliberately: peak transient memory is approximately + /// max_concurrent_sweeps × query_store_text_budget_mb, so an operator who moves one needs the + /// other in front of them, and shipping them together keeps the documented product of the two honest. + /// + /// Defaults reproduce today's hardcoded behavior exactly (64 MB budget from + /// QueryStoreCollector.MaxTextBytesPerDatabase, 4-wide sweep from the #1553 gate), so an upgraded + /// store changes nothing until someone turns a dial. Both are clamped on READ (budget [4,256] MB, + /// sweeps [1,16]) rather than by CHECK constraints, matching the sibling knobs' posture: a bad value + /// degrades to a sane one instead of failing the service's config load. + /// + private const string V59Sql = @" +ALTER TABLE config.config_service + ADD COLUMN IF NOT EXISTS query_store_text_budget_mb integer NOT NULL DEFAULT 64; +ALTER TABLE config.config_service + ADD COLUMN IF NOT EXISTS max_concurrent_sweeps integer NOT NULL DEFAULT 4;"; + + /// + /// V60 — restart-surviving edge memory for the database-state alert (#2166). Two nullable columns on + /// config.database_state_expected, which is already keyed per (server, database) and already + /// exists for this alert, so the memory lives beside the config it belongs with rather than in a new + /// table or smuggled into config_edge_trigger_watermarks' metric_name (that column feeds alert + /// history and mute matching; a compound key hidden in a label is a trap). + /// + /// Why it must persist: the reporter's case is a database deliberately parked OFFLINE for a + /// month. Edge-triggering on in-memory state would re-fire every parked database on every service + /// restart — worse than the cooldown-repeat it replaces. NULL means never alerted, so an upgraded + /// store's first evaluation fires once per deviating database and then goes quiet. + /// + private const string V60Sql = @" +ALTER TABLE config.database_state_expected + ADD COLUMN IF NOT EXISTS last_alerted_state text; +ALTER TABLE config.database_state_expected + ADD COLUMN IF NOT EXISTS last_alerted_at timestamp;"; + + /// + /// V61 — the monotonic per-fingerprint occurrence counters (#2216). The rolling-window count that rides + /// on an alert incident is a GAUGE: it rises as events arrive and falls as they age out of the groupers' + /// read window, so a consumer that only sees throttled deliveries (one per #1154 per-fingerprint + /// cooldown) cannot recover how many events actually happened between two of them. This table is the + /// accumulator's memory, keyed by the #1140 dedup fingerprint. + /// + /// A NEW table rather than columns on config_edge_trigger_watermarks, for two independent + /// reasons. The key is wrong: watermarks are per (server, metric) while occurrences are per (server, + /// metric, FINGERPRINT) — a deadlock on one table and a deadlock on another are separate incidents with + /// separate totals, and folding them into one row would report their sum under both. And the cross-store + /// twin cannot take the columns: Lite writes that same row with INSERT OR REPLACE and a PARTIAL + /// column list, so any column added there is silently reset to its default every time an alert fires — + /// the counter would zero itself precisely when it was being read. + /// + /// config schema because it joins the alert-coordination family (the V8 remarks put the + /// edge-trigger watermarks there for the same reason): service-written, operator-visible, keyed by + /// server. Schema-qualified per the V17 rule — the migrate session's search_path would otherwise resolve + /// a bare name into collect. No per-table grant (provisioning re-runs + /// GRANT … ON ALL TABLES IN SCHEMA config after the migration pass) and no + /// ViewerRestrictedConfigTables carve: the only identity stored is the fingerprint HASH, never + /// the involved object names it was computed from, so there is nothing here for the network-reachable + /// mcp role to read that it should not. + /// + /// NO config_bump_version trigger, per the V32 precedent: this is the service's own + /// coordination state, written on the alert path, and nothing reloads on it. A beacon bump would make + /// every delivered alert trigger a needless fleet reconcile. + /// + /// last_observed_at is not display data — it is what makes a row's staleness decidable. The + /// service deletes a fingerprint's row when its incident ends, but a host that dies mid-incident leaves + /// one behind, and a stranded row trusted on the fingerprint's NEXT incident would decay its + /// already-counted mark to the new window count, read the recurrence as nothing new, and report a stale + /// total under a stale start time. The accumulator therefore ignores rows older than its read window. + /// Row growth needs no separate GC: each delivery REPLACES the set for its (server, metric), so the + /// table holds the live fingerprints plus whatever a crash stranded until that metric next fires. + /// + private const string V61Sql = @" +CREATE TABLE IF NOT EXISTS config.incident_occurrences ( + server_id integer NOT NULL, + metric_name text NOT NULL, + dedup_key text NOT NULL, + total_occurrences bigint NOT NULL, + observed_window_count integer NOT NULL, + incident_started_at timestamp NOT NULL, + last_observed_at timestamp NOT NULL DEFAULT (now() AT TIME ZONE 'UTC'), + PRIMARY KEY (server_id, metric_name, dedup_key) +);"; + + /// + /// V62 — the #2171 plan-XML codec knob for direct-SQL store consumers. 'gzip' (default) keeps + /// today's write path; 'none' makes the dim writer store plain text in query_plan_xml (lz4 TOAST + /// compresses, ~8.9x measured vs gzip's 14.0x) so Grafana-class readers get plans back with plain + /// SQL — PostgreSQL exposes no inflate, so gzip bytes are unreadable without an untrusted-language + /// UDF, which is the contract failure #2171 reports. Rides config_service like V58/V59 so the + /// config_version trigger makes a flip visible to the next reload poll. The CHECK mirrors the + /// provider's normalization; both fail toward 'gzip'. Rides directly above #2216's V61 — the + /// merge-order gate this PR carried (never land 62 over a vacant 61; ascent-only applier) was + /// satisfied when that rung merged; the #2227 density pin now enforces the rule mechanically. + /// + private const string V62Sql = @" +ALTER TABLE config.config_service + ADD COLUMN IF NOT EXISTS plan_xml_compression text NOT NULL DEFAULT 'gzip'; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'config_service_plan_xml_compression_check' + ) THEN + ALTER TABLE config.config_service + ADD CONSTRAINT config_service_plan_xml_compression_check + CHECK (plan_xml_compression IN ('gzip', 'none')); + END IF; +END $$;"; + + /// + /// V63 — pg_wait_stats, the first PostgreSQL collector table: cumulative Aurora wait + /// counters with deltas computed on write, the Postgres counterpart of wait_stats. + /// Columns are spelled out here in the generator's exact emission order (the four standard + /// prefix columns, then 's payload + /// in its declared order) so a fresh store — where V1 generates this from the catalog — and an + /// upgraded store, where this rung creates it, end up with an identical physical column order for + /// the binary COPY. No PRIMARY KEY, and the (server_id, collection_time) index, per the + /// convention every collector table follows. + /// wait_time_us is MICROSECONDS. The AWS documentation contradicts itself on the unit + /// — microseconds for aurora_stat_system_waits, milliseconds for + /// aurora_stat_backend_waits — so it was settled by measurement instead: read as + /// milliseconds, the observed totals imply tens of thousands of concurrently waiting sessions + /// against a max_connections of 5,000, which is impossible. The name carries the unit so a + /// reader never has to relitigate it. + /// Both the numeric id and the decoded name are stored. The name is what an operator reads, + /// but the id is the stable key: wait-event name casing differs between Aurora majors + /// (AutoVacuumMain on 16.11 versus AutovacuumMain on 17.7), so anything keyed on the + /// name breaks its own history across an upgrade. Nullable because the type/event lookups are LEFT + /// JOINed — an event Aurora reports but does not name is still recorded. + /// + private const string V63Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_wait_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + wait_type_id integer, + wait_event_id bigint, + wait_type text, + wait_event text, + waits bigint, + wait_time_us bigint, + delta_waits bigint, + delta_wait_time_us bigint +); + +CREATE INDEX IF NOT EXISTS idx_pg_wait_stats_time + ON collect.pg_wait_stats(server_id, collection_time);"; + + /// + /// V64 — pg_statement_stats, per-query-shape execution statistics from Aurora's extended + /// aurora_stat_statements(): the Postgres counterpart of query_stats. + /// Two column groups exist nowhere on the SQL Server side. The Aurora I/O source split + /// (storage_blks_read / orcache_blks_hit and their times) decomposes what is otherwise + /// an opaque block read into "came from the storage volume" versus "hit the local NVMe tier" — which + /// is why a cache-hit ratio computed the community way is arithmetically misleading on Aurora. And + /// total_exec_peakmem_bytes / max_exec_peakmem_bytes are the nearest thing PostgreSQL + /// has to memory-grant data, which core PostgreSQL has no concept of at all. Per-query + /// wal_bytes likewise has no SQL Server DMV equivalent. + /// No query text column, deliberately. Text belongs in the shared + /// query_text_dim rather than inline — inline payload was 94% of a 250 GB field store — but + /// registering a new dim-feeding table cannot be done from a rung this late: V38 is GENERATED from + /// PayloadDimensions.All, so adding an entry makes V38 emit + /// ALTER TABLE pg_statement_stats ADD COLUMN query_text_digest, and on an upgraded store V38 + /// runs long before this rung creates the table — the ALTER would hit a nonexistent table and fail + /// the entire migration. Retrofitting a dim-feeding table therefore needs either a + /// existence-guarded V38 or a rung-aware dimension registry, which is a design change and not a + /// drive-by. Until then queryid is the identity, which is the join key anyway, and text + /// arrives with a dedicated low-cadence text collector that stores each statement once instead of + /// once per snapshot. + /// + private const string V64Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_statement_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + queryid bigint, + database_id bigint, + user_id bigint, + toplevel boolean, + calls bigint, + total_exec_time_ms double precision, + min_exec_time_ms double precision, + max_exec_time_ms double precision, + mean_exec_time_ms double precision, + rows_returned bigint, + shared_blks_hit bigint, + shared_blks_read bigint, + shared_blks_dirtied bigint, + shared_blks_written bigint, + temp_blks_read bigint, + temp_blks_written bigint, + blk_read_time_ms double precision, + blk_write_time_ms double precision, + storage_blks_read bigint, + orcache_blks_hit bigint, + storage_blk_read_time_ms double precision, + orcache_blk_read_time_ms double precision, + wal_records bigint, + wal_fpi bigint, + wal_bytes bigint, + total_exec_peakmem_bytes bigint, + max_exec_peakmem_bytes bigint, + delta_calls bigint, + delta_total_exec_time_ms bigint, + delta_rows bigint +); + +CREATE INDEX IF NOT EXISTS idx_pg_statement_stats_time + ON collect.pg_statement_stats(server_id, collection_time);"; + + /// + /// V65 — pg_wraparound_stats: transaction id and MultiXact id freeze headroom per database. + /// Both counters are stored, because they are independent and each is separately fatal. + /// MultiXact exhaustion is the one almost nobody monitors: ids are consumed when a row is locked by + /// several transactions at once, so a SELECT FOR UPDATE-heavy or foreign-key-heavy workload + /// burns them much faster than plain transaction ids, and a server can look comfortable on XID age + /// while being in trouble on MultiXact age. + /// The percentages are STORED rather than derived on read because their denominators are + /// per-server settings. Recomputing later against whatever autovacuum_freeze_max_age happens + /// to be then would silently rewrite history the moment someone tunes it; a stored percentage stays + /// true to the configuration in force when it was measured. + /// The first PostgreSQL collector table that is not Aurora-specific — it reads only core + /// catalog surfaces, so it populates on any PostgreSQL target. + /// + private const string V65Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_wraparound_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + frozen_xid_age bigint, + min_multixid_age bigint, + autovacuum_freeze_max_age bigint, + autovacuum_multixact_freeze_max_age bigint, + pct_toward_emergency_vacuum double precision, + pct_toward_wraparound double precision, + pct_toward_multixact_emergency double precision, + pct_toward_multixact_wraparound double precision, + xids_remaining bigint, + multixids_remaining bigint, + allows_connections boolean +); + +CREATE INDEX IF NOT EXISTS idx_pg_wraparound_stats_time + ON collect.pg_wraparound_stats(server_id, collection_time);"; + + /// + /// V66 — pg_xmin_horizon: what is holding back the xmin horizon, attributed by cause. + /// Four unrelated causes produce an identical picture — dead tuples accumulate, autovacuum + /// runs and reports success, nothing shrinks — and the fix differs completely for each: kill a + /// session, drop a replication slot, disable standby feedback, or resolve an orphaned prepared + /// transaction. That is why this table stores one row per SOURCE with the oldest holder for that + /// source, plus an is_winner flag, rather than a single horizon age. Attribution is the whole + /// value; an aggregate would leave a reader exactly where they started. + /// is_winner is stamped at collection rather than derived on read, so a stored row names + /// the winner as of the moment it was measured — deriving it later would depend on which rows a + /// query happened to select, and a filtered read could crown a holder that never held the horizon. + /// Zero rows is the HEALTHY state and must never be read as a collection failure. Note also + /// that standby_feedback is expected to be absent on Aurora, whose replicas read the same + /// storage volume instead of streaming WAL. + /// + private const string V66Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_xmin_horizon ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + source text, + xmin_age bigint, + holder text, + detail text, + is_winner boolean +); + +CREATE INDEX IF NOT EXISTS idx_pg_xmin_horizon_time + ON collect.pg_xmin_horizon(server_id, collection_time);"; + + /// + /// V67 — collect.pg_replication_slot_stats: slot state, including the two independent ways an abandoned + /// slot can take a server down. + /// retained_wal_bytes is the disk-exhaustion measure and is COMPUTED rather than read, + /// because the column that would answer it directly — safe_wal_size — is NULL whenever + /// max_slot_wal_keep_size is -1, which is the default. Reading only that column would + /// mean reporting nothing on a stock server, exactly where retention is unbounded. -1 is + /// stored as the not-applicable sentinel so a consumer cannot mistake "no limit configured" for "no + /// data collected". + /// inactive_since and invalidation_reason are PostgreSQL 17+ and + /// conflicting is 16+; on older majors the collector substitutes NULL/false so the table shape + /// stays constant across a mixed-version fleet and a chart does not change shape at an upgrade. + /// inactive_since is the column that distinguishes a consumer between polls from a slot + /// orphaned three weeks ago. + /// + private const string V67Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_replication_slot_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + slot_name text, + slot_type text, + plugin text, + database_name text, + is_active boolean, + active_pid bigint, + is_temporary boolean, + two_phase boolean, + wal_status text, + safe_wal_size_bytes bigint, + retained_wal_bytes bigint, + xmin_age bigint, + catalog_xmin_age bigint, + inactive_since timestamp, + invalidation_reason text, + conflicting boolean +); + +CREATE INDEX IF NOT EXISTS idx_pg_replication_slot_stats_time + ON collect.pg_replication_slot_stats(server_id, collection_time);"; + + /// + /// V68 — collect.pg_autovacuum_stats, per-table autovacuum state, and the first PostgreSQL + /// collector on the per-database fan-out path. + /// The threshold columns are what make the table worth having. Dead-tuple counts alone are not + /// actionable — autovacuum fires at autovacuum_vacuum_threshold + scale_factor * reltuples, so + /// the same count is routine on a large table and urgent on a small one. The collector computes each + /// table's OWN threshold, honouring per-table reloptions overrides rather than only the GUCs, + /// because those overrides are common on exactly the big hot tables where the global default is + /// wrong. + /// inserts_since_vacuum / insert_vacuum_threshold are PostgreSQL 13+ and carry + /// -1 on older majors so the table shape stays constant across a mixed-version fleet. They + /// cover the append-only case, which has no dead tuples at all and is therefore invisible to the + /// dead-tuple rule — and an append-only table that is never vacuumed is never frozen either. + /// database_name comes from the per-database loop's connection rather than the result + /// set: pg_stat_user_tables shows only the connected database, so the connection IS the + /// authoritative answer. Additive and view-less exactly like V63–V67 — a fresh store gets the table + /// from V1's generated schema, and this rung is what an already-existing store gets. + /// + private const string V68Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_autovacuum_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + schema_name text, + table_name text, + live_tuples bigint, + dead_tuples bigint, + mods_since_analyze bigint, + inserts_since_vacuum bigint, + vacuum_threshold bigint, + insert_vacuum_threshold bigint, + analyze_threshold bigint, + autovacuum_disabled boolean, + total_bytes bigint, + last_vacuum timestamp, + last_autovacuum timestamp, + last_analyze timestamp, + last_autoanalyze timestamp, + vacuum_count bigint, + autovacuum_count bigint, + analyze_count bigint, + autoanalyze_count bigint +); + +CREATE INDEX IF NOT EXISTS idx_pg_autovacuum_stats_time + ON collect.pg_autovacuum_stats(server_id, collection_time);"; + + /// + /// V69 — collect.pg_io_stats, I/O attributed to a (backend_type, object, context) triple rather + /// than to a file, from pg_stat_io (PostgreSQL 16+). + /// Every counter column is NULLABLE and that is load-bearing, not incidental. PostgreSQL uses + /// NULL for "this counter does not apply to this combination" — the checkpointer performs no reads, + /// bulkread never extends, the normal context has no ring buffer to reuse — and on Aurora + /// the entire write side is NULL because backends there do not write data files. A NOT NULL column with + /// a 0 default would claim measurements that were never taken, and a consumer averaging write latency + /// would divide by them. + /// Cumulative counters stored raw, with the windowed change computed at read time. Additive and + /// view-less exactly like V63-V68: a fresh store gets the table from V1's generated schema, and this + /// rung is what an already-existing store gets. + /// + private const string V69Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_io_stats ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + backend_type text, + object_type text, + context text, + reads bigint, + read_time_ms double precision, + writes bigint, + write_time_ms double precision, + writebacks bigint, + writeback_time_ms double precision, + extends bigint, + extend_time_ms double precision, + op_bytes bigint, + hits bigint, + evictions bigint, + reuses bigint, + fsyncs bigint, + fsync_time_ms double precision, + stats_reset timestamp +); + +CREATE INDEX IF NOT EXISTS idx_pg_io_stats_time + ON collect.pg_io_stats(server_id, collection_time);"; + + /// + /// V70 — config.config_monitored_servers.engine and .port, the two columns without which a + /// PostgreSQL target cannot survive its own registration. + /// The registry is store-authoritative after the first seed: darling.json seeds it once, and from + /// then on the worker's server list comes from this table. Every other MonitoredServer field had a + /// column here; these two did not, so a PostgreSQL entry round-tripped through the store as + /// "sqlserver" on the driver's default port (both property defaults) and the service then opened a + /// SqlConnection to it. That happens on the FIRST start, not a later one, because the seed is + /// immediately followed by the load that replaces the file's list with the store's. + /// Both defaults are what make this safe on an existing store. Every row already there is a SQL + /// Server target, and port is consumed only by the PostgreSQL connection builder (the SQL Server + /// path carries a port in the host string), so 0 means "the driver's default" exactly as the + /// property does. The SQL-Server-only writers — the Viewer's Add / Manage Servers dialogs — keep + /// inserting without naming either column and keep meaning the same thing. + /// + private const string V70Sql = @" +ALTER TABLE config.config_monitored_servers + ADD COLUMN IF NOT EXISTS engine text NOT NULL DEFAULT 'sqlserver'; + +ALTER TABLE config.config_monitored_servers + ADD COLUMN IF NOT EXISTS port integer NOT NULL DEFAULT 0;"; + + /// + /// V71 — collect.pg_blocking_edges: who is blocked, by whom, and what state each side was in. + /// An EDGE LIST, which is why the table is named for edges rather than for chains. One row per + /// (blocked, blocking) pair, so a chain of four is four rows and a blocker with thirty victims is thirty. + /// Storing a rendered tree instead would bake in one traversal and make root-blocker, depth, and fan-out + /// queries string work; from edges they are ordinary SQL. + /// Both sides carry their own state because the remedy depends on it: a chain rooted in + /// idle in transaction is an application defect, one rooted in a long-running query is a tuning + /// problem, and the pid alone does not distinguish them. That doubling of columns is the point of the + /// table. + /// Reader beware — sparse by design. This table is empty on a healthy instance, and unlike + /// SQL Server's blocked_process_report there is no engine-side recorder behind it: PostgreSQL + /// materialises nothing unless something asks, so a gap means "not sampled", not "not blocked". A count + /// over this table measures how often blocking was CAUGHT. + /// Additive and view-less exactly like V63-V69: a fresh store gets the table from V1's generated + /// schema, and this rung is what an already-existing store gets. + /// + private const string V71Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_blocking_edges ( + collection_id bigint NOT NULL, + collection_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + blocked_backend_id bigint, + blocked_pid integer, + blocking_backend_id bigint, + blocking_pid integer, + database_name text, + blocked_username text, + blocked_application_name text, + blocked_client_addr text, + blocked_state text, + blocked_wait_event_type text, + blocked_wait_event text, + blocked_query text, + blocked_xact_duration_ms bigint, + blocked_query_duration_ms bigint, + blocking_username text, + blocking_application_name text, + blocking_client_addr text, + blocking_state text, + blocking_wait_event_type text, + blocking_wait_event text, + blocking_query text, + blocking_xact_duration_ms bigint, + blocking_query_duration_ms bigint, + blocked_pid_count integer, + blocking_is_idle_in_transaction boolean, + query_text_may_be_truncated boolean +); + +CREATE INDEX IF NOT EXISTS idx_pg_blocking_edges_time + ON collect.pg_blocking_edges(server_id, collection_time);"; + + /// + /// V73 — collect.pg_statement_text: one row per (server_id, queryid) holding the statement text + /// for a PostgreSQL target, so get_pg_top_queries can return something a human can read (#2219). + /// + /// The gap this closes. pg_statement_stats identifies queries by queryid and stores + /// no text, because aurora_stat_statements's showtext costs real money per collection and + /// normalized text is highly repetitive. But queryid is NOT stable across a major version upgrade, so + /// after one the stored history joins to nothing readable — a list of integers that used to be your slowest + /// queries. Keying text on (server_id, queryid) is what preserves the OLD ids' text when the live view + /// re-keys, which is the whole point: no live fetch can recover it afterwards. + /// + /// Inline text, not a query_text_dim digest, and that is a deliberate reversal of what V64's + /// comment promised. The dimension route is blocked and would stay expensive to unblock: V38 is GENERATED + /// from PayloadDimensions.All, so registering pg_statement_stats makes V38 emit an + /// ALTER TABLE against a table it has not created yet on every upgraded store, and it would also break + /// V64's own ladder diff, which asserts each rung equals the generated schema. More importantly the dimension + /// needs the liveness interlock documents at length — the GC sweeps on + /// last_seen rather than counting references, so a dim row can be collected while live facts still + /// point at it, and the failure mode is SILENTLY missing text. Inline cannot dangle. It costs cross-server + /// dedup — one row per server per queryid rather than one per distinct text — which on a 52-server fleet of + /// pg_stat_statements.max = 5000 is a few hundred MB against a store whose Query Store plan XML alone + /// measured 43 GB. Paying that to make a silent-loss mode impossible is the trade. + /// + /// Not a hypertable and not a collector table: one row per statement per server, near-static once a + /// workload is warm, so it is dimension-shaped and pruned on last_seen rather than by + /// drop_chunks. That also keeps it out of the generated-schema ladder diff, exactly as V72's + /// query_store_plan_map is — the established shape for content keyed to facts rather than collected + /// as facts. + /// + /// first_seen is kept alongside last_seen because they answer different questions: when a + /// statement shape first appeared on this server (which survives the upgrade re-key and is the only record of + /// it) versus whether the text is still live enough to keep. NUMBERED max(dev) + 1 without a gap, for + /// the reason V72's comment gives — a gap is skipped silently on every upgraded store. + /// + private const string V73Sql = @" +CREATE TABLE IF NOT EXISTS collect.pg_statement_text ( + server_id integer NOT NULL, + queryid bigint NOT NULL, + query_text text NOT NULL, + first_seen timestamp NOT NULL, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, queryid) +); +CREATE INDEX IF NOT EXISTS idx_pg_statement_text_last_seen + ON collect.pg_statement_text(last_seen);"; + + /// + /// V76 — the per-database Query Store health table (#2319): what database_config's single + /// is_query_store_on bit cannot say — actual vs desired state (the cap-hit READ_ONLY transition + /// and its readonly_reason), current vs max storage, cleanup mode and thresholds, and the + /// runtime-stats interval length. Body matches PgSchemaGenerator's emission for the definition + /// (verified by generating it) plus the rung's explicit collect. prefix, so fresh stores (which + /// generate from the catalog) and upgraded stores (which run this rung) agree byte-for-byte. + /// Hypertable conversion is automatic from CollectorCatalog on the next service start, the same + /// path pvs_stats took in V47. The v_ passthrough keeps the two viewers' SQL byte-identical. + /// + private const string V76Sql = @" +CREATE TABLE IF NOT EXISTS collect.query_store_health ( + config_id bigint NOT NULL, + capture_time timestamp NOT NULL, + server_id integer NOT NULL, + server_name text NOT NULL, + database_name text, + actual_state text, + desired_state text, + readonly_reason integer, + current_storage_size_mb bigint, + max_storage_size_mb bigint, + size_based_cleanup_mode text, + stale_query_threshold_days bigint, + max_plans_per_query bigint, + interval_length_minutes bigint +); + +CREATE INDEX IF NOT EXISTS idx_query_store_health_time ON collect.query_store_health(server_id, capture_time); + +CREATE OR REPLACE VIEW v_query_store_health AS SELECT * FROM query_store_health;"; + + /// + /// V75 — the plan-content retention knob (#2316). The payload dimensions' GC horizon is coupled to + /// the WIDEST dim-feeding fact retention (90 days) so a raised override can never orphan a reader — + /// which also means a store younger than that horizon has an UNBOUNDED plan dimension: measured on + /// the 42-server dogfood fleet, query_plan_dim reached 127 GB (63% of the store) in its first + /// 22 days, growing ~6 GB/day of parameter-sniffing recompile churn (65 distinct XMLs per plan SHAPE + /// per day; the worst single shape produced 57k in one day), with the coupled GC unable to delete a + /// single row until the horizon crossed the dim's birth date — a month AFTER the projected disk-full. + /// This knob decouples plan CONTENT lifetime from fact lifetime: facts keep their full retention + /// (metrics, hashes and text stay analyzable); stored plan XML older than this many days since last + /// sighting becomes unfetchable, which every reader already renders as a missing plan. Clamped on + /// READ like the V59 knobs ([7,365]; 0 = disabled, restoring the fact-coupled horizon alone). + /// + private const string V75Sql = @" +ALTER TABLE config.config_service + ADD COLUMN IF NOT EXISTS plan_content_retention_days integer NOT NULL DEFAULT 21;"; + + /// + /// V74 — where the query-text fetch lands statement text (#2150), keyed + /// (server_id, database_name, query_id). + /// + /// The runtime-stats payload carried query_sql_text (nvarchar(max)) inside a + /// TOP ... WITH TIES ... ORDER BY last_execution_time, and a Top-N Sort carries every output + /// column through the sort while reading ALL of its input first — so choosing the rows to ship + /// materialized text for the entire qualifying set. With #2210's plan XML already gone and that column + /// as the only difference, time-to-first-row measured 4.67s against 0.45s at 1,505 rows and 5.02s + /// against 0.57s at 4,037. + /// + /// Keyed on query_id rather than query_text_id because query_id is ALREADY a + /// stored column on the fact table — so this rung adds a table and touches nothing existing, and readers + /// get the join key for free. Text is stored INLINE rather than as a digest into query_plan_dim's + /// sibling: Query Store already de-duplicates it one row per statement per database, so there is nothing + /// to squeeze, and inline removes the dimension GC liveness interlock whose failure mode is silently + /// missing text. Not a hypertable — it has a PRIMARY KEY and no time dimension — so it is pruned on + /// last_seen rather than by drop_chunks, exactly like query_store_plan_map. + /// + private const string V74Sql = @"CREATE TABLE IF NOT EXISTS collect.query_store_text ( + server_id integer NOT NULL, + database_name text NOT NULL, + query_id bigint NOT NULL, + query_sql_text text, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, database_name, query_id) +); +CREATE INDEX IF NOT EXISTS idx_query_store_text_last_seen + ON collect.query_store_text(last_seen);"; + + /// + /// V72 — the Query Store plan map (#2210): (server_id, database_name, plan_id) → digest, so Query + /// Store facts can reference plan XML they no longer carry once the cutover moves that content into the + /// shared query_plan_dim. Plan XML was stored INLINE on query_store_stats at roughly 5x + /// redundancy — the same plans re-shipped pass after pass — which is what this replaces. + /// + /// plan_hash is the re-verification key and is nullable on purpose: rows written before it + /// existed re-verify once and self-heal. last_seen is the liveness column the map prune sweeps and + /// the batch touch refreshes — load-bearing, because the dimension GC decides what to collect from + /// last_seen rather than by counting references, and ending the re-shipping ends the signal that + /// used to keep those dim rows alive. + /// + /// NUMBERED max(dev) + 1, WITHOUT a gap, and that is the load-bearing part. The runner skips + /// any rung at or below the store's stamped version, so a gap left for another in-flight branch is skipped + /// SILENTLY on every upgraded store the moment this one stamps a higher number. Gapping is only safe when + /// the gap-filler lands first, which a branch cannot guarantee about another branch. + /// + /// The race this comment was written to survive HAPPENED: it was V61 while #2213 and the PostgreSQL + /// collector rungs were in flight, they merged first and took the ladder to V71, and the collision surfaced + /// as a conflict on the migration list — loudly, on the merge, exactly as intended — so this renumbered to + /// sit immediately above them. A collision is loud; a gap is silent, and a map table that was never created + /// reads as "plan not yet collected" on every lookup, so the cutover would look healthy and hold nothing. + /// + private const string V72Sql = @" +CREATE TABLE IF NOT EXISTS collect.query_store_plan_map ( + server_id integer NOT NULL, + database_name text NOT NULL, + plan_id bigint NOT NULL, + digest bytea NOT NULL, + plan_hash text, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, database_name, plan_id) +); +CREATE INDEX IF NOT EXISTS idx_query_store_plan_map_last_seen + ON collect.query_store_plan_map(last_seen);"; + /// /// V9 — the FinOps copy-parity fields that were user-input config or previously live-only: /// server_properties gains the three inventory columns the shared ServerPropertiesCollector now @@ -1168,8 +1915,11 @@ ALTER TABLE query_plan_dim /* --- A. Config plane: the Viewer writes desired state, the service reads + honors it. --- */ /* 1. config_monitored_servers — the desired-state twin of the collect.servers observed registry. - server_id = ServerIdHelper.GetDeterministicHashCode(BuildStorageName(host,database,ro)), the - SAME identity the collectors stamp, so it JOINs collected data. is_enabled drives collection; + server_id is THIS ROW'S identity and this table owns it: the service reads it here rather than + recomputing it, so a stored id keeps working when the fields below no longer produce it (#2218). + It is minted from the storage identity host[:database][:RO] — the same value the collectors + stamp, which is why it JOINs collected data and why no existing store needs migrating — but that + is now the ALLOCATION rule, not a definition anything re-derives. is_enabled drives collection; the connection fields reconstruct a MonitoredServer for the service's connect path. */ CREATE TABLE IF NOT EXISTS config.config_monitored_servers ( server_id integer NOT NULL PRIMARY KEY, diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgSchemaGenerator.cs b/Darling/PerformanceMonitor.Darling.Storage/PgSchemaGenerator.cs index 66b86f153..daeb0d3a9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgSchemaGenerator.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgSchemaGenerator.cs @@ -190,6 +190,7 @@ public static string CreateTable(ICollectorSchemaInfo schema) SessionSummaryStatsCollector.Instance, SystemHealthEventsCollector.Instance, PvsStatsCollector.Instance, + QueryStoreHealthCollector.Instance, }; /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgStatementText.cs b/Darling/PerformanceMonitor.Darling.Storage/PgStatementText.cs new file mode 100644 index 000000000..4882630e4 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/PgStatementText.cs @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The (server_id, queryid) → statement text store for PostgreSQL targets (#2219), so +/// get_pg_top_queries returns something a human can read. +/// +/// The gap. pg_statement_stats identifies queries by queryid and stores no text — +/// aurora_stat_statements's showtext is a real per-collection cost and normalized text is highly +/// repetitive, so storing it per snapshot would be almost entirely duplication. But queryid is NOT stable +/// across a major version upgrade, so afterwards the stored history joins to nothing readable: a list of integers +/// that used to be your slowest queries. Keying text on (server_id, queryid) preserves the OLD ids' text +/// when the live view re-keys, which no on-demand fetch can recover — the live view no longer has the entry, and +/// anything else reading pg_stat_statements on the instance may have reset it (pganalyze's collector calls +/// pg_stat_statements_reset() on a size budget). +/// +/// Written here rather than by a collector, on purpose. The collector framework writes append-only +/// binary COPY into a hypertable, once per collection — which is exactly the duplication being avoided. This is +/// an UPSERT of one row per statement, so it takes the shape established for +/// content keyed to facts rather than collected as facts: a plain table, a bespoke write path, pruned on +/// last_seen. Being outside the catalog also keeps it out of the generated-schema ladder diff, which +/// compares rungs against PgSchemaGenerator output for collector tables only. +/// +/// Idempotent by construction, which is what makes the cadence a free choice. Every fetch upserts +/// the same rows, so re-fetching costs one statement and no growth — there is no "which queryids do I already +/// have" bookkeeping to get wrong, and no watermark to corrupt. The cadence therefore only trades freshness +/// against the showtext cost, and asks the STORE when it last wrote rather than +/// keeping state in the service, so a restart cannot re-fetch the fleet. +/// +public static class PgStatementText +{ + /// The table. Not a hypertable: one row per statement per server, near-static once a workload is + /// warm, so it is dimension-shaped and pruned on rather than by drop_chunks. + public const string TableName = "collect.pg_statement_text"; + + /// The liveness column the prune sweeps and every upsert refreshes. + public const string LastSeenColumn = "last_seen"; + + /// + /// How often text is re-fetched for a server. One hour, and the reasoning is a cost trade rather than a + /// preference: showtext is paid for the WHOLE aurora_stat_statements call regardless of how few + /// rows want text, so the only lever is cadence — and the thing being collected barely changes, because a + /// statement's normalized text is a property of its queryid. Hourly means a newly-appeared statement + /// is unreadable for at most an hour, against 24 calls per server per day. + /// + public static readonly TimeSpan RefreshInterval = TimeSpan.FromHours(1); + + public const string CreateTableSql = @"CREATE TABLE IF NOT EXISTS collect.pg_statement_text ( + server_id integer NOT NULL, + queryid bigint NOT NULL, + query_text text NOT NULL, + first_seen timestamp NOT NULL, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, queryid) +); +CREATE INDEX IF NOT EXISTS idx_pg_statement_text_last_seen ON collect.pg_statement_text(last_seen);"; + + /// + /// The text fetch, against the monitored PostgreSQL server. $1 caps the rows. + /// + /// showtext = true is the entire point of this query and the reason it is separate from + /// pg_statement_stats's: that one passes false every minute and must keep doing so. Ordered by + /// total_exec_time descending and capped, so if a catalog holds more statements than the cap the text + /// that lands is the text for the queries anyone would look at — a truncation that keeps the useful half + /// rather than an arbitrary one. + /// + /// Aliased explicitly, per the house rule: an unaliased expression comes back named after the function + /// and the query stops being debuggable in psql, which is the one tool anyone reaches for. toplevel is + /// filtered rather than grouped — a nested statement shares its parent's text and would only duplicate the + /// row it upserts into. + /// + public const string FetchSql = @" +SELECT + s.queryid AS queryid, + s.query AS query_text +FROM aurora_stat_statements(true) AS s +WHERE s.queryid IS NOT NULL +AND s.query IS NOT NULL +AND s.toplevel +ORDER BY s.total_exec_time DESC +LIMIT $1"; + + /// + /// Whether this server is due a text fetch — asked of the STORE rather than remembered in the service, so a + /// restart does not re-fetch the whole fleet and two hosts cannot disagree about when they last wrote. + /// + /// Returns true when the server has no rows at all (the first fetch) or its newest row is older than + /// the interval. $2 is the caller's naive-UTC now, passed in rather than read from now() so the + /// decision uses the same clock as the timestamps it writes — mixing the store's clock with the service's is + /// how a cadence check drifts by exactly the offset nobody measures. + /// + public const string IsDueSql = @"SELECT COALESCE( + (SELECT max(last_seen) FROM collect.pg_statement_text WHERE server_id = $1) < $2::timestamp, + TRUE) AS is_due"; + + /// + /// Upserts a batch. first_seen is preserved on conflict — it records when this statement shape was + /// first seen on this server, which is the one fact that survives a major-version re-key and cannot be + /// recovered afterwards; overwriting it would quietly turn every row's age into "since the last fetch". + /// + /// query_text IS advanced on conflict. A queryid is derived from the parse tree so its + /// text is stable in practice, but not by guarantee across versions — and if it ever differs, the newer text + /// is the one that matches the stats being collected now. + /// + /// Ordered by the conflict key for the #1801 reason gives: + /// concurrent batch upserts taking row locks in different relative orders deadlock, and this runs per server + /// across a fleet. Cheap to keep, expensive to rediscover. + /// + public const string UpsertSql = @"INSERT INTO collect.pg_statement_text + (server_id, queryid, query_text, first_seen, last_seen) +SELECT server_id, queryid, query_text, stamped, stamped +FROM unnest($1::integer[], $2::bigint[], $3::text[], $4::timestamp[]) + AS batch(server_id, queryid, query_text, stamped) +ORDER BY server_id, queryid +ON CONFLICT (server_id, queryid) DO UPDATE SET + query_text = EXCLUDED.query_text, + last_seen = EXCLUDED.last_seen +WHERE EXCLUDED.last_seen >= pg_statement_text.last_seen"; + + /// + /// Strips the before binding to any ::timestamp parameter here — the #1969 + /// trap, and it is silent: Npgsql infers timestamptz from a Utc or Local Kind, PostgreSQL then + /// converts into the session zone on the way into a naive column, and the row lands at the wrong hour with no + /// error. For last_seen that means text ageing out ahead of the facts that reference it, which is the + /// silently-missing-text outcome this design exists to prevent, arrived at through a timezone. + /// + public static DateTime Naive(DateTime utc) => DateTime.SpecifyKind(utc, DateTimeKind.Unspecified); + + /// + /// Days of margin the prune adds past the fact-retention horizon, so text outlives the statistics rows that + /// reference it. Strictly greater than zero for the reason + /// explains for its own pair: the two bad end-states are not symmetric. Text kept past its facts is some dead + /// bytes; facts kept past their text is a reader resolving a live row to nothing, which is the failure this + /// whole table exists to prevent. + /// + public const int PruneMarginDays = 2; + + /// + /// Retires text whose statements have all aged out: an index range scan on , + /// time-sliced like every sibling purge so one sweep cannot take an unbounded lock. + /// + /// Timestamp-driven, NOT an anti-join against pg_statement_stats. The anti-join is the cost this + /// shape avoids, and it is unnecessary because every fetch refreshes last_seen for everything still in + /// the target's pg_stat_statements — so a statement that falls out of the view stops being touched and + /// ages out on its own. + /// + public static string PruneSql(int chunkIntervalDays) => + "DELETE FROM collect.pg_statement_text WHERE " + LastSeenColumn + " < $1" + + " AND " + LastSeenColumn + " >= (SELECT min(" + LastSeenColumn + ") FROM collect.pg_statement_text WHERE " + + LastSeenColumn + " < $1)" + + " AND " + LastSeenColumn + " < (SELECT min(" + LastSeenColumn + ") FROM collect.pg_statement_text WHERE " + + LastSeenColumn + " < $1) + INTERVAL '" + + chunkIntervalDays.ToString(CultureInfo.InvariantCulture) + " days'"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs b/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs index cc128f392..7059a840a 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs @@ -51,7 +51,7 @@ conversion uses. */ /// COVERING composer indexes (INCLUDE the aggregate columns the Procedures / Queries / Query Store measures /// SUM/AVG, for an Index Only Scan), three (server_id, handle/hash/id, collection_time DESC) lookup /// indexes for the single-row analyze_*_plan reads (no INCLUDE — one heap fetch is cheap), then the per-table - /// autovacuum-insert override on exactly the three growing tables. Bare collect-qualified names; every + /// autovacuum-insert override on exactly the four growing tables. Bare collect-qualified names; every /// identifier is a compile-time constant, never user input, so interpolation is not a concern. /// public static IReadOnlyList Statements { get; } = new[] @@ -69,6 +69,12 @@ above. Bounded by raw retention (4 days of chunks), so the build is cheap on any "ALTER TABLE collect.procedure_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", "ALTER TABLE collect.query_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", "ALTER TABLE collect.query_store_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", + /* pg_statement_stats is query_stats' per-minute PostgreSQL twin — same shape, same cadence, same + pure-insert hypertable chunks — so the identical reasoning applies: the stock 0.2 scale factor + leaves the day's hot chunk stale before the TimescaleDB rollover and the Index Only Scan degrades + to heap fetches. It was simply missed when the PostgreSQL collectors landed, since this list is + hand-maintained rather than derived from the catalog. */ + "ALTER TABLE collect.pg_statement_stats SET (autovacuum_vacuum_insert_scale_factor = 0.02, autovacuum_vacuum_insert_threshold = 10000)", }; /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanMap.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanMap.cs new file mode 100644 index 000000000..7669f5c72 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanMap.cs @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The (server_id, database_name, plan_id) → digest map that lets Query Store facts reference plan XML +/// they no longer carry (#2210). Query Store plan XML was stored INLINE on query_store_stats — measured +/// at 43 GB total, 3,743 MB of plan XML per day, and 871,196 XML-carrying rows against 175,328 distinct +/// (database_name, plan_id), so **5.0x** of it was the same plans re-shipped. The plan fetch now lands +/// each plan once, in plan_id order, and this map is how a fact row finds its content. +/// +/// Facts deliberately do NOT gain a digest column, which is why this table exists rather than a +/// entry: a fact row is written when the runtime stats arrive, and under the +/// new ordering that is potentially several budgeted cycles BEFORE its plan's XML is fetched, so there is no +/// digest to write at fact time. The map's absence of a row IS the pending state — distinguishable from "never +/// collected" by whether the plan_id sits above the database's watermark — and a reader with no map row renders +/// "plan not yet collected" instead of resolving to nothing. +/// +/// THIS TABLE'S last_seen IS LOAD-BEARING, AND IT IS THE ONLY PROTECTION QUERY STORE DIGESTS HAVE. +/// The dimension GC does not enumerate references — an anti-join per dim row against two hypertables is not +/// affordable at this size — so it sweeps on last_seen, which the write path refreshes on every cycle +/// that references a digest. That worked precisely BECAUSE plan XML was re-shipped every pass; ending the +/// re-shipping ends the liveness signal, and a plan fetched once would have its dim row collected while live +/// facts still referenced it. Hence , which asserts liveness for plans the batch no +/// longer carries. +/// +/// The GC's second belt does not cover this either: its cutoff is clamped to one day before the oldest +/// surviving DIGEST-CARRYING fact (DarlingRetention.ComputeDimensionCutoff), and Query Store facts carry +/// no digest, so the measured floor is blind to them. Do not add a Query Store entry to +/// to "fix" that — the entry means "this fact column holds a digest", +/// which is not true here, and the clamp would still be measuring a column that does not exist. +/// is the whole of the protection. +/// +public static class QueryStorePlanMap +{ + /// The map table. Not a hypertable: one row per distinct plan per database, ~175k rows/day of + /// churn on the measured fleet and near-static once a catalog is warm, so it is dimension-shaped rather + /// than time-series and is pruned on rather than by drop_chunks. + public const string TableName = "collect.query_store_plan_map"; + + /// The liveness column, swept by the prune and stamped by . + public const string LastSeenColumn = "last_seen"; + + public const string CreateTableSql = @"CREATE TABLE IF NOT EXISTS collect.query_store_plan_map ( + server_id integer NOT NULL, + database_name text NOT NULL, + plan_id bigint NOT NULL, + digest bytea NOT NULL, + plan_hash text, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, database_name, plan_id) +); +CREATE INDEX IF NOT EXISTS idx_query_store_plan_map_last_seen ON collect.query_store_plan_map(last_seen);"; + + /* Deliberately NO index on digest. Nothing reads this table by digest: readers resolve + (server_id, database_name, plan_id) -> digest through the primary key, the liveness touch joins on that + same key, and the dimension GC sweeps its OWN last_seen rather than asking who references a digest. An + index nothing queries is pure write tax on a table every plan fetch upserts into. If a reverse lookup + ("which plans share this content") is ever wanted, add it with the query that needs it. */ + + /// + /// Records what the plan fetch landed: one row per plan, carrying the digest of the content written to + /// query_plan_dim. The conflict arm advances digest as well as last_seen, because a + /// plan whose XML is rewritten in place keeps its plan_id while its content digest changes — the + /// case the watermark's refresh horizon exists to catch, and this is where the corrected content gets + /// pointed at. + /// + /// plan_hash is what makes re-verification cheap, and it is why it is stored here rather than + /// derived: sys.query_store_plan.query_plan_hash reads WITHOUT decompressing the plan, so the + /// re-verify cursor can walk [0..watermark] comparing hashes on cheap columns alone and re-fetch XML + /// only where a hash DIFFERS or a map row is ABSENT. That turns in-place rewrites from a full catalog walk + /// per horizon into per-changed-plan work — 0 of 38,420 plan_ids changed hash across a day of fleet data — + /// and dormant plans fall out of the same pass with no heuristic to separate them from a reset, because mass + /// absence is caught wholesale by the runtime stream's reset arm within one cycle. + /// + /// Ordered by the conflict key. Same reason as : concurrent + /// batch upserts that take row locks in different relative orders deadlock (#1801), and a plan fetch runs + /// per database against a fleet of servers. Checked, not assumed — do not drop the ORDER BY on the belief + /// that a single-row-per-plan insert cannot conflict with anything. + /// + public const string UpsertSql = @"INSERT INTO collect.query_store_plan_map + (server_id, database_name, plan_id, digest, plan_hash, last_seen) +SELECT server_id, database_name, plan_id, digest, plan_hash, stamped +FROM unnest($1::integer[], $2::text[], $3::bigint[], $4::bytea[], $5::text[], $6::timestamp[]) + AS batch(server_id, database_name, plan_id, digest, plan_hash, stamped) +ORDER BY server_id, database_name, plan_id +ON CONFLICT (server_id, database_name, plan_id) DO UPDATE SET + digest = EXCLUDED.digest, + plan_hash = EXCLUDED.plan_hash, + last_seen = EXCLUDED.last_seen +WHERE EXCLUDED.last_seen >= query_store_plan_map.last_seen"; + + /// + /// The liveness assertion, and the reason this whole design is safe: for the distinct + /// (database_name, plan_id) a runtime-stats batch just wrote, refresh BOTH this map row's + /// last_seen and the dimension row's, so neither can age out while facts still point at the plan. + /// The batch already carries those two columns, so nothing extra is collected to make this work. + /// + /// Both timestamps are stamped by the SAME pass, which is what makes the map-prune-versus-dim-GC race + /// structurally impossible rather than carefully avoided: a map row's last_seen can never be older + /// than the newest fact batch that referenced it, so the prune cannot take a row that live facts are + /// touching. + /// + /// It also returns the RESOLVED-ness of every batch row, in the same round trip, because the batch + /// join it already does is where the reset signal lives: a plan the store has never resolved cannot be + /// produced by "no new plans this window". What it deliberately does NOT do is decide that a reset happened. + /// Two reasons, both of which bit the first version of this query: + /// + /// One dormant plan is not a reset. Filtering to absent rows at or below a watermark fires on + /// a SINGLE dormant plan resuming execution, which would zero that database's watermark and trigger a full + /// refetch — the opposite of what this design is for. The reset case is MASS absence, and "mass" is a + /// judgement the caller makes across the batch. A lone absence is the CURSOR's job (it fetches that plan and + /// moves on), which is what RefreshAfter's comment already says owns dormancy.
+ /// • Watermarks are per database. These array parameters can carry rows for several databases in one + /// call, so comparing them all against one scalar watermark is wrong for every database but one. The caller + /// already holds the per-database watermarks; it applies them.
+ /// + /// So this returns facts — (server_id, database_name, plan_id, resolved) — and the host decides. + /// When it does conclude a reset it zeroes that database's watermark and logs loudly, recovering in one + /// cycle rather than waiting on a refresh sweep. + /// + /// The three preceding CTEs still run. Postgres executes data-modifying WITH statements exactly + /// once and to completion whether or not the primary query reads their output, so making the final statement + /// a SELECT does not turn the liveness stamping into a no-op. That is a load-bearing detail: if it were not + /// true, this restructure would silently stop refreshing last_seen and reintroduce the GC hazard the + /// touch exists to prevent. + /// + /// Guarded at one hour like the dimension upsert's own conflict arm, and for the same reason — the + /// horizons are multi-day, so an update per row per hour is enough freshness and the write amplification + /// stays bounded on a hot catalog. The margin arithmetic already accounts for this trailing hour. + /// + /// The CTE is ordered by the map's primary key for the #1801 reason above — this is the one statement + /// in the design that touches many rows across two tables on every cycle of every server, so it is the most + /// likely place for an unordered-batch deadlock to form. Being precise about how much that buys, because + /// can claim more than this can: an ORDER BY inside a CTE + /// feeding UPDATE ... FROM is NOT a guaranteed lock-acquisition order in Postgres the way ordering an + /// INSERT ... ON CONFLICT's input is — the planner may reorder. It makes the common plan deterministic + /// rather than making the deadlock impossible. If one is observed, the fix is to drive the update from an + /// explicitly ordered SELECT ... FOR UPDATE, not to widen this comment. + ///
+ public const string TouchSql = @"WITH touched AS ( + SELECT m.server_id, m.database_name, m.plan_id, m.digest + FROM collect.query_store_plan_map AS m + JOIN unnest($1::integer[], $2::text[], $3::bigint[]) + AS batch(server_id, database_name, plan_id) + ON batch.server_id = m.server_id + AND batch.database_name = m.database_name + AND batch.plan_id = m.plan_id + WHERE m.last_seen < $4::timestamp - interval '1 hour' + ORDER BY m.server_id, m.database_name, m.plan_id +), +map_touch AS ( + UPDATE collect.query_store_plan_map AS m + SET last_seen = $4::timestamp + FROM touched AS t + WHERE m.server_id = t.server_id + AND m.database_name = t.database_name + AND m.plan_id = t.plan_id + RETURNING t.digest +), +dim_touch AS ( + UPDATE collect.query_plan_dim AS d + SET last_seen = $4::timestamp + WHERE d.digest IN (SELECT digest FROM map_touch) + AND d.last_seen < $4::timestamp - interval '1 hour' + RETURNING d.digest +) +SELECT batch.server_id, batch.database_name, batch.plan_id, (m.plan_id IS NOT NULL) AS resolved +FROM unnest($1::integer[], $2::text[], $3::bigint[]) + AS batch(server_id, database_name, plan_id) +LEFT JOIN collect.query_store_plan_map AS m + ON m.server_id = batch.server_id + AND m.database_name = batch.database_name + AND m.plan_id = batch.plan_id +ORDER BY batch.server_id, batch.database_name, batch.plan_id"; + + /// + /// Strips the off a timestamp before it is bound to any of this class's + /// ::timestamp parameters. Every call site that binds a here must go through + /// this — 's stamp array and 's $4, plus the prune's + /// cutoff. + /// + /// This is the #1969 trap, and it is silent. Npgsql infers the parameter type from the value's Kind: + /// a Utc or Local DateTime infers timestamptz, Postgres then converts it into the + /// session time zone on the way into a naive timestamp column, and the row lands at the wrong hour + /// with no error anywhere. For the liveness columns that is worse than a visible failure: a + /// last_seen written hours early ages a map or dimension row out ahead of the facts that reference + /// it, which is the silent-missing-plans outcome this whole design is built to prevent, arrived at through + /// a timezone rather than through a GC bug. + /// + /// A helper rather than a convention because a convention is what fails at the one call site somebody + /// adds later. The value is not shifted, only relabelled: callers are expected to pass UTC already, and + /// changes the Kind without touching the ticks. + /// + public static DateTime Naive(DateTime utc) => DateTime.SpecifyKind(utc, DateTimeKind.Unspecified); + + /// + /// The map rows a re-verify cursor slice needs to judge, for one database over a bounded + /// plan_id range: what content the store believes each plan has. + /// + /// Returns plan_id and plan_hash only — never the digest, never content. The caller + /// pairs this against the same id range read from sys.query_store_plan (also hash-only, which reads + /// without decompressing) and re-fetches XML for exactly three cases: a hash that DIFFERS (the plan was + /// rewritten in place while keeping its id), a map row that is ABSENT (a plan dormant through every + /// collected window, so the watermark passed it without its content ever landing), and a stored + /// plan_hash that is NULL (written by a build before the hash column existed — re-verify once, then + /// it self-heals). + /// + /// This is the whole reason the horizon stopped being a full refetch. The old expiry dropped the + /// watermark to zero and re-walked every plan's XML, which the walk-cost measurement showed cannot even + /// complete inside a day on the larger catalogs (2.2-15.1 GB of plan XML per catalog; 15.9 to 107.5 hours at + /// a 12 MB budget and 5-minute cadence), so those catalogs restarted forever and never reached their own + /// newest plans. A hash-only sweep over the same id range is bounded by ROW count instead of BYTE volume — + /// 77k ids at ~270 per pass — and re-fetches only what actually changed, which across a day of fleet data + /// was 0 of 38,420 plans. + /// + public const string CursorSliceSql = @"SELECT m.plan_id, m.plan_hash +FROM collect.query_store_plan_map AS m +WHERE m.server_id = $1 + AND m.database_name = $2 + AND m.plan_id > $3 + AND m.plan_id <= $4 +ORDER BY m.plan_id"; + + /// + /// The cursor's slice width for one pass: the id range divided by how many passes fit in the sweep period. + /// is no longer an expiry — it is the target period for ONE full + /// re-verification sweep — and this is where that meaning is applied. + /// + /// Floored at one so a cursor always makes progress, and floored again by + /// so a tiny catalog does not crawl an id at a time. Bounded by the range + /// itself, so a sweep never claims to cover ids that do not exist. + /// + public static long CursorSliceWidth(long watermark, TimeSpan refreshAfter, TimeSpan cadence, long minimumSlice = 64) + { + if (watermark <= 0) + { + return 0; + } + + var passes = cadence > TimeSpan.Zero ? refreshAfter.Ticks / cadence.Ticks : 1; + if (passes < 1) + { + passes = 1; + } + + /* CEILING, not floor. Truncating divides a sweep that never completes: Redstone's 77,176 ids over 288 + five-minute passes floors to 267, and 267 * 288 = 76,896 — 280 ids short, every sweep, forever. The + cursor would walk almost the whole catalog and then restart, which is a quieter version of the exact + failure this design replaced. */ + var slice = (watermark + passes - 1) / passes; + if (slice < minimumSlice) + { + slice = minimumSlice; + } + + return slice > watermark ? watermark : slice; + } + + /// + /// Days of margin the map prune adds past the fact-retention horizon. **Strictly less than the dimension + /// GC's margin**, which is ChunkIntervalDays + 1 — see for why + /// that direction is the safe one and not merely a convention. + /// + public const int PruneMarginDays = 1; + + /// + /// The invariant the two horizons must satisfy: the DIMENSION must outlive the MAP, because the two bad + /// end-states are not symmetric. + /// + /// A pruned map row whose dim row survives is a plan rendering "not collected" plus some dim bytes + /// that go unreclaimed until the dim's own horizon passes — visibly degraded, self-correcting, no wrong + /// answers. A pruned DIM row whose map row survives is a reader resolving a live fact to absent content, + /// which is the silent-missing-plans failure this entire design exists to prevent. Ordering the margins + /// makes the recoverable end-state the only reachable one. + /// + /// Pinned as a function rather than asserted in a comment so a future change to either margin — or to + /// ChunkIntervalDays, which is where the dim's margin comes from — fails a test instead of quietly + /// inverting the ordering. + /// + public static bool MarginOrderingHolds(int chunkIntervalDays) => + PruneMarginDays < chunkIntervalDays + 1; + + /// + /// Retires map rows whose facts have all aged out: an index range scan on + /// against the fact horizon plus , time-sliced like every sibling purge. + /// + /// Timestamp-driven, NOT an existence check against query_store_stats. An anti-join against a + /// 43 GB hypertable per map row is exactly the cost this architecture avoids, and it is unnecessary here + /// because keeps last_seen current for anything live — the same argument the + /// dimension GC already rests on, applied to one more timestamped table. + /// + /// A plan whose query goes quiet needs no special handling: it stops being touched, its facts age out + /// within retention, and the margin ordering retires the map row before the dim row it points at. + /// + public static string PruneSql(int chunkIntervalDays) => + "DELETE FROM collect.query_store_plan_map WHERE " + LastSeenColumn + " < $1" + + " AND " + LastSeenColumn + " >= (SELECT min(" + LastSeenColumn + ") FROM collect.query_store_plan_map WHERE " + + LastSeenColumn + " < $1)" + + " AND " + LastSeenColumn + " < (SELECT min(" + LastSeenColumn + ") FROM collect.query_store_plan_map WHERE " + + LastSeenColumn + " < $1) + INTERVAL '" + + chunkIntervalDays.ToString(CultureInfo.InvariantCulture) + " days'"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanWriter.cs new file mode 100644 index 000000000..1e89638af --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStorePlanWriter.cs @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Storage; + +/// One plan the plan-XML fetch landed, before it has been given a content digest. +/// The Query Store plan id, unique within its database. +/// The plan XML, or null — a plan too large to persist reads NULL and still ships, so the +/// watermark can advance past a plan whose content will never exist. +/// SQL Server's query_plan_hash, readable without decompressing the plan, which is +/// what lets the re-verify cursor detect in-place rewrites on cheap columns alone. +public readonly record struct FetchedPlan(long PlanId, string? PlanXml, string? PlanHash); + +/// +/// Writes what the plan fetch landed: the XML into the shared query_plan_dim (gzip, content-keyed, +/// fleet-wide deduplicated) and one map row per plan pointing at it (#2210). +/// +/// Deliberately reuses and rather +/// than writing the dimension directly. That is the whole argument for putting Query Store plans in the shared +/// dimension instead of a table of their own: compression, content dedup across collectors, the GC, the +/// recompression pass and every reader already exist and are exercised by two other collectors. A Query Store +/// plan byte-identical to the same plan collected through query_stats costs nothing new. +/// +/// Both writes go in ONE transaction, and the order inside it matters: the dimension row lands before the +/// map row that points at it. A map row referencing a digest that is not yet in the dimension is the +/// resolves-to-missing-content state this design exists to prevent — the transaction makes it unobservable, and +/// doing the dimension first means even a torn write leaves the recoverable side (content with nothing pointing +/// at it, which the GC reclaims on its own horizon) rather than the unrecoverable one. +/// +public static class QueryStorePlanWriter +{ + /// + /// Lands a fetch's plans for one database. Returns the plan_ids whose content actually stored, in the order + /// they were supplied, which is what the caller feeds to + /// QueryStorePlanXmlState.AdvanceWatermark — the watermark must reflect what LANDED, not what was + /// selected, or a torn pass advances past content that never arrived. + /// + /// A plan with NULL XML counts as landed and gets NO dimension row and NO map row: there is no content + /// to key, and inventing a digest for absent content would make the map point at nothing. The watermark + /// still advances past it, which is correct — that plan's XML will never exist, and stalling on it forever is + /// the failure the budget predicate already had to be fixed for twice. + /// + public static async Task> WriteAsync( + NpgsqlConnection connection, + int serverId, + string databaseName, + IReadOnlyList plans, + DateTime collectionTimeUtc, + CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + if (plans is null) + { + throw new ArgumentNullException(nameof(plans)); + } + + var landed = new List(plans.Count); + if (plans.Count == 0) + { + return landed; + } + + var batch = new PayloadDimensionBatch(); + var mapServerIds = new List(plans.Count); + var mapDatabases = new List(plans.Count); + var mapPlanIds = new List(plans.Count); + var mapDigests = new List(plans.Count); + var mapHashes = new List(plans.Count); + + /* Naive() on the stamp, not the raw UTC value: these are ::timestamp parameters, and Npgsql would infer + timestamptz from a Utc Kind and let Postgres convert into the session zone on the way in. See + QueryStorePlanMap.Naive — a last_seen written at the wrong hour ages rows out ahead of the facts that + reference them, which is silent. */ + var stamp = QueryStorePlanMap.Naive(collectionTimeUtc); + + foreach (var plan in plans) + { + landed.Add(plan.PlanId); + + if (string.IsNullOrEmpty(plan.PlanXml)) + { + continue; + } + + var digest = PayloadDimensions.Digest(plan.PlanXml!); + batch.Add(PayloadDimensions.QueryPlanDimTable, digest, plan.PlanXml!); + + mapServerIds.Add(serverId); + mapDatabases.Add(databaseName); + mapPlanIds.Add(plan.PlanId); + mapDigests.Add(digest); + mapHashes.Add(plan.PlanHash); + } + + if (mapPlanIds.Count == 0) + { + return landed; + } + + using var transaction = await connection.BeginTransactionAsync(cancellationToken); + + /* Dimension FIRST — see the class comment on why the torn-write side matters. */ + await PayloadDimensionWriter.FlushAsync(connection, transaction, batch, stamp, cancellationToken); + + using (var upsert = new NpgsqlCommand(QueryStorePlanMap.UpsertSql, connection, transaction)) + { + upsert.Parameters.AddWithValue(mapServerIds.ToArray()); + upsert.Parameters.AddWithValue(mapDatabases.ToArray()); + upsert.Parameters.AddWithValue(mapPlanIds.ToArray()); + upsert.Parameters.AddWithValue(mapDigests.ToArray()); + upsert.Parameters.AddWithValue(mapHashes.ToArray()); + upsert.Parameters.AddWithValue(CreateStamps(stamp, mapPlanIds.Count)); + await upsert.ExecuteNonQueryAsync(cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + return landed; + } + + private static DateTime[] CreateStamps(DateTime stamp, int count) + { + var stamps = new DateTime[count]; + for (var i = 0; i < count; i++) + { + stamps[i] = stamp; + } + + return stamps; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSliceRepair.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSliceRepair.cs index 0e356f92d..7b24d9ba9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSliceRepair.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSliceRepair.cs @@ -8,7 +8,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Text; using System.Threading; @@ -249,12 +248,32 @@ USING qs_slice_repair AS r """; } + /// + /// Per-statement timeout for the repair's heavy statements (#2105 field failure): Npgsql's + /// default 30s killed the STAGE aggregation on a store fresh off a large catch-up — a day + /// slice's GROUP BY spools every row of the day including the query-text payloads, and the + /// verb runs beside the live service (a managed store cannot stop it — stopping the service + /// stops Postgres), so collector writes and compression jobs contend for the same chunks. + /// The failure read as "Exception while reading from stream" after 0 rows, which is how an + /// Npgsql command timeout surfaces — nothing in the message says timeout. Fifteen minutes is + /// deliberately generous-but-bounded: the slice transaction holds chunk locks, so infinite + /// (the VACUUM precedent) is wrong here. + /// + public const int SliceStatementTimeoutSeconds = 900; + /// /// Runs the collapse over one half-open collection-time slice and returns how many rows it removed. /// /// One transaction per slice: the DELETE and the INSERT must not be separable, or an abort between /// them destroys the interval outright rather than leaving it split. Slicing keeps that transaction — and /// the locks it takes on chunks a compression job may also want — short. + /// + /// The removed count is DERIVED from the statements' own affected-row counts — the DELETE removes + /// every row of every split group and the INSERT restores one combined row per group, so + /// deleted − reinserted IS the net removal. The previous shape bracketed the work with two + /// window-wide COUNT(*) scans to compute the same number, which on the stores this verb exists + /// for (measured: ~12 s per day-wide scan, hash-aggregate spill + a backward index scan over the + /// uncompressed hot chunk) paid the slice's dominant cost twice more per slice for pure bookkeeping. /// public static async Task CollapseSliceAsync( NpgsqlConnection connection, DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken) @@ -263,43 +282,42 @@ public static async Task CollapseSliceAsync( await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - long before; - await using (var count = new NpgsqlCommand($"SELECT count(*) FROM collect.{Table} WHERE collection_time >= $1 AND collection_time < $2", connection, transaction)) + /* #2105 field failure round two: the repair's DELETE touches COMPRESSED chunks (a store old + enough to need this repair has had its compression policy running the whole time), and + TimescaleDB caps decompression at 100k tuples per DML transaction by default — the field + run died at `53400: tuple decompression limit exceeded` four minutes in. Lift it for this + transaction only (SET LOCAL dies with the transaction), the same rail-lift the retention + purge's fallback DELETE already does — deliberate bulk decompression is this verb's job. + On a store without the extension the qualified name is a placeholder GUC, safe everywhere. */ + await using (var lift = new NpgsqlCommand( + "SET LOCAL timescaledb.max_tuples_decompressed_per_dml_transaction = 0", connection, transaction)) { - count.Parameters.AddWithValue(fromUtc); - count.Parameters.AddWithValue(toUtc); - before = Convert.ToInt64(await count.ExecuteScalarAsync(cancellationToken) ?? 0L, CultureInfo.InvariantCulture); + await lift.ExecuteNonQueryAsync(cancellationToken); } var statements = BuildCollapseStatements(); - await using (var stage = new NpgsqlCommand(statements.Stage, connection, transaction)) + await using (var stage = new NpgsqlCommand(statements.Stage, connection, transaction) { CommandTimeout = SliceStatementTimeoutSeconds }) { stage.Parameters.AddWithValue(fromUtc); stage.Parameters.AddWithValue(toUtc); await stage.ExecuteNonQueryAsync(cancellationToken); } - await using (var delete = new NpgsqlCommand(statements.Delete, connection, transaction)) - { - await delete.ExecuteNonQueryAsync(cancellationToken); - } - - await using (var insert = new NpgsqlCommand(statements.Insert, connection, transaction)) + long deleted; + await using (var delete = new NpgsqlCommand(statements.Delete, connection, transaction) { CommandTimeout = SliceStatementTimeoutSeconds }) { - await insert.ExecuteNonQueryAsync(cancellationToken); + deleted = await delete.ExecuteNonQueryAsync(cancellationToken); } - long after; - await using (var count = new NpgsqlCommand($"SELECT count(*) FROM collect.{Table} WHERE collection_time >= $1 AND collection_time < $2", connection, transaction)) + long reinserted; + await using (var insert = new NpgsqlCommand(statements.Insert, connection, transaction) { CommandTimeout = SliceStatementTimeoutSeconds }) { - count.Parameters.AddWithValue(fromUtc); - count.Parameters.AddWithValue(toUtc); - after = Convert.ToInt64(await count.ExecuteScalarAsync(cancellationToken) ?? 0L, CultureInfo.InvariantCulture); + reinserted = await insert.ExecuteNonQueryAsync(cancellationToken); } await transaction.CommitAsync(cancellationToken); - return before - after; + return deleted - reinserted; } /// The survey result: what a dry run reports and what a real run plans from. @@ -315,7 +333,9 @@ public static async Task SurveyAsync(NpgsqlConnection connection, Cancel { ArgumentNullException.ThrowIfNull(connection); - await using var command = new NpgsqlCommand(SurveySql, connection); + /* Same #2105 timeout treatment: the survey aggregates the whole table's key columns, and a + dry run must not die on the store size the repair exists to handle. */ + await using var command = new NpgsqlCommand(SurveySql, connection) { CommandTimeout = SliceStatementTimeoutSeconds }; await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (!await reader.ReadAsync(cancellationToken)) diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextStore.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextStore.cs new file mode 100644 index 000000000..3039ddc94 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextStore.cs @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Globalization; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// Where the query-text fetch lands statement text (#2150): one row per +/// (server_id, database_name, query_id), with the text stored directly. +/// +/// Why this table exists. The runtime-stats payload used to carry query_sql_text +/// inside a TOP ... WITH TIES ... ORDER BY last_execution_time projection, and a Top-N Sort carries +/// every output column through the sort while reading ALL of its input before emitting row one — so +/// choosing the rows to ship materialized nvarchar(max) text for the entire qualifying set. +/// Measured with #2210's plan XML already gone and that column as the only difference: time-to-first-row +/// 4.67s against 0.45s at 1,505 rows, 5.02s against 0.57s at 4,037. +/// +/// Deliberately NOT modeled on , and this is the design choice +/// worth understanding before extending it. The plan side is a MAP into a content-addressed +/// query_plan_dim, with a digest per row and a liveness interlock so the dimension GC cannot delete +/// content that live facts still reference. That machinery is bought by plan XML being enormous and +/// heavily duplicated across plans. Query text is neither: Query Store has already de-duplicated it, one +/// row per distinct statement per database, so there is nothing left to squeeze — and storing it inline +/// removes the interlock entirely. That matters because the interlock's failure mode is text that is +/// silently missing, which no reader can distinguish from a statement that never had text. +/// +/// PostgreSQL TOASTs the column transparently, the same property that made storing plan text +/// acceptable on this store in the first place. +/// +public static class QueryStoreTextStore +{ + public const string TableName = "collect.query_store_text"; + + /// + /// The liveness column. This table is a keyed store rather than a time series — it has a PRIMARY KEY + /// and no time dimension to partition on — so it is pruned on this column rather than by + /// drop_chunks, exactly as is. + /// + public const string LastSeenColumn = "last_seen"; + + /// + /// Days ADDED to the widest fact retention before text is eligible to go. + /// + /// Added rather than subtracted, and the direction is the whole point: text must outlive the rows + /// that reference it. Retiring it early leaves facts whose statement reads as absent — a query with no + /// text at all, which is worse than a stale one because nothing distinguishes it from a statement that + /// never had text. The cost of being late is a few rows of text nobody reads. + /// + public const int PruneMarginDays = 2; + + public const string CreateTableSql = @"CREATE TABLE IF NOT EXISTS collect.query_store_text ( + server_id integer NOT NULL, + database_name text NOT NULL, + query_id bigint NOT NULL, + query_sql_text text, + last_seen timestamp NOT NULL, + PRIMARY KEY (server_id, database_name, query_id) +); +CREATE INDEX IF NOT EXISTS idx_query_store_text_last_seen + ON collect.query_store_text(last_seen);"; + + /// + /// Records what a text fetch landed. + /// + /// The conflict arm overwrites the TEXT, not just the stamp, and that is load-bearing + /// rather than defensive. query_id is unique within a database only until Query Store is reset: + /// a reset renumbers from the start, so id 5 afterwards is a DIFFERENT statement than id 5 before. The + /// refresh horizon on the watermark is what brings us back to re-read it, and this is where the + /// corrected text has to land. Touching only last_seen would leave the old statement's text + /// attached to the new id forever, which reads as a plausible wrong answer rather than as missing + /// data. + /// + /// ORDER BY on the conflict key because concurrent batches that touch overlapping keys in + /// different orders deadlock (#1801) — the same reason the plan map's upsert carries one. The + /// WHERE EXCLUDED.last_seen >= guard keeps the stamp monotonic so an out-of-order write + /// cannot age a row backwards into the prune's reach. + /// + public const string UpsertSql = @"INSERT INTO collect.query_store_text + (server_id, database_name, query_id, query_sql_text, last_seen) +SELECT server_id, database_name, query_id, query_sql_text, stamped +FROM unnest($1::integer[], $2::text[], $3::bigint[], $4::text[], $5::timestamp[]) + AS batch(server_id, database_name, query_id, query_sql_text, stamped) +ORDER BY server_id, database_name, query_id +ON CONFLICT (server_id, database_name, query_id) DO UPDATE SET + query_sql_text = EXCLUDED.query_sql_text, + last_seen = EXCLUDED.last_seen +WHERE EXCLUDED.last_seen >= query_store_text.last_seen"; + + /// + /// Retires text whose facts have all aged out, bounded to roughly one chunk-width of the oldest rows + /// per call so a single sweep cannot take an unbounded row lock — the same shape and the same reason as + /// . + /// + /// Safe to run against live data because last_seen is refreshed by every pass that + /// re-observes a statement: a row can only fall behind the cutoff once nothing has referenced it for + /// the retention window, and re-fetching text for a statement that comes back is one row through a + /// watermark that has already expired. + /// + public static string PruneSql(int chunkIntervalDays) => + "DELETE FROM collect.query_store_text WHERE " + LastSeenColumn + " < $1" + + " AND " + LastSeenColumn + " >= (SELECT min(" + LastSeenColumn + ") FROM collect.query_store_text WHERE " + + LastSeenColumn + " < $1)" + + " AND " + LastSeenColumn + " < (SELECT min(" + LastSeenColumn + ") FROM collect.query_store_text WHERE " + + LastSeenColumn + " < $1) + INTERVAL '" + + chunkIntervalDays.ToString(CultureInfo.InvariantCulture) + " days'"; +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextWriter.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextWriter.cs new file mode 100644 index 000000000..09aad9edc --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTextWriter.cs @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Storage; + +/// One statement's text as the fetch returned it. +public readonly record struct FetchedQueryText(long QueryId, string? QueryText); + +/// +/// Lands what the query-text fetch returned into (#2150). +/// +/// Much simpler than , and the difference is the whole point of +/// keying this store the way it is keyed: there is no content dimension to write first, so there is no +/// torn-write ordering to reason about, no digest, and no transaction — a single upsert either lands or it +/// does not. +/// +public static class QueryStoreTextWriter +{ + /// + /// Lands a fetch's statement text for one database, returning the query_ids that stored, in the + /// order supplied — which is what the caller feeds to + /// . The watermark must + /// reflect what LANDED rather than what was selected, or a torn pass advances past text that never + /// arrived. + /// + /// Rows with null text are stored as null rather than skipped. Query Store does not produce them + /// in practice, so this is about not having a special case to get wrong: a null in this store means "we + /// fetched and there was nothing", the readers already COALESCE onto the fact row's own column, + /// and the watermark advances either way — stalling on a statement whose text will never exist is the + /// failure the budget predicate had to be fixed for twice on the plan side. + /// + public static async Task> WriteAsync( + NpgsqlConnection connection, + int serverId, + string databaseName, + IReadOnlyList texts, + DateTime collectionTimeUtc, + CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + if (texts is null) + { + throw new ArgumentNullException(nameof(texts)); + } + + var landed = new List(texts.Count); + if (texts.Count == 0) + { + return landed; + } + + var serverIds = new int[texts.Count]; + var databases = new string[texts.Count]; + var queryIds = new long[texts.Count]; + var bodies = new string?[texts.Count]; + var stamps = new DateTime[texts.Count]; + + /* Naive() on the stamp, not the raw UTC value: last_seen is a ::timestamp parameter, and Npgsql + infers timestamptz from a Kind=Utc value and lets Postgres convert it into the session zone on the + way in (#1969). A last_seen written at the wrong hour ages rows out ahead of the facts that + reference them, and does it silently. */ + var stamp = QueryStorePlanMap.Naive(collectionTimeUtc); + + for (var i = 0; i < texts.Count; i++) + { + var text = texts[i]; + landed.Add(text.QueryId); + + serverIds[i] = serverId; + databases[i] = databaseName; + queryIds[i] = text.QueryId; + bodies[i] = text.QueryText; + stamps[i] = stamp; + } + + using var upsert = new NpgsqlCommand(QueryStoreTextStore.UpsertSql, connection); + upsert.Parameters.AddWithValue(serverIds); + upsert.Parameters.AddWithValue(databases); + upsert.Parameters.AddWithValue(queryIds); + upsert.Parameters.AddWithValue(bodies); + upsert.Parameters.AddWithValue(stamps); + await upsert.ExecuteNonQueryAsync(cancellationToken); + + return landed; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index 998a27744..13c4693ab 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; ///
public static class StorageVersion { - public const int SchemaVersion = 54; + public const int SchemaVersion = 76; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs b/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs index aaa9bb6f0..28f08e0da 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs @@ -52,6 +52,23 @@ namespace PerformanceMonitor.Darling.Storage; ///
public static class StoreSelfMetrics { + /// + /// Per-statement command timeout for the sweep (#2317) — and, at the worker's call site, the + /// budget for the WHOLE sweep via a linked CTS (see SweepStoreSelfMetricsAsync: this sweep is + /// awaited on the main loop, so five sequential per-statement timeouts must not stack). The + /// sizing queries call hypertable_detailed_size across every hypertable (whose inner + /// hypertable_local_size is the frame the server log names when it cancels) and + /// pg_database_size over the whole + /// store, and on the dogfood fleet (141 objects, a 100+ GB dimension) they outgrew Npgsql's default + /// 30 seconds ~5x/day under load — surfacing as "Exception while reading from stream" (Npgsql + /// cancels the statement; the server logs 'canceling statement due to user request'; the client + /// holds a torn stream), an ERROR that reads as a network fault and pollutes the count every health + /// check watches. Five minutes matches DarlingRetention's destructive-statement budget: this sweep + /// runs hourly on its own connection, so a slow sizing pass costs patience, not correctness — and a + /// sweep that cannot finish in five minutes should skip the tick (one-hour series gap, self-healing) + /// rather than retry into the same load. + /// + public const int SweepTimeoutSeconds = 300; /// How long the series is kept — 400 days, so a year-over-year forecast always has a full /// prior year plus headroom. Enforced by the sweep's own DELETE, not a retention policy. public const int RetentionDays = 400; @@ -87,6 +104,35 @@ LEFT JOIN LATERAL ( FROM chunk_compression_stats(format('%I.%I', h.hypertable_schema, h.hypertable_name)::regclass) ) c ON true"; + /// + /// The background-job rows (#2136) — TimescaleDB stores only, like the hypertable arm (the + /// timescaledb_information views do not exist on plain PostgreSQL). The store's own background jobs + /// (CAGG refreshes, compression, retention) are its heaviest recurring work, their runtimes scale + /// SERIALLY with raw volume (the finalize hash-aggregate runs in one process — measured in #2136: + /// the four most expensive jobs are all the query_store_stats family, compression at 157s and the + /// interval_hourly refresh at 96s on a 52-server store), and a job that outgrows its own schedule + /// interval compounds refresh lag silently. One row per job per sweep makes that a queryable series: + /// object_name is proc_name plus the hypertable/CAGG it serves (the telemetry job has + /// neither) plus a [job_id] suffix — the uniqueness guarantee (review catch): two user-added + /// jobs sharing a proc_name, or two hypertable-less jobs, would otherwise collide into one + /// object_name and the readers' DISTINCT ON would silently drop one job's telemetry. job_id is + /// stable for a job's lifetime, so per-job series continuity holds. schedule_interval_ms + /// rides along so "duration vs cadence" — the honest tripwire — is one division. $1 metric_time. + /// + public const string BackgroundJobInsertSql = @" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, last_run_duration_ms, schedule_interval_ms, total_runs, total_failures) +SELECT + $1, + j.proc_name || coalesce(' ' || j.hypertable_name, '') || ' [' || j.job_id || ']', + 'background_job', + (EXTRACT(EPOCH FROM js.last_run_duration) * 1000)::bigint, + (EXTRACT(EPOCH FROM j.schedule_interval) * 1000)::bigint, + js.total_runs, + js.total_failures +FROM timescaledb_information.job_stats AS js +JOIN timescaledb_information.jobs AS j USING (job_id)"; + /// /// The payload dimension rows — every store shape (the dims are plain tables everywhere). Table names /// are the compile-time constants, so interpolation is safe (the @@ -158,24 +204,28 @@ public static async Task SweepAsync( if (timescaleAvailable) { - using var hypertables = new NpgsqlCommand(HypertableInsertSql, connection); + using var hypertables = new NpgsqlCommand(HypertableInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }; hypertables.Parameters.AddWithValue(metricTime); written += await hypertables.ExecuteNonQueryAsync(cancellationToken); + + using var jobs = new NpgsqlCommand(BackgroundJobInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }; + jobs.Parameters.AddWithValue(metricTime); + written += await jobs.ExecuteNonQueryAsync(cancellationToken); } - using (var dimensions = new NpgsqlCommand(DimensionInsertSql, connection)) + using (var dimensions = new NpgsqlCommand(DimensionInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }) { dimensions.Parameters.AddWithValue(metricTime); written += await dimensions.ExecuteNonQueryAsync(cancellationToken); } - using (var store = new NpgsqlCommand(StoreInsertSql, connection)) + using (var store = new NpgsqlCommand(StoreInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }) { store.Parameters.AddWithValue(metricTime); written += await store.ExecuteNonQueryAsync(cancellationToken); } - using (var retention = new NpgsqlCommand(RetentionDeleteSql, connection)) + using (var retention = new NpgsqlCommand(RetentionDeleteSql, connection) { CommandTimeout = SweepTimeoutSeconds }) { retention.Parameters.AddWithValue(metricTime.AddDays(-RetentionDays)); await retention.ExecuteNonQueryAsync(cancellationToken); diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index c887a5970..c85fff039 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -2800,14 +2800,25 @@ public static TimeSpan StuckRunningBound(TimeSpan? scheduleInterval) /// /// The pure stuck-compression-job decision (#1581). A compression policy job is STUCK when either: /// - /// its next_start is -infinity — the scheduler will NEVER re-fire it (the dead-job - /// bug that let uncompressed data grow without bound until the disk filled), or + /// its next_start is -infinity while the job is NOT currently running — the scheduler + /// abandoned it and will NEVER re-fire it (the dead-job bug that let uncompressed data grow without bound + /// until the disk filled), or /// it has been in the Running state since a last_run_started_at older than /// — a run that began long ago and never finished (a hung run). /// /// A job with neither condition is healthy and is NOT flagged. No I/O, so it pins directly with a /// controllable clock. Scoping to compression jobs happens in the query — this decides only "stuck". /// + /// -infinity is ALSO the engine's mid-run marker, measured live on TimescaleDB + /// 2.x (pg17): from the moment the scheduler picks up a due job until its run completes, + /// job_stats.next_start reads -infinity with job_status = 'Running', and the real + /// next start is only computed at completion. So -infinity alone cannot mean "dead" — an + /// unconditioned first arm flagged every healthy job the check happened to catch mid-run, alerted it as + /// stuck, and "self-healed" it with a pointless re-arm (the field's transient stuck→self-healed noise; + /// the CI flake was the live test catching its own re-arm-triggered run). A running job is therefore + /// left to the second arm, whose elapsed bound is what actually distinguishes a hung run from a + /// healthy one. + /// /// A of counts as NEVER RAN, /// not as "started in year 1" (#1760). already NULLIFs TimescaleDB's /// -infinity never-ran sentinel, so this is the second line of defence: the sentinel maps to @@ -2822,13 +2833,15 @@ public static bool IsCompressionJobStuck( DateTime nowUtc, out string reason) { - if (nextStartIsNegativeInfinity) + var isRunning = string.Equals(jobStatus, "Running", StringComparison.OrdinalIgnoreCase); + + if (nextStartIsNegativeInfinity && !isRunning) { reason = "next_start is -infinity — the scheduler will never run it again"; return true; } - if (string.Equals(jobStatus, "Running", StringComparison.OrdinalIgnoreCase) + if (isRunning && lastRunStartedAtUtc is DateTime startedUtc && startedUtc != DateTime.MinValue) { @@ -2933,6 +2946,55 @@ public static async Task> ReadStuckCompressio return stuck; } + /// + /// Every background job's last-run duration against its own schedule interval (#2136) — the readings + /// the Store Job Over Cadence self-alert judges. job_stats for the same reason the #1778 + /// observability path uses it (maintained unconditionally; the per-execution history table is empty + /// unless job-execution logging is on). Only a SUCCESSFUL last run judges: a failed run's duration is + /// not a cadence signal, and job failures are their own condition (total_failures rides the + /// V56 telemetry). Tolerant like — a plain-PG store or a + /// hiccup yields no readings, never an exception. + /// + public static async Task> ReadJobCadenceReadingsAsync( + NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + const string sql = @" +SELECT + j.job_id, + j.proc_name || coalesce(' ' || j.hypertable_name, ''), + (EXTRACT(EPOCH FROM js.last_run_duration) * 1000)::bigint, + (EXTRACT(EPOCH FROM j.schedule_interval) * 1000)::bigint +FROM timescaledb_information.job_stats AS js +JOIN timescaledb_information.jobs AS j USING (job_id) +WHERE js.last_run_status = 'Success'"; + + var readings = new List(); + try + { + using var command = new NpgsqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + readings.Add(new StoreJobCadenceReading( + Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture), + reader.IsDBNull(1) ? "" : reader.GetString(1), + reader.IsDBNull(2) ? null : Convert.ToInt64(reader.GetValue(2), CultureInfo.InvariantCulture), + reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3), CultureInfo.InvariantCulture))); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogDebug("Store-job cadence check: could not read job stats: {Message}", ex.Message); + } + + return readings; + } + /* ---------------- compression-run observability (#1778) ---------------- */ /// @@ -3112,6 +3174,14 @@ public static async Task TryRearmJobAsync( /// public sealed record StuckCompressionJob(long JobId, string? HypertableName, string Reason); +/// +/// One background job's cadence reading (#2136): the last SUCCESSFUL run's duration against the job's own +/// schedule interval, from . +/// is proc_name plus the hypertable/CAGG it serves — the V56 telemetry's naming, minus the +/// [job_id] suffix (the id rides separately as the alert key). +/// +public sealed record StoreJobCadenceReading(long JobId, string JobName, long? LastRunDurationMs, long ScheduleIntervalMs); + /// /// One hypertable's compression-policy activity (#1778): whether a run is in progress, when it started, how /// long the last COMPLETED run took, and how many chunks are past the eligibility delay but still uncompressed. diff --git a/Darling/PerformanceMonitor.Darling.Viewer/AddServerDialog.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/AddServerDialog.xaml.cs index 1aff2f54f..c09b84a45 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/AddServerDialog.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/AddServerDialog.xaml.cs @@ -221,8 +221,37 @@ will block. (A profile-backed Azure identity is caught at resolve time.) */ { StatusText.Text = ""; } + /* #2279: SQL auth means a password, and a password is stored as a DPAPI LocalMachine blob that ONLY the + machine writing it can decrypt. The service is what has to decrypt it, so a credential saved from a + viewer on another PC can never be used and the server fails to connect on every sweep afterwards — + the #2255 report. Said as soon as the mode is picked, in the same place and the same way the Azure + arm above says its piece, so it lands before the password is typed rather than after the save. + + WARNED, not refused: a non-loopback store does not prove this viewer is remote (a BYO store on + another host with the service local reads the same), and refusing would block a legitimate first-run + Add. Silent for a loopback store, which is the managed single-box deploy and the overwhelmingly + common case — a hint that fires for everyone is a hint nobody reads. */ + else if (SqlAuthRadio.IsChecked == true && _dataService is { StoreIsOnThisMachine: false }) + { + StatusText.Text = SqlCredentialMachineBoundHint; + } + else if (StatusText.Text == SqlCredentialMachineBoundHint) + { + StatusText.Text = ""; + } } + /// + /// The #2279 hint. A const so can clear exactly its own message when the mode + /// changes away — the same self-clearing discipline the Azure arm uses, which is what stops a stale hint + /// sitting under an unrelated mode. + /// + private const string SqlCredentialMachineBoundHint = + "This viewer's store is not on this machine. A SQL-auth password is encrypted for THIS machine only, " + + "so if the Darling service runs elsewhere it will not be able to decrypt it and the server will fail " + + "to connect. Add it from a viewer on the service's host, run --add-server there, or use an env:/file: " + + "reference instead."; + private string GetSelectedEncryptMode() => EncryptModeComboBox.SelectedIndex switch { 1 => "Mandatory", @@ -348,7 +377,13 @@ authoring convenience the store needs no table for. */ return new MonitoredServerRow { - ServerId = ViewerDataService.ComputeServerId(host, database, readOnlyIntent), + /* #2158: an EDIT keeps the row's identity; only an Add derives one. Changing a server's address + does not make it a different server — it is the same monitored instance — and every collect.* + row is keyed by this id, so re-deriving it abandons the whole of that server's history. The old + shape wrote a row under the new hash and deleted the old one, which left the REGISTRY tidy and + the history orphaned with nothing pointing at it: the failure looked like a server that had + never been monitored. Derivation now runs only where there is no history to lose. */ + ServerId = _originalServerId ?? ViewerDataService.ComputeServerId(host, database, readOnlyIntent), Name = displayName, Host = host, Database = database, @@ -397,24 +432,28 @@ private async void SaveButton_Click(object sender, RoutedEventArgs e) return; } - /* Refuse to silently overwrite a DIFFERENT existing server that shares this identity — the upsert's - ON CONFLICT DO UPDATE would clobber its excluded databases / capture override. Covers Add and an - edit that re-points host/database/read-only-intent onto another server's identity. */ - if (_originalServerId != row.ServerId - && await _dataService.GetMonitoredServerAsync(row.ServerId) is not null) + /* Refuse to point this definition at an address another server already monitors: on Add the + upsert's ON CONFLICT DO UPDATE would clobber that row's excluded databases / capture override, + and on Edit it would leave two registrations collecting the same real instance under two + identities — #2228's shape, arrived at from the registry side. + + #2158: checked against the ADDRESS rather than against a derived id. Now that an edit preserves + its identity, a row's server_id no longer has to equal the hash of its own address, so the old + id-based lookup would miss exactly the row it exists to protect. Comparing ids afterwards is + what excludes "collided with myself" — an edit that leaves the address alone, or that only + renames or re-credentials the server. */ + var occupant = await _dataService.GetMonitoredServerByAddressAsync(row.Host, row.Database, row.ReadOnlyIntent); + if (occupant is not null && occupant.ServerId != row.ServerId) { StatusText.Text = "A server with this address (and database / read-only intent) is already monitored. Edit it from Manage Servers instead."; SaveButton.IsEnabled = true; return; } - /* Write the NEW row first, THEN drop the old identity on an edit that moved it — if the delete - fails we leave a recoverable duplicate rather than losing the definition entirely. */ + /* One row, one identity, in place — no delete. The upsert's ON CONFLICT (server_id) arm rewrites + the address on the row that already owns this id, so the server's collected history stays + attached to it. */ await _dataService.UpsertMonitoredServerAsync(row); - if (_originalServerId is int original && original != row.ServerId) - { - await _dataService.DeleteMonitoredServerAsync(original); - } /* Favorites are viewer-local (the service never reads them) — keyed by the server address. */ _serverStore.SetFavorite(row.Host, FavoriteCheckBox.IsChecked == true); diff --git a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.IndexAnalysis.cs b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.IndexAnalysis.cs index 729e3cde0..8e1b7b6b1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.IndexAnalysis.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.IndexAnalysis.cs @@ -45,7 +45,7 @@ private async Task LoadFinOpsIndexAnalysisAsync() _finopsIndexAnalysisRollupFilterMgr!.UpdateData(new List()); _finopsIndexAnalysisFilterMgr!.UpdateData(new List()); FinOpsIndexAnalysisBannerPanel.Children.Clear(); - FinOpsIndexAnalysisBannerPanel.Visibility = Visibility.Collapsed; + FinOpsIndexAnalysisBannerScroller.Visibility = Visibility.Collapsed; FinOpsIndexAnalysisNoDataMessage.Visibility = Visibility.Visible; FinOpsIndexAnalysisCountIndicator.Text = ""; return; @@ -89,7 +89,9 @@ private void BuildIndexAnalysisBanners(IndexCleanupAnalysisResult result) AddIndexAnalysisBanner(note, "#7F8C8D", "#202628"); } - FinOpsIndexAnalysisBannerPanel.Visibility = + /* #2300: visibility lives on the scroller — the element that occupies the layout row. The panel + inside it stays visible; collapsing only the panel would leave an empty scroller holding the row. */ + FinOpsIndexAnalysisBannerScroller.Visibility = FinOpsIndexAnalysisBannerPanel.Children.Count > 0 ? Visibility.Visible : Visibility.Collapsed; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.Loaders.cs b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.Loaders.cs index 842dd4053..3dab90dff 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.Loaders.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.Loaders.cs @@ -299,9 +299,13 @@ private void UpdateUtilizationSummary(UtilizationEfficiencyRow? data) { "RIGHT_SIZED" => $"CPU is moderately loaded (avg {data.AvgCpuPct:N1}%, p95 {data.P95CpuPct:N1}%) and memory is well-utilized (buffer pool uses {bpPct:N0}% of physical RAM). No action needed.", "OVER_PROVISIONED" => $"CPU is lightly loaded (avg {data.AvgCpuPct:N1}%, max {data.MaxCpuPct}%) and buffer pool uses only {bpPct:N0}% of physical RAM. This server may have more resources than it needs.", - "UNDER_PROVISIONED" => data.P95CpuPct > 85 - ? $"CPU p95 is {data.P95CpuPct:N1}% (threshold: 85%). This server may need more CPU capacity." - : $"Buffer pool uses {bpPct:N0}% of physical RAM and memory ratio is {data.MemoryRatio:N2} (threshold: 0.95). Memory pressure is high.", + /* The reason comes from the same place as the verdict. This branch used to read + "P95CpuPct > 85 ? CPU : memory ratio is {x} (threshold: 0.95)", so a server flagged for grant + pressure or worker saturation would have been explained as a memory ratio that no longer + decides anything, citing a threshold the code does not check (#2246). */ + "UNDER_PROVISIONED" => ProvisioningVerdict.UnderProvisionedReason( + data.P95CpuPct, data.MaxGrantWaiters, data.GrantTimeouts, data.ForcedGrants, + data.MaxWorkersCount, data.CurrentWorkersCount), _ => "" }; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml index 89c0763d9..41a8371f3 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml @@ -1021,7 +1021,16 @@ - + + + + @@ -1065,7 +1074,9 @@ - + + private bool TryReadServiceFlags( - out bool capturePlans, out bool mcpEnabled, out int mcpPort, out bool webEnabled, out int webPort) + out bool capturePlans, out bool mcpEnabled, out int mcpPort, out bool webEnabled, out int webPort, + out int textBudgetMb, out int maxSweeps) { capturePlans = CapturePlansCheckBox.IsChecked == true; + textBudgetMb = 64; + maxSweeps = 4; mcpEnabled = McpEnabledCheckBox.IsChecked == true; mcpPort = _appSettings.McpPort; webEnabled = WebEnabledCheckBox.IsChecked == true; @@ -422,6 +431,33 @@ private bool TryReadServiceFlags( return false; } + /* #2164 / #2170 collector memory knobs. Unlike the ports these are always in force (there is no + "disabled" state to excuse a bad value), so a bad entry always blocks the save rather than + silently keeping a last-known value. The service clamps on read as defense in depth. */ + if (int.TryParse(QueryStoreTextBudgetMbTextBox.Text, out var parsedBudget) && parsedBudget is >= 4 and <= 256) + { + textBudgetMb = parsedBudget; + } + else + { + MessageBox.Show( + "Query Store text budget must be between 4 and 256 MB.", + "Validation", MessageBoxButton.OK, MessageBoxImage.Warning); + return false; + } + + if (int.TryParse(MaxConcurrentSweepsTextBox.Text, out var parsedSweeps) && parsedSweeps is >= 1 and <= 16) + { + maxSweeps = parsedSweeps; + } + else + { + MessageBox.Show( + "Concurrent server sweeps must be between 1 and 16.", + "Validation", MessageBoxButton.OK, MessageBoxImage.Warning); + return false; + } + return true; } @@ -703,6 +739,12 @@ private void SeedAlertControlsFrom(AlertSettingsRow r) AlertLowDiskCheckBox.IsChecked = r.LowDiskEnabled; AlertLowDiskThresholdPercentBox.Text = r.LowDiskThresholdPercent.ToString(CultureInfo.InvariantCulture); AlertLowDiskThresholdGbBox.Text = r.LowDiskThresholdGb.ToString(CultureInfo.InvariantCulture); + AlertDiskCriticalPercentBox.Text = r.DiskCriticalFreePercent.ToString(CultureInfo.InvariantCulture); + AlertDiskCriticalGbBox.Text = r.DiskCriticalFreeGb.ToString(CultureInfo.InvariantCulture); + AlertSelfDiskWarnPercentBox.Text = r.SelfDiskFreeWarnPercent.ToString(CultureInfo.InvariantCulture); + AlertCollectionStaleMinutesBox.Text = r.CollectionStaleMinutes.ToString(CultureInfo.InvariantCulture); + AlertCollectionFailureThresholdBox.Text = r.CollectionFailureThreshold.ToString(CultureInfo.InvariantCulture); + AlertStoreJobCadenceWarnPercentBox.Text = r.StoreJobCadenceWarnPercent.ToString(CultureInfo.InvariantCulture); AlertPvsCheckBox.IsChecked = r.PvsEnabled; AlertPvsThresholdPercentBox.Text = r.PvsThresholdPercent.ToString(CultureInfo.InvariantCulture); AlertPvsFloorGbBox.Text = r.PvsFloorGb.ToString(CultureInfo.InvariantCulture); @@ -716,6 +758,7 @@ private void SeedAlertControlsFrom(AlertSettingsRow r) AnalysisIntervalBox.Text = r.AnalysisIntervalMinutes.ToString(CultureInfo.InvariantCulture); AnalysisNotificationsCheckBox.IsChecked = r.AnalysisNotificationsEnabled; AnalysisNotifySeverityBox.Text = r.AnalysisNotifySeverity.ToString("0.0", CultureInfo.InvariantCulture); + AnalysisNotifyCooldownBox.Text = r.AnalysisNotifyCooldownMinutes.ToString(CultureInfo.InvariantCulture); /* #1141/#1236: the delivery mode + per-event cap are now STORE-backed (the service honors them), seeded from the row like every other alert-engine control. */ AlertDeliveryModeBox.SelectedIndex = r.DeliveryMode == "PerEvent" ? 1 : 0; @@ -794,6 +837,20 @@ make the gate impossible to turn back off once enabled. */ row.LowDiskThresholdPercent = lowDiskPct; if (int.TryParse(AlertLowDiskThresholdGbBox.Text, out var lowDiskGb) && lowDiskGb >= 0) row.LowDiskThresholdGb = lowDiskGb; + /* #2107: the previously-hardcoded knobs, validated to the same ranges the service clamps. */ + if (int.TryParse(AlertDiskCriticalPercentBox.Text, out var critPct) && critPct is >= 0 and <= 100) + row.DiskCriticalFreePercent = critPct; + if (int.TryParse(AlertDiskCriticalGbBox.Text, out var critGb) && critGb >= 0) + row.DiskCriticalFreeGb = critGb; + if (int.TryParse(AlertSelfDiskWarnPercentBox.Text, out var selfDiskPct) && selfDiskPct is >= 0 and <= 100) + row.SelfDiskFreeWarnPercent = selfDiskPct; + if (int.TryParse(AlertCollectionStaleMinutesBox.Text, out var staleMin) && staleMin is >= 5 and <= 1440) + row.CollectionStaleMinutes = staleMin; + if (int.TryParse(AlertCollectionFailureThresholdBox.Text, out var failThresh) && failThresh is >= 1 and <= 1000) + row.CollectionFailureThreshold = failThresh; + /* #2136: validated to the same range DarlingAlertSettings clamps ([5, 100]). */ + if (int.TryParse(AlertStoreJobCadenceWarnPercentBox.Text, out var cadencePct) && cadencePct is >= 5 and <= 100) + row.StoreJobCadenceWarnPercent = cadencePct; if (int.TryParse(AlertPvsThresholdPercentBox.Text, out var pvsPct) && pvsPct is >= 0 and <= 100) row.PvsThresholdPercent = pvsPct; if (int.TryParse(AlertPvsFloorGbBox.Text, out var pvsFloor) && pvsFloor >= 0) @@ -819,6 +876,11 @@ make the gate impossible to turn back off once enabled. */ else errors.Add("Analysis notify severity must be between 0.0 and 2.0."); + if (int.TryParse(AnalysisNotifyCooldownBox.Text, out var analysisCooldown) && analysisCooldown is >= 30 and <= 10080) + row.AnalysisNotifyCooldownMinutes = analysisCooldown; + else + errors.Add("Analysis re-notify cooldown must be between 30 and 10080 minutes."); + /* #1141/#1236: delivery mode + per-event cap (store-backed). */ row.DeliveryMode = AlertDeliveryModeBox.SelectedIndex == 1 ? "PerEvent" : "Summary"; if (int.TryParse(AlertPerEventMaxBox.Text, out var perEventMax) && perEventMax is >= 1 and <= 100) @@ -860,6 +922,14 @@ private void RestoreAlertDefaultsButton_Click(object sender, RoutedEventArgs e) AlertTempDbSpaceThresholdBox.Text = "80"; AlertLowDiskThresholdPercentBox.Text = "10"; AlertLowDiskThresholdGbBox.Text = "5"; + /* #2107: the previously-hardcoded knobs reset to the constants they replaced. */ + AlertDiskCriticalPercentBox.Text = "3"; + AlertDiskCriticalGbBox.Text = "2"; + AlertSelfDiskWarnPercentBox.Text = "10"; + AlertCollectionStaleMinutesBox.Text = "30"; + AlertCollectionFailureThresholdBox.Text = "10"; + AlertStoreJobCadenceWarnPercentBox.Text = "25"; + AnalysisNotifyCooldownBox.Text = "360"; AlertPvsThresholdPercentBox.Text = "40"; AlertPvsFloorGbBox.Text = "1"; AlertLongRunningJobMultiplierBox.Text = "3"; @@ -961,6 +1031,13 @@ private void UpdateAlertControlStates() AlertPvsThresholdPercentBox.IsEnabled = enabled; AlertPvsFloorGbBox.IsEnabled = enabled; AlertLowDiskThresholdGbBox.IsEnabled = enabled; + /* #2107: the new threshold boxes follow the master switch like every sibling. */ + AlertDiskCriticalPercentBox.IsEnabled = enabled; + AlertDiskCriticalGbBox.IsEnabled = enabled; + AlertSelfDiskWarnPercentBox.IsEnabled = enabled; + AlertCollectionStaleMinutesBox.IsEnabled = enabled; + AlertCollectionFailureThresholdBox.IsEnabled = enabled; + AlertStoreJobCadenceWarnPercentBox.IsEnabled = enabled; AlertLongRunningJobCheckBox.IsEnabled = enabled; AlertLongRunningJobMultiplierBox.IsEnabled = enabled; AlertFailedJobCheckBox.IsEnabled = enabled; @@ -1385,7 +1462,8 @@ private async void SaveButton_Click(object sender, RoutedEventArgs e) { var errors = new List(); var mcpValid = TryReadServiceFlags( - out var capturePlans, out var mcpEnabled, out var mcpPort, out var webEnabled, out var webPort); + out var capturePlans, out var mcpEnabled, out var mcpPort, out var webEnabled, out var webPort, + out var textBudgetMb, out var maxSweeps); var alertRow = BuildAlertRowFromControls(errors); SaveViewerLocalAlertFields(errors); var notifyRow = BuildNotificationRowFromControls(errors); @@ -1444,7 +1522,8 @@ A read-only seat surfaces the friendly message and the window stays open. */ { await _dataService.UpsertAlertSettingsAsync(alertRow); await _dataService.UpsertNotificationAsync(notifyRow); - await _dataService.UpdateServiceFlagsAsync(capturePlans, mcpEnabled, mcpPort, webEnabled, webPort); + await _dataService.UpdateServiceFlagsAsync(capturePlans, mcpEnabled, mcpPort, webEnabled, webPort, + QueryStoreBackfillCheckBox.IsChecked == true, textBudgetMb, maxSweeps); } catch (ViewerReadOnlyException ex) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerControlPlaneMigration.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerControlPlaneMigration.cs index 6b6c14d8e..a50de0e94 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerControlPlaneMigration.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerControlPlaneMigration.cs @@ -120,9 +120,12 @@ public async Task MigrateAsync(ViewerDataService? dataService, Cancellation var storeService = await dataService.GetServiceConfigAsync(cancellationToken); if (storeService is not null && ShouldImportMcp(storeService, _appSettings)) { + /* #2167: carry the store's current backfill flag through unchanged — this migration imports + MCP/web settings and must never flip an operator's backfill switch as a side effect. */ await dataService.UpdateServiceFlagsAsync( storeService.CapturePlans, _appSettings.McpEnabled, _appSettings.McpPort, - _appSettings.WebEnabled, _appSettings.WebPort, cancellationToken); + _appSettings.WebEnabled, _appSettings.WebPort, storeService.QueryStoreBackfillEnabled, + storeService.QueryStoreTextBudgetMb, storeService.MaxConcurrentSweeps, cancellationToken); imported++; } } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs index 43621d79a..2e69f1ff6 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.AlertSettings.cs @@ -57,7 +57,10 @@ notify toggle (V20) are appended so the existing ordinals stay pinned. */ "notify_connection_down_at_startup, connection_refire_minutes, " + "notify_ag_health, ag_lag_alert_seconds, ag_redo_queue_alert_kb, " + "ag_disconnect_refire_minutes, blocking_wait_seconds_threshold, " + - "pvs_enabled, pvs_threshold_percent, pvs_floor_gb, database_state_enabled"; + "pvs_enabled, pvs_threshold_percent, pvs_floor_gb, database_state_enabled, " + + "self_disk_free_warn_percent, collection_stale_minutes, collection_failure_threshold, " + + "disk_critical_free_percent, disk_critical_free_gb, analysis_notify_cooldown_minutes, " + + "store_job_cadence_warn_percent"; /// The single global alert-settings row (id=1), for the Settings window prefill + the migrate-in /// defaults check. Column order matches . @@ -71,7 +74,7 @@ notify toggle (V20) are appended so the existing ordinals stay pinned. */ INSERT INTO config_alert_settings (id, " + AlertSettingsColumns + @", modified_at) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, - $44, $45, $46, $47, + $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, (now() AT TIME ZONE 'UTC')) ON CONFLICT (id) DO UPDATE SET enabled = EXCLUDED.enabled, @@ -121,6 +124,13 @@ ON CONFLICT (id) DO UPDATE SET pvs_threshold_percent = EXCLUDED.pvs_threshold_percent, pvs_floor_gb = EXCLUDED.pvs_floor_gb, database_state_enabled = EXCLUDED.database_state_enabled, + self_disk_free_warn_percent = EXCLUDED.self_disk_free_warn_percent, + collection_stale_minutes = EXCLUDED.collection_stale_minutes, + collection_failure_threshold = EXCLUDED.collection_failure_threshold, + disk_critical_free_percent = EXCLUDED.disk_critical_free_percent, + disk_critical_free_gb = EXCLUDED.disk_critical_free_gb, + analysis_notify_cooldown_minutes = EXCLUDED.analysis_notify_cooldown_minutes, + store_job_cadence_warn_percent = EXCLUDED.store_job_cadence_warn_percent, modified_at = (now() AT TIME ZONE 'UTC')"; /// The two cpu_mode values the service honors (it compares case-insensitively against @@ -197,6 +207,13 @@ private static void BindAlertSettings(NpgsqlCommand command, AlertSettingsRow r) command.Parameters.Add(new NpgsqlParameter { TypedValue = r.PvsThresholdPercent }); // $45 (#1984, V48) command.Parameters.Add(new NpgsqlParameter { TypedValue = r.PvsFloorGb }); // $46 (#1984, V48) command.Parameters.Add(new NpgsqlParameter { TypedValue = r.DatabaseStateEnabled }); // $47 (database-state alert, V49) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.SelfDiskFreeWarnPercent }); // $48 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.CollectionStaleMinutes }); // $49 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.CollectionFailureThreshold }); // $50 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.DiskCriticalFreePercent }); // $51 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.DiskCriticalFreeGb }); // $52 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.AnalysisNotifyCooldownMinutes }); // $53 (#2107, V55) + command.Parameters.Add(new NpgsqlParameter { TypedValue = r.StoreJobCadenceWarnPercent }); // $54 (#2136, V57) } private static AlertSettingsRow ReadAlertSettingsRow(NpgsqlDataReader reader) => new() @@ -254,6 +271,14 @@ private static void BindAlertSettings(NpgsqlCommand command, AlertSettingsRow r) PvsFloorGb = reader.GetInt32(45), /* database-state alert master switch appended (V49) at ordinal 46. */ DatabaseStateEnabled = reader.GetBoolean(46), + /* #2107 threshold knobs appended (V55) at ordinals 47–52. */ + SelfDiskFreeWarnPercent = reader.GetInt32(47), + CollectionStaleMinutes = reader.GetInt32(48), + CollectionFailureThreshold = reader.GetInt32(49), + DiskCriticalFreePercent = reader.GetInt32(50), + DiskCriticalFreeGb = reader.GetInt32(51), + AnalysisNotifyCooldownMinutes = reader.GetInt32(52), + StoreJobCadenceWarnPercent = reader.GetInt32(53), }; /// Maps the Settings window's CPU-mode combo tag ("Total"/"SqlOnly") to the store value. @@ -305,6 +330,15 @@ public sealed class AlertSettingsRow /// Master switch for the baseline-deviation database-state alert (V40 DDL default true). public bool DatabaseStateEnabled { get; set; } = true; + /* #2107 (V55): the previously-hardcoded thresholds; defaults are the constants they replaced. */ + public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int CollectionStaleMinutes { get; set; } = 30; + public int CollectionFailureThreshold { get; set; } = 10; + public int DiskCriticalFreePercent { get; set; } = 3; + public int DiskCriticalFreeGb { get; set; } = 2; + public int AnalysisNotifyCooldownMinutes { get; set; } = 360; + public int StoreJobCadenceWarnPercent { get; set; } = 25; + public bool CpuEnabled { get; set; } = true; public int CpuThresholdPercent { get; set; } = 80; @@ -394,6 +428,12 @@ public bool ValueEquals(AlertSettingsRow other) && AgRedoQueueAlertKb == other.AgRedoQueueAlertKb && AgDisconnectRefireMinutes == other.AgDisconnectRefireMinutes && DatabaseStateEnabled == other.DatabaseStateEnabled + && SelfDiskFreeWarnPercent == other.SelfDiskFreeWarnPercent + && CollectionStaleMinutes == other.CollectionStaleMinutes + && CollectionFailureThreshold == other.CollectionFailureThreshold + && DiskCriticalFreePercent == other.DiskCriticalFreePercent + && DiskCriticalFreeGb == other.DiskCriticalFreeGb + && AnalysisNotifyCooldownMinutes == other.AnalysisNotifyCooldownMinutes && CpuEnabled == other.CpuEnabled && CpuThresholdPercent == other.CpuThresholdPercent && string.Equals(CpuMode, other.CpuMode, StringComparison.OrdinalIgnoreCase) diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Config.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Config.cs index 457799bba..4544d98d6 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Config.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Config.cs @@ -63,6 +63,16 @@ FROM v_database_scoped_config ORDER BY database_name, configuration_name """; + + public const string QueryStoreHealthSql = """ + SELECT database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes + FROM v_query_store_health + WHERE server_id = $1 + AND capture_time = (SELECT MAX(capture_time) FROM v_query_store_health WHERE server_id = $1) + AND ($2::text[] IS NULL OR database_name = ANY($2)) + ORDER BY database_name + """; + public const string TraceFlagsSql = """ SELECT trace_flag, status, is_global, is_session FROM v_trace_flags @@ -167,6 +177,36 @@ public async Task> GetLatestDatabaseScopedConfigAs return items; } + + /// Latest per-database Query Store health snapshot for one server (Query Store grid, #2319). + public async Task> GetLatestQueryStoreHealthAsync(int serverId, IReadOnlyList? databaseNames = null, CancellationToken cancellationToken = default) + { + var items = new List(); + + await using var command = _dataSource.CreateCommand(QueryStoreHealthSql); + command.Parameters.Add(new NpgsqlParameter { TypedValue = serverId }); + command.Parameters.Add(DatabaseFilterParameter(databaseNames)); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + items.Add(new QueryStoreHealthRow + { + DatabaseName = reader.GetString(0), + ActualState = reader.IsDBNull(1) ? "" : reader.GetString(1), + DesiredState = reader.IsDBNull(2) ? "" : reader.GetString(2), + ReadonlyReason = reader.IsDBNull(3) ? 0 : reader.GetInt32(3), + CurrentStorageMb = reader.IsDBNull(4) ? 0L : reader.GetInt64(4), + MaxStorageMb = reader.IsDBNull(5) ? 0L : reader.GetInt64(5), + SizeBasedCleanupMode = reader.IsDBNull(6) ? "" : reader.GetString(6), + StaleQueryThresholdDays = reader.IsDBNull(7) ? 0L : reader.GetInt64(7), + MaxPlansPerQuery = reader.IsDBNull(8) ? 0L : reader.GetInt64(8), + IntervalLengthMinutes = reader.IsDBNull(9) ? 0L : reader.GetInt64(9), + }); + } + + return items; + } + /// Latest trace-flags snapshot for one server (Trace Flags grid). public async Task> GetLatestTraceFlagsAsync(int serverId, CancellationToken cancellationToken = default) { @@ -257,6 +297,41 @@ public class DatabaseConfigRow public string OptimizedLockingDisplay => IsOptimizedLockingOn ? "Yes" : "No"; } + +/// +/// One database's Query Store health row (#2319) — the latest collected +/// sys.database_query_store_options snapshot. folds the classic silent +/// failure into one glanceable cell: actual and desired agreeing shows one state; disagreeing shows +/// both, because desired READ_WRITE with actual READ_ONLY is precisely the condition this collector +/// exists to surface. decodes the bitmask values an operator +/// actually meets; unknown bits fall back to the raw number rather than guessing. +/// +public class QueryStoreHealthRow +{ + public string DatabaseName { get; set; } = ""; + public string ActualState { get; set; } = ""; + public string DesiredState { get; set; } = ""; + public int ReadonlyReason { get; set; } + public long CurrentStorageMb { get; set; } + public long MaxStorageMb { get; set; } + public string SizeBasedCleanupMode { get; set; } = ""; + public long StaleQueryThresholdDays { get; set; } + public long MaxPlansPerQuery { get; set; } + public long IntervalLengthMinutes { get; set; } + + public string StateDisplay => + string.Equals(ActualState, DesiredState, StringComparison.OrdinalIgnoreCase) + ? ActualState + : $"{ActualState} (wanted {DesiredState})"; + + /// Percent of the storage cap in use; blank when the cap is 0 (unlimited/unknown). + public string PercentOfCapDisplay => + MaxStorageMb > 0 ? $"{100.0 * CurrentStorageMb / MaxStorageMb:F0}%" : ""; + + /// The shared bit-by-bit decode — one label table for every surface that shows this value. + public string ReadonlyReasonDisplay => PerformanceMonitor.Common.QueryStoreReadonlyReason.Decode(ReadonlyReason); +} + public class DatabaseScopedConfigRow { public string DatabaseName { get; set; } = ""; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DatabaseStates.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DatabaseStates.cs index 7d9e47ee7..ac0b9edeb 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DatabaseStates.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.DatabaseStates.cs @@ -27,18 +27,44 @@ namespace PerformanceMonitor.Darling.Viewer; public sealed partial class ViewerDataService { /* Matches the service alert read's seed: effective state (STANDBY for a log-shipping secondary, else - state_desc), and a critical effective state is NOT baselined so it stays pending — otherwise opening - the editor mid-outage would baseline SUSPECT and silence the alert. */ - private const string DatabaseStateSeedSql = @" + state_desc), and an integrity or transient effective state is NOT baselined so it stays pending — + otherwise opening the editor mid-outage would baseline SUSPECT and silence the alert, or opening it + mid-restore would baseline RESTORING and invert it (#2189). The state list is shared with the service + rather than spelled twice: this project cannot reference the service's, and two hand-kept copies of + "what must never be learned" is the drift that would let the editor seed what the alert refuses to. */ + private const string DatabaseStateSeedSql = $@" INSERT INTO config.database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at) SELECT $1, ds.database_name, CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END, false, (now() AT TIME ZONE 'UTC') FROM database_states ds WHERE ds.server_id = $1 AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) AND ds.state_desc IS NOT NULL -AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) NOT IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY') +AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) NOT IN ({DatabaseStateTokens.NeverBaselinedSqlList}) ON CONFLICT (server_id, database_name) DO NOTHING"; + /* The service adapter's #2189 heal, run for the same reason the seed is: the editor must not SHOW a + stale auto-baseline the alert has already stopped honouring, and a viewer seat pointed at a store + whose service is down would otherwise display — and keep — an expectation the database outgrew. + Auto-baselines only, and only ones recording a state the seed would refuse to learn: a user override + is the operator's declaration, and OFFLINE/STANDBY are steady states whose departure is real news. */ + private const string DatabaseStateHealToOnlineSql = $@" +UPDATE config.database_state_expected e +SET expected_state = 'ONLINE', + updated_at = (now() AT TIME ZONE 'UTC'), + last_alerted_state = NULL, + last_alerted_at = NULL +WHERE e.server_id = $1 +AND e.is_user_override = false +AND e.expected_state IN ({DatabaseStateTokens.NeverBaselinedSqlList}) +AND EXISTS ( + SELECT 1 + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND ds.database_name = e.database_name + AND (CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END) = 'ONLINE' +)"; + private const string DatabaseStateExpectationsSql = @" WITH latest AS ( SELECT ds.database_name, CASE WHEN ds.is_in_standby THEN 'STANDBY' ELSE ds.state_desc END AS eff @@ -80,6 +106,10 @@ public async Task> GetDatabaseStateExpectationsAs await using var seed = _dataSource.CreateCommand(DatabaseStateSeedSql); seed.Parameters.Add(new NpgsqlParameter { TypedValue = serverId }); await seed.ExecuteNonQueryAsync(cancellationToken); + + await using var heal = _dataSource.CreateCommand(DatabaseStateHealToOnlineSql); + heal.Parameters.Add(new NpgsqlParameter { TypedValue = serverId }); + await heal.ExecuteNonQueryAsync(cancellationToken); } var rows = new List(); diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Inventory.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Inventory.cs index 510b152dd..279abcf21 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Inventory.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Inventory.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Viewer; @@ -37,9 +38,12 @@ FROM v_cpu_utilization_stats WHERE server_id = $1 AND collection_time >= $2 ), +/* Only the worker counts are consumed now: memory_ratio used to feed this read's own CASE, and that + CASE was the #2246 bug. The verdict comes from ProvisioningVerdict, so the division would be dead. */ mem_latest AS ( SELECT - CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0) AS memory_ratio + max_workers_count, + current_workers_count FROM v_memory_stats WHERE server_id = $1 AND (server_id, collection_time) IN ( @@ -49,6 +53,19 @@ FROM v_memory_stats GROUP BY server_id ) ), +/* Same workspace-memory pressure signals as the drill-down read, so the INVENTORY GRID cannot classify a + server by a rule the drill-down no longer uses (#2246 — this grid is the screen the field report was + looking at). */ +grants AS ( + SELECT + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 +), storage_totals AS ( SELECT SUM(total_size_mb) / 1024.0 AS total_storage_gb @@ -87,18 +104,20 @@ AND delta_execution_count > 0 c.avg_cpu_pct, st.total_storage_gb, id.idle_db_count, - CASE - WHEN c.avg_cpu_pct < 15 AND c.max_cpu_pct < 40 AND COALESCE(m.memory_ratio, 0) < 0.5 - THEN 'OVER_PROVISIONED' - WHEN c.p95_cpu_pct > 85 OR COALESCE(m.memory_ratio, 0) > 0.95 - THEN 'UNDER_PROVISIONED' - ELSE 'RIGHT_SIZED' - END AS provisioning_status + c.max_cpu_pct, + c.p95_cpu_pct, + COALESCE(m.max_workers_count, 0), + COALESCE(m.current_workers_count, 0), + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0) FROM (SELECT 1) AS anchor LEFT JOIN cpu_24h c ON true LEFT JOIN mem_latest m ON true LEFT JOIN storage_totals st ON true -LEFT JOIN idle_dbs id ON true"; +LEFT JOIN idle_dbs id ON true +LEFT JOIN grants g ON true"; public async Task<(decimal? AvgCpuPct, decimal? StorageTotalGb, int? IdleDbCount, string? ProvisioningStatus)> GetServerMetricsAsync(int serverId, CancellationToken cancellationToken = default) { @@ -113,11 +132,25 @@ LEFT JOIN storage_totals st ON true await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { + /* The verdict is computed HERE rather than as a SQL CASE, so this grid and the drill-down + cannot disagree — they now call the same predicate. The old inline CASE was copies 5 and 6 + of the ratio bug, on the screen the field report was actually looking at (#2246). */ + var status = ProvisioningVerdict.Evaluate( + avgCpuPercent: reader.IsDBNull(0) ? 0m : Convert.ToDecimal(reader.GetValue(0)), + maxCpuPercent: reader.IsDBNull(3) ? 0m : Convert.ToDecimal(reader.GetValue(3)), + p95CpuPercent: reader.IsDBNull(4) ? 0m : Convert.ToDecimal(reader.GetValue(4)), + maxGrantWaiters: reader.IsDBNull(7) ? 0L : Convert.ToInt64(reader.GetValue(7)), + grantTimeouts: reader.IsDBNull(8) ? 0L : Convert.ToInt64(reader.GetValue(8)), + forcedGrants: reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9)), + grantUtilizationPercent: reader.IsDBNull(10) ? 0m : Convert.ToDecimal(reader.GetValue(10)), + maxWorkers: reader.IsDBNull(5) ? 0 : Convert.ToInt32(reader.GetValue(5)), + currentWorkers: reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6))); + return ( reader.IsDBNull(0) ? null : Convert.ToDecimal(reader.GetValue(0)), reader.IsDBNull(1) ? null : Convert.ToDecimal(reader.GetValue(1)), reader.IsDBNull(2) ? null : Convert.ToInt32(reader.GetValue(2)), - reader.IsDBNull(3) ? null : reader.GetString(3) + status ); } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Utilization.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Utilization.cs index 105c3f13e..6988afa59 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Utilization.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.Utilization.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Viewer; @@ -59,6 +60,20 @@ FROM server_properties WHERE server_id = $1 ORDER BY collection_time DESC LIMIT 1 +), +/* Workspace-memory pressure, which is what being short of memory actually looks like: a query asked the + resource semaphore for a grant and did not simply get it. Counts of events, so no threshold to tune + (#2246). The utilization peak rides along to stop a CPU-quiet server that is straining its semaphore + from being called idle. */ +grants AS ( + SELECT + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 ) SELECT c.avg_cpu_pct, @@ -72,10 +87,15 @@ LIMIT 1 m.memory_ratio, m.max_workers_count, m.current_workers_count, - s.cpu_count + s.cpu_count, + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0) FROM cpu_stats c CROSS JOIN mem_latest m -LEFT JOIN server_info s ON true"; +LEFT JOIN server_info s ON true +LEFT JOIN grants g ON true"; public async Task GetUtilizationEfficiencyAsync(int serverId, CancellationToken cancellationToken = default) { @@ -93,11 +113,20 @@ CROSS JOIN mem_latest m var p95Cpu = reader.IsDBNull(2) ? 0m : Convert.ToDecimal(reader.GetValue(2)); var memRatio = reader.IsDBNull(8) ? 0m : Convert.ToDecimal(reader.GetValue(8)); - var status = "RIGHT_SIZED"; - if (avgCpu < 15 && maxCpu < 40 && memRatio < 0.5m) - status = "OVER_PROVISIONED"; - else if (p95Cpu > 85 || memRatio > 0.95m) - status = "UNDER_PROVISIONED"; + var maxWorkers = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)); + var currentWorkers = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)); + + /* memory_ratio is still SELECTed and still displayed — it is a real fact about the instance — but it + is no longer part of the verdict: Total over Target Server Memory converges at 1.0 on any warmed + server, so it reported every server as under-provisioned (#2246). */ + var status = ProvisioningVerdict.Evaluate( + avgCpu, maxCpu, p95Cpu, + maxGrantWaiters: reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12)), + grantTimeouts: reader.IsDBNull(13) ? 0L : Convert.ToInt64(reader.GetValue(13)), + forcedGrants: reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14)), + grantUtilizationPercent: reader.IsDBNull(15) ? 0m : Convert.ToDecimal(reader.GetValue(15)), + maxWorkers: maxWorkers, + currentWorkers: currentWorkers); return new UtilizationEfficiencyRow { @@ -111,8 +140,12 @@ CROSS JOIN mem_latest m BufferPoolMb = reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7)), MemoryRatio = memRatio, ProvisioningStatus = status, - MaxWorkersCount = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)), - CurrentWorkersCount = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)), + MaxGrantWaiters = reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12)), + GrantTimeouts = reader.IsDBNull(13) ? 0L : Convert.ToInt64(reader.GetValue(13)), + ForcedGrants = reader.IsDBNull(14) ? 0L : Convert.ToInt64(reader.GetValue(14)), + GrantUtilizationPct = reader.IsDBNull(15) ? 0m : Convert.ToDecimal(reader.GetValue(15)), + MaxWorkersCount = maxWorkers, + CurrentWorkersCount = currentWorkers, CpuCount = reader.IsDBNull(11) ? 0 : Convert.ToInt32(reader.GetValue(11)) }; } @@ -133,20 +166,43 @@ GROUP BY CAST(collection_time AS DATE) daily_mem AS ( SELECT CAST(collection_time AS DATE) AS day, - AVG(CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0)) AS avg_memory_ratio + AVG(CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0)) AS avg_memory_ratio, + MAX(max_workers_count) AS max_workers_count, + MAX(current_workers_count) AS current_workers_count FROM v_memory_stats WHERE server_id = $1 AND collection_time >= $2 GROUP BY CAST(collection_time AS DATE) +), +/* Same pressure signals as the point-in-time read, per day, so a day cannot be classified by a rule + the current verdict does not use (#2246). */ +daily_grants AS ( + SELECT + CAST(collection_time AS DATE) AS day, + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 + GROUP BY CAST(collection_time AS DATE) ) SELECT c.day, c.avg_cpu_pct, c.max_cpu_pct, c.p95_cpu_pct, - COALESCE(m.avg_memory_ratio, 0) + COALESCE(m.avg_memory_ratio, 0), + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0), + COALESCE(m.max_workers_count, 0), + COALESCE(m.current_workers_count, 0) FROM daily_cpu c LEFT JOIN daily_mem m ON m.day = c.day +LEFT JOIN daily_grants g ON g.day = c.day ORDER BY c.day"; public async Task> GetProvisioningTrendAsync(int serverId, CancellationToken cancellationToken = default) @@ -166,11 +222,14 @@ public async Task> GetProvisioningTrendAsync(int serv var p95Cpu = reader.IsDBNull(3) ? 0m : Convert.ToDecimal(reader.GetValue(3)); var memRatio = reader.IsDBNull(4) ? 0m : Convert.ToDecimal(reader.GetValue(4)); - var status = "RIGHT_SIZED"; - if (avgCpu < 15 && maxCpu < 40 && memRatio < 0.5m) - status = "OVER_PROVISIONED"; - else if (p95Cpu > 85 || memRatio > 0.95m) - status = "UNDER_PROVISIONED"; + var status = ProvisioningVerdict.Evaluate( + avgCpu, maxCpu, p95Cpu, + maxGrantWaiters: reader.IsDBNull(5) ? 0L : Convert.ToInt64(reader.GetValue(5)), + grantTimeouts: reader.IsDBNull(6) ? 0L : Convert.ToInt64(reader.GetValue(6)), + forcedGrants: reader.IsDBNull(7) ? 0L : Convert.ToInt64(reader.GetValue(7)), + grantUtilizationPercent: reader.IsDBNull(8) ? 0m : Convert.ToDecimal(reader.GetValue(8)), + maxWorkers: reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)), + currentWorkers: reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10))); items.Add(new ProvisioningTrendRow { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs index 6d95d34f6..c1b2418ec 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.FinOps.cs @@ -95,6 +95,21 @@ public sealed class UtilizationEfficiencyRow public int PhysicalMemoryMb { get; set; } public int BufferPoolMb { get; set; } public decimal MemoryRatio { get; set; } + + /// Peak resource-semaphore waiters over the window. Any waiter at all means a query asked for + /// workspace memory and did not simply get it — the signal the verdict uses in place of the ratio that + /// pinned at 1.0 (#2246). + public long MaxGrantWaiters { get; set; } + + /// Grant timeouts accrued over the window (delta, not cumulative). + public long GrantTimeouts { get; set; } + + /// Grants forced through below what was requested, over the window. + public long ForcedGrants { get; set; } + + /// Peak granted-over-target workspace memory, as a percentage. Fleet max is 18.8%. + public decimal GrantUtilizationPct { get; set; } + public int MaxWorkersCount { get; set; } public int CurrentWorkersCount { get; set; } public int CpuCount { get; set; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.JobHistory.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.JobHistory.cs index 36d9f0454..59d69b468 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.JobHistory.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.JobHistory.cs @@ -28,7 +28,11 @@ namespace PerformanceMonitor.Darling.Viewer; public sealed class ViewerJobHistoryRow { public int ServerId { get; init; } + + /// The operator's display alias when one is registered, the raw collected name otherwise + /// (#2126 — the Server column and filter combo show the same names every other tab does). public string ServerName { get; init; } = ""; + public long InstanceId { get; init; } public string JobId { get; init; } = ""; public string JobName { get; init; } = ""; @@ -94,7 +98,9 @@ public sealed partial class ViewerDataService /// Long-runtime is computed reader-side via a per-job window function (a step_id 0 outcome exceeding 2x /// its job's average successful-outcome duration, floored at 60s), and each row carries its job's last /// successful outcome run. With no it aggregates ALL servers (the tab - /// default); with one it scopes to that server (the Server filter combo). + /// default); with one it scopes to that server (the Server filter combo). server_name resolves through + /// the servers registry to the operator's display alias when one exists (#2126), so the tab + /// speaks the same names as the rest of the viewer. /// /// public async Task> GetJobHistoryAsync( @@ -115,7 +121,7 @@ WHERE utc_offset_minutes IS NOT NULL base AS ( SELECT jh.server_id, - jh.server_name, + COALESCE(reg.display_name, jh.server_name) AS server_name, jh.instance_id, jh.job_id, jh.job_name, @@ -136,6 +142,7 @@ base AS ( OVER (PARTITION BY jh.server_id, jh.job_id) AS last_success_run_utc FROM job_history AS jh LEFT JOIN svr ON svr.server_id = jh.server_id + LEFT JOIN servers AS reg ON reg.server_id = jh.server_id WHERE jh.run_datetime - make_interval(mins => COALESCE(svr.utc_offset_minutes, 0)) >= $1 {serverFilter} ) @@ -232,7 +239,7 @@ WHERE utc_offset_minutes IS NOT NULL latest AS ( SELECT a.server_id, - a.server_name, + COALESCE(reg.display_name, a.server_name) AS server_name, a.agent_running, a.agent_status_desc, a.agent_startup_desc, @@ -240,6 +247,7 @@ latest AS ( ROW_NUMBER() OVER (PARTITION BY a.server_id ORDER BY a.collection_time DESC) AS rn FROM agent_status AS a LEFT JOIN svr ON svr.server_id = a.server_id + LEFT JOIN servers AS reg ON reg.server_id = a.server_id {serverFilter} ) SELECT diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.MonitoredServers.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.MonitoredServers.cs index aea0fd354..51ebbd46a 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.MonitoredServers.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.MonitoredServers.cs @@ -131,6 +131,32 @@ FROM config_monitored_servers FROM config_monitored_servers WHERE server_id = $1"; + /// + /// One configured server by its ADDRESS — host, database and read-only intent (#2158). The collision check + /// the Add/Edit save runs before writing. + /// + /// Why by address and not by derived id. The guard used to look the address's + /// hash up by server_id, which only works while every row's id still + /// equals the hash of its own address. Once an edit PRESERVES a row's identity — which is the point of + /// #2158, so a re-addressed server keeps its collected history — that stops being true, and a hash lookup + /// would miss the very row it is meant to protect: two registrations would end up pointing at one real + /// instance, which is #2228's shape. Matching the address columns asks the question the guard actually + /// means. + /// + /// IS NOT DISTINCT FROM for database because it is nullable and NULL = NULL is unknown + /// in SQL: a plain = would never match the server-scoped registrations (the common case), so every + /// one of them would read as "address free". Secret-free projection — the caller only needs to know whether + /// a row exists and which id it has, so this runs for a read-only seat too. + /// + public const string MonitoredServerByAddressSql = @" +SELECT server_id, name, host, database, auth, username, encrypt_mode, + trust_server_certificate, read_only_intent, multi_subnet_failover, excluded_databases, + monthly_cost_usd, capture_plans, is_enabled, created_at, alert_delivery_mode_override +FROM config_monitored_servers +WHERE host = $1 +AND database IS NOT DISTINCT FROM $2 +AND read_only_intent = $3"; + /// Row count — the migrate-in / reconcile "is the config-server set seeded yet?" guard. public const string MonitoredServersCountSql = "SELECT COUNT(*) FROM config_monitored_servers"; @@ -262,6 +288,24 @@ public async Task> GetMonitoredServersAsync(Cancellatio return await reader.ReadAsync(cancellationToken) ? ReadMonitoredServerRow(reader) : null; } + /// + /// The server already registered at this address, or null when the address is free (#2158) — what the + /// Add/Edit save checks before writing, so one real instance cannot end up under two identities. + /// + /// Secret-free by design: the caller compares ids and shows a message, so there is no reason to + /// read the DPAPI blob, and skipping it means a read-only seat gets the same answer instead of 42501. + /// + public async Task GetMonitoredServerByAddressAsync( + string host, string? database, bool readOnlyIntent, CancellationToken cancellationToken = default) + { + await using var command = _dataSource.CreateCommand(MonitoredServerByAddressSql); + command.Parameters.Add(new NpgsqlParameter { TypedValue = host }); + command.Parameters.Add(new NpgsqlParameter { Value = (object?)database ?? DBNull.Value }); + command.Parameters.Add(new NpgsqlParameter { TypedValue = readOnlyIntent }); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + return await reader.ReadAsync(cancellationToken) ? ReadMonitoredServerRowNoSecret(reader) : null; + } + /// How many servers the config-server registry holds (the migrate-in / reconcile guard). public async Task GetMonitoredServerCountAsync(CancellationToken cancellationToken = default) { diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Perfmon.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Perfmon.cs index 74872186c..37608cb19 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Perfmon.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Perfmon.cs @@ -17,14 +17,19 @@ namespace PerformanceMonitor.Darling.Viewer; /// /// One point on a selected perfmon counter's trend line: the raw counter value and the per-interval -/// delta, both summed across the counter's instances at each collection_time. Copied from Lite's +/// delta, both summed across the counter's instances at each collection_time, plus the wall-clock +/// seconds that delta covers — MAX, not SUM, because the interval is one measured sweep gap repeated +/// per instance rather than a per-instance quantity (#2234; Transactions/sec carries a median of 12 +/// instance rows). The picker does not derive a rate today, so the field rides along unplotted so that +/// whoever adds one has a real denominator instead of a fabricated cadence. Copied from Lite's /// PerfmonTrendPoint (LocalDataService.Perfmon.cs). The picker plots ; /// rides along for parity with Lite's row shape. /// public sealed record PerfmonTrendPoint( DateTime CollectionTime, long Value, - long DeltaValue); + long DeltaValue, + long SampleIntervalSeconds); public sealed partial class ViewerDataService { @@ -63,7 +68,8 @@ public static string PerfmonTrendsSql(int counterCount) counter_name, collection_time, CAST(SUM(cntr_value) AS bigint) AS cntr_value, - CAST(SUM(delta_cntr_value) AS bigint) AS delta_cntr_value + CAST(SUM(delta_cntr_value) AS bigint) AS delta_cntr_value, + CAST(MAX(sample_interval_seconds) AS bigint) AS sample_interval_seconds FROM v_perfmon_stats WHERE server_id = $1 AND collection_time >= $2 @@ -143,7 +149,8 @@ public async Task>> GetPerfmonTrendsB list.Add(new PerfmonTrendPoint( reader.GetDateTime(1), reader.IsDBNull(2) ? 0 : reader.GetInt64(2), - reader.IsDBNull(3) ? 0 : reader.GetInt64(3))); + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + reader.IsDBNull(4) ? 0 : reader.GetInt64(4))); } return result; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs index d697ca4c3..e59485d4e 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs @@ -276,15 +276,33 @@ ORDER BY SUM(execution_count) * AVG(CAST(avg_duration_us AS double precision)) D r.max_num_physical_io_reads, r.replica_role FROM ranked AS r + /* #2150: resolve the text ONCE, inside the lateral, so everything downstream still reads a single + t.query_text — the projection above and the WAITFOR self-exclusion below both get the resolved + value without either having to know where it came from. + First arm: collect.query_store_text, one row per (server, database, query_id), which is where the + collector lands text once the separate fetch is on. Second arm: the newest inline query_text on + the fact row, which is where text lived BEFORE the cutover — it stays because dropping it would + blank the text on all existing history. */ LEFT JOIN LATERAL ( - SELECT query_text - FROM query_store_stats - WHERE server_id = $1 - AND query_id = r.query_id - AND database_name = r.database_name - AND query_text IS NOT NULL - ORDER BY collection_time DESC - LIMIT 1 + SELECT COALESCE( + ( + SELECT x.query_sql_text + FROM query_store_text AS x + WHERE x.server_id = $1 + AND x.database_name = r.database_name + AND x.query_id = r.query_id + ), + ( + SELECT s.query_text + FROM query_store_stats AS s + WHERE s.server_id = $1 + AND s.query_id = r.query_id + AND s.database_name = r.database_name + AND s.query_text IS NOT NULL + ORDER BY s.collection_time DESC + LIMIT 1 + ) + ) AS query_text ) AS t ON TRUE WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' ORDER BY r.total_executions * r.avg_duration_ms DESC @@ -390,6 +408,9 @@ recent window than in the baseline window (recent windows hold more still-open i so both arms are treated identically. */ SELECT database_name, + /* #2150: query_id is projected (it was already a partition key) so the period CTEs can + resolve text from collect.query_store_text, which is keyed on it. */ + query_id, query_hash, query_text, execution_count, @@ -409,6 +430,9 @@ FROM query_store_stats deduped_baseline AS ( SELECT database_name, + /* #2150: query_id is projected (it was already a partition key) so the period CTEs can + resolve text from collect.query_store_text, which is keyed on it. */ + query_id, query_hash, query_text, execution_count, @@ -457,11 +481,21 @@ current_period AS ( SUM(qs.execution_count * qs.avg_duration_us::double precision) / NULLIF(SUM(qs.execution_count), 0) / 1000.0 AS avg_duration_ms, SUM(qs.execution_count * qs.avg_cpu_time_us::double precision) / NULLIF(SUM(qs.execution_count), 0) / 1000.0 AS avg_cpu_ms, SUM(qs.execution_count * qs.avg_logical_io_reads::double precision) / NULLIF(SUM(qs.execution_count), 0) AS avg_reads, - MAX(qs.query_text) AS query_text + /* #2150: this comparison groups by query_hash, but text is stored per query_id, so the + side table is joined on the finer key and MAX still picks one member's text for the + group — the same arbitrary-but-deterministic choice MAX(qs.query_text) made before. + The join cannot fan out (query_store_text is one row per server/database/query_id, by + primary key), so the execution-count SUMs above are unaffected. The COALESCE keeps + pre-cutover rows, whose text is still inline, reading exactly as they used to. */ + MAX(COALESCE(x.query_sql_text, qs.query_text)) AS query_text FROM top_hashes th INNER JOIN deduped_current qs ON qs.query_hash IS NOT DISTINCT FROM th.query_hash AND qs.database_name IS NOT DISTINCT FROM th.database_name + LEFT JOIN query_store_text AS x + ON x.server_id = $1 + AND x.database_name = qs.database_name + AND x.query_id = qs.query_id WHERE qs.rn = 1 AND qs.execution_count > 0 GROUP BY th.database_name, th.query_hash @@ -472,11 +506,18 @@ baseline_period AS ( SUM(qs.execution_count * qs.avg_duration_us::double precision) / NULLIF(SUM(qs.execution_count), 0) / 1000.0 AS avg_duration_ms, SUM(qs.execution_count * qs.avg_cpu_time_us::double precision) / NULLIF(SUM(qs.execution_count), 0) / 1000.0 AS avg_cpu_ms, SUM(qs.execution_count * qs.avg_logical_io_reads::double precision) / NULLIF(SUM(qs.execution_count), 0) AS avg_reads, - MAX(qs.query_text) AS query_text + /* #2150 — same resolution as current_period above. Both arms need it because the final + projection takes COALESCE(c.query_text, b.query_text): converting only one arm would + leave a GONE row (present in baseline only) with no text to fall back to. */ + MAX(COALESCE(x.query_sql_text, qs.query_text)) AS query_text FROM top_hashes th INNER JOIN deduped_baseline qs ON qs.query_hash IS NOT DISTINCT FROM th.query_hash AND qs.database_name IS NOT DISTINCT FROM th.database_name + LEFT JOIN query_store_text AS x + ON x.server_id = $1 + AND x.database_name = qs.database_name + AND x.query_id = qs.query_id WHERE qs.rn = 1 AND qs.execution_count > 0 GROUP BY th.database_name, th.query_hash diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs index 292170c69..7f63677a4 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs @@ -82,8 +82,10 @@ public sealed partial class ViewerDataService /// split on collection_time (naive UTC) — the column EVERY other Darling Query Store read windows /// on, and the frame the toolbar's [start,end] bounds are in (Darling's last_execution_time is the /// server's LOCAL wall clock, so windowing on it against UTC bounds would be a timezone bug). - /// (2) The recent query-text sample is MAX(query_text) from Darling's already-decompressed - /// query_text column (the TVF DECOMPRESS()es a compressed query_sql_text). + /// (2) The recent query-text sample comes from collect.query_store_text (#2150), falling back to + /// MAX(query_text) on the fact rows for history collected before that table existed — either way + /// Darling's text is already decompressed, where the TVF DECOMPRESS()es a compressed + /// query_sql_text. /// (3) The stale INTENT comment on the Dashboard's C# caller (bounded/mirrored baseline, weighted averages, /// multi-metric, absolute minimums) does NOT match what the TVF actually runs — this ports the ACTUAL TVF /// (unbounded baseline, plain AVG, CPU-only > 25% gate, no minimums), which is what the Dashboard grid shows. @@ -194,12 +196,20 @@ FROM deduped_recent WHEN (r.avg_duration_ms - b.avg_duration_ms) * 100.0 / NULLIF(b.avg_duration_ms, 0) > 25 THEN 'MEDIUM' ELSE 'LOW' END AS severity, - r.query_text_sample, + /* #2150: text comes from collect.query_store_text now, and this projection's grain is exactly + that table's key, so it resolves here with a keyed join rather than inside the aggregate. + The MAX(query_text) sample below it stays as the fallback: it is where text lived before the + cutover, and it is what keeps the regression grid readable for existing history. */ + COALESCE(x.query_sql_text, r.query_text_sample) AS query_text_sample, r.last_execution_time FROM recent_performance AS r JOIN baseline_performance AS b ON b.database_name = r.database_name AND b.query_id = r.query_id + LEFT JOIN query_store_text AS x + ON x.server_id = $1 + AND x.database_name = r.database_name + AND x.query_id = r.query_id WHERE (r.avg_cpu_time_ms - b.avg_cpu_time_ms) * 100.0 / NULLIF(b.avg_cpu_time_ms, 0) > 25 ORDER BY additional_duration_ms DESC LIMIT 50 diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ServiceConfig.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ServiceConfig.cs index 585da6cee..91fd14b86 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ServiceConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.ServiceConfig.cs @@ -38,14 +38,14 @@ public sealed partial class ViewerDataService /// The service-wide flags (id=1): paused + the three viewer-owned toggles. Column order matches /// the service's ReadServiceRowAsync prefix. public const string ServiceConfigSelectSql = - "SELECT paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port FROM config_service WHERE id = 1"; + "SELECT paused, capture_plans, mcp_enabled, mcp_port, web_enabled, web_port, query_store_backfill_enabled, query_store_text_budget_mb, max_concurrent_sweeps FROM config_service WHERE id = 1"; /// Updates ONLY the viewer-owned service flags on the seeded row (never paused — a /// command). The self-bump trigger fires config_version. $1 capture_plans, $2 mcp_enabled, $3 mcp_port, - /// $4 web_enabled, $5 web_port. + /// $4 web_enabled, $5 web_port, $6 query_store_backfill_enabled (#2167). public const string ServiceConfigUpdateFlagsSql = @" UPDATE config_service -SET capture_plans = $1, mcp_enabled = $2, mcp_port = $3, web_enabled = $4, web_port = $5 +SET capture_plans = $1, mcp_enabled = $2, mcp_port = $3, web_enabled = $4, web_port = $5, query_store_backfill_enabled = $6, query_store_text_budget_mb = $7, max_concurrent_sweeps = $8 WHERE id = 1"; /// Reads the service-wide flags, or null when the store has not seeded config_service yet @@ -68,6 +68,9 @@ UPDATE config_service McpPort = reader.GetInt32(3), WebEnabled = reader.GetBoolean(4), WebPort = reader.GetInt32(5), + QueryStoreBackfillEnabled = reader.GetBoolean(6), + QueryStoreTextBudgetMb = reader.GetInt32(7), + MaxConcurrentSweeps = reader.GetInt32(8), }; } @@ -92,7 +95,7 @@ public async Task IsServicePausedAsync(CancellationToken cancellationToken /// Updates the viewer-owned service flags (Settings window Save — MCP + web dashboard + global plan /// capture). A no-op on an unseeded store (zero rows). Read-only seats throw . public async Task UpdateServiceFlagsAsync( - bool capturePlans, bool mcpEnabled, int mcpPort, bool webEnabled, int webPort, CancellationToken cancellationToken = default) + bool capturePlans, bool mcpEnabled, int mcpPort, bool webEnabled, int webPort, bool queryStoreBackfillEnabled, int queryStoreTextBudgetMb, int maxConcurrentSweeps, CancellationToken cancellationToken = default) { await using var command = _dataSource.CreateCommand(ServiceConfigUpdateFlagsSql); command.Parameters.Add(new NpgsqlParameter { TypedValue = capturePlans }); @@ -100,6 +103,9 @@ public async Task UpdateServiceFlagsAsync( command.Parameters.Add(new NpgsqlParameter { TypedValue = mcpPort }); command.Parameters.Add(new NpgsqlParameter { TypedValue = webEnabled }); command.Parameters.Add(new NpgsqlParameter { TypedValue = webPort }); + command.Parameters.Add(new NpgsqlParameter { TypedValue = queryStoreBackfillEnabled }); + command.Parameters.Add(new NpgsqlParameter { TypedValue = queryStoreTextBudgetMb }); + command.Parameters.Add(new NpgsqlParameter { TypedValue = maxConcurrentSweeps }); await ExecuteWriteAsync(command, cancellationToken); } @@ -134,4 +140,13 @@ public sealed class ServiceConfigRow public int McpPort { get; set; } = 5152; public bool WebEnabled { get; set; } public int WebPort { get; set; } = 5153; + + /// The #2167 Query Store backfill off switch — service reads it live; default on. + public bool QueryStoreBackfillEnabled { get; set; } = true; + + /// The #2164 per-database query_store text budget in MB — service reads it live; default 64. + public int QueryStoreTextBudgetMb { get; set; } = 64; + + /// The #2170 fleet sweep width — service reads it live; default 4. + public int MaxConcurrentSweeps { get; set; } = 4; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index 621eb4466..376e83618 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -263,6 +263,80 @@ public ViewerDataService(string connectionString, int? connectionTimeoutSeconds ? ApplyConnectionTimeout(connectionString, seconds) : connectionString; _dataSource = NpgsqlDataSource.Create(effectiveConnectionString); + StoreIsOnThisMachine = StoreHostIsLoopback(connectionString); + } + + /// + /// Whether this viewer's store is reached over loopback — the best available proxy for "the Darling service + /// runs on THIS machine" (#2279). + /// + /// Why it is a proxy for that at all. A DPAPI blob is LocalMachine-scoped, so a + /// credential this viewer encrypts is decryptable only on this machine, and the service is the thing that + /// has to decrypt it. The managed deploy builds its store connection on literal 127.0.0.1 + /// (ViewerSettings, matching the service's own DarlingManagedPostgres.BuildConnectionString), + /// and the service runs where its managed store runs. So a loopback store means viewer and service share a + /// machine and a saved credential will work — which is the single-box deploy the DPAPI design targets, and + /// where a warning would be pure noise. + /// + /// What it deliberately does not claim. A non-loopback store does NOT prove the viewer is + /// remote — a bring-your-own store on another host with the service local is a real configuration, and it + /// reads as false here. That is why #2279 warns rather than refuses: this signal is good enough to decide + /// whether to SAY something, and not good enough to decide whether to BLOCK. Getting that backwards would + /// refuse a legitimate first-run Add on the service host. + /// + public bool StoreIsOnThisMachine { get; } + + /// + /// True when a store connection string names a loopback host, or names none at all (#2279). + /// + /// Static and pure so the rule is testable without a store — it is the whole basis of the warning, and + /// a viewer cannot be stood up in a unit test. Uses the base rather + /// than Npgsql's, for the reason documented on : the Npgsql builder + /// answers ContainsKey for every KNOWN key rather than only the present ones, so it cannot tell an + /// omitted host from a specified one. + /// + /// An omitted host counts as loopback because that is what it MEANS — Npgsql defaults to localhost, so + /// a string with no Host is a local store and must not warn. ::1 and localhost are + /// included alongside 127.0.0.1 even though the managed path always writes the literal IPv4 form, + /// because a hand-written BYO string pointing at the local box is still local and warning about it would be + /// wrong. + /// + internal static bool StoreHostIsLoopback(string? connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + return true; + } + + string? host = null; + try + { + var builder = new DbConnectionStringBuilder { ConnectionString = connectionString }; + if (builder.ContainsKey("Host")) + { + host = builder["Host"]?.ToString(); + } + else if (builder.ContainsKey("Server")) + { + host = builder["Server"]?.ToString(); + } + } + catch (ArgumentException) + { + /* An unparseable string is not evidence the store is remote, and this decides only whether to show + a hint — so fail toward silence rather than toward a warning nobody can act on. */ + return true; + } + + if (string.IsNullOrWhiteSpace(host)) + { + return true; + } + + var trimmed = host.Trim(); + return trimmed.Equals("127.0.0.1", StringComparison.Ordinal) + || trimmed.Equals("::1", StringComparison.Ordinal) + || trimmed.Equals("localhost", StringComparison.OrdinalIgnoreCase); } /// @@ -436,7 +510,22 @@ OR NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'query_stats' AND column_name = 'host_object_name'), EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'analysis_findings' AND column_name = 'drill_down_json'), EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'store_metrics'), - EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'query_plan_dim' AND column_name = 'query_plan_gz')"; + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'query_plan_dim' AND column_name = 'query_plan_gz'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_alert_settings' AND column_name = 'self_disk_free_warn_percent'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'store_metrics' AND column_name = 'last_run_duration_ms'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_alert_settings' AND column_name = 'store_job_cadence_warn_percent'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_service' AND column_name = 'query_store_backfill_enabled'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_service' AND column_name = 'query_store_text_budget_mb'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'database_state_expected' AND column_name = 'last_alerted_state'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'incident_occurrences'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_service' AND column_name = 'plan_xml_compression'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_monitored_servers' AND column_name = 'engine'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'pg_blocking_edges'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'query_store_plan_map'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'pg_statement_text'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'query_store_text'), + EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'config_service' AND column_name = 'plan_content_retention_days'), + EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'query_store_health')"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -457,7 +546,7 @@ OR NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51)); } return null; @@ -482,8 +571,189 @@ OR NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'timescaledb') /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false) { + /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. + A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP + rung always needs one, whatever it happens to be, because RequiredStoreSchemaVersion is + StorageVersion.SchemaVersion and a fully-migrated store must map to EXACTLY that or the version + banner reports a mismatch on a store that is current. Being a collector table makes it no less + reliable as a sentinel; it simply is not interesting for any other reason. + + Deliberately NOT spelled with its collect. name here, and this is the only place in the + file where that matters. ViewerCollectorCoverageTests scans this reader layer for collector table + names by plain substring to decide which tables the viewer actually reads; it strips the probe's + information_schema lines precisely so a migration sentinel cannot fake coverage, but a PROSE + mention has no such line to strip. Naming the table in this comment made the table look read, + which silently exempted it from the coverage ratchet — the exact failure that pin exists to + catch, reported as a stale allow-list entry. */ + /* V72 (the Query Store plan map, #2210): table-existence sentinel, and it has to sit ABOVE the V71 + arm because these are evaluated newest-first — a V72 store also has V71's table, so testing V71 + first would report every V72 store as V71 and the viewer would show a spurious upgrade banner + against a store that is actually current. + + The gate matters rather than being bookkeeping: the SERVICE writes this map on every plan fetch and + resolves plan XML through it, so a viewer pointed at a V71 store while the service expects V72 would + find no map rows and render every plan as "not yet collected" — indistinguishable from a healthy + store that simply has not fetched yet, which is the one failure mode this design works hardest to + avoid being silent about. + + The table is named in the probe line above and deliberately NOT in this prose, per the V71 arm's + finding: the coverage ratchet strips information_schema lines but cannot strip a comment, so a + prose mention would exempt the table from it. */ + /* #2150: newest first, and the arm below STAYS for the same reason every previous one did — a store + migrated to exactly 73 must map to 73 rather than falling through. The table is named only in the + probe line, not in this prose, per the V71 finding: the coverage ratchet strips + information_schema lines but cannot strip a comment, so a prose mention would exempt the table + from it. */ + /* #2316: newest first. The arm below STAYS — a store migrated to exactly 74 must map to 74 + rather than falling through. The column is named only in the probe line, not in this prose, + per the V71 finding: the coverage ratchet strips information_schema lines but cannot strip a + comment, so a prose mention would exempt it. */ + /* #2319: newest first. The arm below STAYS — a store migrated to exactly 75 must map to 75 + rather than falling through. The table is named only in the probe line, not in this prose, + per the V71 finding: the coverage ratchet strips information_schema lines but cannot strip + a comment, so a prose mention would exempt it. */ + if (hasQueryStoreHealth) + { + return 76; + } + + if (hasPlanContentRetentionKnob) + { + return 75; + } + + if (hasQueryStoreText) + { + return 74; + } + + /* #2219: newest first. The previous arm STAYS — a store migrated to exactly 72 must still map to 72 + rather than falling through to the pre-PostgreSQL floor. */ + if (hasPgStatementText) + { + return 73; + } + + if (hasQueryStorePlanMap) + { + return 72; + } + + if (hasPgBlockingEdges) + { + return 71; + } + + /* V70 (the monitored-server engine + port columns): column-existence sentinel, newest-first arm, and + the reason to gate is the standing invariant rather than a 42703 — RequiredStoreSchemaVersion is + StorageVersion.SchemaVersion, so a fully-migrated store must map to EXACTLY that or the version + banner reports a mismatch on a store that is current. + + V63-V69 (the seven PostgreSQL collector tables) get no arms of their own deliberately: they add + tables in `collect` that no viewer read names, so a store between them is only ever transient + mid-migration, and the invariant that has to hold is "fully migrated maps to the top rung" — which + V70, the top rung, is what senses. */ + if (hasMonitoredServerEngine) + { + return 70; + } + + /* V62 (the #2171 plan-XML codec knob): column-existence sentinel, newest-first arm. + config_service.plan_xml_compression exists only at V62 or later. Sits directly above the + V61 arm it merged over - a store carrying both sentinels is V62 and must map there, not to + the first older arm that happens to match. */ + if (hasPlanXmlCompressionKnob) + { + return 62; + } + + /* V61 (per-fingerprint occurrence counters, #2216): table-existence sentinel, newest-first arm. + config.incident_occurrences exists only at V61 or later. + + Same reason to gate as V60 rather than V59: nothing in the VIEWER names this table (it is the + alert engine's accumulator memory, written and read by the service), so the viewer would not + 42703 against a V60 store. The gate exists to keep the standing invariant — + RequiredStoreSchemaVersion is StorageVersion.SchemaVersion, so a fully-migrated store has to map + to exactly that or the version banner reports a mismatch on a store that is current. The + consequence of a V60 store is on the SERVICE side: the occurrence load/save would fail, the + accumulator would fall back to reporting the total as the window count, and #2216's counter would + silently be a gauge again. */ + if (hasIncidentOccurrences) + { + return 61; + } + + /* V60 (database-state edge memory, #2166): column-existence sentinel, newest-first arm. + config.database_state_expected.last_alerted_state exists only at V60 or later. + + Unlike the V59 arm below, the reason to gate here is NOT that the viewer would 42703: the + expected-state editor names only expected_state / is_user_override, so nothing in the viewer + touches either V60 column. The gate is the standing invariant instead — RequiredStoreSchemaVersion + is StorageVersion.SchemaVersion, so a fully-migrated store must map to exactly that or the version + banner reports a mismatch on a store that is actually current. The columns are written by the + SERVICE (the alert engine's edge memory) and read by its deviation query, so it is the service that + would fail on a V59 store, which is precisely why the viewer must not report V59 as good. Stated + explicitly because the V59 wording does not transfer, and someone deciding later whether this gate + can be relaxed needs the real reason. */ + if (hasDatabaseStateEdgeMemory) + { + return 60; + } + + /* V59 (the collector memory knobs, #2164 + #2170): column-existence sentinel, newest-first arm. + config_service.query_store_text_budget_mb exists only at V59 or later. The viewer NAMES both new + columns in ServiceConfigSelectSql/UpdateFlagsSql, so against a V58 store the Settings read would + fail 42703 — the gate must refuse it, and a fully-migrated V59 store must map to exactly + RequiredStoreSchemaVersion. */ + if (hasCollectorMemoryKnobs) + { + return 59; + } + + /* V58 (the Query Store backfill off switch, #2167): column-existence sentinel, newest-first arm. + config_service.query_store_backfill_enabled exists only at V58 or later. The viewer NAMES it in + ServiceConfigSelectSql/UpdateFlagsSql, so against a V57 store the Settings read would fail + 42703 — the gate must refuse it, and a fully-migrated V58 store must map to exactly + RequiredStoreSchemaVersion. */ + if (hasBackfillSwitch) + { + return 58; + } + + /* V57 (the Store Job Over Cadence warning knob, #2136): column-existence sentinel, newest-first + arm. config_alert_settings.store_job_cadence_warn_percent exists only at V57 or later. The + viewer NAMES it in AlertSettingsSelectSql/Upsert, so against a V56 store the Settings read + would fail 42703 — the gate must refuse it, and a fully-migrated V57 store must map to + exactly RequiredStoreSchemaVersion. */ + if (hasJobCadenceKnob) + { + return 57; + } + + /* V56 (background-job self-metrics columns, #2136): column-existence sentinel, newest-first arm. + store_metrics.last_run_duration_ms exists only at V56 or later. The viewer never reads these + columns — they feed the service's own capacity series (job duration vs schedule cadence) over + MCP/REST — so like V53's rung nothing in the viewer would fail against a V55 store. The rung + exists so a fully-migrated store maps to exactly RequiredStoreSchemaVersion instead of capping + at 55 and tripping the connect-time gate against a healthy store. Under-reporting is the + guarded failure. */ + if (hasJobMetricsColumns) + { + return 56; + } + + /* V55 (self-alert threshold knobs, #2107): column-existence sentinel, newest-first arm. + config_alert_settings.self_disk_free_warn_percent exists only at V55 or later. The viewer + NAMES the V55 columns in AlertSettingsSelectSql/Upsert, so against a V54 store the + Settings read would fail 42703 — the gate must refuse it, and a fully-migrated V55 store + must map to exactly RequiredStoreSchemaVersion. */ + if (hasSelfAlertKnobs) + { + return 55; + } + /* V54 (gzip plan-dim content, #2069): column-existence sentinel, newest-first arm. query_plan_dim.query_plan_gz exists only at V54 or later. The viewer NAMES the column in its plan-fetch reads (gz-else-text coalesce), so against a V53 store those reads would @@ -961,7 +1231,10 @@ public sealed class ViewerStoreUnreachableException : Exception public ViewerStoreUnreachableException(Exception innerException) : base( "Can't reach the Darling store — is the Darling service running? Check the postgres section of " + - "darling.json (the host, port, and database must point at the running service's store).", + "darling.json (the host, port, and database must point at the running service's store). " + + /* #2117: the swallowed detail cost a field operator hours — a TLS chain rejection, a wrong + password, a pg_hba refusal, and a dead host all read identically without it. */ + $"Underlying error: {innerException?.Message?.Split('\n')[0].TrimEnd('\r') ?? "(none)"}", innerException) { } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Filters.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Filters.cs index b06116c1e..fde758f9f 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Filters.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Filters.cs @@ -34,6 +34,7 @@ public partial class ViewerServerTab : UserControl private DataGridFilterManager? _serverConfigFilterMgr; private DataGridFilterManager? _databaseConfigFilterMgr; private DataGridFilterManager? _dbScopedConfigFilterMgr; + private DataGridFilterManager? _queryStoreHealthFilterMgr; private DataGridFilterManager? _automaticTuningFilterMgr; private DataGridFilterManager? _traceFlagsFilterMgr; private DataGridFilterManager? _runningJobsFilterMgr; @@ -59,6 +60,7 @@ private void InitializeFilterManagers() _serverConfigFilterMgr = new DataGridFilterManager(ServerConfigGrid); _databaseConfigFilterMgr = new DataGridFilterManager(DatabaseConfigGrid); _dbScopedConfigFilterMgr = new DataGridFilterManager(DatabaseScopedConfigGrid); + _queryStoreHealthFilterMgr = new DataGridFilterManager(QueryStoreHealthGrid); _automaticTuningFilterMgr = new DataGridFilterManager(AutomaticTuningGrid); _traceFlagsFilterMgr = new DataGridFilterManager(TraceFlagsGrid); _runningJobsFilterMgr = new DataGridFilterManager(RunningJobsGrid); @@ -80,6 +82,7 @@ private void InitializeFilterManagers() _filterManagers[ServerConfigGrid] = _serverConfigFilterMgr; _filterManagers[DatabaseConfigGrid] = _databaseConfigFilterMgr; _filterManagers[DatabaseScopedConfigGrid] = _dbScopedConfigFilterMgr; + _filterManagers[QueryStoreHealthGrid] = _queryStoreHealthFilterMgr; _filterManagers[AutomaticTuningGrid] = _automaticTuningFilterMgr; _filterManagers[TraceFlagsGrid] = _traceFlagsFilterMgr; _filterManagers[RunningJobsGrid] = _runningJobsFilterMgr; @@ -111,14 +114,16 @@ private async Task LoadConfigurationAsync() var serverConfigTask = _dataService.GetLatestServerConfigAsync(_server.ServerId); var databaseConfigTask = _dataService.GetLatestDatabaseConfigAsync(_server.ServerId, databaseNames: SelectedDatabaseFilter); var databaseScopedConfigTask = _dataService.GetLatestDatabaseScopedConfigAsync(_server.ServerId, databaseNames: SelectedDatabaseFilter); + var queryStoreHealthTask = _dataService.GetLatestQueryStoreHealthAsync(_server.ServerId, databaseNames: SelectedDatabaseFilter); var automaticTuningTask = _dataService.GetLatestAutomaticTuningAsync(_server.ServerId, databaseNames: SelectedDatabaseFilter); var traceFlagsTask = _dataService.GetLatestTraceFlagsAsync(_server.ServerId); - await Task.WhenAll(serverConfigTask, databaseConfigTask, databaseScopedConfigTask, automaticTuningTask, traceFlagsTask); + await Task.WhenAll(serverConfigTask, databaseConfigTask, databaseScopedConfigTask, queryStoreHealthTask, automaticTuningTask, traceFlagsTask); _serverConfigFilterMgr!.UpdateData(serverConfigTask.Result); _databaseConfigFilterMgr!.UpdateData(databaseConfigTask.Result); _dbScopedConfigFilterMgr!.UpdateData(databaseScopedConfigTask.Result); + _queryStoreHealthFilterMgr!.UpdateData(queryStoreHealthTask.Result); _automaticTuningFilterMgr!.UpdateData(automaticTuningTask.Result); _traceFlagsFilterMgr!.UpdateData(traceFlagsTask.Result); } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.TimeRange.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.TimeRange.cs index ec86947db..6be2da51d 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.TimeRange.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.TimeRange.cs @@ -231,6 +231,16 @@ below drive the reload. Suppress so the two picker writes coalesce into one load ToDatePicker.SelectedDate = DateTime.Today; _suppressRangeEvents = false; } + + if (!isCustom) + { + /* #2154: a DatePicker's calendar dropdown is a POPUP, which lives outside the visual + tree's visibility — collapsing the picker does not close an already-open dropdown, + so backing out of Custom Range without picking a date left an orphaned floating + calendar on screen. Close them explicitly alongside the collapse (Lite twin fix). */ + FromDatePicker.IsDropDownOpen = false; + ToDatePicker.IsDropDownOpen = false; + } } /* Presets reload here; a custom range reloads off the picker changes (below), except the seeded diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml index d8fb01a73..d3afe94a4 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml @@ -1405,8 +1405,14 @@ +
` passthrough views so the shared analysis SQL runs verbatim against this store | @@ -510,9 +613,53 @@ The service migrates the store itself at startup — plain versioned SQL scripts | **V30** — web dashboard config | `config_service.web_enabled` + `web_port` — the read-only web dashboard's live enable/port toggle, the twin of `mcp_enabled`/`mcp_port` (#1562) | | **V46** — automatic plan correction | `collect.plan_correction` + its index — the #1952 collector's store table (FORCE_LAST_GOOD_PLAN enablement plus the engine's live recommendation set). Additive and view-less, so a fresh store gets it from V1's generated schema and V46 is what an already-existing store gets | | **V47** — ADR persistent version store | `collect.pvs_stats` + its index + the `v_pvs_stats` passthrough view — the #1951 ADR version-store collector's store table. A fresh store gets the table from V1's generated schema; V47 is what an already-existing store gets, and the view is what keeps the Darling viewer's FinOps read byte-identical to Lite's | +| **V61** — per-fingerprint occurrence counters | `config.config_incident_occurrences` — the accumulator’s memory for the monotonic count behind an alert incident (#2216). The count that rides on an incident is a GAUGE (it falls as events age out of the read window), so a consumer seeing only throttled deliveries cannot recover how many events happened between two of them. A NEW table, not columns on `config_edge_trigger_watermarks`: the key is wrong (per (server, metric) vs per (server, metric, fingerprint)) and Lite writes that row with a PARTIAL `INSERT OR REPLACE` column list, so an added column would zero itself every time an alert fired | +| **V62** — plan-XML codec knob | `config.config_service.plan_xml_compression` (#2171). `gzip` (default) keeps today’s write path; `none` stores plain text in `query_plan_xml` so direct-SQL readers get plans back — PostgreSQL exposes no inflate, so gzip bytes are unreadable without an untrusted-language UDF. Rides `config_service` like V58/V59 so the `config_version` trigger makes a flip visible to the next reload poll | +| **V63–V69** — PostgreSQL collector tables | `collect.pg_wait_stats`, `collect.pg_statement_stats`, `collect.pg_wraparound_stats`, `collect.pg_xmin_horizon`, `collect.pg_replication_slot_stats`, `collect.pg_autovacuum_stats`, and `collect.pg_io_stats`, each with its time index — one rung per PostgreSQL collector. Additive and view-less, exactly like V46/V47: a fresh store gets all seven from V1's generated schema, and these rungs are what an already-existing store gets. They add tables only, so a store that monitors no PostgreSQL target carries seven empty tables and nothing else changes | +| **V70** — monitored-server engine + port | `config.config_monitored_servers.engine` (`NOT NULL DEFAULT 'sqlserver'`) and `.port` (`NOT NULL DEFAULT 0` = the driver's default). The registry is authoritative for the server list once seeded, and these were the two `MonitoredServer` fields with no column — so a PostgreSQL target round-tripped as a SQL Server one and was connected to with `SqlConnection`. Every existing row means exactly what it meant before, and the SQL-Server-only writers keep inserting without naming either column | +| **V71** — PostgreSQL blocking edges | `collect.pg_blocking_edges` + its time index — the eighth PostgreSQL collector's store table. One row per (blocked, blocking) pair rather than a rendered tree, which is what lets the read layer compute root blocker, chain depth and fan-out in SQL instead of parsing a string. Additive and view-less exactly like V63–V69. **Sparse by design**: empty on a healthy instance, and because PostgreSQL has no engine-side blocked-process recorder, a gap means "not sampled" rather than "not blocked" — a count over this table measures how often blocking was *caught* | +| **V72** — Query Store plan map | `collect.query_store_plan_map` — `(server_id, database_name, plan_id)` → digest, so Query Store facts can reference plan XML they no longer carry once that content moves into the shared `query_plan_dim`. Plan XML was stored INLINE on `query_store_stats` at roughly 5x redundancy. Not a hypertable: one row per distinct plan per database, so it is dimension-shaped and pruned on `last_seen` rather than by `drop_chunks`. Its `last_seen` is load-bearing — the dimension GC sweeps on timestamps rather than counting references, so ending the re-shipping also ends the liveness signal that used to keep those dim rows alive | +| **V73** — PostgreSQL statement text | `collect.pg_statement_text` — `(server_id, queryid)` → statement text, refreshed hourly, so `get_pg_top_queries` returns something readable (#2219). `pg_statement_stats` stores no text because `showtext` is a real per-collection cost and normalized text is highly repetitive; but `queryid` is NOT stable across a major version upgrade, so without this the stored history joins to nothing after one — a list of integers that used to be your slowest queries, unrecoverable because the live view no longer holds the old ids. Text is INLINE rather than a `query_text_dim` digest: the dimension route needs the GC liveness interlock whose failure mode is silently missing text, and inline cannot dangle. Not a hypertable and not a collector table, exactly like V72 — a bespoke upsert path, pruned on `last_seen` with a margin that makes text OUTLIVE the statistics referencing it | +| **V76** — Query Store health | `collect.query_store_health` + its index + the `v_query_store_health` passthrough view — the #2319 per-database `sys.database_query_store_options` collector's store table: actual vs desired state (the cap-hit READ_ONLY transition and its readonly_reason), current vs max storage, cleanup thresholds, and the runtime-stats interval length. A fresh store gets the table from V1's generated schema; V76 is what an already-existing store gets | All timestamps in the store are **naive-UTC** `timestamp` columns — the product-wide cross-store contract (Lite's DuckDB does the same). +### Reading the store directly (plan XML is compressed) + +The store is deliberately queryable — it is documented PostgreSQL with named tables, and people build +panels and reports straight off it. One thing will surprise you if you do that: **execution-plan XML is +stored gzip-compressed**, and has been since v3.4.0. + +`collect.query_plan_dim` holds plan content once, keyed by a content digest, in one of two columns: + +| Column | Meaning | +|---|---| +| `query_plan_gz` (`bytea`) | The plan XML, **gzip-compressed** (magic bytes `1f 8b`). This is where new plans go. | +| `query_plan_xml` (`text`) | Uncompressed plan XML. Nullable since v3.4.0; only rows written by older builds still carry it. | + +So a consumer that reads only `query_plan_xml` silently returns nothing for anything collected by a +current build. **`query_plan_xml IS NULL` does not mean "no plan" — it means look at `query_plan_gz`.** + +Both apps and every MCP tool decompress client-side, so nothing in the product is affected; this note +exists because the change altered the contract for direct SQL consumers and the v3.4.0 release notes did +not say so. That omission is on us. + +**Getting the XML back.** PostgreSQL has no built-in gunzip for arbitrary `bytea`, so a plain-SQL +consumer cannot decompress in the database without an extension. Practical options, in the order most +people should try them: + +1. **Ask the product for the plan** rather than the store — `get_plan_xml` over MCP, or the Viewer's + plan surfaces. Both hand back decompressed XML and neither cares how it is stored. +2. **Decompress in your client.** Any language's gzip library reads the bytes directly. Python: + `gzip.decompress(row['query_plan_gz']).decode('utf-8')`. PowerShell: a `GZipStream` over a + `MemoryStream` of the bytes. C#: the same, which is exactly what the apps do. +3. **Ship a UDF into your own store** if your tooling is SQL-only (Grafana, a reporting view). A + `plpython3u` function works and has been used in the field, at the cost of an untrusted-language + extension in a monitoring database — weigh that against how much you need it. + +Why compressed at all: plan XML dominates store size, and gzip took a production dim table from 885 GB +of raw text to 64 GB — a 14x reduction. That is the tradeoff being made on your behalf. + ### TimescaleDB (Optional, Auto-Adopted) At startup, right after migration, the service attempts `CREATE EXTENSION IF NOT EXISTS timescaledb` and checks `pg_extension`: @@ -535,13 +682,13 @@ timescaledb.max_background_workers = + 2 max_worker_processes = 3 + timescaledb.max_background_workers + 8 ``` -Today that is **41** and **52** for 39 hypertables (the 38 collector tables plus `collection_log`). The `+ 2` is not slack — it is exactly TimescaleDB's own two built-in jobs, `policy_telemetry` and `policy_job_stat_history_retention`, so a fully migrated store holds precisely one job per worker: +Today that is **51** and **62** for 49 hypertables (the 48 collector tables plus `collection_log`). The `+ 2` is not slack — it is exactly TimescaleDB's own two built-in jobs, `policy_telemetry` and `policy_job_stat_history_retention`, so a fully migrated store holds precisely one job per worker: ```sql SELECT proc_name, count(*) FROM timescaledb_information.jobs GROUP BY proc_name; ``` -Both settings need a **server restart** (`max_worker_processes` is restart-only — a reload leaves the old value serving), and the hypertable count grows as collectors are added, so re-check it after a major upgrade rather than pinning 41/52 forever. +Both settings need a **server restart** (`max_worker_processes` is restart-only — a reload leaves the old value serving), and the hypertable count grows as collectors are added, so re-check it after a major upgrade rather than pinning 52/63 forever. **One store per cluster is the assumption.** `timescaledb.max_background_workers` is a **cluster-wide** pool shared by every database, while the derivation above is **per-store**. Managed mode puts one store on one cluster so the two coincide, but if you run **N Darling stores on one PostgreSQL cluster** — or share the cluster with any other TimescaleDB database — multiply both numbers by N. Each database with the extension loaded also permanently holds a scheduler slot out of that same pool, so the sharing starts before any policy fires. @@ -751,7 +898,7 @@ It decrypts the `network.role` credential and prints a paste-ready connection st **"out of background workers" / "failed to start a background worker" in the postmaster log, or the store keeps growing despite compression** — bring-your-own stores only: the cluster has fewer worker slots than the store has policies, so compression and retention jobs are being skipped. An occasional one is benign (the job retries on its next schedule); persistent ones mean the store is effectively uncompressed. Size the two settings and restart the server — see [Background workers](#background-workers-sizing-an-unmanaged-store-and-what-happens-if-you-dont), and multiply them if the cluster hosts more than one store. `timescaledb_information.job_stats` tells you whether jobs are actually succeeding. -**"Why are there 40+ postgres.exe processes?"** — the count is three populations, and only one is client connections: (1) PostgreSQL's own system processes (postmaster, checkpointer, WAL/background writers, autovacuum, stats); (2) **TimescaleDB background workers** — the managed conf sizes `timescaledb.max_background_workers` to the hypertable count + 2 (≈38), and every RUNNING compression/retention policy job is its own process, so the count legitimately surges during checkpoint/compression waves and falls back when they finish; (3) client backends — the service's pools are capped at 24, the co-located viewer's at 10. Decompose it live with: `SELECT backend_type, count(*) FROM pg_stat_activity GROUP BY backend_type ORDER BY 2 DESC;` — and remember Windows charges the shared buffer segment to every attached process's working set, so per-process memory numbers cannot be summed. +**"Why are there 40+ postgres.exe processes?"** — the count is three populations, and only one is client connections: (1) PostgreSQL's own system processes (postmaster, checkpointer, WAL/background writers, autovacuum, stats); (2) **TimescaleDB background workers** — the managed conf sizes `timescaledb.max_background_workers` to the hypertable count + 2 (≈52), and every RUNNING compression/retention policy job is its own process, so the count legitimately surges during checkpoint/compression waves and falls back when they finish; (3) client backends — the service's pools are capped at 24, the co-located viewer's at 10. Decompose it live with: `SELECT backend_type, count(*) FROM pg_stat_activity GROUP BY backend_type ORDER BY 2 DESC;` — and remember Windows charges the shared buffer segment to every attached process's working set, so per-process memory numbers cannot be summed. **query_store bursts every ~15 minutes** — two or three near-empty cycles, then one large one, is Query Store's own behavior, not a collector bug: the engine buffers in memory and flushes to its persisted tables on `DATA_FLUSH_INTERVAL_SECONDS` (default 900s), so the collector genuinely sees nothing new between flushes. Narrowing the collection interval will not smooth it. The per-database log lines show which database drove a burst. @@ -793,7 +940,7 @@ With `postgres.managed = true` (the sample's default), the service runs its own } ``` -**What first run does.** The service looks for `pg-runtime\pgsql\` beside its binary, extracting it from `pg-runtime.zip` when only the zip is present (deleting the extracted directory is therefore always safe — it self-heals). If the data directory has no cluster, it generates a 32-character random password, protects it with DPAPI LocalMachine into `pg-credential.dpapi` beside the data directory (credential first, so a crash mid-initdb never strands a cluster nobody can log into), then runs `initdb` with `scram-sha-256` auth, data checksums, and UTF8/C locale. A marker-guarded block appended to `postgresql.conf` preloads TimescaleDB, sets the port, and restricts listening to `127.0.0.1`; a second versioned block sizes background workers up for the per-hypertable compression jobs, DERIVED from the live hypertable count so it cannot go stale as collectors are added (`timescaledb.max_background_workers = hypertables + 2`, `max_worker_processes = 3 + that + 8` — today 41 and 52 for 39 hypertables; PostgreSQL's default of 8 workers cannot launch them); a third versioned block sizes memory from the host's physical RAM for the up-to-500-servers case (`shared_buffers = min(25% RAM, 1GB)`, `effective_cache_size = 75% RAM`, `maintenance_work_mem = min(max(5% RAM, 1536MB), 25% RAM, 2048MB)`, and a deliberately-modest per-connection `work_mem = clamp(RAM/512, 16MB, 64MB)` — on an 8 GB box that is `shared_buffers 1024MB` / `work_mem 16MB`; the stock 128 MB / 4 MB defaults are fine at small scale but bottleneck at fleet scale). Later blocks re-state single settings that field measurement moved: a fifth caps `shared_buffers` for the co-located store, a sixth turns on the log-rotation ring, and a seventh carries the `maintenance_work_mem` floor that TimescaleDB's compression sort runs on (measured at ~+70% compression throughput on a 16 GB-class host, plateauing by 1536 MB). `postgresql.conf` takes the LAST assignment of a setting, so these override without rewriting anything. Every append is re-checked on every start, so a crash between initdb and the append heals itself instead of silently degrading — and clusters initialized before a given block existed gain it on their next start (effective at the next PostgreSQL restart). Then `pg_ctl start`, `CREATE DATABASE darling`, and the normal startup path (migrations, TimescaleDB adoption — you should see `N/N collector table(s) are hypertables`, both numbers equal and equal to the collector count; a converted count BELOW the total means some table stayed plain and the line above it says which) continues exactly as in bring-your-own mode. The connection string is derived from the stored credential; the Viewer and the MCP host on the same machine derive it the same way, so nothing needs configuring there either. +**What first run does.** The service looks for `pg-runtime\pgsql\` beside its binary, extracting it from `pg-runtime.zip` when only the zip is present (deleting the extracted directory is therefore always safe — it self-heals). If the data directory has no cluster, it generates a 32-character random password, protects it with DPAPI LocalMachine into `pg-credential.dpapi` beside the data directory (credential first, so a crash mid-initdb never strands a cluster nobody can log into), then runs `initdb` with `scram-sha-256` auth, data checksums, and UTF8/C locale. A marker-guarded block appended to `postgresql.conf` preloads TimescaleDB, sets the port, and restricts listening to `127.0.0.1`; a second versioned block sizes background workers up for the per-hypertable compression jobs, DERIVED from the live hypertable count so it cannot go stale as collectors are added (`timescaledb.max_background_workers = hypertables + 2`, `max_worker_processes = 3 + that + 8` — today 52 and 63 for 50 hypertables; PostgreSQL's default of 8 workers cannot launch them); a third versioned block sizes memory from the host's physical RAM for the up-to-500-servers case (`shared_buffers = min(25% RAM, 1GB)`, `effective_cache_size = 75% RAM`, `maintenance_work_mem = min(max(5% RAM, 1536MB), 25% RAM, 2048MB)`, and a deliberately-modest per-connection `work_mem = clamp(RAM/512, 16MB, 64MB)` — on an 8 GB box that is `shared_buffers 1024MB` / `work_mem 16MB`; the stock 128 MB / 4 MB defaults are fine at small scale but bottleneck at fleet scale). Later blocks re-state single settings that field measurement moved: a fifth caps `shared_buffers` for the co-located store, a sixth turns on the log-rotation ring, and a seventh carries the `maintenance_work_mem` floor that TimescaleDB's compression sort runs on (measured at ~+70% compression throughput on a 16 GB-class host, plateauing by 1536 MB). `postgresql.conf` takes the LAST assignment of a setting, so these override without rewriting anything. Every append is re-checked on every start, so a crash between initdb and the append heals itself instead of silently degrading — and clusters initialized before a given block existed gain it on their next start (effective at the next PostgreSQL restart). Then `pg_ctl start`, `CREATE DATABASE darling`, and the normal startup path (migrations, TimescaleDB adoption — you should see `N/N collector table(s) are hypertables`, both numbers equal and equal to the collector count; a converted count BELOW the total means some table stayed plain and the line above it says which) continues exactly as in bring-your-own mode. The connection string is derived from the stored credential; the Viewer and the MCP host on the same machine derive it the same way, so nothing needs configuring there either. **Why scram and not trust, even loopback-only.** Trust auth would hand superuser to any local code that can open a loopback socket — every other local user, and network-capable-but-not-filesystem-capable attack primitives like SSRF from a co-hosted app. With scram the credential travels on the wire, failed attempts are auditable, and access is confined to what can read the DPAPI-protected credential file. `listen_addresses = '127.0.0.1'` keeps the server unreachable off the machine on top — unless you deliberately opt into a LAN endpoint (see [Opt-in Network Endpoints (LAN)](#opt-in-network-endpoints-lan)), which reconciles `listen_addresses`, a `hostssl` pg_hba rule, and TLS on every start and is otherwise off. diff --git a/Darling/tools/generate-ladder-fixture/Program.cs b/Darling/tools/generate-ladder-fixture/Program.cs new file mode 100644 index 000000000..744e1630a --- /dev/null +++ b/Darling/tools/generate-ladder-fixture/Program.cs @@ -0,0 +1,22 @@ +using System.Text; +using PerformanceMonitor.Darling.Storage; + +var sb = new StringBuilder(); +sb.Append("-- Migration ladder as RESOLVED by the release this fixture is named for — every rung's\n"); +sb.Append("-- SQL frozen as that era's code (including its generators) emitted it, plus the same\n"); +sb.Append("-- version-table DDL and stamps MigrateLockedAsync writes. Generated by tools/generate-\n"); +sb.Append("-- ladder-fixture; regenerate at each release cut from the release tag. DO NOT HAND-EDIT.\n"); +sb.Append("-- ===BATCH=== bootstrap\n"); +sb.Append("SET search_path = collect, config, public;\n"); +sb.Append("CREATE TABLE IF NOT EXISTS darling_schema_version (\n version integer NOT NULL PRIMARY KEY,\n name text NOT NULL,\n applied_at timestamp NOT NULL\n);\n"); + +foreach (var m in PgMigrations.Scripts) +{ + sb.Append("-- ===BATCH=== V").Append(m.Version).Append(' ').Append(m.Name).Append('\n'); + sb.Append(m.Sql.TrimEnd()).Append('\n'); + sb.Append("INSERT INTO darling_schema_version (version, name, applied_at) VALUES (") + .Append(m.Version).Append(", '").Append(m.Name.Replace("'", "''")).Append("', now()::timestamp);\n"); +} + +File.WriteAllText(args[0], sb.ToString()); +Console.WriteLine($"wrote {args[0]}: {PgMigrations.Scripts.Count} rungs, top V{PgMigrations.Scripts[^1].Version}"); diff --git a/Darling/tools/generate-ladder-fixture/generate-ladder-fixture.csproj b/Darling/tools/generate-ladder-fixture/generate-ladder-fixture.csproj new file mode 100644 index 000000000..010cd931e --- /dev/null +++ b/Darling/tools/generate-ladder-fixture/generate-ladder-fixture.csproj @@ -0,0 +1,23 @@ + + + + Exe + net10.0 + enable + enable + + + + + diff --git a/Darling/tools/install-darling.ps1 b/Darling/tools/install-darling.ps1 index c47652358..69a155d59 100644 --- a/Darling/tools/install-darling.ps1 +++ b/Darling/tools/install-darling.ps1 @@ -10,6 +10,11 @@ C:\PerformanceMonitorDarling). What it does, in order: 1. Verifies elevation, the service exe, and darling.json (offers to copy darling.sample.json and stops so you can edit it — the service is not installed with an unedited sample). + 1b. REFUSES an install directory the service account can never read: anywhere under a user profile + (C:\Users\...), or a UNC / mapped-drive path. The service runs as an unprivileged virtual account + that is neither you nor an administrator, and a profile folder grants nothing to it, so the + service installs cleanly and then dies at the bundled PostgreSQL's first step (#2185, #2187). + Extract to a machine-scoped local path such as C:\PerformanceMonitorDarling instead. 2. Optional pre-flight: runs `--test-connection` and shows the per-server PASS/FAIL lines (continue-or-abort prompt on failure; -SkipPreflight to skip). 3. Registers the Windows Event Log source 'PerformanceMonitor Darling' (requires elevation — @@ -61,6 +66,88 @@ $samplePath = Join-Path $root 'darling.sample.json' function Fail([string]$message) { Write-Host "ERROR: $message" -ForegroundColor Red; exit 1 } +# True when $candidate IS $parent or sits underneath it. +# +# The separator is appended before the prefix test on purpose: a bare StartsWith reads C:\UsersData as +# living under C:\Users and would refuse a perfectly good install directory. A false refusal is a worse +# failure than the one this check exists to catch - it strands someone whose install would have worked - +# so the boundary is the one thing here worth being exact about. Equality counts as under: an install +# root sitting AT the profile root is exactly as unreadable as one below it. +function Test-PathIsAtOrUnder([string]$candidate, [string]$parent) { + if ([string]::IsNullOrWhiteSpace($candidate) -or [string]::IsNullOrWhiteSpace($parent)) { return $false } + + try { + # Normalizes separators, resolves . and .., and makes the comparison independent of how the path + # was typed. Anything GetFullPath rejects is not a path we can reason about, so it is not matched. + $c = [IO.Path]::GetFullPath($candidate).TrimEnd('\') + $p = [IO.Path]::GetFullPath($parent).TrimEnd('\') + } + catch { + return $false + } + + if ($c.Equals($p, [StringComparison]::OrdinalIgnoreCase)) { return $true } + return $c.StartsWith($p + '\', [StringComparison]::OrdinalIgnoreCase) +} + +# The machine's profile root, read from where Windows actually keeps it rather than assumed to be +# C:\Users. ProfilesDirectory is relocatable, and a hardcoded literal would quietly stop matching on +# precisely the box that moved it - the one box where a missed check costs the most. +function Get-ProfilesDirectory { + try { + $configured = (Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList' -Name 'ProfilesDirectory' -ErrorAction Stop).ProfilesDirectory + $expanded = [Environment]::ExpandEnvironmentVariables($configured) + if (-not [string]::IsNullOrWhiteSpace($expanded)) { return $expanded } + } + catch { + # An unreadable ProfileList is not a reason to skip the check - fall back to the default location. + } + + return (Join-Path $env:SystemDrive 'Users') +} + +# 'UNC', 'mapped drive', or $null. Named separately from the profile case because the reason differs: +# a virtual service account reaches the network as the COMPUTER account rather than as the operator who +# typed the path, and a mapped drive letter belongs to one logon session, which a service never shares. +function Get-NetworkPathKind([string]$path) { + if ([string]::IsNullOrWhiteSpace($path)) { return $null } + + # \\?\ is the long-path prefix on a LOCAL path, not a server name - excluded so an extended-length + # local path is not mistaken for a share. + if ($path.StartsWith('\\', [StringComparison]::Ordinal) -and -not $path.StartsWith('\\?\', [StringComparison]::Ordinal)) { + return 'UNC' + } + + $qualifier = $null + try { $qualifier = Split-Path -Qualifier $path -ErrorAction Stop } catch { } + if (-not $qualifier) { return $null } + + try { + $drive = Get-CimInstance -ClassName Win32_LogicalDisk -Filter "DeviceID='$qualifier'" -ErrorAction Stop + # DriveType 4 = network drive. + if ($drive -and $drive.DriveType -eq 4) { return 'mapped drive' } + # A DEFINITE answer is trusted, including "local" - so the fallback below is not consulted and + # cannot second-guess WMI on a box where WMI works. DriveType 0 is "unknown", which is NOT an + # answer: 0 is falsy here, so an unknown row falls through to the probe below instead of being + # read as "local". That matters because a partial WMI response is likeliest on exactly the + # restricted images this fallback exists for (review catch on #2248). + if ($drive -and $drive.DriveType) { return $null } + } + catch { + # WMI unavailable (locked-down or Server Core images). Fall through to the probe below rather + # than answering "not network", which is the fail-open in #2201. + } + + # No answer from WMI. DisplayRoot is populated ONLY for a drive letter mapped to a share, and comes + # from .NET rather than WMI, so it survives a restricted image. That keeps the rule this function has + # always followed - unknown is not network, and a refusal needs evidence - while no longer MISSING the + # one case the guard exists to catch. Unknown still returns $null two lines down. + $psDrive = Get-PSDrive -Name $qualifier.TrimEnd(':') -ErrorAction SilentlyContinue + if ($psDrive -and -not [string]::IsNullOrWhiteSpace($psDrive.DisplayRoot)) { return 'mapped drive' } + + return $null +} + # -- 1. Environment checks ------------------------------------------------------------------------ $identity = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() if (-not $identity.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { @@ -71,6 +158,108 @@ if (-not (Test-Path $serviceExe)) { Fail "PerformanceMonitor.Darling.Service.exe not found beside this script. Extract the full Darling zip and run install-darling.ps1 from the extracted folder." } +# -- 1b. Refuse an install root the service account can never read (#2187) ------------------------- +# Reported as #2185: a zip extracted to C:\Users\\Desktop\PerformanceMonitorDarling-3.2.0\ - +# a completely reasonable thing to do with a download - installed without a complaint and then failed, +# and nothing in the product said why. +# +# The mechanism is the deliberate account choice made at step 4. The service runs as the virtual account +# 'NT SERVICE\' and NEVER as LocalSystem, because the bundled PostgreSQL refuses to run with +# administrative privileges. That account is therefore not the installing user, not SYSTEM, and not +# Administrators - and a user profile grants access to approximately those three and nobody else. Measured +# on a clean Windows 11 box, a directory created under a profile inherits exactly: +# NT AUTHORITY\SYSTEM:(I)(OI)(CI)(F) BUILTIN\Administrators:(I)(OI)(CI)(F) :(I)(OI)(CI)(F) +# with no BUILTIN\Users, no Authenticated Users, and no CREATOR OWNER - so the account cannot read the +# program files it was pointed at, and cannot even read back what it writes there itself. The documented +# location inherits BUILTIN\Users:(I)(OI)(CI)(RX) from the volume root instead, which every service +# account is a member of, which is why C:\PerformanceMonitorDarling works and this does not. +# +# Step 4b is not a substitute: its ACL work is scoped to darling.json and its .bak-* copies, so pg-runtime +# keeps whatever the profile gave it. Fixing the tree's ACLs instead of refusing was considered and +# rejected in #2187 - it means the product starts silently ACLing directories inside somebody's profile. +# +# The residual, seen and accepted rather than discovered later: C:\Users\Public sits under the profile root +# and is refused, but an install there would actually WORK - it grants NT AUTHORITY\SERVICE:(OI)(CI)(IO)(M,DC), +# which every service account holds. It is deliberately not carved out. Nobody installs a Windows service +# into the shared documents profile, a carve-out would amount to documenting it as a reasonable place to +# install, and the refusal is not a dead end - it names C:\PerformanceMonitorDarling. One rule, no +# exceptions, and the one location it costs is one nobody wants. +# +# This runs BEFORE the pre-flight, the Event Log source, and service creation, so a doomed location costs +# nothing and leaves nothing behind. +$existing = Get-Service -Name $serviceName -ErrorAction SilentlyContinue +$networkKind = Get-NetworkPathKind $root +# $env:USERPROFILE as well as the machine's profile root: a profile redirected outside ProfilesDirectory +# is still a profile, and it is the profile whose owner is most likely to be running this script. +$underProfile = (Test-PathIsAtOrUnder $root (Get-ProfilesDirectory)) -or (Test-PathIsAtOrUnder $root $env:USERPROFILE) + +if ($underProfile -or $networkKind) { + if ($underProfile) { + $why = @" +This folder is under a user profile: + + $root + +The service runs as the unprivileged virtual account 'NT SERVICE\$serviceName' - never LocalSystem, +because the bundled PostgreSQL refuses to run with administrative privileges. That account is not you, +not SYSTEM, and not Administrators, and a user profile grants access to about those three and nobody +else. So the service installs cleanly and then cannot read its own program files: the bundled +PostgreSQL's initdb.exe dies at exit code -1073741515 (0xC0000135, STATUS_DLL_NOT_FOUND) before it can +write a word of output, because the DLLs sitting beside it are unreadable (#2185). +"@ + } + else { + $why = @" +This folder is on a network location ($networkKind): + + $root + +The service runs as the unprivileged virtual account 'NT SERVICE\$serviceName' - never LocalSystem, +because the bundled PostgreSQL refuses to run with administrative privileges. That account reaches the +network as the COMPUTER account rather than as you, so a share that opens for you is not open for it; +and a mapped drive letter belongs to YOUR logon session, which a service does not share and cannot see +at all. Either way the service installs cleanly and then cannot read its own program files. +"@ + } + + $fix = @" +Move the extracted folder to a machine-scoped local path and run this script again from there: + + C:\PerformanceMonitorDarling + +That is the documented location, and a folder created there inherits read + execute for BUILTIN\Users, +which the service's virtual account is a member of. Your darling.json can move with it. +"@ + + if (-not $existing) { + Fail @" +$why + +$fix + +Nothing was installed or changed. +"@ + } + + # An UPGRADE is a question rather than a refusal, and only here. This script's upgrade path exists to + # preserve installs operators have customized - it deliberately touches only binPath so a re-homed + # logon account survives (#1802, #1823) - and #2187's rejected option 2 (grant the service account + # read + execute on the tree) is exactly the thing an operator may already have done by hand here. + # Refusing outright would strand a deployment that works. The diagnosis is identical and unmissable; + # only the verdict is theirs. A fresh install in the same folder is still refused outright above. + Write-Host '' + Write-Host 'WARNING: a FRESH install would be refused in this location.' -ForegroundColor Red + Write-Host $why -ForegroundColor Red + Write-Host '' + Write-Host $fix -ForegroundColor Yellow + Write-Host '' + Write-Host "Service '$serviceName' already exists, so this is an upgrade, and this folder may have been made" -ForegroundColor Yellow + Write-Host 'to work by hand (granting the service account read + execute on the tree). That is the only reason' -ForegroundColor Yellow + Write-Host 'this is a question. If it has NOT been, the service will stop working the moment it restarts.' -ForegroundColor Yellow + $answer = Read-Host 'Point the service at this folder anyway? [y/N]' + if ($answer -notmatch '^[Yy]') { exit 4 } +} + if (-not (Test-Path $configPath)) { if (Test-Path $samplePath) { Copy-Item $samplePath $configPath @@ -109,7 +298,9 @@ catch [System.InvalidOperationException] { } # -- 4. Create or upgrade the service ------------------------------------------------------------- -$existing = Get-Service -Name $serviceName -ErrorAction SilentlyContinue +# $existing was resolved back at 1b, which needs to know fresh-versus-upgrade before it decides whether a +# bad install location is a refusal or a question. Nothing between there and here creates or removes the +# service, so it is the same answer. if ($existing) { Write-Host "Service already exists - upgrading its binPath in place (config, store data, and credentials untouched)." if ($existing.Status -ne 'Stopped') { diff --git a/Darling/tools/provision-roles.sql b/Darling/tools/provision-roles.sql index 1bcb8fa22..6ed8ff792 100644 --- a/Darling/tools/provision-roles.sql +++ b/Darling/tools/provision-roles.sql @@ -100,7 +100,9 @@ GRANT SELECT ON ALL TABLES IN SCHEMA config TO admin, viewer; REVOKE SELECT ON config.config_monitored_servers FROM viewer; GRANT SELECT (server_id, name, host, database, auth, username, encrypt_mode, trust_server_certificate, read_only_intent, multi_subnet_failover, excluded_databases, monthly_cost_usd, capture_plans, - is_enabled, created_at, modified_at, alert_delivery_mode_override) + is_enabled, created_at, modified_at, alert_delivery_mode_override, + -- V68: engine + port. Non-secret, exactly like host. + engine, port) ON config.config_monitored_servers TO viewer; REVOKE SELECT ON config.config_command FROM viewer; GRANT SELECT (command_id, created_at, requested_by, command_type, target_server_id, status, claimed_at, diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 000000000..cd477d4d3 --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,34 @@ + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Lite.Tests/AbandonableStepTests.cs b/Lite.Tests/AbandonableStepTests.cs new file mode 100644 index 000000000..7dcfb556c --- /dev/null +++ b/Lite.Tests/AbandonableStepTests.cs @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the #2148 abandonment discipline — the primitive that keeps one wedged ladder step from +/// stopping ALL collection (the field failure: a step hung on an Azure elastic pool after the 3.4.0 +/// upgrade and every chart went silent, permanently, with all exception armor intact). The arms that +/// matter most are the guard's: an abandoned run must QUARANTINE the step (no relaunch on top of the +/// wedged task) and must RELEASE it the moment the wedged task truly ends — both directions, because a +/// guard that never releases turns one hang into a permanently dead step, which is the bug again with +/// extra steps. +/// +public sealed class AbandonableStepTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromMilliseconds(200); + private static readonly TimeSpan Generous = TimeSpan.FromSeconds(10); + + [Fact] + public async Task CompletedWithinDeadline_ReportsCompleted_AndReleasesTheGuard() + { + var step = new AbandonableStep(); + + var result = await step.RunAsync(() => Task.CompletedTask, Generous); + + Assert.Equal(AbandonableStepOutcome.Completed, result.Outcome); + Assert.Null(result.Exception); + Assert.False(step.IsInFlight); + } + + [Fact] + public async Task Fault_ReportsFaulted_WithTheException_AndReleasesTheGuard() + { + var step = new AbandonableStep(); + + var result = await step.RunAsync( + () => Task.FromException(new InvalidOperationException("boom")), Generous); + + Assert.Equal(AbandonableStepOutcome.Faulted, result.Outcome); + Assert.IsType(result.Exception); + Assert.False(step.IsInFlight); + } + + [Fact] + public async Task SynchronousThrow_IsAFault_NotAnEscape_AndReleasesTheGuard() + { + /* The loop calls RunAsync inline — a delegate that throws BEFORE returning its task must not + blow through the ladder; it is a fault like any other. */ + var step = new AbandonableStep(); + + var result = await step.RunAsync( + () => throw new InvalidOperationException("sync boom"), Generous); + + Assert.Equal(AbandonableStepOutcome.Faulted, result.Outcome); + Assert.IsType(result.Exception); + Assert.False(step.IsInFlight); + } + + [Fact] + public async Task DeadlineElapsed_ReportsAbandoned_AndTheLoopGetsControlBack() + { + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + + var result = await step.RunAsync(() => wedge.Task, Deadline); + + Assert.Equal(AbandonableStepOutcome.Abandoned, result.Outcome); + /* The wedged task is still running — the guard holds. */ + Assert.True(step.IsInFlight); + + wedge.SetResult(); + } + + [Fact] + public async Task WhileWedged_NextRunIsSkipped_NeverOverlapped() + { + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + var secondRan = false; + + await step.RunAsync(() => wedge.Task, Deadline); + + var second = await step.RunAsync( + () => { secondRan = true; return Task.CompletedTask; }, Generous); + + /* THE quarantine: the wedged task must never be overlapped by a relaunch — on the real ladder + that would stack a second hung backfill slice (and its connection) on top of the first. */ + Assert.Equal(AbandonableStepOutcome.SkippedStillRunning, second.Outcome); + Assert.False(secondRan); + + wedge.SetResult(); + } + + [Fact] + public async Task WhenTheWedgedTaskFinallyEnds_TheStepRunsAgain() + { + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + + await step.RunAsync(() => wedge.Task, Deadline); + wedge.SetResult(); + + /* The guard clears via the task's own completion — poll briefly for the continuation. */ + for (var i = 0; i < 100 && step.IsInFlight; i++) + { + await Task.Delay(10); + } + Assert.False(step.IsInFlight); + + var next = await step.RunAsync(() => Task.CompletedTask, Generous); + Assert.Equal(AbandonableStepOutcome.Completed, next.Outcome); + } + + [Fact] + public async Task AbandonedTaskThatLaterFaults_IsObserved_AndReleasesTheGuard() + { + /* The nasty double: abandoned first, THEN faults. The fault must be observed (no + UnobservedTaskException tearing anything down) and the guard must still release. */ + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + + var result = await step.RunAsync(() => wedge.Task, Deadline); + Assert.Equal(AbandonableStepOutcome.Abandoned, result.Outcome); + + wedge.SetException(new InvalidOperationException("late boom")); + + for (var i = 0; i < 100 && step.IsInFlight; i++) + { + await Task.Delay(10); + } + Assert.False(step.IsInFlight); + + var next = await step.RunAsync(() => Task.CompletedTask, Generous); + Assert.Equal(AbandonableStepOutcome.Completed, next.Outcome); + } + + [Fact] + public async Task AbandonedThenFaulted_SurfacesTheLateFault_ThroughTheCallback() + { + /* Review catch: without the callback, the one exception that explains a wedge was observed + and DISCARDED — abandoned at the deadline, faulted a minute later, nothing in any log. */ + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + Exception? lateFault = null; + + var result = await step.RunAsync( + () => wedge.Task, Deadline, onLateFault: ex => lateFault = ex); + Assert.Equal(AbandonableStepOutcome.Abandoned, result.Outcome); + + wedge.SetException(new InvalidOperationException("the wedge's own exception")); + + for (var i = 0; i < 100 && lateFault is null; i++) + { + await Task.Delay(10); + } + Assert.IsType(lateFault); + Assert.Equal("the wedge's own exception", lateFault!.Message); + } + + [Fact] + public async Task FaultWithinDeadline_DoesNotAlsoFireTheLateCallback() + { + /* The awaited path already returned the exception to the caller — the callback firing too + would double-log every ordinary failure. */ + var step = new AbandonableStep(); + var fired = false; + + var result = await step.RunAsync( + () => Task.FromException(new InvalidOperationException("boom")), Generous, + onLateFault: _ => fired = true); + + Assert.Equal(AbandonableStepOutcome.Faulted, result.Outcome); + await Task.Delay(50); + Assert.False(fired); + } + + [Fact] + public async Task ThrowingLateFaultCallback_StillReleasesTheGuard() + { + /* A logging callback that itself throws must not leave the step permanently wedged. */ + var step = new AbandonableStep(); + var wedge = new TaskCompletionSource(); + + await step.RunAsync(() => wedge.Task, Deadline, + onLateFault: _ => throw new InvalidOperationException("logger boom")); + wedge.SetException(new InvalidOperationException("late")); + + for (var i = 0; i < 100 && step.IsInFlight; i++) + { + await Task.Delay(10); + } + Assert.False(step.IsInFlight); + } + + [Fact] + public async Task CallerCancellation_ReportsCancelled_NotAbandoned() + { + /* Shutdown must read as shutdown — an Abandoned logged at ERROR during a clean exit would + train operators to ignore the one line that matters in the field. */ + var step = new AbandonableStep(); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + var wedge = new TaskCompletionSource(); + + var result = await step.RunAsync(() => wedge.Task, Generous, cancellationToken: cts.Token); + + Assert.Equal(AbandonableStepOutcome.Cancelled, result.Outcome); + + wedge.SetResult(); + } +} diff --git a/Lite.Tests/AgAlertEvaluatorTests.cs b/Lite.Tests/AgAlertEvaluatorTests.cs index 20ef5c960..aee0c8154 100644 --- a/Lite.Tests/AgAlertEvaluatorTests.cs +++ b/Lite.Tests/AgAlertEvaluatorTests.cs @@ -8,7 +8,6 @@ using System; using System.Collections.Generic; -using System.Linq; using PerformanceMonitor.Common; using PerformanceMonitorLite.Services; using Xunit; @@ -123,6 +122,39 @@ public void Suspended_FiresOnTheEdgeWithTheReason_ThenResumeIsAResolution() Assert.True(resumed.IsResolution); } + [Fact] + public void DatabaseScopedAlerts_CarryTheDiscreteDatabaseFacts() + { + /* #2109: the database-scoped AG alerts carry Database / Availability Group / Replica as + discrete fields (the wire contract downstream automation routes on), via the SAME shared + builder Darling's evaluator uses — the fact names cannot drift between the SKUs. */ + var e = new AgAlertEvaluator(); + + /* Suspension is edge-triggered with first-sighting-silent semantics — establish the healthy + baseline first, exactly like the edge test above. */ + Assert.Empty(e.EvaluateDatabases(ServerId, new[] { Database(suspended: false) }, 300, 0, Cooldown)); + + var suspended = Assert.Single(e.EvaluateDatabases( + ServerId, new[] { Database(suspended: true, suspendReason: "SUSPEND_FROM_USER") }, 300, 0, Cooldown)); + var item = Assert.Single(suspended.Context!.Details); + Assert.Contains(item.Fields, f => f.Label == "Database"); + Assert.Contains(item.Fields, f => f.Label == "Availability Group"); + Assert.Contains(item.Fields, f => f.Label == "Replica"); + Assert.Contains(("Suspend Reason", "SUSPEND_FROM_USER"), item.Fields); + + var behind = Assert.Single( + e.EvaluateDatabases( + ServerId, new[] { Database(suspended: false, lagSeconds: 600) }, 300, 0, Cooldown), + a => a.MetricName == AgAlertPolicy.SyncFellBehindMetric); + Assert.Contains(Assert.Single(behind.Context!.Details).Fields, f => f.Label == "Database"); + + /* Resolutions stay context-less — they carry no database-scoped payload to route on. */ + var resumed = Assert.Single( + e.EvaluateDatabases(ServerId, new[] { Database(suspended: false) }, 300, 0, Cooldown), + a => a.IsResolution); + Assert.Null(resumed.Context); + } + /* ---------------- sync fell behind ---------------- */ [Fact] diff --git a/Lite.Tests/AlertIncidentRenderTests.cs b/Lite.Tests/AlertIncidentRenderTests.cs index c1ff10b82..d65795bd8 100644 --- a/Lite.Tests/AlertIncidentRenderTests.cs +++ b/Lite.Tests/AlertIncidentRenderTests.cs @@ -66,6 +66,51 @@ public void Apply_UnresolvedObjects_RendersPlaceholder() Assert.Contains(incident.Fields, f => f.Label == "Involved Objects" && f.Value == "(unresolved)"); } + [Fact] + public void TeamsPayload_EachFieldsItem_GetsItsOwnLabeledSection() + { + /* #2108: the association between an item's fields IS the item boundary — a flat fact list + loses it. Each Fields-carrying detail item becomes its own MessageCard section, titled by + its heading and carrying ONLY its own facts; advice prose stays folded into the lead + section, because it is commentary on the whole alert. */ + var ctx = new AlertContext(); + ctx.Details.Add(new AlertDetailItem + { + Heading = "Deadlock 1 of 2", + Fields = new() { ("Database", "SalesDB"), ("Dedup Key", "aaa") } + }); + ctx.Details.Add(new AlertDetailItem + { + Heading = "Deadlock 2 of 2", + Fields = new() { ("Database", "OtherDb"), ("Dedup Key", "bbb") } + }); + ctx.Details.Add(new AlertDetailItem { Heading = "Check the graph", Body = "Advice prose." }); + + var payload = WebhookAlertService.BuildTeamsPayload("Deadlocks Detected", "S1", "2", "n/a", Branding, context: ctx); + + using var doc = System.Text.Json.JsonDocument.Parse(payload); + var sections = doc.RootElement.GetProperty("sections").EnumerateArray().ToList(); + /* lead + one per Fields item + snooze-hint text section (Branding has none here → 3). */ + Assert.Equal(3, sections.Count); + + var lead = sections[0]; + Assert.Contains("Deadlocks Detected", lead.GetProperty("activityTitle").GetString()); + Assert.Contains(lead.GetProperty("facts").EnumerateArray(), + f => f.GetProperty("name").GetString() == "Advice"); + + var first = sections[1]; + Assert.Equal("Deadlock 1 of 2", first.GetProperty("activityTitle").GetString()); + var firstFacts = first.GetProperty("facts").EnumerateArray().ToList(); + Assert.Equal(2, firstFacts.Count); + Assert.Equal("SalesDB", firstFacts[0].GetProperty("value").GetString()); + Assert.Equal("aaa", firstFacts[1].GetProperty("value").GetString()); + + var second = sections[2]; + Assert.Equal("Deadlock 2 of 2", second.GetProperty("activityTitle").GetString()); + Assert.Contains(second.GetProperty("facts").EnumerateArray(), + f => f.GetProperty("value").GetString() == "bbb"); + } + [Fact] public void DedupKey_RendersOnTeamsSlackAndBothEmailBodies() { diff --git a/Lite.Tests/AnomalyDetectorTests.cs b/Lite.Tests/AnomalyDetectorTests.cs index 37cf17da6..073cd6f80 100644 --- a/Lite.Tests/AnomalyDetectorTests.cs +++ b/Lite.Tests/AnomalyDetectorTests.cs @@ -31,6 +31,26 @@ public class AnomalyDetectorTests : IClassFixture, IDisposa private static readonly DateTime _analysisEnd = _now; private static readonly DateTime _analysisStart = _now.AddHours(-4); + /* #2177: the start of a seeded baseline day, floored to the HOUR. + + Every seed helper below writes several samples spanning ~21 minutes from this point. They used to + start at whatever time-of-day _analysisStart inherited from the wall clock, so when a CI run put + _analysisStart within 21 minutes of midnight the span crossed a date boundary and each intended + 'day' contributed TWO distinct dates — doubling the distinct-day count the baseline-quality gate + counts, which flipped a deliberately-thin (2-day) baseline into a trustworthy one and sent the + detector down the z-path instead of the absolute fallback. Deterministic failure for runs between + 03:39 and 04:00 UTC. + + Flooring to the hour rather than midday-anchoring (#1972's discipline elsewhere) is deliberate: + the Full baseline tier buckets by hour AND day-of-week, so the seeds must keep _analysisStart's + hour and weekday to land in the same bucket the analysis window reads. Starting at :00 keeps both + while making a 21-minute span unable to leave the hour, let alone the date. */ + private static DateTime SeedDayStart(int daysBack) + { + var day = _analysisStart.AddDays(-daysBack); + return day.Date.AddHours(day.Hour); + } + private long _nextId = -1; public AnomalyDetectorTests(SharedDuckDbFixture fixture) @@ -457,7 +477,7 @@ private async Task SeedBaselineCpu(int avgCpu, int variance) var rng = new Random(42); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) { var cpu = Math.Clamp(avgCpu + rng.Next(-variance, variance + 1), 0, 100); @@ -479,7 +499,7 @@ private async Task SeedThinBaselineCpu(int avgCpu, int variance) var rng = new Random(42); foreach (var day in new[] { 7, 14 }) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 8; i++) { var cpu = Math.Clamp(avgCpu + rng.Next(-variance, variance + 1), 0, 100); @@ -495,7 +515,7 @@ private async Task SeedBaselinePerfmon(string counterName, long avgValue, int va var rng = new Random(42); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) { var value = Math.Max(0, avgValue + rng.Next(-variance, variance + 1)); @@ -511,7 +531,7 @@ private async Task SeedBaselineSessions(int avgConnections, int variance) var rng = new Random(42); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) { var count = Math.Max(1, avgConnections + rng.Next(-variance, variance + 1)); @@ -527,7 +547,7 @@ private async Task SeedBaselineQueryStats(long avgElapsed, int variance) var rng = new Random(42); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) { var elapsed = Math.Max(0, avgElapsed + rng.Next(-variance, variance + 1)); @@ -542,7 +562,7 @@ private async Task SeedBaselineWaits() await ExecuteSeedAsync("BEGIN TRANSACTION"); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) await SeedWaitStatAsync(baseDay.AddMinutes(i * 3), "SOS_SCHEDULER_YIELD", 100); } @@ -554,7 +574,7 @@ private async Task SeedBaselineMemory(double avgTotalServerMb, double targetMb) await ExecuteSeedAsync("BEGIN TRANSACTION"); for (int day = 1; day <= 14; day++) { - var baseDay = _analysisStart.AddDays(-day); + var baseDay = SeedDayStart(day); for (int i = 0; i < 4; i++) await SeedMemoryStatAsync(baseDay.AddMinutes(i * 3), avgTotalServerMb, targetMb); } diff --git a/Lite.Tests/AzureSweepScopeTests.cs b/Lite.Tests/AzureSweepScopeTests.cs new file mode 100644 index 000000000..b89472573 --- /dev/null +++ b/Lite.Tests/AzureSweepScopeTests.cs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2220: which databases one Azure SQL DB registration's per-database sweep covers. +/// +/// The field report. A single real deadlock in one Azure SQL Database produced near-identical +/// "Deadlocks Detected" alerts on every OTHER monitored database sharing the same logical server — and the +/// stored data matched: byte-identical deadlock graphs and the same top query, with counters within ~1%, +/// under six unrelated server_ids. Azure SQL DB engines are isolated per database, so one database's +/// sessions cannot block another's; the rows were not cross-talk, they were the same rows collected six +/// times. +/// +/// The cause, and why it is not a typo. The enumeration read master unconditionally and +/// swept every online database on the logical server, storing all of it under whichever registration ran the +/// sweep. Two parts of the product hold incompatible ideas of what a registration IS, both deliberate: the +/// enumeration assumes one registration = one LOGICAL SERVER (#857's shape), while identity assumes one +/// registration = one DATABASE (server_id hashes host[:database][:RO], and the Azure +/// query_store path needs a per-database connection anyway, #1836). The second shape silently behaved like +/// the first, N times over — N registrations of N databases is N² collection. +/// +/// Pinned in BOTH suites against the shared implementation — this is Lite's half. A scoping rule that +/// disagrees between Lite and Darling is the same class of defect as the one being fixed, and both runners +/// previously carried their own private copy of it. +/// +public sealed class AzureSweepScopeTests +{ + /// + /// THE FIX. A registration naming a database sweeps exactly that database — the reported case, where + /// fifteen registrations on one logical server each swept all fifteen. + /// + [Theory] + [InlineData("db1")] + [InlineData("AdventureWorks")] + [InlineData("Sibling-A")] + public void ARegistrationThatNamesADatabase_SweepsOnlyThatDatabase(string catalog) + { + Assert.Equal(new[] { catalog }, AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// A registration naming NO database is a registration of the logical server, so it must enumerate — + /// signalled by the empty list rather than by a separate flag, because the caller's next step is a list + /// either way. This is the behaviour #857 was written for and it is deliberately unchanged. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + public void ARegistrationThatNamesNoDatabase_StillEnumeratesTheServer(string? catalog) + { + Assert.Empty(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// master counts as naming none, in any casing. A connection string with no + /// Initial Catalog lands in master on Azure SQL DB, so treating it as a named database + /// would scope such a registration to the one database holding none of the user's data — collecting + /// nothing and looking healthy while doing it. + /// + [Theory] + [InlineData("master")] + [InlineData("MASTER")] + [InlineData("Master")] + public void MasterIsNotADatabaseAnyoneRegisteredFor(string catalog) + { + Assert.Empty(AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// A database that merely CONTAINS "master" is a real database and is swept. Guards the obvious + /// over-match, which would silently stop collecting from it. + /// + [Theory] + [InlineData("mastermind")] + [InlineData("paymaster")] + [InlineData("master_archive")] + public void ADatabaseNamedLikeMasterIsStillItsOwnDatabase(string catalog) + { + Assert.Equal(new[] { catalog }, AzureSweepScope.OwnDatabaseOrEmpty(catalog)); + } + + /// + /// The returned list is the caller's to keep: both runners hand it straight to their per-database loop, + /// and a shared or cached instance would let one sweep's mutation reach another's. + /// + [Fact] + public void EachCallReturnsItsOwnList() + { + var first = AzureSweepScope.OwnDatabaseOrEmpty("db1"); + var second = AzureSweepScope.OwnDatabaseOrEmpty("db1"); + + Assert.NotSame(first, second); + first.Add("mutated"); + Assert.Single(second); + } +} diff --git a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs index 1f76ff84a..ae2d4df8d 100644 --- a/Lite.Tests/BlockingDeadlockContextBuilderTests.cs +++ b/Lite.Tests/BlockingDeadlockContextBuilderTests.cs @@ -192,49 +192,105 @@ public void BuildDeadlockContext_EmptyOrNull_ReturnsNull() } [Fact] - public void BuildDeadlockContext_RendersVictim_WithParsedProcessSummary_AndAttachment() + public void BuildDeadlockContext_FingerprintedDeadlock_IsOneSelfContainedItem() { var context = AlertContextBuilders.BuildDeadlockContext( Server, new List { Deadlock() }, NoExclusions); Assert.NotNull(context); - /* 1 victim item + 1 appended incident item. */ - Assert.Equal(2, context!.Details.Count); - Assert.Equal("Deadlock Victim", context.Details[0].Heading); + /* #2108: a fingerprinted deadlock renders as ONE self-contained item — its Database + (#2109), forensic fields, and dedup metadata together — no separate victim item whose + association with the fingerprint a multi-incident card would lose. */ + var item = Assert.Single(context!.Details); + Assert.Equal("Deadlock", item.Heading); + var expected = AlertFingerprint.ForObjects(Server, AlertFingerprint.Deadlock, new[] { "StackOverflow.dbo.Users" }); Assert.Equal( new List<(string, string)> { + ("Database", "StackOverflow"), ("Victim SQL", "UPDATE Users SET Reputation = 1"), - ("Processes", "SPID 55 (victim) vs SPID 60") + ("Processes", "SPID 55 (victim) vs SPID 60"), + ("Dedup Key", expected!.DedupKey), + ("Involved Objects", "StackOverflow.dbo.Users") }, - context.Details[0].Fields); + item.Fields); Assert.Equal(DeadlockGraph, context.AttachmentXml); Assert.Equal("deadlock_graph.xml", context.AttachmentFileName); /* #1140: involved-object fingerprint from the graph's resource list. */ var incident = Assert.Single(context.Incidents!); - var expected = AlertFingerprint.ForObjects(Server, AlertFingerprint.Deadlock, new[] { "StackOverflow.dbo.Users" }); - Assert.Equal(expected!.DedupKey, incident.DedupKey); + Assert.Equal(expected.DedupKey, incident.DedupKey); } [Fact] - public void BuildDeadlockContext_ShowsThreeVictims_ButFingerprintsAllDeadlocks() + public void BuildDeadlockContext_RecurrencesCollapse_ToOneItemWithTheCount() { - /* 4 deadlocks over the same object set: 3 rendered (cap), ONE incident carrying the - occurrence count across ALL of them — the pre-extraction #1140 semantics. */ + /* 4 deadlocks over the same object set: ONE self-contained incident item carrying the + occurrence count across ALL of them — the #1140 collapse, now without the three raw + victim repeats beside it (#2108). */ var rows = new List { Deadlock(), Deadlock(), Deadlock(), Deadlock() }; var context = AlertContextBuilders.BuildDeadlockContext(Server, rows, NoExclusions); Assert.NotNull(context); - /* 3 victim items + 1 incident item. */ - Assert.Equal(4, context!.Details.Count); - Assert.Equal("Deadlock Victim", context.Details[2].Heading); + var item = Assert.Single(context!.Details); + Assert.Equal("Deadlock", item.Heading); + Assert.Contains(("Occurrences", "4"), item.Fields); var incident = Assert.Single(context.Incidents!); Assert.Equal(4, incident.OccurrenceCount); } + [Fact] + public void BuildDeadlockContext_DistinctFingerprints_EachSelfContained_AndIndexed() + { + /* #2108's core: two different deadlocks on one alert must read as two labeled units, each + carrying its OWN victim + database + dedup metadata — the victim→fingerprint association + the flat list lost. */ + var otherGraph = DeadlockGraph + .Replace("StackOverflow.dbo.Users", "OtherDb.dbo.T1") + .Replace(@"currentdbname=""StackOverflow""", @"currentdbname=""OtherDb"""); + var rows = new List + { + Deadlock(), + Deadlock(xml: otherGraph, victimSql: "UPDATE T1 SET x = 1") + }; + + var context = AlertContextBuilders.BuildDeadlockContext(Server, rows, NoExclusions); + + Assert.NotNull(context); + Assert.Equal(2, context!.Details.Count); + Assert.Equal("Deadlock 1 of 2", context.Details[0].Heading); + Assert.Equal("Deadlock 2 of 2", context.Details[1].Heading); + Assert.Contains(("Database", "StackOverflow"), context.Details[0].Fields); + Assert.Contains(("Victim SQL", "UPDATE Users SET Reputation = 1"), context.Details[0].Fields); + Assert.Contains(("Database", "OtherDb"), context.Details[1].Fields); + Assert.Contains(("Victim SQL", "UPDATE T1 SET x = 1"), context.Details[1].Fields); + Assert.Equal(2, context.Incidents!.Count); + } + + [Fact] + public void BuildDeadlockContext_UnfingerprintableDeadlock_KeepsTheStandaloneVictimItem() + { + /* A graph with no parseable lock objects has no fingerprint identity — under incident-only + rendering it would vanish, so it keeps the classic victim item (#1140's "the builder + still displays them", scoped to exactly these). */ + var noObjects = DeadlockGraph.Replace( + @"", + ""); + + var context = AlertContextBuilders.BuildDeadlockContext( + Server, new List { Deadlock(xml: noObjects) }, NoExclusions); + + Assert.NotNull(context); + var item = Assert.Single(context!.Details); + Assert.Equal("Deadlock Victim", item.Heading); + /* #2109: the Database fact comes from the processes' currentdbname, so even the + unfingerprintable form names where it happened. */ + Assert.Contains(("Database", "StackOverflow"), item.Fields); + Assert.Null(context.Incidents); + } + [Fact] public void BuildDeadlockContext_AttachmentComesFromFirstRowWithXml() { diff --git a/Lite.Tests/ChartStyleGapFillTests.cs b/Lite.Tests/ChartStyleGapFillTests.cs new file mode 100644 index 000000000..0f3d4261f --- /dev/null +++ b/Lite.Tests/ChartStyleGapFillTests.cs @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Ui; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// The #2324 field regression: a series carrying #1944's injected NaN gap markers must never take the +/// gradient area fill. ScottPlot 5.1.59's Scatter.Render builds the fill's clip/gradient rect as +/// new PixelRect(linePixels) over pixels that still contain the NaN markers — the rect is +/// NaN-poisoned, the axis-gradient shader built for it is invalid, and Skia falls back to the base fill +/// paint, which ScottPlot hardcodes to Colors.Black whenever ColorPositions is in use. On +/// 3.4.0 (the first release carrying gap markers) every gapped chart rendered its ribbon as an opaque +/// black polygon burying the other series; the reporter's one healthy tab was the one whose data had no +/// gaps. Lines and markers handle NaN contours fine, so line-only is the honest degradation for a gapped +/// series — and a gapless series must KEEP the ribbon, or the fix would quietly repeal the feature. +/// +public class ChartStyleGapFillTests +{ + [Fact] + public void GapMarkedSeries_GetsNoFill_LineOnly() + { + var plot = new ScottPlot.Plot(); + // The #1944 shape: a real gap carries a mid-gap point with a real X and a NaN Y. + var sc = plot.Add.Scatter( + new double[] { 1, 2, 3, 4, 5 }, + new double[] { 0, 5, double.NaN, 7, 10 }); + sc.Color = ScottPlot.Color.FromHex("#4E79A7"); + + ChartStyle.StyleScatter(sc); + + Assert.False(sc.FillY); + Assert.Empty(sc.ColorPositions); + Assert.Equal(2f, sc.LineWidth); // still styled as a line — only the ribbon is withheld + } + + [Fact] + public void GaplessSeries_KeepsTheGradientRibbon() + { + var plot = new ScottPlot.Plot(); + var sc = plot.Add.Scatter( + new double[] { 1, 2, 3, 4, 5 }, + new double[] { 0, 5, 6, 7, 10 }); + sc.Color = ScottPlot.Color.FromHex("#4E79A7"); + + ChartStyle.StyleScatter(sc); + + Assert.True(sc.FillY); + Assert.Equal(2, sc.ColorPositions.Count); + } +} diff --git a/Lite.Tests/CollectorGateSurfacePinTests.cs b/Lite.Tests/CollectorGateSurfacePinTests.cs index 08d7f7f53..6e8c34b9a 100644 --- a/Lite.Tests/CollectorGateSurfacePinTests.cs +++ b/Lite.Tests/CollectorGateSurfacePinTests.cs @@ -61,6 +61,26 @@ public void ServerConfig_AppliesTo_SkipsOnlyAzureSqlDb() Assert.True(ServerConfigCollector.Instance.AppliesTo(Unknown)); } + /// + /// #2150 field report: this fired 11x consecutive on an Azure SQL DB elastic pool with error 262, + /// "VIEW DATABASE PERFORMANCE STATE permission denied in database 'tempdb'". The query reads + /// tempdb.sys.dm_db_file_space_usage three-part, which a non-administrative login on Azure + /// SQL DB cannot be granted, so the collector could only ever fail there. + /// Managed Instance must KEEP collecting — it has a real tempdb — which is why this asserts + /// both directions rather than just the skip. + /// + [Fact] + public void TempDbStats_AppliesTo_SkipsOnlyAzureSqlDb() + { + Assert.False(TempDbStatsCollector.Instance.AppliesTo(AzureSqlDb)); /* error 262 in tempdb */ + Assert.True(TempDbStatsCollector.Instance.AppliesTo(AzureMi)); + Assert.True(TempDbStatsCollector.Instance.AppliesTo(AwsRds)); + Assert.True(TempDbStatsCollector.Instance.AppliesTo(OnPrem2016)); + Assert.True(TempDbStatsCollector.Instance.AppliesTo(OnPrem2014)); + Assert.True(TempDbStatsCollector.Instance.AppliesTo(NoMsdb)); + Assert.True(TempDbStatsCollector.Instance.AppliesTo(Unknown)); + } + [Fact] public void TraceFlags_AppliesTo_SkipsOnlyAzureSqlDb() { @@ -127,17 +147,30 @@ public void AgentStatus_AppliesTo_SkipsAzureSqlDbRdsAndNoMsdb() [Fact] public void CatalogByNameGate_AgreesWithDefinitionAppliesTo_ForEveryCollectorAndTarget() { - /* The parity crux: Lite consults CollectorCatalog.AppliesTo(name, target) pre-dispatch; Darling's - runner calls definition.AppliesTo(target). If those ever disagreed the two SKUs would gate - differently — exactly the drift this collapse removes. Pin that they are identical for every - catalog collector across every target dimension. */ + /* The parity crux: Lite consults CollectorCatalog.AppliesTo(NAME, target) pre-dispatch, Darling's + runner calls CollectorCatalog.AppliesTo(DEFINITION, target). If those ever disagreed the two SKUs + would gate differently — exactly the drift this collapse removes. + + Both COMPOSED forms, deliberately. This used to compare the raw definition.AppliesTo(target) + against the by-name form, which was equivalent only while every collector was SQL Server: the + composed overload also requires definition.TargetEngine == target.Engine, so a PostgreSQL + definition whose own AppliesTo returns true unconditionally (the slots collector) legitimately + disagrees with its raw gate when handed a SQL Server target. Comparing raw-to-composed would force + either a wrong assertion or a filtered loop; comparing composed-to-composed is the claim that + actually matters and holds for every collector against every target. */ foreach (var definition in CollectorCatalog.All) { foreach (var target in AllTargets) { - Assert.Equal( - definition.AppliesTo(target), - CollectorCatalog.AppliesTo(definition.Name, target)); + /* The expectation is SPELLED OUT rather than delegated to the other overload. Comparing + AppliesTo(definition, t) against AppliesTo(name, t) is nearly circular — the by-name + overload just looks the name up and calls the by-definition one, so only a corrupt + name->definition map could fail it, and a bug in the composed rule itself would pass. + Stating the rule independently means BOTH the lookup and the composition are pinned. */ + var expected = definition.TargetEngine == target.Engine && definition.AppliesTo(target); + + Assert.Equal(expected, CollectorCatalog.AppliesTo(definition, target)); + Assert.Equal(expected, CollectorCatalog.AppliesTo(definition.Name, target)); } } } diff --git a/Lite.Tests/CollectorTargetEngineGateTests.cs b/Lite.Tests/CollectorTargetEngineGateTests.cs new file mode 100644 index 000000000..00047e41d --- /dev/null +++ b/Lite.Tests/CollectorTargetEngineGateTests.cs @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the target-engine half of the dispatch gate: a definition written in one engine's dialect +/// must never be dispatched at the other engine, and adding that dimension must not have moved a +/// single existing dispatch decision. +/// +public class CollectorTargetEngineGateTests +{ + /// + /// The drift guard, keyed on the naming convention rather than on a count: a definition whose name + /// starts with pg_ must declare itself PostgreSQL, and everything else must declare SQL + /// Server. + /// The failure this prevents is silent. + /// defaults to SQL Server so the existing definitions needed no edit, which means a new Postgres + /// definition that forgot to derive from would + /// be advertised as T-SQL and dispatched at SQL Server targets, failing every cycle. The reverse + /// mistake — a T-SQL definition marked PostgreSql — would make it disappear from every target + /// instead, which is quieter still. + /// + [Fact] + public void EveryCatalogDefinitionDeclaresTheEngineItsNameImplies() + { + var mismatched = CollectorCatalog.All + .Where(d => d.TargetEngine != ExpectedEngine(d.Name)) + .Select(d => $"{d.Name} declares {d.TargetEngine} but its name implies {ExpectedEngine(d.Name)}") + .ToList(); + + Assert.True( + mismatched.Count == 0, + "Engine declaration does not match the naming convention. A pg_-prefixed collector must " + + "derive from PostgresCollectorDefinitionBase; anything else must not: " + + string.Join("; ", mismatched)); + + static CollectorTargetEngine ExpectedEngine(string name) => + name.StartsWith("pg_", StringComparison.Ordinal) + ? CollectorTargetEngine.PostgreSql + : CollectorTargetEngine.SqlServer; + } + + /// + /// Both engines are actually represented in the catalog. Without this, the convention check above + /// would still pass on a catalog that had lost every Postgres definition. + /// + [Fact] + public void CatalogContainsBothEngines() + { + Assert.Contains(CollectorCatalog.All, d => d.TargetEngine == CollectorTargetEngine.SqlServer); + Assert.Contains(CollectorCatalog.All, d => d.TargetEngine == CollectorTargetEngine.PostgreSql); + } + + /// A target with no engine specified is SQL Server, so nothing existing changes. + [Fact] + public void TargetDefaultsToSqlServer() + { + Assert.Equal(CollectorTargetEngine.SqlServer, new CollectorTargetInfo().Engine); + } + + /// + /// The whole point: a T-SQL definition is gated off a PostgreSQL target even though its own + /// AppliesTo would say yes. + /// + [Fact] + public void SqlServerDefinitionDoesNotApplyToPostgresTarget() + { + var pgTarget = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql }; + + Assert.True(WaitStatsCollector.Instance.AppliesTo(pgTarget)); + Assert.False(CollectorCatalog.AppliesTo(WaitStatsCollector.Instance, pgTarget)); + Assert.False(CollectorCatalog.AppliesTo(WaitStatsCollector.Instance.Name, pgTarget)); + Assert.False(CollectorCatalog.EngineMatches(WaitStatsCollector.Instance.Name, pgTarget)); + } + + /// And the composed gate is a no-op for a SQL Server target — the pre-existing behaviour. + [Fact] + public void SqlServerDefinitionStillAppliesToSqlServerTarget() + { + var sqlTarget = new CollectorTargetInfo(); + + Assert.True(CollectorCatalog.AppliesTo(WaitStatsCollector.Instance, sqlTarget)); + Assert.True(CollectorCatalog.AppliesTo(WaitStatsCollector.Instance.Name, sqlTarget)); + Assert.True(CollectorCatalog.EngineMatches(WaitStatsCollector.Instance.Name, sqlTarget)); + } + + /// + /// The composed gate must still honour a definition's own within-engine gate, not just the + /// engine: agent_status is off on RDS regardless of dialect. + /// + [Fact] + public void ComposedGateStillHonoursWithinEngineGating() + { + var rds = new CollectorTargetInfo { IsAwsRds = true }; + + Assert.False(CollectorCatalog.AppliesTo(AgentStatusCollector.Instance, rds)); + Assert.True(CollectorCatalog.EngineMatches(AgentStatusCollector.Instance.Name, rds)); + } + + /// A Postgres definition is dispatched at Postgres targets and nowhere else. + [Fact] + public void PostgresDefinitionAppliesOnlyToPostgresTarget() + { + var definition = new FakePostgresCollector(); + + Assert.Equal(CollectorTargetEngine.PostgreSql, definition.TargetEngine); + Assert.True(CollectorCatalog.AppliesTo(definition, new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql })); + Assert.False(CollectorCatalog.AppliesTo(definition, new CollectorTargetInfo())); + } + + /// + /// An unknown name is not filtered, so a typo surfaces as the dispatch switch's unknown-collector + /// path rather than a collector that silently never runs. + /// + [Fact] + public void UnknownCollectorNameIsNotEngineFiltered() + { + Assert.True(CollectorCatalog.EngineMatches("no_such_collector", new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql })); + Assert.True(CollectorCatalog.AppliesTo("no_such_collector", new CollectorTargetInfo())); + } + + private sealed class FakePostgresCollector : PostgresCollectorDefinitionBase + { + public override string Name => "pg_fake"; + + public override string TargetTable => "pg_fake"; + + public override IReadOnlyList PayloadColumns => []; + + public override CollectorQuery BuildQuery(CollectorContext context) => new("SELECT 1"); + + public override ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + => new(new List()); + + public override void WritePayload(object row, ICollectorRowWriter writer, CollectorContext context) + { + } + } +} diff --git a/Lite.Tests/CollectorViewerCoverageTests.cs b/Lite.Tests/CollectorViewerCoverageTests.cs index 9bf3b6c36..40910a01e 100644 --- a/Lite.Tests/CollectorViewerCoverageTests.cs +++ b/Lite.Tests/CollectorViewerCoverageTests.cs @@ -12,6 +12,7 @@ using System.Linq; using System.Runtime.CompilerServices; using PerformanceMonitor.Collectors; +using PerformanceMonitorLite.Database; using Xunit; namespace Lite.Tests; @@ -58,7 +59,11 @@ public void EveryCollectorTable_HasALiteReader_OrIsAllowListed() var readerText = ReaderLayerText(); var uncovered = new List(); - foreach (var definition in CollectorCatalog.All) + /* StoredCollectors, not CollectorCatalog.All: the shared catalog is engine-mixed, and Lite neither + collects nor creates a table for the PostgreSQL definitions — it has no PostgreSQL target and the + engine gate means it can never dispatch one. A reader for a table this SKU does not have would be + dead code, so they are out of scope here rather than allow-listed exceptions. */ + foreach (var definition in DuckDbSchemaGenerator.StoredCollectors) { var table = definition.TargetTable; if (ReferencedIn(readerText, table)) diff --git a/Lite.Tests/CpuAttributionTests.cs b/Lite.Tests/CpuAttributionTests.cs new file mode 100644 index 000000000..a6cc50ba4 --- /dev/null +++ b/Lite.Tests/CpuAttributionTests.cs @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Decision-table pins for the shared (#2320) — the attributed-CPU +/// disclosure both SKUs' get_top_queries_by_cpu / get_top_procedures_by_cpu serve. The contract under +/// pin: the ratio is measured-or-omitted (never invented — missing samples, missing core count, or +/// thin coverage all degrade to null + a reason), the low note fires under half, and above the +/// process's own measured CPU the note calls the number impossible rather than presenting it — +/// the 137%-of-the-box claim is the whole reason the marker exists. This SAME table is pinned +/// identically in Darling.Tests so the two SKUs cannot drift. +/// +public sealed class CpuAttributionTests +{ + private static readonly DateTime Start = new(2026, 8, 18, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime End = Start.AddHours(1); + + /// Full coverage, healthy ratio: 25% of 8 cores over an hour = 7,200 CPU-seconds; + /// 5,000 ranked seconds is 0.694 — present, rounded to 3, no note. + [Fact] + public void HealthyRatio_NoNote() + { + var result = CpuAttribution.Compute( + rankedCpuSeconds: 5000, Start, End, + sampleCount: 60, firstSampleUtc: Start, lastSampleUtc: End, avgSqlCpuPercent: 25, cpuCount: 8); + + Assert.Equal(5000, result.RankedCpuSeconds); + Assert.Equal(7200, result.SqlCpuSecondsInWindow); + Assert.Equal(0.694, result.AttributedCpuRatio); + Assert.Null(result.Note); + } + + /// The pre-#2290 shape this feature exists for: the ranking explains ~10% of the box, + /// and now something says so instead of letting the caller chase the visible tenth. + [Fact] + public void LowRatio_SaysNotTheWholeStory() + { + var result = CpuAttribution.Compute(720, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(0.1, result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("10%", result.Note, StringComparison.Ordinal); + Assert.Contains("not the whole story", result.Note, StringComparison.Ordinal); + } + + /// The 137% case — worker_time summing to more CPU than the process consumed is an + /// impossible claim, and the note must say to distrust the numbers, not decorate them. + [Fact] + public void OverAttribution_IsFlaggedImpossible() + { + var result = CpuAttribution.Compute(9864, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1.37, result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("137%", result.Note, StringComparison.Ordinal); + Assert.Contains("impossible-claim", result.Note, StringComparison.Ordinal); + } + + /// Just above 1.0 is sampling noise between two independent series, not a lie — + /// the impossible flag waits for the slack threshold. + [Fact] + public void SlightlyOverOne_CarriesNoNote() + { + var result = CpuAttribution.Compute(7500, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1.042, result.AttributedCpuRatio); + Assert.Null(result.Note); + } + + [Fact] + public void NoSamples_OmitsRatio_AndSaysWhy() + { + var result = CpuAttribution.Compute(5000, Start, End, 0, null, null, null, 8); + + Assert.Equal(5000, result.RankedCpuSeconds); + Assert.Null(result.SqlCpuSecondsInWindow); + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("no cpu_utilization samples", result.Note, StringComparison.Ordinal); + } + + [Fact] + public void NoCoreCount_OmitsRatio_AndSaysWhy() + { + var result = CpuAttribution.Compute(5000, Start, End, 60, Start, End, 25, cpuCount: 0); + + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("core count unavailable", result.Note, StringComparison.Ordinal); + } + + /// #2320's explicit degrade rule: a server whose CPU series starts mid-window (added, + /// or monitoring resumed) would deflate the denominator and inflate the ratio — omit instead. + [Fact] + public void PartialCoverage_OmitsRatio_WithThePercentage() + { + var result = CpuAttribution.Compute(5000, Start, End, 30, Start.AddMinutes(30), End, 25, 8); + + Assert.Null(result.AttributedCpuRatio); + Assert.NotNull(result.Note); + Assert.Contains("50%", result.Note, StringComparison.Ordinal); + Assert.Contains("partial denominator", result.Note, StringComparison.Ordinal); + } + + /// Samples straddling the window edges clamp to full coverage — a series wider than the + /// window is the NORMAL case (the store holds more history than any one read). + [Fact] + public void SamplesBeyondTheWindow_ClampToFullCoverage() + { + var result = CpuAttribution.Compute( + 5000, Start, End, 120, Start.AddHours(-1), End.AddHours(1), 25, 8); + + Assert.Equal(0.694, result.AttributedCpuRatio); + } + + /// An idle box measures zero CPU-seconds; a ratio against zero is undefined, and the + /// measured zero is still reported so the caller sees WHY. + [Fact] + public void ZeroMeasuredCpu_OmitsRatio_ReportsTheZero() + { + var result = CpuAttribution.Compute(5000, Start, End, 60, Start, End, avgSqlCpuPercent: 0, cpuCount: 8); + + Assert.Equal(0, result.SqlCpuSecondsInWindow); + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("zero", result.Note, StringComparison.Ordinal); + } + + [Fact] + public void EmptyWindow_OmitsRatio() + { + var result = CpuAttribution.Compute(5000, Start, Start, 60, Start, End, 25, 8); + + Assert.Null(result.AttributedCpuRatio); + Assert.Contains("window is empty", result.Note, StringComparison.Ordinal); + } + + /// The numerator is rounded for emission but the ratio divides the RAW value — rounding + /// before dividing would move the third decimal on big windows. + [Fact] + public void RankedSecondsRoundToOneDecimal_RatioToThree() + { + var result = CpuAttribution.Compute(1234.5678, Start, End, 60, Start, End, 25, 8); + + Assert.Equal(1234.6, result.RankedCpuSeconds); + Assert.Equal(0.171, result.AttributedCpuRatio); + } +} diff --git a/Lite.Tests/CrossAppMcpToolInventoryPinTests.cs b/Lite.Tests/CrossAppMcpToolInventoryPinTests.cs index 9041f99ad..983236a30 100644 --- a/Lite.Tests/CrossAppMcpToolInventoryPinTests.cs +++ b/Lite.Tests/CrossAppMcpToolInventoryPinTests.cs @@ -55,6 +55,25 @@ private static readonly (string Lite, string Darling)[] KnownNamingDrift = // system_health parser tools). A NEW Darling-only tool must be either ported to Lite or added here. private static readonly HashSet KnownLiteMissingMcpTools = new(StringComparer.Ordinal) { + /* The eight PostgreSQL reads. Darling-ONLY by architecture, not "not ported yet", so these are the + same kind of entry as get_store_metrics rather than a to-do: Lite has no PostgreSQL target and + cannot acquire one (the engine gate never dispatches a PostgreSQL definition there), and Lite does + not even create the tables — DuckDbSchemaGenerator.StoredCollectors filters them out, so there is + nothing for a Lite twin to read. If Lite ever gains a PostgreSQL target, port these and delete + them from here; the ratchet only shrinks. */ + "get_pg_wait_stats", + "get_pg_top_queries", + "get_pg_wraparound_risk", + "get_pg_xmin_horizon", + "get_pg_replication_slots", + "get_pg_autovacuum_health", + "get_pg_io_stats", + /* get_pg_blocking is the one whose NAME collides with a Lite tool that already exists — Lite has + get_blocking over blocked_process_report. They are not twins and must not be conflated: Lite's + reads an engine-recorded event with a graph, this reads periodic samples of an edge list. Porting + this to Lite would require a PostgreSQL target Lite cannot have, so it belongs here with the rest. */ + "get_pg_blocking", + /* #2068: the store self-metrics read (get_store_metrics) over collect.store_metrics — the central Postgres store measuring ITSELF (hypertable sizes/compression, payload dims, whole-store growth) for capacity forecasting. Darling-ONLY by architecture, not a "not ported yet" item: Lite is a @@ -170,6 +189,32 @@ public void LiteMissingMcpTools_MatchTheRatchetAllowList() $" Listed but no longer Darling-only: [{Format(KnownLiteMissingMcpTools.Except(darlingOnly))}]"); } + /// + /// The Darling MCP instructions' tool census must match the real inventory. It is prose an LLM plans + /// against, and nothing pinned it — which is exactly how it sat at "ninety tools" while the server + /// exposed one hundred (ten tools landed without the sentence moving, the PostgreSQL reads among + /// them). The census now uses digits so this pin can parse it; a new tool on either side fails here + /// until the sentence is updated. + /// + [Fact] + public void DarlingInstructionsCensus_MatchesTheScannedInventory() + { + var lite = ExtractToolNames(LiteMcpDir); + var darling = ExtractToolNames(DarlingMcpDir); + var shared = lite.Count(t => darling.Contains(t)); + + var instructions = ParitySource.ReadFile(DarlingMcpDir + "/DarlingMcpInstructions.cs"); + var census = Regex.Match( + instructions, + @"This server exposes (\d+) tools\. (\d+) are the same names .*?The remaining (\d+) are unique to Darling", + RegexOptions.Singleline); + Assert.True(census.Success, "The census sentence ('This server exposes N tools. M are the same names ... The remaining K are unique to Darling') was not found in DarlingMcpInstructions.cs — keep it parseable so this pin can hold it to the real inventory."); + + Assert.Equal(darling.Count, int.Parse(census.Groups[1].Value)); + Assert.Equal(shared, int.Parse(census.Groups[2].Value)); + Assert.Equal(darling.Count - shared, int.Parse(census.Groups[3].Value)); + } + private static string Format(IEnumerable names) => string.Join(", ", names.OrderBy(n => n, StringComparer.Ordinal)); diff --git a/Lite.Tests/DatabaseSizeCollectorDefinitionTests.cs b/Lite.Tests/DatabaseSizeCollectorDefinitionTests.cs index a0ead6e6c..9fa2e9a68 100644 --- a/Lite.Tests/DatabaseSizeCollectorDefinitionTests.cs +++ b/Lite.Tests/DatabaseSizeCollectorDefinitionTests.cs @@ -46,6 +46,58 @@ public void BuildQuery_OnPrem_SplicesExclusionAtBothSites_ParamsOnce() Assert.Equal("SO", Assert.Single(plan.Parameters).Value); } + [Fact] + public void OnPrem_TotalSize_PrefersTheInDatabaseCurrentSize_SoTempdbCannotExceed100Percent() + { + /* #2169: the viewer computes used% as used_size_mb / total_size_mb. Used comes from FILEPROPERTY + read INSIDE each database; total used to come from sys.master_files.size, which records the size + at configuration time and does NOT track autogrowth for tempdb. A grown tempdb therefore + reported current usage against its startup size and rendered above 100%. The probe now captures + the in-database current size in the SAME round trip, and the payload prefers it, so both operands + come from one snapshot. */ + var plan = DatabaseSizeStatsCollector.Instance.BuildQuery(new CollectorContext + { + ServerId = 1, + ServerName = "test-server", + CollectionTime = DateTime.UtcNow, + Deltas = s_deltas, + }); + + Assert.Contains("current_size_mb decimal(19,2) NULL", plan.Text, StringComparison.Ordinal); + Assert.Contains("INSERT #file_space (database_id, file_id, used_size_mb, current_size_mb)", plan.Text, StringComparison.Ordinal); + Assert.Contains("CONVERT(decimal(19,2), df.size * 8.0 / 1024.0)", plan.Text, StringComparison.Ordinal); + + /* The fallback is load-bearing: a database whose probe failed (mid-restore, permissions) still + reports a total from master_files rather than NULL, so it degrades in precision and never + disappears from the grid. */ + Assert.Contains("COALESCE(fs.current_size_mb, mf.size * 8.0 / 1024.0)", plan.Text, StringComparison.Ordinal); + + /* The stale source must no longer be the total on its own. */ + Assert.DoesNotContain("total_size_mb =" + Environment.NewLine + " CONVERT(decimal(19,2), mf.size * 8.0 / 1024.0),", plan.Text, StringComparison.Ordinal); + } + + [Fact] + public void AzureSqlDb_AlreadyUsedInDatabaseSizes_SoItNeverHadTheTempdbSkew() + { + /* The Azure SQL DB path reads BOTH size and SpaceUsed from sys.database_files in the connected + database, so its used% was always internally consistent — #2169 was specific to the path that + mixes master_files with in-database reads (on-prem, RDS, and Managed Instance, which honors the + cross-database reference and therefore takes that path). Pinned so a future refactor does not + 'unify' the two by moving Azure onto the stale source. */ + var plan = DatabaseSizeStatsCollector.Instance.BuildQuery(new CollectorContext + { + ServerId = 1, + ServerName = "test-server", + CollectionTime = DateTime.UtcNow, + Deltas = s_deltas, + Target = new CollectorTargetInfo { IsAzureSqlDb = true }, + }); + + Assert.Contains("total_size_mb =", plan.Text, StringComparison.Ordinal); + Assert.Contains("df.size * 8.0 / 1024.0", plan.Text, StringComparison.Ordinal); + Assert.DoesNotContain("sys.master_files", plan.Text, StringComparison.Ordinal); + } + [Fact] public void AzureDmvPermissionHint_ExplainsError300_OnAzureOnly() { diff --git a/Lite.Tests/DatabaseStateExpectedStoreTests.cs b/Lite.Tests/DatabaseStateExpectedStoreTests.cs index 79ddb699d..fb0214c3d 100644 --- a/Lite.Tests/DatabaseStateExpectedStoreTests.cs +++ b/Lite.Tests/DatabaseStateExpectedStoreTests.cs @@ -7,8 +7,10 @@ */ using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using PerformanceMonitor.Alerting; using DuckDB.NET.Data; using PerformanceMonitorLite.Database; using PerformanceMonitorLite.Services; @@ -79,6 +81,64 @@ INSERT INTO database_states (collection_id, collection_time, server_id, server_n private static readonly DateTime T0 = new(2026, 8, 1, 9, 0, 0, DateTimeKind.Unspecified); + /// + /// Runs the deviation sweep until holds, up to times + /// (#2266). Returns the LAST result either way, so a real regression still fails on its own assertion with + /// its own message. + /// + /// Why any retry is correct here rather than a tolerance hack. + /// GetDatabaseStateDeviationsAsync does its seeding, the #2189 heal, the #2203 forget and the prune + /// inside a BEST-EFFORT block: it opens the write connection with a 5-second lock acquisition and, on + /// TimeoutException, skips the whole maintenance block and runs the deviation read anyway. That is + /// deliberate and documented — skipping is the only lossless option when archival holds the lock. The write + /// lock is static, shared by the whole process (the method's own comment says so), and xunit runs + /// test classes in parallel, so another class can hold it long enough for a cycle here to skip its + /// maintenance. + /// + /// The observed flake is exactly that: ExpectedState = SUSPECT with StateDesc = ONLINE — + /// a combination only reachable by skipping the heal while completing the read. So asserting the heal lands + /// in ONE cycle asserts something the design does not promise; asserting it lands within a FEW cycles is the + /// contract. No tuned number is involved: the semantics are "eventually", and the count only needs to + /// exceed one. + /// + /// It cannot mask a regression, which is the property that makes it acceptable in 23 tests' worth of + /// company: a heal that is genuinely broken never settles, all cycles run, and the caller's own assertion + /// fails on the final result exactly as it does today. + /// + private static async Task> SweepUntilAsync( + LocalDataService service, + Func, bool> settled, + int cycles = 5) + { + List result; + do + { + result = await service.GetDatabaseStateDeviationsAsync(ServerId); + } + while (!settled(result) && --cycles > 0); + + return result; + } + + /// + /// Writes an expectation row directly, bypassing the service's writers — the only way to reproduce a + /// row the OLD seed wrote (#2189), since no current code path can produce one any more. + /// + private async Task SeedExpectedAsync(string database, string expected, bool isOverride) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" +INSERT INTO config_database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at) +SELECT $1, $2, $3, $4, now()::TIMESTAMP"; + cmd.Parameters.Add(new DuckDBParameter { Value = ServerId }); + cmd.Parameters.Add(new DuckDBParameter { Value = database }); + cmd.Parameters.Add(new DuckDBParameter { Value = expected }); + cmd.Parameters.Add(new DuckDBParameter { Value = isOverride }); + await cmd.ExecuteNonQueryAsync(); + } + [Fact] public async Task FirstObservation_SeedsBaseline_AndReportsNoDeviation() { @@ -219,6 +279,327 @@ public async Task PendingCritical_AutoAcceptsBaseline_WhenItRecoversToNonCritica Assert.False(row.IsUserOverride); } + /* ---------------- #2189: transient states are never learned, and ONLINE un-learns a stale one ---------------- */ + + [Fact] + public async Task OnboardedMidRestore_NeverLearnsRestoring_AndIsSilentBeforeAndAfterTheRestoreCompletes() + { + /* #2189, the reported bug, end to end. A database swept into monitoring during a consolidation is + mid-restore, not in a steady state anybody chose — so nothing is learned while it restores, and + when the restore finishes the STEADY state is what gets learned. The old seed learned RESTORING + here and the database then "deviated" by being healthy, forever. */ + var service = new LocalDataService(_duckDb); + + await SeedSnapshotAsync(T0, ("App", "RESTORING", false)); + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + await SeedSnapshotAsync(T0.AddMinutes(1), ("App", "RESTORING", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("App", "RESTORING", false)); + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); // a restore in progress is not news + + var pending = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("", pending.ExpectedState); + + await SeedSnapshotAsync(T0.AddMinutes(3), ("App", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(4), ("App", "ONLINE", false)); + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + + var settled = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("ONLINE", settled.ExpectedState); + Assert.False(settled.IsUserOverride); + } + + [Fact] + public async Task TransientStates_AreNeverLearnedAsABaseline_RestoringAndRecovering() + { + /* Both halves of "mid-something": RESTORING is a restore in flight, RECOVERING a database still + coming up. Neither is a steady state, and the states around them are unaffected — a healthy + database still baselines on its first observation. */ + await SeedSnapshotAsync(T0, + ("Restoring", "RESTORING", false), ("Recovering", "RECOVERING", false), ("Healthy", "ONLINE", false)); + var service = new LocalDataService(_duckDb); + await service.GetDatabaseStateDeviationsAsync(ServerId); + + var rows = await service.GetDatabaseStateExpectationsAsync(ServerId); + Assert.Equal("", rows.Single(r => r.DatabaseName == "Restoring").ExpectedState); + Assert.Equal("", rows.Single(r => r.DatabaseName == "Recovering").ExpectedState); + Assert.Equal("ONLINE", rows.Single(r => r.DatabaseName == "Healthy").ExpectedState); + } + + [Fact] + public async Task PoisonedRestoringBaseline_HealsToOnline_RatherThanAlertingForeverOnBeingHealthy() + { + /* The other half of #2189, and the half the widened seed cannot reach: rows that were ALREADY + written. Five databases on the reporting fleet sat like this — baselined RESTORING during a + consolidation, then ~127 identical alerts each in 24 hours for the crime of being ONLINE. No + current code path can write this row any more, so it is planted directly. */ + await SeedExpectedAsync("pecan", "RESTORING", isOverride: false); + await SeedSnapshotAsync(T0, ("pecan", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(1), ("pecan", "ONLINE", false)); + var service = new LocalDataService(_duckDb); + + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + + var row = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("ONLINE", row.ExpectedState); + Assert.False(row.IsUserOverride); + } + + [Fact] + public async Task RebaselinedByHandMidRestore_StillHealsOnceTheRestoreCompletes() + { + /* The seed is not the only way into a transient baseline: "reset to current" pressed while a restore + is running writes exactly the same poisoned row, and always will. The heal is what makes that a + self-correcting mistake instead of a trap armed for the next operator. */ + await SeedSnapshotAsync(T0, ("App", "RESTORING", false)); + var service = new LocalDataService(_duckDb); + await service.ResetDatabaseStateExpectedToCurrentAsync(ServerId, "App"); + Assert.Equal("RESTORING", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + + await SeedSnapshotAsync(T0.AddMinutes(1), ("App", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("App", "ONLINE", false)); + + /* #2266: same best-effort maintenance exposure as the outage sibling below — structurally identical + test, so it can flake the same way even though only the SUSPECT one has been seen to. */ + Assert.Empty(await SweepUntilAsync(service, deviations => deviations.Count == 0)); + Assert.Equal("ONLINE", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + } + + [Fact] + public async Task RebaselinedByHandDuringAnOutage_HealsOnceTheDatabaseRecovers() + { + /* The seed is not the only writer of inferred baselines, and never will be: "reset to current" + records whatever it sees with NO state filter, so pressing it during an outage writes SUSPECT as + the accepted normal. That silences the database while it is corrupt (the operator's own doing) and + then, without this, makes it deviate by RECOVERING. Same shape as the restore case, integrity + flavour — which is why the heal keys off the seed's refusal list rather than RESTORING alone. */ + await SeedSnapshotAsync(T0, ("Payments", "SUSPECT", false)); + var service = new LocalDataService(_duckDb); + await service.ResetDatabaseStateExpectedToCurrentAsync(ServerId, "Payments"); + var planted = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("SUSPECT", planted.ExpectedState); + Assert.False(planted.IsUserOverride); + + await SeedSnapshotAsync(T0.AddMinutes(1), ("Payments", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("Payments", "ONLINE", false)); + + /* #2266: the heal rides a best-effort maintenance block, so a cycle whose write-lock acquisition times + out against another parallel test class skips it silently. Sweep until it lands. */ + Assert.Empty(await SweepUntilAsync(service, deviations => deviations.Count == 0)); + Assert.Equal("ONLINE", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + } + + [Fact] + public async Task AutoBaselinedOffline_IsNeverHealed_SoParkingItAgainStaysQuiet() + { + /* The heal is deliberately NOT "ONLINE overwrites anything the machine inferred". OFFLINE is a steady + state the seed is happy to learn, so it is a legitimate baseline, and rewriting it on the first + ONLINE sighting would be this bug inverted: bring a parked database up for an hour of maintenance, + re-park it, and it now deviates forever against a baseline it never had - which in Lite, with no + persisted alerted-state memory to edge-trigger against, means an alert every cooldown for good. + + Coming UP still alerts, because that is a real departure from the accepted normal. Going back to it + is silence, and the baseline is the same one it started with. */ + await SeedSnapshotAsync(T0, ("Parked", "OFFLINE", false)); + var service = new LocalDataService(_duckDb); + await service.GetDatabaseStateDeviationsAsync(ServerId); // learns OFFLINE + + await SeedSnapshotAsync(T0.AddMinutes(1), ("Parked", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("Parked", "ONLINE", false)); + var up = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("ONLINE", up.StateDesc); + Assert.Equal("OFFLINE", up.ExpectedState); + + await SeedSnapshotAsync(T0.AddMinutes(3), ("Parked", "OFFLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(4), ("Parked", "OFFLINE", false)); + + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + var row = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("OFFLINE", row.ExpectedState); + Assert.False(row.IsUserOverride); + } + + [Fact] + public async Task StandbySecondaryRecoveredOutOfStandby_StillAlerts_BecauseItHasStoppedBeingASecondary() + { + /* The other state the heal must not touch, and the sharper of the two. A STANDBY secondary that turns + up truly ONLINE (is_in_standby now 0) has been RECOVERED - log shipping is broken and that is + exactly what this alert exists to say. Healing it would swap that alert for silence and then fire + when the operator re-established standby, announcing the repair instead of the break. */ + await SeedSnapshotAsync(T0, ("LogShip", "ONLINE", true)); + var service = new LocalDataService(_duckDb); + await service.GetDatabaseStateDeviationsAsync(ServerId); // learns STANDBY + + await SeedSnapshotAsync(T0.AddMinutes(1), ("LogShip", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("LogShip", "ONLINE", false)); + + var fired = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("ONLINE", fired.StateDesc); + Assert.Equal("STANDBY", fired.ExpectedState); + Assert.Equal("STANDBY", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + } + + [Fact] + public async Task OperatorParkedOffline_StillAlertsWhenTheDatabaseComesBackOnline() + { + /* #2166's composition contract, which the heal must not eat. The heal second-guesses only what the + machine inferred; an operator who DECLARED an expected state meant it, so a parked database coming + back ONLINE is a deviation from a real intent and still fires — and the override survives the sweep + rather than being quietly rewritten to ONLINE underneath them. */ + await SeedSnapshotAsync(T0, ("Parked", "OFFLINE", false)); + var service = new LocalDataService(_duckDb); + await service.SetDatabaseStateExpectedAsync(ServerId, "Parked", PerformanceMonitor.Alerting.DatabaseStateTokens.Offline); + + await SeedSnapshotAsync(T0.AddMinutes(1), ("Parked", "ONLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("Parked", "ONLINE", false)); + + var fired = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("ONLINE", fired.StateDesc); + Assert.Equal("OFFLINE", fired.ExpectedState); + + var row = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.True(row.IsUserOverride); + Assert.Equal("OFFLINE", row.ExpectedState); + } + + [Fact] + public async Task NorecoveryLogShippingSecondary_StaysQuietForever_AndCanBeOptedIntoCoverageByHand() + { + /* The #1986 property that must survive #2189: a log-shipping secondary restored WITH NORECOVERY sits + in RESTORING permanently (is_in_standby is 0 — only a read-only STANDBY secondary sets that), and + must never page. It now gets there by staying PENDING rather than by learning RESTORING, which is + silent for the same reason: the no-baseline arm only alerts on the integrity states. + + The cost is that a pending database has no baseline to deviate FROM, so the operator who wants + deviation coverage on a permanent secondary declares it — and that being an override is the point, + since it is a genuine choice about a database only they can classify. */ + var service = new LocalDataService(_duckDb); + for (int minute = 0; minute < 6; minute++) + { + await SeedSnapshotAsync(T0.AddMinutes(minute), ("Secondary", "RESTORING", false)); + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + } + + Assert.Equal("", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + + await service.SetDatabaseStateExpectedAsync(ServerId, "Secondary", PerformanceMonitor.Alerting.DatabaseStateTokens.Restoring); + await SeedSnapshotAsync(T0.AddMinutes(6), ("Secondary", "OFFLINE", false)); + await SeedSnapshotAsync(T0.AddMinutes(7), ("Secondary", "OFFLINE", false)); + + var fired = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("OFFLINE", fired.StateDesc); + Assert.Equal("RESTORING", fired.ExpectedState); + } + + [Fact] + public async Task StandbySecondary_IsNeverHealedToOnline_BecauseItsEffectiveStateIsStandby() + { + /* The trap inside the heal. A standby secondary reports state_desc = 'ONLINE' with is_in_standby set, + so a heal written against the RAW column would re-baseline every log-shipping secondary from + STANDBY to ONLINE and then alert it forever for being STANDBY — #2189 recreated for exactly the + database family #1986 works hardest to keep quiet. Matching the EFFECTIVE state is what prevents it. */ + await SeedSnapshotAsync(T0, ("LogShip", "ONLINE", true)); + var service = new LocalDataService(_duckDb); + await service.GetDatabaseStateDeviationsAsync(ServerId); // baselines STANDBY + + await SeedSnapshotAsync(T0.AddMinutes(1), ("LogShip", "ONLINE", true)); + await SeedSnapshotAsync(T0.AddMinutes(2), ("LogShip", "RESTORING", true)); + + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("STANDBY", (await service.GetDatabaseStateExpectationsAsync(ServerId)).Single().ExpectedState); + } + + [Fact] + public async Task AlertedState_RoundTripsToTheDeviationRead_SoTheEdgeTriggerCanEngage() + { + // #2203: until Lite persisted this, `alreadyAnnounced` was always false here and a database parked + // OFFLINE for a month alerted every cooldown forever — the original #2166 complaint, still live in + // Lite after the Darling half shipped. The whole feature depends on this value surviving the round + // trip, so pin the trip rather than the write. + var service = new LocalDataService(_duckDb); + await DriveAppToStableStateAsync(service, "OFFLINE"); + + var before = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("", before.LastAlertedState); // never announced yet + + var store = new DuckDbAlertHistoryStore(_duckDb); + await store.SaveDatabaseStateAlertedAsync(ServerId, "App", "OFFLINE"); + + var after = Assert.Single(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Equal("OFFLINE", after.LastAlertedState); + Assert.Equal("OFFLINE", after.StateDesc); // still deviating — the engine is what goes quiet, not the read + } + + [Fact] + public async Task RecoveredDatabase_HasItsAlertedStateClearedByTheStore_NotJustByTheEngine() + { + // The restart-gap invariant, and the reason this clear is store-derived rather than engine-derived. + // The engine also clears on the falling edge it witnesses, but that path runs off an in-memory active + // set that empties on restart — so a restart landing between an alert and the recovery would leave the + // memory sticky forever and swallow the next parking entirely. Nothing is held in memory here: the + // deviation read alone must heal it. + var service = new LocalDataService(_duckDb); + await DriveAppToStableStateAsync(service, "OFFLINE"); + await service.GetDatabaseStateDeviationsAsync(ServerId); + + var store = new DuckDbAlertHistoryStore(_duckDb); + await store.SaveDatabaseStateAlertedAsync(ServerId, "App", "OFFLINE"); + Assert.Equal("OFFLINE", (await service.GetDatabaseStateDeviationsAsync(ServerId)).Single().LastAlertedState); + + // Operator brings it back. It stops deviating, so it drops out of the read entirely... + await SeedSnapshotAsync(T0.AddMinutes(3), ("App", "ONLINE", false)); + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + + // ...and the memory must be gone, or a second parking weeks later reads as already-announced. + Assert.Null(await AlertedStateAsync("App")); + } + + [Fact] + public async Task IgnoredDatabase_AlsoHasItsAlertedStateCleared() + { + // An operator silencing a database should not leave a memory behind that outlives the silence. + var service = new LocalDataService(_duckDb); + await DriveAppToStableStateAsync(service, "OFFLINE"); + await service.GetDatabaseStateDeviationsAsync(ServerId); + + var store = new DuckDbAlertHistoryStore(_duckDb); + await store.SaveDatabaseStateAlertedAsync(ServerId, "App", "OFFLINE"); + await service.SetDatabaseStateExpectedAsync(ServerId, "App", PerformanceMonitor.Alerting.DatabaseStateTokens.Ignore); + + Assert.Empty(await service.GetDatabaseStateDeviationsAsync(ServerId)); + Assert.Null(await AlertedStateAsync("App")); + } + + [Fact] + public async Task ClearAlertedState_IsTheImmediatePath_AndLeavesTheBaselineIntact() + { + // The engine's own falling-edge call. It must forget the announcement WITHOUT disturbing the baseline — + // clearing the expected state instead would re-baseline the database and silence a real deviation. + var service = new LocalDataService(_duckDb); + await DriveAppToStableStateAsync(service, "OFFLINE"); + await service.GetDatabaseStateDeviationsAsync(ServerId); + + var store = new DuckDbAlertHistoryStore(_duckDb); + await store.SaveDatabaseStateAlertedAsync(ServerId, "App", "OFFLINE"); + await store.ClearDatabaseStateAlertedAsync(ServerId, "App"); + + Assert.Null(await AlertedStateAsync("App")); + var row = Assert.Single(await service.GetDatabaseStateExpectationsAsync(ServerId)); + Assert.Equal("ONLINE", row.ExpectedState); // baseline untouched + } + + /// Reads the raw memory column, so a test can distinguish "cleared" from "never set". + private async Task AlertedStateAsync(string database) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT last_alerted_state FROM config_database_state_expected WHERE server_id = $1 AND database_name = $2"; + cmd.Parameters.Add(new DuckDBParameter { Value = ServerId }); + cmd.Parameters.Add(new DuckDBParameter { Value = database }); + var value = await cmd.ExecuteScalarAsync(); + return value is null or DBNull ? null : (string)value; + } + [Fact] public async Task StandbySecondary_BaselinesAsStandby_AndDoesNotChurnOnLogRestores() { diff --git a/Lite.Tests/DatabaseStateWriteLockTests.cs b/Lite.Tests/DatabaseStateWriteLockTests.cs new file mode 100644 index 000000000..d22ccfd97 --- /dev/null +++ b/Lite.Tests/DatabaseStateWriteLockTests.cs @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitorLite.Services; +using PerformanceMonitorLite.Tests; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2208: the database-state write paths take the WRITE lock. They used the read lock, which let their +/// INSERT/UPDATE/DELETE interleave with archival and starved writers process-wide. +/// +/// These exploit the lock's own recursion policy rather than its timeout, which makes them both fast and +/// genuinely watched-red. s_dbLock is built with LockRecursionPolicy.NoRecursion, so a thread +/// that already holds the write lock and calls TryEnterWriteLock again gets a +/// immediately. AcquireReadLock, by contrast, deliberately CATCHES +/// that exception and hands back a no-op disposable. So from a thread holding the write lock: +/// +/// • a method that takes the WRITE lock throws — which is what these assert;
+/// • a method that takes the READ lock proceeds silently, which is exactly what these methods did before +/// #2212, so the pre-fix behaviour is a FAILED assertion rather than a hang or a pass.
+/// +/// Deliberately NOT the 5-second timeout route, and not a serialised collection. Both were in the first +/// draft of this file and both were wrong: calling on the holding thread never reaches the timeout (the +/// recursion check fires first), and DisableParallelization only orders collections inside the +/// non-parallel bucket — it cannot stop other classes from contending on a static lock. Holding the write lock +/// for five seconds to force a timeout would therefore have blocked arbitrary neighbours for five seconds with +/// no isolation to show for it. The recursion route holds the lock for the duration of one synchronous call. +/// +/// What is still NOT covered: the deviation read's best-effort skip path. Forcing its prologue to fail +/// means making its write-lock acquisition fail, and the read that follows takes the read lock — which under +/// this same recursion policy would be handed a no-op disposable and proceed, so the skip is unobservable from +/// here. That arm needs the timeout injected, i.e. production shape changed for testability. Left visible +/// rather than implied-covered. +///
+public sealed class DatabaseStateWriteLockTests : IClassFixture +{ + private readonly SharedDuckDbFixture _fixture; + + public DatabaseStateWriteLockTests(SharedDuckDbFixture fixture) => _fixture = fixture; + + [Fact] + public async Task SetDatabaseStateExpected_TakesTheWriteLock() + { + var service = new LocalDataService(_fixture.DuckDb); + + using (_fixture.DuckDb.AcquireWriteLock()) + { + /* Same thread, so NoRecursion rejects the second write-lock entry. A read-lock implementation — + what this was before #2212 — would be handed a no-op disposable and complete normally. */ + await Assert.ThrowsAsync( + () => service.SetDatabaseStateExpectedAsync(1, "probedb", "OFFLINE")); + } + } + + [Fact] + public async Task ResetDatabaseStateExpectedToCurrent_TakesTheWriteLock() + { + var service = new LocalDataService(_fixture.DuckDb); + + using (_fixture.DuckDb.AcquireWriteLock()) + { + await Assert.ThrowsAsync( + () => service.ResetDatabaseStateExpectedToCurrentAsync(1, "probedb")); + } + } +} diff --git a/Lite.Tests/DuckDbSchemaEquivalenceTests.cs b/Lite.Tests/DuckDbSchemaEquivalenceTests.cs index b96e07350..401b124f3 100644 --- a/Lite.Tests/DuckDbSchemaEquivalenceTests.cs +++ b/Lite.Tests/DuckDbSchemaEquivalenceTests.cs @@ -108,15 +108,20 @@ private static List TableInfo(DuckDBConnection conn, string ddl, str [Fact] public void Golden_CoversExactlyTheCatalogCollectorTables() { - var catalogTables = CollectorCatalog.All.Select(c => c.TargetTable).OrderBy(t => t, StringComparer.Ordinal); + /* The oracle describes the tables LITE stores, which is the SQL Server subset of the shared + engine-mixed catalog — Lite has no PostgreSQL target and creates no table for those definitions. */ + var catalogTables = DuckDbSchemaGenerator.StoredCollectors + .Select(c => c.TargetTable).OrderBy(t => t, StringComparer.Ordinal); var goldenTables = GoldenCollectorSchema.Tables.Keys.OrderBy(t => t, StringComparer.Ordinal); - /* The frozen oracle must describe exactly the 41 catalog collector tables — no more, no fewer. */ + /* The frozen oracle must describe exactly those tables — no more, no fewer. 41 stays a literal here + BECAUSE it is the frozen historical shape: if a SQL Server collector is added, this is supposed to + fail until the oracle is extended by hand. That is the whole point of an oracle. */ Assert.Equal(catalogTables, goldenTables); - Assert.Equal(41, GoldenCollectorSchema.Tables.Count); + Assert.Equal(42, GoldenCollectorSchema.Tables.Count); /* Only server_config and database_config lack an index (matches DuckDbSchemaGenerator.CreateIndex). */ - var goldenIndexless = CollectorCatalog.All + var goldenIndexless = DuckDbSchemaGenerator.StoredCollectors .Select(c => c.TargetTable) .Where(t => !GoldenCollectorSchema.Indexes.ContainsKey(t)) .OrderBy(t => t, StringComparer.Ordinal) @@ -172,7 +177,7 @@ public void GeneratedCollectorTables_AreStorageEquivalentToPreChangeHandWritten( using var conn = new DuckDBConnection($"Data Source={_dbPath}"); conn.Open(); - foreach (var schema in CollectorCatalog.All) + foreach (var schema in DuckDbSchemaGenerator.StoredCollectors) { var table = schema.TargetTable; @@ -226,7 +231,7 @@ public void GeneratedCollectorIndexes_MatchPreChangeHandWritten() { var failures = new List(); - foreach (var schema in CollectorCatalog.All) + foreach (var schema in DuckDbSchemaGenerator.StoredCollectors) { var table = schema.TargetTable; var generated = DuckDbSchemaGenerator.CreateIndex(schema); @@ -254,7 +259,7 @@ public void GeneratedSchema_TablesAndIndexes_AllExecuteAgainstDuckDb() using var conn = new DuckDBConnection($"Data Source={_dbPath}"); conn.Open(); - foreach (var schema in CollectorCatalog.All) + foreach (var schema in DuckDbSchemaGenerator.StoredCollectors) { using (var t = conn.CreateCommand()) { @@ -275,8 +280,12 @@ public void GeneratedSchema_TablesAndIndexes_AllExecuteAgainstDuckDb() using var count = conn.CreateCommand(); count.CommandText = "SELECT COUNT(*) FROM information_schema.tables WHERE table_name IN (" + - string.Join(",", CollectorCatalog.All.Select(c => $"'{c.TargetTable}'")) + ")"; - Assert.Equal(41, Convert.ToInt32(count.ExecuteScalar())); + string.Join(",", DuckDbSchemaGenerator.StoredCollectors.Select(c => $"'{c.TargetTable}'")) + ")"; + /* The generated schema executes into DuckDB and must produce exactly the tables Lite stores. + Derived rather than pinned at 41: the golden below stays a frozen literal on purpose (it is the + historical shape), but this side tracks the generator, so adding a SQL Server collector updates it + and adding a PostgreSQL one correctly does not. */ + Assert.Equal(DuckDbSchemaGenerator.StoredCollectors.Count(), Convert.ToInt32(count.ExecuteScalar())); } private static string BuildTableDiff(string table, List golden, List generated) diff --git a/Lite.Tests/DuckDbSchemaGeneratorTests.cs b/Lite.Tests/DuckDbSchemaGeneratorTests.cs index a6efb885c..aac7827c7 100644 --- a/Lite.Tests/DuckDbSchemaGeneratorTests.cs +++ b/Lite.Tests/DuckDbSchemaGeneratorTests.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using PerformanceMonitor.Collectors; using PerformanceMonitorLite.Database; using PerformanceMonitorLite.Services; @@ -165,11 +166,27 @@ public void CreateIndex_MirrorsLitesIrregularNamesAndColumns() [Fact] public void Generated_EmitsEveryCatalogTable_AndThirtyNineIndexes() { - Assert.Equal(41, DuckDbSchemaGenerator.CreateTableStatements().Count()); + /* Counting the filtered sequence against itself could not fail. What matters is WHICH tables are + emitted, so the names are compared as sets — and that no PostgreSQL table leaks into Lite's DuckDB, + which is the actual invariant this file now guards. */ + var emitted = DuckDbSchemaGenerator.CreateTableStatements() + .Select(s => Regex.Match(s, @"CREATE TABLE IF NOT EXISTS (\w+)").Groups[1].Value) + .OrderBy(n => n, StringComparer.Ordinal) + .ToArray(); + var expected = DuckDbSchemaGenerator.StoredCollectors + .Select(c => c.TargetTable) + .OrderBy(n => n, StringComparer.Ordinal) + .ToArray(); + + Assert.Equal(expected, emitted); + Assert.DoesNotContain(emitted, n => n.StartsWith("pg_", StringComparison.Ordinal)); + Assert.All( + CollectorCatalog.All.Where(c => c.TargetEngine == CollectorTargetEngine.PostgreSql), + c => Assert.DoesNotContain(c.TargetTable, emitted)); - /* 41 collectors minus the two index-less config tables = 39 indexes (database_states is a + /* The stored collectors minus the two index-less config tables (database_states is a time-series collector and gets the default retrieval index). */ - Assert.Equal(39, DuckDbSchemaGenerator.CreateIndexStatements().Count()); + Assert.Equal(DuckDbSchemaGenerator.StoredCollectors.Count() - 2, DuckDbSchemaGenerator.CreateIndexStatements().Count()); } /// @@ -184,7 +201,7 @@ public void GeneratedColumnsAndTypes_MatchTheCatalog_ForEveryCollector() { var failures = new List(); - foreach (var schema in CollectorCatalog.All) + foreach (var schema in DuckDbSchemaGenerator.StoredCollectors) { var expected = new List<(string Name, string Type)>(); if (schema.IncludesCollectionId) @@ -222,7 +239,7 @@ public void GeneratedColumnsAndTypes_MatchTheCatalog_ForEveryCollector() [Fact] public void ArchivableTables_AreCatalogDriven_AndMirrorEachOther() { - var expected = CollectorCatalog.All.Select(c => c.TargetTable) + var expected = DuckDbSchemaGenerator.StoredCollectors.Select(c => c.TargetTable) .Concat(new[] { "config_alert_log", "collection_log" }) .OrderBy(t => t, StringComparer.Ordinal) .ToArray(); @@ -236,7 +253,7 @@ public void ArchivableTables_AreCatalogDriven_AndMirrorEachOther() /* The time column for every archivable collector table is its catalog prefix-time column; the two non-collector tables carry their own. */ var timeByTable = ArchiveService.ArchivableTables.ToDictionary(t => t.Table, t => t.TimeColumn); - foreach (var schema in CollectorCatalog.All) + foreach (var schema in DuckDbSchemaGenerator.StoredCollectors) { Assert.Equal(schema.PrefixTimeColumnName, timeByTable[schema.TargetTable]); } diff --git a/Lite.Tests/DuckDbSchemaTests.cs b/Lite.Tests/DuckDbSchemaTests.cs index f9e2cd173..691e949a5 100644 --- a/Lite.Tests/DuckDbSchemaTests.cs +++ b/Lite.Tests/DuckDbSchemaTests.cs @@ -80,7 +80,8 @@ public async Task InitializeAsync_CreatesAllTables() "agent_status", "ag_replica_states", "ag_database_replica_states", - "pvs_stats" + "pvs_stats", + "query_store_health" }; using var connection = new DuckDBConnection($"Data Source={_dbPath}"); @@ -148,8 +149,9 @@ public void SchemaStatements_MatchTableCount() foreach (var _ in Schema.GetAllTableStatements()) tableCount++; - /* 52 tables from Schema (schema_version is created separately by DuckDbInitializer). - Includes config_edge_trigger_watermarks (#1145), dmv_blocking_snapshots (always-on + /* 53 tables from Schema (schema_version is created separately by DuckDbInitializer). + Includes config_edge_trigger_watermarks (#1145), config_incident_occurrences (#2216's + per-fingerprint occurrence counters), dmv_blocking_snapshots (always-on blocking fallback), latch_stats/spinlock_stats, cpu_scheduler_stats/plan_cache_stats, session_summary_stats, system_health_events, default_trace_events (#1262 shared DMV/XE/Default-Trace collectors), job_history + agent_status (#1433 Job History tab), @@ -159,7 +161,7 @@ Includes config_edge_trigger_watermarks (#1145), dmv_blocking_snapshots (always- (#1952 automatic plan correction), pvs_stats (#1951 ADR persistent version store), the database-state alert's database_states collector + config_database_state_expected control table, and the fleet-tag tables server_tags + server_tag_map (#2020 2b-i). */ - Assert.Equal(52, tableCount); + Assert.Equal(54, tableCount); } [Fact] diff --git a/Lite.Tests/EmptyEnumerationInventoryTests.cs b/Lite.Tests/EmptyEnumerationInventoryTests.cs index 428290367..36558f1c5 100644 --- a/Lite.Tests/EmptyEnumerationInventoryTests.cs +++ b/Lite.Tests/EmptyEnumerationInventoryTests.cs @@ -113,7 +113,7 @@ quietly turn this pin into a comparison of two empty sets. */ /* Named outright as well as compared, so the failure message names the drift rather than a set. */ Assert.Equal( - new HashSet(StringComparer.OrdinalIgnoreCase) { "query_store", "database_scoped_config", "index_object_stats", "plan_correction" }, + new HashSet(StringComparer.OrdinalIgnoreCase) { "query_store", "database_scoped_config", "index_object_stats", "plan_correction", "query_store_health" }, enumerators); foreach (var name in CollectorCatalog.All.Select(c => c.Name)) diff --git a/Lite.Tests/EntraInteractiveAuthTests.cs b/Lite.Tests/EntraInteractiveAuthTests.cs new file mode 100644 index 000000000..b57014bb6 --- /dev/null +++ b/Lite.Tests/EntraInteractiveAuthTests.cs @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitorLite.Services; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #2184: interactive Entra auth needs a parent window handle because SqlClient routes it through the +/// WAM broker. These pin the wiring contracts that are verifiable without a tenant — argument +/// validation and the idempotent process-wide registration. Whether the picker actually authenticates +/// is a real-tenant step: the Studio twin of this seam (PerformanceStudio#426) was verified by this +/// issue's reporter against his Entra-MFA-on-Azure-VM environment. +/// +public class EntraInteractiveAuthTests +{ + [Fact] + public void Register_RejectsANullHandleProvider() + { + /* The whole point of the type is supplying a handle; accepting null would register a provider + that fails at prompt time instead of at wiring time, which is the harder bug to find. */ + Assert.Throws(() => EntraInteractiveAuth.Register(null!)); + } + + [Fact] + public void Register_IsProcessWideAndIdempotent() + { + /* SqlAuthenticationProvider.SetProvider is process-wide and a second registration would + silently replace the first, so "first one wins, later ones are no-ops" is the contract worth + pinning — the app registers at startup and nothing else should be able to swap the provider + out from under it. The reset makes registration order observable regardless of what ran + earlier in the test process. */ + EntraInteractiveAuth.ResetRegistrationForTests(); + + var first = EntraInteractiveAuth.Register(() => IntPtr.Zero); + var second = EntraInteractiveAuth.Register(() => new IntPtr(1234)); + + Assert.True(first, "the first registration after reset must install the provider"); + Assert.False(second, "a second Register must be a no-op rather than replacing the provider"); + } +} diff --git a/Lite.Tests/EnumeratedCollectorDriverTests.cs b/Lite.Tests/EnumeratedCollectorDriverTests.cs index b7ccf4461..5aaaa47e5 100644 --- a/Lite.Tests/EnumeratedCollectorDriverTests.cs +++ b/Lite.Tests/EnumeratedCollectorDriverTests.cs @@ -200,4 +200,249 @@ exactly the batch size. */ Assert.Equal((3, 1), (completed.Single(c => c.Item == "a").Count, completed.Single(c => c.Item == "b").Count)); Assert.Equal(4, result.Rows); } + + /* ---------------- #2150: the per-item wall-clock budget ---------------- */ + + /// + /// A single slow database is abandoned and the rest of the cycle continues. THE property the field + /// report needs: two Azure SQL DB databases produced per-database passes of up to 99.8 minutes, and + /// because a host's live collectors run one after another, that one pass starved every other collector + /// on the server (#2148's "all collection stopped"). + /// + [Fact] + public async Task RunAsync_AnItemThatExceedsItsBudget_IsAbandoned_AndTheRestStillCollect() + { + var errors = new List<(string Item, string Message)>(); + var written = new List(); + + var result = await EnumeratedCollectorDriver.RunAsync( + new[] { "fast-a", "slow", "fast-b" }, + perItemWatermark: null, + readItem: async (item, ct) => + { + if (item == "slow") + { + /* Longer than any test would tolerate, so the budget is what ends it — and awaited on + the token so the wait is what gets cancelled rather than the delay elapsing. */ + await Task.Delay(TimeSpan.FromMinutes(5), ct); + } + + return new List { item.Length }; + }, + writeBatch: (batch, ct) => { written.AddRange(batch); return Task.CompletedTask; }, + onItemComplete: (item, count, sqlMs, storageMs) => { }, + onItemError: (item, ex) => errors.Add((item, ex.Message)), + CancellationToken.None, + perItemBudget: TimeSpan.FromMilliseconds(150)); + + /* The two healthy databases collected; the slow one did not, and nothing threw. */ + Assert.Equal(new[] { 6, 6 }, written); + Assert.Equal(2, result.Rows); + + /* And it reported ITSELF, with the budget named — a skipped database that says only "cancelled" + sends an operator looking for a shutdown that did not happen. */ + var failure = Assert.Single(errors); + Assert.Equal("slow", failure.Item); + Assert.Contains("wall-clock budget", failure.Message, StringComparison.Ordinal); + Assert.Contains("re-read next cycle", failure.Message, StringComparison.Ordinal); + /* And it names the actual NUMBER. Asserted because "contains 'wall-clock budget'" passed happily + against a first cut that rendered this 150 ms budget as "0.0-minute" — a message with no number + in it at all, on the one line an operator works from. */ + Assert.Contains("0.15-second", failure.Message, StringComparison.Ordinal); + } + + /// + /// The budget renders in the unit it was set in. The shipped value is 10 minutes, so a minutes-only + /// format would never have looked wrong in the field — but small values are what a person types while + /// diagnosing, which is exactly when the message matters. Found by a scratch harness, not by reading. + /// + [Theory] + [InlineData(600, "10-minute")] + [InlineData(90, "1.5-minute")] + [InlineData(60, "1-minute")] + [InlineData(59.5, "59.5-second")] + [InlineData(30, "30-second")] + [InlineData(0.15, "0.15-second")] + public void DescribeBudget_NamesANumberAtEveryScale(double seconds, string expected) + { + Assert.Equal(expected, EnumeratedCollectorDriver.DescribeBudget(TimeSpan.FromSeconds(seconds))); + } + + /// + /// Shutdown still propagates. The budget's whole risk is misreading a real cancellation as a per-item + /// skip, which would have the loop keep collecting through a service stop — so this is the arm that + /// makes the feature safe rather than the one that makes it work. + /// + [Fact] + public async Task RunAsync_HostShutdown_StillPropagates_EvenWithABudgetSet() + { + using var cts = new CancellationTokenSource(); + var errors = new List(); + + await Assert.ThrowsAsync(async () => + await EnumeratedCollectorDriver.RunAsync( + new[] { "a", "b" }, + perItemWatermark: null, + readItem: async (item, ct) => + { + await cts.CancelAsync(); + ct.ThrowIfCancellationRequested(); + return new List { 1 }; + }, + writeBatch: (batch, ct) => Task.CompletedTask, + onItemComplete: (item, count, sqlMs, storageMs) => { }, + onItemError: (item, ex) => errors.Add(item), + cts.Token, + /* Generous, so the ONLY cancelled token is the outer one — the ambiguous case is covered by + ItemBudgetExpired's own tests below. */ + perItemBudget: TimeSpan.FromMinutes(10))); + + Assert.Empty(errors); + } + + /// + /// No budget means the loop is what it always was: the delegates get the caller's own token, not a + /// linked one. Asserted by IDENTITY rather than by behaviour, because "no wrapper" is the property — + /// a linked token with no timer behaves identically and would hide an unnecessary allocation per item. + /// + [Fact] + public async Task RunAsync_WithNoBudget_PassesTheCallersOwnToken() + { + using var cts = new CancellationTokenSource(); + var seen = new List(); + + await EnumeratedCollectorDriver.RunAsync( + new[] { "a" }, + perItemWatermark: (item, ct) => { seen.Add(ct); return Task.CompletedTask; }, + readItem: (item, ct) => { seen.Add(ct); return Task.FromResult(new List()); }, + writeBatch: (batch, ct) => Task.CompletedTask, + onItemComplete: (item, count, sqlMs, storageMs) => { }, + onItemError: (item, ex) => { }, + cts.Token); + + Assert.Equal(2, seen.Count); + Assert.All(seen, token => Assert.Equal(cts.Token, token)); + } + + /// + /// The WRITE is outside the budget. Abandoning a flush already in flight would trade a slow cycle for a + /// partially-written one, which is the worse of the two — so the write gets the caller's token even when + /// the read was bounded. + /// + [Fact] + public async Task RunAsync_TheWrite_IsNotSubjectToTheItemBudget() + { + using var cts = new CancellationTokenSource(); + CancellationToken readToken = default, writeToken = default; + + await EnumeratedCollectorDriver.RunAsync( + new[] { "a" }, + perItemWatermark: null, + readItem: (item, ct) => { readToken = ct; return Task.FromResult(new List { 1 }); }, + writeBatch: (batch, ct) => { writeToken = ct; return Task.CompletedTask; }, + onItemComplete: (item, count, sqlMs, storageMs) => { }, + onItemError: (item, ex) => { }, + cts.Token, + perItemBudget: TimeSpan.FromMinutes(10)); + + Assert.NotEqual(cts.Token, readToken); // the read was bounded + Assert.Equal(cts.Token, writeToken); // the write was not + } + + /// + /// An OutOfMemoryException that fires while the budget has ALREADY expired must still propagate. + /// + /// The reason it is not obvious: + /// classifies on the TOKENS and never looks at the exception type — which is deliberate, because a + /// cancelled SqlClient command does not reliably arrive as an OperationCanceledException. The cost is + /// that the budget arm would happily claim an unrelated fatal exception, so its ORDERING behind the + /// OOM rethrow is load-bearing rather than stylistic. Review found both hosts' per-database loops + /// missing that ordering; this pins the shared driver's. + /// + [Fact] + public async Task RunAsync_AnOomWithAnExpiredBudget_StillPropagates() + { + var errors = new List(); + + await Assert.ThrowsAsync(async () => + await EnumeratedCollectorDriver.RunAsync( + new[] { "a" }, + perItemWatermark: null, + readItem: async (item, ct) => + { + /* Let the budget expire FIRST, then throw something unrelated and fatal. */ + try + { + await Task.Delay(TimeSpan.FromSeconds(5), ct); + } + catch (OperationCanceledException) + { + } + + throw new OutOfMemoryException("unrelated to the budget"); + }, + writeBatch: (batch, ct) => Task.CompletedTask, + onItemComplete: (item, count, sqlMs, storageMs) => { }, + onItemError: (item, ex) => errors.Add(item), + CancellationToken.None, + perItemBudget: TimeSpan.FromMilliseconds(100))); + + /* Not swallowed as a routine per-database timeout. */ + Assert.Empty(errors); + } + + /// + /// The classifier, all four combinations. It decides whether an exception is a per-item skip or a + /// shutdown, and it is deliberately asked of the TOKENS rather than of the exception type: cancelling a + /// SqlClient command does not reliably surface as an OperationCanceledException, so the type cannot + /// answer this. Shutdown must win the ambiguous case, or the loop keeps collecting through a stop. + /// + [Fact] + public void ItemBudgetExpired_TellsABudgetApartFromAShutdown() + { + using var outer = new CancellationTokenSource(); + using var budget = CancellationTokenSource.CreateLinkedTokenSource(outer.Token); + + /* Neither: an ordinary per-item failure, which the generic catch owns. */ + Assert.False(EnumeratedCollectorDriver.ItemBudgetExpired(budget, outer.Token)); + + /* No budget at all (every collector but query_store) can never be a budget expiry. */ + Assert.False(EnumeratedCollectorDriver.ItemBudgetExpired(null, outer.Token)); + + /* The budget alone: a per-item skip. */ + budget.Cancel(); + Assert.True(EnumeratedCollectorDriver.ItemBudgetExpired(budget, outer.Token)); + + /* Both — the race between a budget firing and a service stop. Shutdown wins. */ + outer.Cancel(); + Assert.False(EnumeratedCollectorDriver.ItemBudgetExpired(budget, outer.Token)); + } + + /// A zero or negative budget is treated as no budget rather than as an instantly-expired one, + /// so a misconfigured value degrades to today's behaviour instead of collecting nothing at all. + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void StartItemBudget_ANonPositiveBudget_IsNoBudget(int minutes) + { + Assert.Null(EnumeratedCollectorDriver.StartItemBudget( + TimeSpan.FromMinutes(minutes), CancellationToken.None)); + } + + /// + /// query_store is the collector that declares a budget, and the only one. The value is pinned because + /// it is a published constant with a measured justification (#2150): 4.8 s median and 31 s max over 198 + /// healthy field passes against 37.6 minutes at the low end of the pathological ones. + /// + [Fact] + public void OnlyQueryStore_DeclaresAWallClockBudget() + { + Assert.Equal(TimeSpan.FromMinutes(10), QueryStoreCollector.Instance.PerItemWallClockBudget); + Assert.Equal(QueryStoreCollector.PerDatabaseWallClockBudget, QueryStoreCollector.Instance.PerItemWallClockBudget); + + /* The siblings that also run per database stay unbounded: neither has an unbounded-input shape, and + a budget on a collector with no field evidence for one is a cut waiting to surprise somebody. */ + Assert.Null(DatabaseSizeStatsCollector.Instance.PerItemWallClockBudget); + Assert.Null(DatabaseScopedConfigCollector.Instance.PerItemWallClockBudget); + } } diff --git a/Lite.Tests/FileIoCollectorDefinitionTests.cs b/Lite.Tests/FileIoCollectorDefinitionTests.cs index 55422be79..287b14ac9 100644 --- a/Lite.Tests/FileIoCollectorDefinitionTests.cs +++ b/Lite.Tests/FileIoCollectorDefinitionTests.cs @@ -115,7 +115,7 @@ public void WritePayload_EmitsSchemaOrder_AndPinsDeltaKeyAndGroups() Assert.Equal(8, deltas.Calls.Count); Assert.All(deltas.Calls, c => Assert.Equal("SO|SO_data", c.Key)); - Assert.All(deltas.Calls, c => Assert.Equal(300, c.MaxGap)); + Assert.All(deltas.Calls, c => Assert.Equal(CollectorDeltaCalculator.DefaultMaxGapSeconds, c.MaxGap)); Assert.Equal( new[] { diff --git a/Lite.Tests/FinOpsVerdictSourcePinTests.cs b/Lite.Tests/FinOpsVerdictSourcePinTests.cs new file mode 100644 index 000000000..4b3394ce0 --- /dev/null +++ b/Lite.Tests/FinOpsVerdictSourcePinTests.cs @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2246, Lite's half. The provisioning verdict used to be duplicated SIX times — Darling's point-in-time, +/// trend and inventory reads, and Lite's three — each testing +/// total_server_memory_mb / target_server_memory_mb > 0.95, a ratio measured at median 1.0000 across +/// 42 production servers. Every server came out UNDER_PROVISIONED and OVER_PROVISIONED was unreachable. +/// +/// Why source pins rather than behavioural ones. Darling exposes its reads as +/// public const string, so its tests assert against the shipped SQL directly. Lite builds the same +/// queries as inline command.CommandText, so there is nothing to reference — the reads are only +/// reachable through a live DuckDB store. These pins therefore read the source, the same technique +/// exists for, and cover the two invariants a broken edit would trip: the pressure +/// inputs must be SELECTed, and the verdict must not be decided in SQL. +/// +/// Without them the identical mistake fails fast on Darling and silently on Lite, which is the drift +/// this whole issue is about — the shared predicate removed the duplicated LOGIC, but the two apps still +/// carry their own copies of the SQL that feeds it. +/// +public sealed class FinOpsVerdictSourcePinTests +{ + private const string Utilization = "Lite/Services/LocalDataService.FinOps.Utilization.cs"; + private const string Inventory = "Lite/Services/LocalDataService.FinOps.ServerProperties.cs"; + + /// Both Lite reads must fetch the workspace-memory pressure signals the shared predicate + /// consumes. A reader wired to ordinals the SELECT list no longer produces is the failure this catches + /// earliest. + [Theory] + [InlineData(Utilization)] + [InlineData(Inventory)] + public void BothLiteReads_FetchThePressureInputs(string path) + { + var source = ParitySource.ReadFile(path); + + Assert.Contains("FROM v_memory_grant_stats", source, StringComparison.Ordinal); + Assert.Contains("waiter_count", source, StringComparison.Ordinal); + Assert.Contains("timeout_error_count_delta", source, StringComparison.Ordinal); + Assert.Contains("forced_grant_count_delta", source, StringComparison.Ordinal); + Assert.Contains("granted_memory_mb", source, StringComparison.Ordinal); + Assert.Contains("max_workers_count", source, StringComparison.Ordinal); + } + + /// Both must classify through the shared predicate rather than deciding for themselves. The + /// inventory read in particular used to carry an inline SQL CASE, and it feeds the Server + /// Inventory grid — the screen the field report was looking at. + [Theory] + [InlineData(Utilization)] + [InlineData(Inventory)] + public void BothLiteReads_ClassifyThroughTheSharedPredicate(string path) + { + var source = ParitySource.ReadFile(path); + + Assert.Contains("ProvisioningVerdict.Evaluate(", source, StringComparison.Ordinal); + } + + /// The verdict must not be decided in SQL. These literals are how the inventory read used to do + /// it, so their absence is the guard against a SQL-side verdict coming back — which would silently + /// disagree with the drill-down for the same server. + [Theory] + [InlineData(Utilization)] + [InlineData(Inventory)] + public void NoLiteRead_DecidesTheVerdictInSql(string path) + { + var source = ParitySource.ReadFile(path); + + Assert.DoesNotContain("'OVER_PROVISIONED'", source, StringComparison.Ordinal); + Assert.DoesNotContain("'UNDER_PROVISIONED'", source, StringComparison.Ordinal); + Assert.DoesNotContain("'RIGHT_SIZED'", source, StringComparison.Ordinal); + } + + /// + /// The retired threshold itself, gone from every Lite FinOps read. + /// + /// 0.95 and 0.5 were the two ratio comparisons that produced the bug. Pinning the NUMBER rather + /// than the expression is deliberate: the defect survived being copied six times precisely because each + /// copy was spelled slightly differently, and a literal is what a copy carries unchanged. + /// + [Theory] + [InlineData(Utilization)] + [InlineData(Inventory)] + public void TheRetiredMemoryRatioThresholdIsGone(string path) + { + var source = ParitySource.ReadFile(path); + + Assert.DoesNotContain("> 0.95", source, StringComparison.Ordinal); + Assert.DoesNotContain("0.95m", source, StringComparison.Ordinal); + Assert.DoesNotContain("< 0.5 ", source, StringComparison.Ordinal); + Assert.DoesNotContain("0.5m)", source, StringComparison.Ordinal); + } +} diff --git a/Lite.Tests/ForcePlanReplicaScopeTests.cs b/Lite.Tests/ForcePlanReplicaScopeTests.cs index 05ad87db5..ae72273a0 100644 --- a/Lite.Tests/ForcePlanReplicaScopeTests.cs +++ b/Lite.Tests/ForcePlanReplicaScopeTests.cs @@ -468,5 +468,97 @@ the same state a non-AG server produces — rather than throwing or inventing a /* A legacy row renders exactly what it always rendered. */ var sql = FactRemediation.RenderCopyPasteCommand(restored); Assert.DoesNotContain("Measured on the", sql!, StringComparison.Ordinal); + /* #2138 gap 3: the flag defaults FALSE off legacy JSON, so no caution appears either. */ + Assert.DoesNotContain("CAUTION", sql!, StringComparison.Ordinal); + } + + /* ---------------- #2138 gap 3: the parameter-sensitivity caution ---------------- */ + + [Fact] + public void PspCoFiredTarget_CarriesTheFlag_AndRendersTheCaution() + { + var row = Row(queryId: 123, bestPlanId: 99, regressionFactor: 12.0, replicaRole: null); + row["parameter_sensitivity_cofired"] = true; + var finding = PlanRegressionFinding(row); + + var target = Assert.Single(FactRemediation.ExtractPlanRegressionTargets(finding)); + Assert.True(target.ParameterSensitivityCoFired); + + var sql = FactRemediation.GenerateForFinding(finding); + Assert.NotNull(sql); + Assert.Contains("CAUTION: this query also shows the parameter-sensitivity signature", sql!, StringComparison.Ordinal); + /* The gentler levers are NAMED — the caution is advice with a next step, not just a wince. */ + Assert.Contains("update statistics", sql, StringComparison.Ordinal); + + /* The caution is comment-only: the runnable statement is byte-identical to the unflagged one. */ + var runnableLines = sql!.Split('\n') + .Select(line => line.Trim()) + .Where(line => line.Length > 0 && !line.StartsWith("--", StringComparison.Ordinal)); + Assert.Contains("EXEC sys.sp_query_store_force_plan @query_id = 123, @plan_id = 99;", runnableLines); + } + + [Fact] + public void UnflaggedTarget_RendersExactly_WhatItRenderedBeforeTheFlagExisted() + { + /* An absent key (an old persisted drill-down, the deprecated Dashboard) and an explicit false + must render byte-identically — and contain no caution — so the render-stability discipline + the #1882 replica disclosure established holds for this flag too. */ + var absent = PlanRegressionFinding( + Row(queryId: 123, bestPlanId: 99, regressionFactor: 12.0, replicaRole: null)); + + var flaggedFalse = Row(queryId: 123, bestPlanId: 99, regressionFactor: 12.0, replicaRole: null); + flaggedFalse["parameter_sensitivity_cofired"] = false; + + var absentSql = FactRemediation.GenerateForFinding(absent); + var falseSql = FactRemediation.GenerateForFinding(PlanRegressionFinding(flaggedFalse)); + + Assert.NotNull(absentSql); + Assert.Equal(absentSql, falseSql); + Assert.DoesNotContain("CAUTION", absentSql!, StringComparison.Ordinal); + } + + [Fact] + public void PspCoFiredFlag_SurvivesThePersistedActionRoundTrip() + { + /* Review catch on #2140: the flag existed on ForcePlanTarget but not on its JSON mirror + (ForcePlanTargetDto), so SerializeAction dropped it on the FIRST write and every read + reconstructed false — and both apps render the copy-paste command from the DESERIALIZED + action, so the caution never reached the pasted surface at all, and a future bot reading + persisted actions would have seen false for every flagged target. This is the test that + was missing: a TRUE flag through the actual persistence path. */ + var row = Row(queryId: 123, bestPlanId: 99, regressionFactor: 12.0, replicaRole: null); + row["parameter_sensitivity_cofired"] = true; + + var action = FactRemediation.BuildAction(PlanRegressionFinding(row)); + Assert.NotNull(action); + + var json = AlertContextSerializer.SerializeAction(action!); + var restored = AlertContextSerializer.DeserializeAction(json); + + Assert.NotNull(restored); + Assert.True(Assert.Single(restored!.Targets).ParameterSensitivityCoFired); + + /* And the surface that gets executed renders the caution from the RESTORED action. */ + var sql = FactRemediation.RenderCopyPasteCommand(restored); + Assert.Contains("CAUTION: parameter-sensitive", sql!, StringComparison.Ordinal); + } + + [Fact] + public void PspCoFiredTarget_CautionAlsoRidesTheCopyPasteSurface() + { + /* The paste surface is the one that gets EXECUTED, so the warning must survive the trip through + the persisted action — flag into BuildAction, out through RenderCopyPasteCommand — in its + compact two-line form, with the runnable statement untouched. */ + var row = Row(queryId: 123, bestPlanId: 99, regressionFactor: 12.0, replicaRole: null); + row["parameter_sensitivity_cofired"] = true; + + var action = FactRemediation.BuildAction(PlanRegressionFinding(row)); + Assert.NotNull(action); + Assert.True(Assert.Single(action!.Targets).ParameterSensitivityCoFired); + + var sql = FactRemediation.RenderCopyPasteCommand(action); + Assert.NotNull(sql); + Assert.Contains("CAUTION: parameter-sensitive", sql!, StringComparison.Ordinal); + Assert.Contains("EXEC sys.sp_query_store_force_plan @query_id = 123, @plan_id = 99;", sql, StringComparison.Ordinal); } } diff --git a/Lite.Tests/GenericWebhookTests.cs b/Lite.Tests/GenericWebhookTests.cs index f905d0ee7..21e515419 100644 --- a/Lite.Tests/GenericWebhookTests.cs +++ b/Lite.Tests/GenericWebhookTests.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Net.Http; using System.Text.Json; +using PerformanceMonitor.Analysis; using PerformanceMonitor.Notifications; using Xunit; @@ -266,4 +267,158 @@ public void ApplyHeaders_AddsAnOrdinaryOperatorHeader() Assert.Equal("Bearer ghp_token", string.Join("", request.Headers.GetValues("Authorization"))); } + + /* ---------------- #2302: the automation tokens ---------------- */ + + private static AlertContext TwoIncidentContext() + { + var context = new AlertContext(); + context.Details.Add(new AlertDetailItem + { + Heading = "Deadlock 1", + /* The motivating delimiter collision: {{context}}'s flattening joins on " | " and ": ", + and this value contains BOTH — plus a quote — so only structure can carry it. */ + Fields = { ("Victim SQL", "SELECT a | b FROM t WHERE x = 'y: \"z\"'") } + }); + context.Incidents = new List + { + new("aa11", new List { "db1.dbo.t1" }, OccurrenceCount: 3, TotalOccurrences: 17, + IncidentStartedUtc: new System.DateTime(2026, 8, 17, 10, 0, 0, System.DateTimeKind.Utc)), + new("bb22", new List { "db2.dbo.t2" }), + }; + return context; + } + + [Fact] + public void ContextJsonToken_SubstitutesRawStructure_InThePersistedContextJsonShape() + { + const string template = """{"metric": "{{metric}}", "context": {{context_json}}}"""; + var payload = WebhookAlertService.BuildGenericPayload( + "Deadlocks Detected", "SRV", "2", "0", Branding, context: TwoIncidentContext(), bodyTemplate: template); + + var root = JsonDocument.Parse(payload).RootElement; + var contextElement = root.GetProperty("context"); + + /* Structure, not a string — the whole point of the token. */ + Assert.Equal(JsonValueKind.Object, contextElement.ValueKind); + var incidents = contextElement.GetProperty("Incidents"); + Assert.Equal(2, incidents.GetArrayLength()); + Assert.Equal("aa11", incidents[0].GetProperty("DedupKey").GetString()); + Assert.Equal(17, incidents[0].GetProperty("TotalOccurrences").GetInt64()); + + /* The hostile Victim SQL arrives as a field VALUE, byte-exact — no delimiter parsing needed. */ + var field = contextElement.GetProperty("Details")[0].GetProperty("Fields")[0]; + Assert.Equal("SELECT a | b FROM t WHERE x = 'y: \"z\"'", field.GetProperty("Value").GetString()); + + /* One shape for every consumer: the embedded JSON is EXACTLY what the alert-history + ContextJson persists, so it must round-trip through the same serializer. */ + Assert.True(AlertContextSerializer.TryDeserialize(contextElement.GetRawText(), out var roundTripped)); + Assert.Equal(2, roundTripped.Incidents!.Count); + Assert.Equal("bb22", roundTripped.Incidents[1].DedupKey); + } + + [Fact] + public void ContextJsonToken_IsAnEmptyObject_WhenTheAlertCarriesNoContext() + { + const string template = """{"context": {{context_json}}, "incidents": {{incidents_json}}}"""; + var payload = Build("High CPU", "SRV", template: template); + + var root = JsonDocument.Parse(payload).RootElement; + Assert.Equal(JsonValueKind.Object, root.GetProperty("context").ValueKind); + Assert.Equal(0, root.GetProperty("incidents").GetArrayLength()); + } + + [Fact] + public void IncidentsJsonToken_IsJustTheArray() + { + const string template = """{"incidents": {{incidents_json}}}"""; + var payload = WebhookAlertService.BuildGenericPayload( + "Deadlocks Detected", "SRV", "2", "0", Branding, context: TwoIncidentContext(), bodyTemplate: template); + + var incidents = JsonDocument.Parse(payload).RootElement.GetProperty("incidents"); + Assert.Equal(2, incidents.GetArrayLength()); + Assert.Equal("db1.dbo.t1", incidents[0].GetProperty("InvolvedObjects")[0].GetString()); + } + + [Fact] + public void DedupKeyToken_MatchesThePagerDutyDerivation_IncludingTheFallback() + { + const string template = """{"dedup": "{{dedup_key}}"}"""; + + /* With an incident: the first incident's fingerprint, same anchor PagerDuty correlates on. */ + var withIncident = WebhookAlertService.BuildGenericPayload( + "Deadlocks Detected", "SRV", "2", "0", Branding, context: TwoIncidentContext(), + bodyTemplate: template, serverId: "37"); + Assert.Equal("aa11", JsonDocument.Parse(withIncident).RootElement.GetProperty("dedup").GetString()); + + /* Without one: the stable metric+server fallback — the key level/threshold alerts (High CPU, + Collection Stopped) never exposed before, keyed on serverId when the caller has one. */ + var fallback = WebhookAlertService.BuildGenericPayload( + "High CPU", "SRV", "97", "90", Branding, bodyTemplate: template, serverId: "37"); + Assert.Equal("37:High CPU", JsonDocument.Parse(fallback).RootElement.GetProperty("dedup").GetString()); + + /* And the serverName stands in when no id exists — the PagerDuty call site's own precedence. */ + var byName = Build("High CPU", "SRV", template: template); + Assert.Equal("SRV:High CPU", JsonDocument.Parse(byName).RootElement.GetProperty("dedup").GetString()); + } + + [Fact] + public void ContextJsonToken_RedactsRemediationTsql_LikeEveryOtherChannel() + { + /* The review catch: every channel replaces copy-paste T-SQL with the "see email / in-app dialog" + hint before anything leaves the process, and the raw token must not be the one exception. */ + var context = TwoIncidentContext(); + context.Details.Add(new AlertDetailItem + { + Heading = "Remediation T-SQL", + Body = "ALTER DATABASE [x] SET READ_COMMITTED_SNAPSHOT ON;", + IsCodeBlock = true, + Remediation = new RemediationAction("RCSI_OFF", "DB_CONFIG", new List()) + }); + + const string template = """{"context": {{context_json}}}"""; + var payload = WebhookAlertService.BuildGenericPayload( + "Deadlocks Detected", "SRV", "2", "0", Branding, context: context, bodyTemplate: template); + + /* The T-SQL never reaches the wire; the hint and the code-block FLAG do, so a consumer still + learns a remediation exists and where to read it. */ + Assert.DoesNotContain("ALTER DATABASE", payload, System.StringComparison.Ordinal); + Assert.DoesNotContain("RCSI_OFF", payload, System.StringComparison.Ordinal); + var details = JsonDocument.Parse(payload).RootElement.GetProperty("context").GetProperty("Details"); + var codeBlock = details[1]; + Assert.True(codeBlock.GetProperty("IsCodeBlock").GetBoolean()); + Assert.Contains("in-app Alert Details", codeBlock.GetProperty("Body").GetString(), System.StringComparison.Ordinal); + Assert.Equal(JsonValueKind.Null, codeBlock.GetProperty("Remediation").ValueKind); + + /* Redaction copies — the SAME context instance flows on to email/Teams afterwards, and those + channels legitimately carry the T-SQL (email) or their own hint. Mutation here would be a + cross-channel bug this pin exists to forbid. */ + Assert.Equal("ALTER DATABASE [x] SET READ_COMMITTED_SNAPSHOT ON;", context.Details[1].Body); + Assert.NotNull(context.Details[1].Remediation); + } + + [Fact] + public void ValidateGenericConfig_RejectsAQuotedRawToken_AtSaveTime() + { + /* The trap this forbids (#2310 review catch): with a null context the raw tokens render to the + quote-free {} / [], so `"context": "{{context_json}}"` — the exact mistake the doc comment + warns about — used to validate clean and test-send clean, then break on the first deadlock + alert with real structure. The stand-in context is quote-bearing by construction, so the + mistake now fails where the operator can see it. */ + Assert.NotNull(WebhookAlertService.ValidateGenericConfig(null, """{"context": "{{context_json}}"}""")); + Assert.NotNull(WebhookAlertService.ValidateGenericConfig(null, """{"incidents": "{{incidents_json}}"}""")); + + /* And the CORRECT unquoted usage still validates. */ + Assert.Null(WebhookAlertService.ValidateGenericConfig(null, """{"context": {{context_json}}, "incidents": {{incidents_json}}, "dedup": "{{dedup_key}}"}""")); + } + + [Fact] + public void DefaultTemplate_DoesNotCarryTheAutomationTokens() + { + /* #2302's compatibility promise: the shipped default stays byte-identical, so no existing + consumer's payload changes shape. The automation tokens are opt-in via a custom template. */ + Assert.DoesNotContain("context_json", WebhookAlertService.DefaultGenericBodyTemplate, System.StringComparison.Ordinal); + Assert.DoesNotContain("incidents_json", WebhookAlertService.DefaultGenericBodyTemplate, System.StringComparison.Ordinal); + Assert.DoesNotContain("dedup_key", WebhookAlertService.DefaultGenericBodyTemplate, System.StringComparison.Ordinal); + } } diff --git a/Lite.Tests/GoldenCollectorSchema.cs b/Lite.Tests/GoldenCollectorSchema.cs index 42cf0c255..30c43ead0 100644 --- a/Lite.Tests/GoldenCollectorSchema.cs +++ b/Lite.Tests/GoldenCollectorSchema.cs @@ -318,6 +318,22 @@ is_session BOOLEAN NOT NULL configuration_name VARCHAR NOT NULL, value VARCHAR, value_for_secondary VARCHAR +)", + ["query_store_health"] = @"CREATE TABLE IF NOT EXISTS query_store_health ( + config_id BIGINT PRIMARY KEY, + capture_time TIMESTAMP NOT NULL, + server_id INTEGER NOT NULL, + server_name VARCHAR NOT NULL, + database_name VARCHAR NOT NULL, + actual_state VARCHAR, + desired_state VARCHAR, + readonly_reason INTEGER, + current_storage_size_mb BIGINT, + max_storage_size_mb BIGINT, + size_based_cleanup_mode VARCHAR, + stale_query_threshold_days BIGINT, + max_plans_per_query BIGINT, + interval_length_minutes BIGINT )", ["session_stats"] = @"CREATE TABLE IF NOT EXISTS session_stats ( collection_id BIGINT PRIMARY KEY, @@ -1004,6 +1020,7 @@ is_in_standby BOOLEAN ["server_properties"] = @"CREATE INDEX IF NOT EXISTS idx_server_properties_time ON server_properties(server_id, collection_time)", ["trace_flags"] = @"CREATE INDEX IF NOT EXISTS idx_trace_flags_time ON trace_flags(server_id, capture_time)", ["database_scoped_config"] = @"CREATE INDEX IF NOT EXISTS idx_database_scoped_config_time ON database_scoped_config(server_id, capture_time)", + ["query_store_health"] = @"CREATE INDEX IF NOT EXISTS idx_query_store_health_time ON query_store_health(server_id, capture_time)", ["session_stats"] = @"CREATE INDEX IF NOT EXISTS idx_session_stats_time ON session_stats(server_id, collection_time)", ["session_summary_stats"] = @"CREATE INDEX IF NOT EXISTS idx_session_summary_stats_time ON session_summary_stats(server_id, collection_time)", ["waiting_tasks"] = @"CREATE INDEX IF NOT EXISTS idx_waiting_tasks_time ON waiting_tasks(server_id, collection_time)", diff --git a/Lite.Tests/Helpers/CollectorDefinitionTestFakes.cs b/Lite.Tests/Helpers/CollectorDefinitionTestFakes.cs index efa469fe9..5d77e2790 100644 --- a/Lite.Tests/Helpers/CollectorDefinitionTestFakes.cs +++ b/Lite.Tests/Helpers/CollectorDefinitionTestFakes.cs @@ -125,6 +125,11 @@ internal sealed class RecordingCollectorDeltaCalculator : ICollectorDeltaCalcula public int LastServerId { get; private set; } + /// What hands back. Left at 0 so every existing + /// pin is unaffected; set it to something distinctive to prove a collector writes the MEASURED + /// interval rather than a constant of its own (#2234, where perfmon wrote a literal 60). + public int ReportedInterval { get; set; } + public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, DateTime? collectionTime = null, int maxGapSeconds = 0) { @@ -136,7 +141,25 @@ public long CalculateDelta(int serverId, string collectorName, string key, long public long CalculateDeltaWithInterval(int serverId, string collectorName, string key, long currentValue, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) { - intervalSeconds = 0; + intervalSeconds = ReportedInterval; return CalculateDelta(serverId, collectorName, key, currentValue, collectionTime, maxGapSeconds); } + + /// + /// Series ages seen by (#2235), in call order. + /// + /// Overridden rather than left to the interface's default implementation on purpose: the default + /// forwards to and DROPS the age, so a collector that stopped + /// passing it would keep every existing pin green. Recording it here is what makes that regression + /// visible. + /// + public List SeriesAges { get; } = new(); + + public long CalculateDeltaWithSeriesAge(int serverId, string collectorName, string key, long currentValue, + int? seriesAgeSeconds, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + { + SeriesAges.Add(seriesAgeSeconds); + return CalculateDeltaWithInterval(serverId, collectorName, key, currentValue, out intervalSeconds, + collectionTime, maxGapSeconds); + } } diff --git a/Lite.Tests/IncidentGroupingTests.cs b/Lite.Tests/IncidentGroupingTests.cs index 247325a10..50e16f5cd 100644 --- a/Lite.Tests/IncidentGroupingTests.cs +++ b/Lite.Tests/IncidentGroupingTests.cs @@ -149,4 +149,31 @@ public void DeadlockObjectExtractor_MalformedOrEmpty_ReturnsEmpty() Assert.Empty(DeadlockObjectExtractor.FromGraphXml("not xml <<<")); Assert.Empty(DeadlockObjectExtractor.FromGraphXml("")); } + + [Fact] + public void DeadlockObjectExtractor_PullsDatabasesFromProcessCurrentDbName() + { + /* #2109: the Database fact's source — the processes' currentdbname, distinct + sorted, + case-insensitively deduped. A cross-database deadlock lists every database a process ran + in, which is the "where did this happen" answer, not the lock list's "what was locked". */ + const string xml = @" + + + + + + +"; + + var databases = DeadlockObjectExtractor.DatabasesFromGraphXml(xml); + Assert.Equal(new[] { "Archive", "SalesDB" }, databases); // distinct (case-insensitive) + sorted + } + + [Fact] + public void DeadlockObjectExtractor_Databases_MalformedOrDatabaseless_ReturnsEmpty() + { + Assert.Empty(DeadlockObjectExtractor.DatabasesFromGraphXml(null)); + Assert.Empty(DeadlockObjectExtractor.DatabasesFromGraphXml("not xml <<<")); + Assert.Empty(DeadlockObjectExtractor.DatabasesFromGraphXml("")); + } } diff --git a/Lite.Tests/LatchStatsCollectorDefinitionTests.cs b/Lite.Tests/LatchStatsCollectorDefinitionTests.cs index 6a21bbf75..e53f62df3 100644 --- a/Lite.Tests/LatchStatsCollectorDefinitionTests.cs +++ b/Lite.Tests/LatchStatsCollectorDefinitionTests.cs @@ -102,11 +102,11 @@ public void WritePayload_EmitsPayloadOrder_AndPinsDeltaGroupsKeysAndGapPolicy() /* Payload order: raw values then the three deltas (recording calculator returns value * 10). */ Assert.Equal(new object?[] { "BUFFER", 7L, 300L, 20L, 70L, 3000L, 200L }, writer.Values); - /* Delta contract: group names, key = latch_class, the host collection time, 300 s gap policy. */ + /* Delta contract: group names, key = latch_class, the host collection time, the shared gap policy. */ Assert.Equal(3, deltas.Calls.Count); - Assert.Equal(("latch_stats_waiting_requests", "BUFFER", 7L, context.CollectionTime, 300), deltas.Calls[0]); - Assert.Equal(("latch_stats_wait_time", "BUFFER", 300L, context.CollectionTime, 300), deltas.Calls[1]); - Assert.Equal(("latch_stats_max_wait", "BUFFER", 20L, context.CollectionTime, 300), deltas.Calls[2]); + Assert.Equal(("latch_stats_waiting_requests", "BUFFER", 7L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[0]); + Assert.Equal(("latch_stats_wait_time", "BUFFER", 300L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[1]); + Assert.Equal(("latch_stats_max_wait", "BUFFER", 20L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[2]); Assert.All(deltas.Calls, _ => Assert.Equal(42, deltas.LastServerId)); } } diff --git a/Lite.Tests/Lite.Tests.csproj b/Lite.Tests/Lite.Tests.csproj index 16372892d..c058a8f42 100644 --- a/Lite.Tests/Lite.Tests.csproj +++ b/Lite.Tests/Lite.Tests.csproj @@ -12,12 +12,12 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Lite.Tests/LiteAlertForwardingTests.cs b/Lite.Tests/LiteAlertForwardingTests.cs index a9555f794..d270ee12b 100644 --- a/Lite.Tests/LiteAlertForwardingTests.cs +++ b/Lite.Tests/LiteAlertForwardingTests.cs @@ -152,6 +152,12 @@ public Task GetAnomalousJobsAsync(string serverKey, int mul public Task> GetDatabaseStatesAsync(string serverKey, CancellationToken cancellationToken = default) => Task.FromResult(new List(DatabaseStates)); + + /* #2157: settable so a forwarding test can plant a risen-counter row. */ + public List ForcePlanFailures { get; } = new(); + + public Task> GetForcePlanFailuresAsync(string serverKey, CancellationToken cancellationToken = default) => + Task.FromResult(new List(ForcePlanFailures)); } private sealed class InMemoryStateStore : IAlertStateStore @@ -178,6 +184,44 @@ public Task SaveFailedJobWatermarkAsync(string serverKey, DateTime watermark) FailedJobWatermarks[serverKey] = watermark; return Task.CompletedTask; } + + /* #2216: replace-the-set, exactly like both real stores — whatever arrives IS the metric's state, so + an empty map clears it. A fake that merged instead would hide the falling-edge bug class. */ + public Dictionary<(string Key, string Metric), Dictionary> Occurrences { get; } = new(); + + public Task> LoadIncidentOccurrencesAsync(string serverKey, string metricName) => + Task.FromResult>( + Occurrences.TryGetValue((serverKey, metricName), out var states) + ? states + : new Dictionary(StringComparer.Ordinal)); + + public Task SaveIncidentOccurrencesAsync(string serverKey, string metricName, IReadOnlyDictionary states) + { + var replacement = new Dictionary(StringComparer.Ordinal); + foreach (var entry in states) + { + replacement[entry.Key] = entry.Value; + } + Occurrences[(serverKey, metricName)] = replacement; + return Task.CompletedTask; + } + + /* #2166 */ + public List<(string Server, string Db, string State)> DatabaseStateAlerted { get; } = new(); + + public Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState) + { + DatabaseStateAlerted.Add((serverKey, databaseName, effectiveState)); + return Task.CompletedTask; + } + + public List<(string Server, string Db)> DatabaseStateCleared { get; } = new(); + + public Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName) + { + DatabaseStateCleared.Add((serverKey, databaseName)); + return Task.CompletedTask; + } } private sealed class RecordingDeliverer : IAlertDeliverer diff --git a/Lite.Tests/LiteDeltaSeederTests.cs b/Lite.Tests/LiteDeltaSeederTests.cs index d99ee5500..2fd4b7170 100644 --- a/Lite.Tests/LiteDeltaSeederTests.cs +++ b/Lite.Tests/LiteDeltaSeederTests.cs @@ -162,7 +162,7 @@ public async Task Seed_RowOlderThanTheLookback_IsNotRead() /// /// The memory-grant baselines carry their collection_time, so the gap policy can reject a stale /// one. Seeded with a null timestamp (as they were before #1772) the policy cannot fire at all, - /// and the row below — inside the 15-minute read window but well outside the 300-second gap — + /// and the row below — inside the 15-minute read window but well outside the gap passed below — /// produces a fabricated spike of 40 instead of 0 on the first cycle after a restart. /// [Fact] diff --git a/Lite.Tests/LiteServerTagsStoreTests.cs b/Lite.Tests/LiteServerTagsStoreTests.cs index 1ff2e9240..e64d9101a 100644 --- a/Lite.Tests/LiteServerTagsStoreTests.cs +++ b/Lite.Tests/LiteServerTagsStoreTests.cs @@ -92,10 +92,10 @@ public async Task Assign_IsIdempotent_AndUnassignAndClearRemoveRows() Assert.Equal(2, assigned.Count(a => a.TagId == tag)); await _service.UnassignServerTagAsync(new[] { 100 }, tag); - Assert.Single((await _service.GetServerTagAssignmentsAsync()).Where(a => a.TagId == tag)); + Assert.Single(await _service.GetServerTagAssignmentsAsync(), a => a.TagId == tag); await _service.ClearServerTagsForServerAsync(200); - Assert.Empty((await _service.GetServerTagAssignmentsAsync()).Where(a => a.TagId == tag)); + Assert.DoesNotContain(await _service.GetServerTagAssignmentsAsync(), a => a.TagId == tag); } [Fact] diff --git a/Lite.Tests/LongQueryCompletionsCollectorDefinitionTests.cs b/Lite.Tests/LongQueryCompletionsCollectorDefinitionTests.cs index 0cc530318..ac0b7b672 100644 --- a/Lite.Tests/LongQueryCompletionsCollectorDefinitionTests.cs +++ b/Lite.Tests/LongQueryCompletionsCollectorDefinitionTests.cs @@ -255,11 +255,16 @@ public void BuildCreateSessionSql_ServerScoped_PredicateOnCompletedOnly_Attentio /* The duration predicate fires on the two COMPLETED events (twice), never on attention. */ Assert.Equal(2, System.Text.RegularExpressions.Regex.Matches(sql, "WHERE duration >= 2000000").Count); - /* Customizable text/object columns must be turned on or they arrive NULL. */ + /* Customizable text columns must be turned on or they arrive NULL. */ Assert.Contains("collect_statement = 1", sql, StringComparison.Ordinal); - Assert.Contains("collect_object_name = 1", sql, StringComparison.Ordinal); Assert.Contains("collect_batch_text = 1", sql, StringComparison.Ordinal); + /* #2129 — the field failure this pin used to ENFORCE: rpc_completed has no customizable + collect_object_name attribute (that one is sp_statement_completed's), so SETting it failed + the CREATE on every server and the session never existed. object_name is one of + rpc_completed's default data fields and arrives with no SET at all. */ + Assert.DoesNotContain("collect_object_name", sql, StringComparison.Ordinal); + /* All 9 QuickSessionStandard actions, including nt_username server-scoped + package0.event_sequence. */ Assert.Contains("sqlserver.nt_username", sql, StringComparison.Ordinal); Assert.Contains("package0.event_sequence", sql, StringComparison.Ordinal); diff --git a/Lite.Tests/McpAnalysisFindingsCommandTests.cs b/Lite.Tests/McpAnalysisFindingsCommandTests.cs index da7d17859..2c2a24dce 100644 --- a/Lite.Tests/McpAnalysisFindingsCommandTests.cs +++ b/Lite.Tests/McpAnalysisFindingsCommandTests.cs @@ -101,15 +101,31 @@ enabling ALTER — proving the risk disclosure rides along through the MCP envel rootFactKey: "SOS_SCHEDULER_YIELD", storyPathHash: "reco_none_hash", remediation: null); + /* #2138: a FORCE finding with a PSP-flagged target. Asserting its verdict at the WIRE — after + the persisted action round-trips the store — proves both the one-line tool wiring (a dropped + field here is invisible to the builder's unit pins) and, incidentally, that the #2140 DTO + mirror really does carry the flag through persistence. */ + var force = MakeFinding( + findingId: 900003, analysisTime, severity: 1.6, + rootFactKey: "PLAN_REGRESSION", storyPathHash: "reco_force_hash", + remediation: new RemediationAction( + "PLAN_REGRESSION", "force", + new[] + { + new ForcePlanTarget( + "MyDb", 123, 99, "0xBEST", "0xLATEST", 9000, 1200, 7.5, + ReplicaRole: null, ParameterSensitivityCoFired: true) + })); + await store.InsertFindingsAsync( - new List { destructive, nonRemediable }, context); + new List { destructive, nonRemediable, force }, context); var json = await McpAnalysisTools.GetAnalysisFindings( new AnalysisService(_duckDb), _serverManager, "TestServer", 24); using var doc = JsonDocument.Parse(json); var findings = doc.RootElement.GetProperty("findings").EnumerateArray().ToList(); - Assert.Equal(2, findings.Count); + Assert.Equal(3, findings.Count); /* The field is present on EVERY finding (JsonOptions does not ignore nulls). */ Assert.All(findings, f => Assert.True(f.TryGetProperty("remediation_command", out _), @@ -133,6 +149,26 @@ then the enabling ALTER — byte-identical to the shared renderer the viewer car /* Non-remediable finding: the field is present but null (no command). */ var none = findings.Single(f => f.GetProperty("story_path_hash").GetString() == "reco_none_hash"); Assert.Equal(JsonValueKind.Null, none.GetProperty("remediation_command").ValueKind); + + /* #2138: structured_remediation rides every finding (present-but-null off the force verb)... */ + Assert.All(findings, f => Assert.True(f.TryGetProperty("structured_remediation", out _), + "every finding must expose structured_remediation")); + Assert.Equal(JsonValueKind.Null, rcsi.GetProperty("structured_remediation").ValueKind); + Assert.Equal(JsonValueKind.Null, none.GetProperty("structured_remediation").ValueKind); + + /* ...and the force finding's verdict crosses the wire intact: ineligible, the blocker NAMED, + the artifacts split. This is the flag surviving FindingStore -> AlertContextSerializer -> + tool projection end to end, not just the builder's unit pins. */ + var forced = findings.Single(f => f.GetProperty("story_path_hash").GetString() == "reco_force_hash"); + var structured = forced.GetProperty("structured_remediation"); + Assert.Equal("force", structured.GetProperty("verb").GetString()); + var target = Assert.Single(structured.GetProperty("force_plan_targets").EnumerateArray()); + Assert.False(target.GetProperty("eligible").GetBoolean()); + Assert.Equal("parameter_sensitivity_cofired", + Assert.Single(target.GetProperty("blockers").EnumerateArray()).GetString()); + Assert.Contains("sp_query_store_force_plan", target.GetProperty("force_sql").GetString()!, StringComparison.Ordinal); + Assert.Contains("force_failure_count", target.GetProperty("verify_sql").GetString()!, StringComparison.Ordinal); + Assert.Equal(7.5, target.GetProperty("evidence").GetProperty("regression_factor").GetDouble()); } /// diff --git a/Lite.Tests/MemoryGrantsCollectorDefinitionTests.cs b/Lite.Tests/MemoryGrantsCollectorDefinitionTests.cs index 904612a69..5a0f54def 100644 --- a/Lite.Tests/MemoryGrantsCollectorDefinitionTests.cs +++ b/Lite.Tests/MemoryGrantsCollectorDefinitionTests.cs @@ -17,7 +17,7 @@ namespace Lite.Tests; /// /// Pins the parity contract of the extracted memory_grant_stats definition: column mapping, -/// the composite "{pool}_{semaphore}" delta key, the two delta groups with the 300 s gap +/// the composite "{pool}_{semaphore}" delta key, the two delta groups with the shared gap /// policy, and the payload order matching the memory_grant_stats schema. /// public sealed class MemoryGrantsCollectorDefinitionTests @@ -84,9 +84,9 @@ public void WritePayload_EmitsSchemaOrder_AndPinsCompositeDeltaKey() new object?[] { (short)1, 2, 100.5m, 200.5m, 90.25m, 80.75m, 10.5m, 8.25m, 3, 4, 5L, 6L, 50L, 60L }, writer.Values); - /* Delta contract: composite key "{pool}_{semaphore}", both groups, 300 s gap policy. */ + /* Delta contract: composite key "{pool}_{semaphore}", both groups, the shared gap policy. */ Assert.Equal(2, deltas.Calls.Count); - Assert.Equal(("memory_grants_timeouts", "2_1", 5L, context.CollectionTime, 300), deltas.Calls[0]); - Assert.Equal(("memory_grants_forced", "2_1", 6L, context.CollectionTime, 300), deltas.Calls[1]); + Assert.Equal(("memory_grants_timeouts", "2_1", 5L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[0]); + Assert.Equal(("memory_grants_forced", "2_1", 6L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[1]); } } diff --git a/Lite.Tests/PerfmonAndDmvBlockingCollectorDefinitionTests.cs b/Lite.Tests/PerfmonAndDmvBlockingCollectorDefinitionTests.cs index 72fa5a4ad..ee3517a90 100644 --- a/Lite.Tests/PerfmonAndDmvBlockingCollectorDefinitionTests.cs +++ b/Lite.Tests/PerfmonAndDmvBlockingCollectorDefinitionTests.cs @@ -65,9 +65,11 @@ public void BuildQuery_OverrideWins_AndEscapesQuotes() } [Fact] - public async Task WritePayload_PinsDeltaContract_AndConstantInterval() + public async Task WritePayload_PinsDeltaContract_AndTheMeasuredInterval() { - var deltas = new RecordingCollectorDeltaCalculator(); + /* A distinctive interval, deliberately neither 0 nor the 60 this collector used to hard-code, + so the payload assertion below can only pass if the MEASURED value is what gets written. */ + var deltas = new RecordingCollectorDeltaCalculator { ReportedInterval = 137 }; var context = CollectorTestContext.Make(deltas); using var reader = new FakeCollectorDataReader( new object[] { "SQLServer:SQL Statistics", "Batch Requests/sec", "", 987654L }); @@ -76,9 +78,9 @@ public async Task WritePayload_PinsDeltaContract_AndConstantInterval() var writer = new RecordingCollectorRowWriter(); PerfmonStatsCollector.Instance.WritePayload(Assert.Single(rows), writer, context); - Assert.Equal(new object?[] { "SQLServer:SQL Statistics", "Batch Requests/sec", "", 987654L, 9876540L, 60 }, writer.Values); + Assert.Equal(new object?[] { "SQLServer:SQL Statistics", "Batch Requests/sec", "", 987654L, 9876540L, 137 }, writer.Values); var call = Assert.Single(deltas.Calls); - Assert.Equal(("perfmon", "SQLServer:SQL Statistics|Batch Requests/sec|", 987654L, context.CollectionTime, 300), call); + Assert.Equal(("perfmon", "SQLServer:SQL Statistics|Batch Requests/sec|", 987654L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), call); } } diff --git a/Lite.Tests/PerfmonIntervalAggregationTests.cs b/Lite.Tests/PerfmonIntervalAggregationTests.cs new file mode 100644 index 000000000..f95ce20ba --- /dev/null +++ b/Lite.Tests/PerfmonIntervalAggregationTests.cs @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2234: Lite's half of the "a zero delta must be readable" contract, and the aggregate that makes it +/// so. +/// +/// cntr_value and delta_cntr_value are additive across a counter's instance rows — +/// summing Transactions/sec over every database is a meaningful total. sample_interval_seconds is +/// NOT: it is one measured sweep gap repeated once per instance, so SUM multiplies the denominator +/// by the instance count. Measured on the production fleet, Transactions/sec, Log Flushes/sec, Log Bytes +/// Flushed/sec and Log Flush Write Time carry a median of 12 and up to 17 rows per collection_time +/// (the Lock* counters 15), so a summed denominator yields rates 12-17x too LOW. +/// +/// Both Darling surfaces carry a dedicated pin for this (DarlingTrendReader and the Viewer's +/// PerfmonTrendsSql); Lite had none, which a review caught. Lite's queries are inline +/// CommandText rather than exposed constants, so this reads the source the way +/// exists to — the alternative is a live multi-instance DuckDB fixture, and +/// McpStatusEnvelopeTests seeds one row per counter, so the case that breaks is exactly the one no +/// existing test exercises. +/// +public sealed class PerfmonIntervalAggregationTests +{ + private const string ReadPath = "Lite/Services/LocalDataService.Perfmon.cs"; + + /// Both trend reads — the single-counter one and its batched sibling — must take the interval + /// as MAX. Two occurrences, because a fix applied to only one of the pair is the likelier mistake. + /// + [Fact] + public void BothPerfmonTrendReads_TakeTheIntervalAsMax() + { + var source = ParitySource.ReadFile(ReadPath); + + var max = CountOccurrences(source, "MAX(sample_interval_seconds)"); + Assert.Equal(2, max); + } + + /// The failure mode itself: a summed interval anywhere in this read path is the 12-17x + /// denominator inflation, and it is silent — the column is populated, the query succeeds, and only + /// the derived rate is wrong. + [Fact] + public void NoPerfmonTrendRead_SumsTheInterval() + { + var source = ParitySource.ReadFile(ReadPath); + + Assert.DoesNotContain("SUM(sample_interval_seconds)", source, StringComparison.Ordinal); + } + + /// Guards the fix from being over-applied: the two columns that ARE additive must stay sums, + /// in both reads. A well-meant "make it consistent" edit that turned these into MAX would silently + /// report one instance's value as the whole counter. + [Fact] + public void TheAdditiveColumnsStaySummed() + { + var source = ParitySource.ReadFile(ReadPath); + + Assert.Equal(2, CountOccurrences(source, "SUM(cntr_value)")); + Assert.Equal(2, CountOccurrences(source, "SUM(delta_cntr_value)")); + Assert.DoesNotContain("MAX(cntr_value)", source, StringComparison.Ordinal); + Assert.DoesNotContain("MAX(delta_cntr_value)", source, StringComparison.Ordinal); + } + + /// The interval has to reach the caller, or the distinction dies at the API boundary and a + /// Lite caller is back to an ambiguous zero — the half of #2234 that only landed for Darling first. + /// + [Fact] + public void LitesPerfmonTrendTool_ProjectsTheInterval() + { + var source = ParitySource.ReadFile("Lite/Mcp/McpPerfmonTools.cs"); + + Assert.Contains("sample_interval_seconds = p.SampleIntervalSeconds", source, StringComparison.Ordinal); + } + + private static int CountOccurrences(string haystack, string needle) + { + var count = 0; + for (var i = haystack.IndexOf(needle, StringComparison.Ordinal); + i >= 0; + i = haystack.IndexOf(needle, i + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Lite.Tests/PgAutovacuumStatsCollectorDefinitionTests.cs b/Lite.Tests/PgAutovacuumStatsCollectorDefinitionTests.cs new file mode 100644 index 000000000..b216386d1 --- /dev/null +++ b/Lite.Tests/PgAutovacuumStatsCollectorDefinitionTests.cs @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the autovacuum collector: the per-table threshold computation that makes a dead-tuple count +/// mean something, the reloptions overrides it honours, the version gating that keeps the row shape +/// constant, and the activity filter — including the append-only case the filter must NOT drop. +/// +public class PgAutovacuumStatsCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext( + int major = 17, string? database = "appdb", ICollectorDeltaCalculator? deltas = null) + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas ?? s_deltas, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = major, + }, + ExcludedDatabases = Array.Empty(), + CurrentDatabaseName = database, + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_autovacuum_stats", PgAutovacuumStatsCollector.Instance.Name); + Assert.Equal("pg_autovacuum_stats", PgAutovacuumStatsCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgAutovacuumStatsCollector.Instance.TargetEngine); + } + + [Fact] + public void AppliesToAnyPostgresWriterButNeverSqlServer() + { + Assert.True(PgAutovacuumStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + Assert.False(CollectorCatalog.AppliesTo( + PgAutovacuumStatsCollector.Instance, new CollectorTargetInfo())); + } + + /// + /// Writers only, and NOT because the view is unreadable on a standby — it reads fine and reports all + /// zeros. Measured on Aurora 17.7, same cluster and database and 15 tables: the writer reported + /// 13,654,458 dead tuples and 150,790,506 live tuples where the reader reported 0 for every one of + /// n_dead_tup, n_mod_since_analyze, n_ins_since_vacuum and n_live_tup. + /// Ungated, a reader would yield zero rows, the activity filter would read that as "nothing has + /// pending work", and the tool would report perfect autovacuum health for a cluster 13 million dead + /// tuples behind. That false negative is the reason this gate exists. + /// + [Fact] + public void DoesNotRunOnAStandbyWhereEveryCounterReadsZero() + { + var reader = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + IsAurora = true, + IsInRecovery = true, + }; + + Assert.False(PgAutovacuumStatsCollector.Instance.AppliesTo(reader)); + Assert.False(CollectorCatalog.AppliesTo(PgAutovacuumStatsCollector.Instance, reader)); + + /* The engine gate is a separate concern and must still agree it is a Postgres collector — a + reader is skipped for the RECOVERY reason, not because it stopped being PostgreSQL. */ + Assert.True(CollectorCatalog.EngineMatches(PgAutovacuumStatsCollector.Instance.Name, reader)); + } + + /// + /// pg_stat_user_tables is scoped to the connected database and PostgreSQL has no cross-database + /// read, so this must fan out on EVERY target — unlike the SQL Server collectors, where per-database + /// is an Azure-only shape. + /// + [Fact] + public void AlwaysRunsPerDatabase() + { + Assert.True(PgAutovacuumStatsCollector.Instance.RunsPerDatabase( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql })); + Assert.True(PgAutovacuumStatsCollector.Instance.RunsPerDatabase( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true })); + } + + /// + /// The threshold is the whole point: a dead-tuple count is not actionable without the line it has to + /// cross. Both terms of autovacuum's formula must be present, and reltuples must be floored — it is + /// -1 on a never-analyzed table (PG14+), which unfloored yields a NEGATIVE threshold and makes such a + /// table read as permanently overdue. + /// + [Fact] + public void ComputesTheAutovacuumThresholdFromBothTerms() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("autovacuum_vacuum_threshold", sql, StringComparison.Ordinal); + Assert.Contains("autovacuum_vacuum_scale_factor", sql, StringComparison.Ordinal); + Assert.Contains("GREATEST(c.reltuples, 0)", sql, StringComparison.Ordinal); + } + + /// + /// Per-table reloptions must win over the GUCs, because ALTER TABLE ... SET (autovacuum_*) is common + /// on exactly the big hot tables where the global default is wrong. Reading only the GUC would report + /// a threshold the server is not using, which is worse than reporting none — it looks authoritative. + /// + [Fact] + public void PrefersPerTableReloptionsOverTheServerSettings() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("pg_options_to_table(c.reloptions)", sql, StringComparison.Ordinal); + /* coalesce(reloption, current_setting(...)) — the override first, the GUC as fallback. */ + Assert.Contains("current_setting('autovacuum_vacuum_threshold')", sql, StringComparison.Ordinal); + Assert.Matches( + @"coalesce\(\s*\(SELECT option_value FROM pg_options_to_table\(c\.reloptions\)\s*WHERE option_name = 'autovacuum_vacuum_threshold'\)::bigint,\s*current_setting", + sql.Replace("\n", " ").Replace("\r", " ")); + } + + /// A table with autovacuum switched off is its own finding, so the flag must be read. + [Fact] + public void DetectsAutovacuumDisabledPerTable() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("'autovacuum_enabled'", sql, StringComparison.Ordinal); + } + + /// + /// The insert-only path is PG13+, substituted with -1 on older majors so the row shape is constant. + /// A version-conditional COLUMN COUNT would change the table shape mid-fleet. + /// + [Fact] + public void GatesTheInsertOnlyPathByVersionWithoutChangingShape() + { + var pg17 = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext(17)).Text; + var pg12 = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext(12)).Text; + + Assert.Contains("t.n_ins_since_vacuum", pg17, StringComparison.Ordinal); + Assert.Contains("autovacuum_vacuum_insert_threshold", pg17, StringComparison.Ordinal); + + Assert.DoesNotContain("n_ins_since_vacuum", pg12, StringComparison.Ordinal); + Assert.DoesNotContain("autovacuum_vacuum_insert_threshold", pg12, StringComparison.Ordinal); + + Assert.Equal(pg17.Split(" AS ").Length, pg12.Split(" AS ").Length); + } + + /// + /// The filter's most important clause. An append-only table has NO dead tuples and NO modifications, + /// so the dead-tuple and analyze predicates both miss it — and an append-only table that is never + /// vacuumed is never frozen either, which is a route into a wraparound emergency. Filtering on dead + /// tuples alone would drop exactly the tables this collector exists to surface. + /// + [Fact] + public void ActivityFilterKeepsAppendOnlyTables() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext(17)).Text; + + Assert.Contains("t.n_dead_tup > 0", sql, StringComparison.Ordinal); + Assert.Contains("t.n_mod_since_analyze > 0", sql, StringComparison.Ordinal); + Assert.Contains("OR t.n_ins_since_vacuum > 0", sql, StringComparison.Ordinal); + } + + /// + /// On a major with no insert counter the filter must still be VALID SQL — the clause is dropped, not + /// left as a dangling OR. + /// + [Fact] + public void ActivityFilterStaysWellFormedWithoutTheInsertCounter() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext(12)).Text; + + Assert.DoesNotContain("OR t.n_ins_since_vacuum", sql, StringComparison.Ordinal); + Assert.DoesNotContain("OR OR", sql, StringComparison.Ordinal); + Assert.DoesNotContain("( OR", sql, StringComparison.Ordinal); + } + + /// + /// The maintenance timestamps are `timestamp with time zone` and must be converted with + /// AT TIME ZONE 'UTC', never `::timestamp`. The cast form renders the instant in the SESSION's + /// TimeZone before dropping the offset, so it agrees with UTC only while every server's timezone GUC + /// says UTC — true across the fleet today, which is precisely what would keep the bug hidden. The + /// store contract is naive UTC product-wide. + /// + [Fact] + public void ConvertsTimestamptzColumnsWithAnExplicitUtcZoneNotACast() + { + var sql = PgAutovacuumStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + foreach (var column in new[] { "last_vacuum", "last_autovacuum", "last_analyze", "last_autoanalyze" }) + { + Assert.Contains($"(t.{column} AT TIME ZONE 'UTC')", sql, StringComparison.Ordinal); + Assert.DoesNotContain($"t.{column}::timestamp", sql, StringComparison.Ordinal); + } + } + + [Fact] + public void PayloadColumns_CountAndKeyTypes_Pinned() + { + var columns = PgAutovacuumStatsCollector.Instance.PayloadColumns; + + Assert.Equal(20, columns.Count); + Assert.Equal("database_name", columns[0].Name); + Assert.Equal("dead_tuples", columns[4].Name); + Assert.Equal("vacuum_threshold", columns[7].Name); + Assert.Equal(CollectorColumnType.Boolean, columns[10].Type); // autovacuum_disabled + Assert.Equal(CollectorColumnType.Timestamp, columns[12].Type); // last_vacuum + } + + /// + /// database_name comes from the per-database loop's connection, not the result set: pg_stat_user_tables + /// shows only the connected database, so the connection IS the authoritative answer. A row written with + /// no database context would be unattributable. + /// + [Fact] + public void StampsTheDatabaseFromTheConnectionNotThePayload() + { + var writer = new RecordingCollectorRowWriter(); + + PgAutovacuumStatsCollector.Instance.WritePayload( + new PgAutovacuumStatsCollector.Row( + "public", "orders", 1_000_000, 250_000, 40_000, 0, 200_050, -1, 100_025, false, + 8_589_934_592, null, new DateTime(2026, 8, 11, 6, 0, 0), null, null, 0, 12, 0, 9), + writer, + MakeContext(database: "warehouse")); + + Assert.Equal("warehouse", writer.Values[0]); + Assert.Equal("public", writer.Values[1]); + Assert.Equal("orders", writer.Values[2]); + } + + /// + /// The reading the collector exists to enable, end to end: dead tuples well past this table's own + /// threshold, with autovacuum having last run hours ago. + /// + [Fact] + public async Task ReadsATableThatIsPastItsOwnThreshold() + { + var reader = new FakeCollectorDataReader( + new object[] + { + "public", "orders", 1_000_000L, 250_000L, 40_000L, 0L, + 200_050L, -1L, 100_025L, false, 8_589_934_592L, + DBNull.Value, new DateTime(2026, 8, 11, 6, 0, 0, DateTimeKind.Unspecified), + DBNull.Value, DBNull.Value, 0L, 12L, 0L, 9L, + }); + + var rows = await PgAutovacuumStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var table = Assert.Single(rows); + Assert.Equal(250_000L, table.DeadTuples); + Assert.Equal(200_050L, table.VacuumThreshold); + Assert.True(table.DeadTuples > table.VacuumThreshold); + Assert.Equal(-1L, table.InsertVacuumThreshold); // pre-13 sentinel, shape preserved + Assert.NotNull(table.LastAutovacuum); + Assert.Null(table.LastVacuum); // never manually vacuumed + } + + /// A clean, quiet database yields no rows — that is the healthy case, not a failure. + [Fact] + public async Task NoTablesWithPendingWorkYieldsNoRows() + { + var rows = await PgAutovacuumStatsCollector.Instance.ReadAsync( + new FakeCollectorDataReader(), MakeContext(), CancellationToken.None); + + Assert.Empty(rows); + } + + /// + /// Levels and lifetime counts read against a threshold — no deltas. The useful reading is how far past + /// the line a table is now, and the timestamps say when maintenance last ran without any arithmetic. + /// + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + + PgAutovacuumStatsCollector.Instance.WritePayload( + new PgAutovacuumStatsCollector.Row( + "public", "t", 1, 1, 0, 0, 50, -1, 50, false, 8192, null, null, null, null, 0, 0, 0, 0), + new RecordingCollectorRowWriter(), + MakeContext(deltas: deltas)); + + Assert.Empty(deltas.Calls); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_autovacuum_stats"); + + var schedule = CollectorScheduleDefaults.All["pg_autovacuum_stats"]; + + /* Hourly, not per-minute: this is a per-database fan-out, and on PostgreSQL that means one + CONNECTION per database per cycle. */ + Assert.Equal(60, schedule.FrequencyMinutes); + Assert.Equal(90, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } +} diff --git a/Lite.Tests/PgBlockingCollectorDefinitionTests.cs b/Lite.Tests/PgBlockingCollectorDefinitionTests.cs new file mode 100644 index 000000000..a0144729e --- /dev/null +++ b/Lite.Tests/PgBlockingCollectorDefinitionTests.cs @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the blocking-chain collector. Two of these tests guard cost rather than correctness, which is +/// unusual and deliberate: pg_blocking_pids() takes ShareLock on the lock manager partitions per +/// call, so an edit that widens where it is evaluated turns the monitor into the contention it reports. +/// That regression would not fail any other test — it would collect perfectly correct data and hurt. +/// +public class PgBlockingCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext() + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 12, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_blocking", PgBlockingCollector.Instance.Name); + Assert.Equal("pg_blocking_edges", PgBlockingCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgBlockingCollector.Instance.TargetEngine); + } + + /// + /// Any PostgreSQL target, standbys INCLUDED — and the standby half is the part worth asserting, because + /// the obvious move is to copy pg_autovacuum_stats's IsInRecovery gate. That gate exists + /// because pg_stat_user_tables reports zeros on a replica. pg_stat_activity does not: it + /// reports the standby's own backends, and recovery conflicts are blocking that happens ONLY on a + /// standby. Inheriting the gate would blind the collector to a condition unique to where it was gated + /// off. + /// + [Fact] + public void AppliesToAnyPostgresTarget_IncludingStandbys() + { + Assert.True(PgBlockingCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + Assert.True(PgBlockingCollector.Instance.AppliesTo( + new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + IsAurora = true, + IsInRecovery = true, + })); + + /* And never against SQL Server, which has its own blocked-process report. */ + Assert.False(CollectorCatalog.AppliesTo( + PgBlockingCollector.Instance, new CollectorTargetInfo())); + } + + /// + /// THE COST PIN. pg_blocking_pids() must be evaluated only for backends already waiting on a + /// lock. On a 5,000-connection instance, calling it for every row is the monitoring query that becomes + /// the incident — it acquires ShareLock on every lock manager partition per call. + /// Asserted as a gated CASE rather than a WHERE filter because both are needed at once: the + /// gate bounds the cost, and keeping it in the select list means one query still returns the full + /// activity snapshot, so the blocker's own state comes back without a second round trip. + /// + [Fact] + public void CallsBlockingPidsOnlyForLockWaiters() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("pg_blocking_pids", sql, StringComparison.Ordinal); + + /* The gate must be on the same expression as the call, not merely present somewhere in the file. */ + var callAt = sql.IndexOf("pg_blocking_pids", StringComparison.Ordinal); + var gateAt = sql.IndexOf("wait_event_type, '') = 'Lock'", StringComparison.Ordinal); + + Assert.True(gateAt > 0, "pg_blocking_pids must be gated on wait_event_type = 'Lock'"); + Assert.True( + gateAt < callAt && callAt - gateAt < 120, + "the 'Lock' gate must be the CASE guarding this pg_blocking_pids call, not an unrelated " + + "occurrence elsewhere in the query — ungated, this call runs once per backend and takes " + + "ShareLock on every lock manager partition each time."); + } + + /// + /// The collector must never attribute blocking to its own backend. Darling's read sits in + /// pg_stat_activity like any other session; without the filter it can appear as a victim of + /// whatever it happens to wait on, and "zero rows when healthy" stops being reachable. + /// + [Fact] + public void ExcludesItsOwnBackend() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("pg_backend_pid()", sql, StringComparison.Ordinal); + } + + /// + /// An EDGE LIST, not a rendered tree: the array must be unnested to one row per pair. Storing a + /// pre-rendered chain would bake in one traversal and turn root-blocker, depth, and fan-out into string + /// work for every future reader. + /// + [Fact] + public void UnnestsToOneRowPerEdge() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("unnest(", sql, StringComparison.Ordinal); + Assert.Contains("blocking_pid", sql, StringComparison.Ordinal); + Assert.Contains("blocked_pid", sql, StringComparison.Ordinal); + } + + /// + /// The blocker's own row must be LEFT joined. A blocker that has already exited still leaves a real + /// edge, and dropping the row would understate the chain — the opposite of what a blocking monitor is + /// for. It reports the edge with the blocker's columns null instead. + /// + [Fact] + public void KeepsEdgesWhoseBlockerHasGone() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("LEFT JOIN activity AS blocker", sql, StringComparison.Ordinal); + } + + /// + /// No bare ::text on a timestamptz, the class of defect + /// probe_timestamptz_render_nonutc.py exists to prove: ts::text renders in the SESSION + /// TimeZone, which is byte-identical to UTC on every instance in the fleet and wrong everywhere else. + /// This collector sidesteps it entirely by shipping durations in milliseconds instead of timestamps — + /// so the assertion is that no activity timestamp column is selected at all. + /// + [Fact] + public void ShipsDurationsRatherThanTimestamps() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("duration_ms", sql, StringComparison.Ordinal); + Assert.Contains("clock_timestamp()", sql, StringComparison.Ordinal); + + /* The two timestamptz columns are consumed only inside the subtraction, never emitted. */ + Assert.DoesNotContain("a.xact_start::text", sql, StringComparison.Ordinal); + Assert.DoesNotContain("a.query_start::text", sql, StringComparison.Ordinal); + } + + /// + /// track_activity_query_size must be read through pg_size_bytes(), never cast with + /// ::int. + /// current_setting() renders a memory GUC WITH ITS UNIT: this one comes back as + /// '8kB' on Aurora 17.7 and '4kB' on 16.11, so ::int raises "invalid input syntax + /// for type integer" and fails the ENTIRE collection, on every cycle, not just this column. The first + /// draft of this collector had the cast; a live probe against both majors found it, and nothing in the + /// C# suite could have. Pinned here so the shorter-looking form cannot come back. + /// + [Fact] + public void ReadsTheQuerySizeGucThroughPgSizeBytes_NotAnIntCast() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains( + "pg_size_bytes(current_setting('track_activity_query_size'))", sql, StringComparison.Ordinal); + Assert.DoesNotContain( + "current_setting('track_activity_query_size')::int", sql, StringComparison.Ordinal); + + /* And the LEFT side of that comparison must be BYTES too — the same unit mistake in a second + disguise, which is how it shipped one line under a comment about getting units right. + track_activity_query_size truncates at a byte boundary; length() counts characters, so on + multi-byte text it undercounts and the flag reads false for a query that really was clipped. + Measured on live Aurora: repeat('あ',100) is length 100, octet_length 300. */ + Assert.Contains("octet_length(coalesce(blocked.query, ''))", sql, StringComparison.Ordinal); + Assert.Contains("octet_length(coalesce(blocker.query, ''))", sql, StringComparison.Ordinal); + /* The negative needs a LEADING SPACE to mean anything: "length(coalesce(" is a substring of + "octet_length(coalesce(", so the bare form is satisfied by the very code it is meant to reject. + With the space it matches only a genuine character-length comparison, since the character before + "length" in the correct form is an underscore. */ + Assert.DoesNotContain(" length(coalesce(blocked.query", sql, StringComparison.Ordinal); + Assert.DoesNotContain(" length(coalesce(blocker.query", sql, StringComparison.Ordinal); + } + + /// + /// The synthetic backend id must combine backend_start with the pid. A pid alone is reused, so a + /// 30-day history keyed on it silently merges two different backends — and the read layer's + /// "has this been the same stuck backend all along" count is computed from exactly this value. + /// + [Fact] + public void BuildsAStableBackendIdentityFromBackendStartAndPid() + { + var sql = PgBlockingCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("backend_start", sql, StringComparison.Ordinal); + Assert.Contains("to_char(a.pid, 'FM0000000')", sql, StringComparison.Ordinal); + /* pg_postmaster_start_time() is the fallback for a backend with no backend_start (background + workers), so the id is never null and never collides with a real one. */ + Assert.Contains("pg_postmaster_start_time()", sql, StringComparison.Ordinal); + } + + [Fact] + public void PayloadColumns_OrderAndTypes_Pinned() + { + var expected = new (string Name, CollectorColumnType Type)[] + { + ("blocked_backend_id", CollectorColumnType.BigInt), + ("blocked_pid", CollectorColumnType.Integer), + ("blocking_backend_id", CollectorColumnType.BigInt), + ("blocking_pid", CollectorColumnType.Integer), + ("database_name", CollectorColumnType.Varchar), + ("blocked_username", CollectorColumnType.Varchar), + ("blocked_application_name", CollectorColumnType.Varchar), + ("blocked_client_addr", CollectorColumnType.Varchar), + ("blocked_state", CollectorColumnType.Varchar), + ("blocked_wait_event_type", CollectorColumnType.Varchar), + ("blocked_wait_event", CollectorColumnType.Varchar), + ("blocked_query", CollectorColumnType.Varchar), + ("blocked_xact_duration_ms", CollectorColumnType.BigInt), + ("blocked_query_duration_ms", CollectorColumnType.BigInt), + ("blocking_username", CollectorColumnType.Varchar), + ("blocking_application_name", CollectorColumnType.Varchar), + ("blocking_client_addr", CollectorColumnType.Varchar), + ("blocking_state", CollectorColumnType.Varchar), + ("blocking_wait_event_type", CollectorColumnType.Varchar), + ("blocking_wait_event", CollectorColumnType.Varchar), + ("blocking_query", CollectorColumnType.Varchar), + ("blocking_xact_duration_ms", CollectorColumnType.BigInt), + ("blocking_query_duration_ms", CollectorColumnType.BigInt), + ("blocked_pid_count", CollectorColumnType.Integer), + ("blocking_is_idle_in_transaction", CollectorColumnType.Boolean), + ("query_text_may_be_truncated", CollectorColumnType.Boolean), + }; + + var actual = PgBlockingCollector.Instance.PayloadColumns; + Assert.Equal(expected.Length, actual.Count); + for (var i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i].Name, actual[i].Name); + Assert.Equal(expected[i].Type, actual[i].Type); + } + } + + /// + /// Both sides of the edge must survive the read, in the order the payload declares. A transposed + /// blocked/blocking pair would invert every remedy the read layer produces — it would send someone to + /// fix the victim. + /// + [Fact] + public async Task ReadsBothSidesOfTheEdge() + { + var reader = new FakeCollectorDataReader( + new object[] + { + 1_754_000_001_234_567L, 4242, 1_754_000_009_876_543L, 9999, + "orders", "app_rw", "checkout-api", "10.0.0.5", "active", "Lock", "transactionid", + "UPDATE orders SET status = $1 WHERE id = $2", 8_400L, 8_100L, + /* The root waits on nothing — it is idle in transaction, so both wait columns are NULL. + That combination IS the signature: holding locks while waiting for a client that has + stopped talking. */ + "app_rw", "nightly-recon", "10.0.0.9", "idle in transaction", DBNull.Value, DBNull.Value, + "SELECT * FROM orders WHERE id = $1", 240_000L, 239_500L, + 3, true, false, + }); + + var rows = await PgBlockingCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var row = Assert.Single(rows); + Assert.Equal(4242, row.BlockedPid); + Assert.Equal(9999, row.BlockingPid); + Assert.Equal("active", row.BlockedState); + Assert.Equal("idle in transaction", row.BlockingState); + Assert.True(row.BlockingIsIdleInTransaction); + Assert.Equal(3, row.BlockedPidCount); + /* The identities are distinct and non-zero — the read must not collapse them onto the pid. */ + Assert.NotEqual(row.BlockedBackendId, row.BlockingBackendId); + } + + /// + /// A missing duration reads as -1, not 0. A backend with no open transaction has no duration to report, + /// and 0 would read as "started this instant" — which for a blocking root inverts the diagnosis from + /// "held for four minutes" to "just arrived". + /// + [Fact] + public async Task AbsentDurationsBecomeMinusOneNotZero() + { + var reader = new FakeCollectorDataReader( + new object[] + { + 1L, 1, 2L, 2, + "db", DBNull.Value, DBNull.Value, DBNull.Value, "active", "Lock", "relation", "SELECT 1", + DBNull.Value, DBNull.Value, + /* The blocker left pg_stat_activity between the two reads, so its whole side is NULL. The + edge is still real and must still be reported — see KeepsEdgesWhoseBlockerHasGone. */ + DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, + DBNull.Value, DBNull.Value, DBNull.Value, + 1, false, false, + }); + + var rows = await PgBlockingCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var row = Assert.Single(rows); + Assert.Equal(-1, row.BlockedXactDurationMs); + Assert.Equal(-1, row.BlockedQueryDurationMs); + Assert.Equal(-1, row.BlockingXactDurationMs); + Assert.Equal(-1, row.BlockingQueryDurationMs); + } + + /// + /// No blocking is the overwhelmingly common case and the HEALTHY one. Zero rows must not be mistaken + /// for a failed collection, so nothing here throws or synthesizes a placeholder. + /// + [Fact] + public async Task NoBlockingYieldsNoRowsRatherThanAPlaceholder() + { + var rows = await PgBlockingCollector.Instance.ReadAsync( + new FakeCollectorDataReader(), MakeContext(), CancellationToken.None); + + Assert.Empty(rows); + } + + /// A blocking snapshot is a state, not a counter — no deltas. + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + var context = new CollectorContext + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 12, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql }, + ExcludedDatabases = Array.Empty(), + }; + + PgBlockingCollector.Instance.WritePayload( + new PgBlockingCollector.Row( + 1L, 1, 2L, 2, "db", "u", "app", "10.0.0.1", "active", "Lock", "relation", "SELECT 1", + 100L, 90L, "u2", "app2", "10.0.0.2", "idle in transaction", null, null, "SELECT 2", + 200L, 190L, 1, true, false), + new RecordingCollectorRowWriter(), + context); + + Assert.Empty(deltas.Calls); + } + + /// + /// Every payload column must be written, in declaration order. The writer is positional, so a payload + /// column added without a matching .Value() call shifts every later column by one and stores + /// data that is silently wrong rather than failing. + /// + [Fact] + public void WritesEveryDeclaredPayloadColumn() + { + var writer = new RecordingCollectorRowWriter(); + + PgBlockingCollector.Instance.WritePayload( + new PgBlockingCollector.Row( + 1L, 1, 2L, 2, "db", "u", "app", "10.0.0.1", "active", "Lock", "relation", "SELECT 1", + 100L, 90L, "u2", "app2", "10.0.0.2", "idle in transaction", null, null, "SELECT 2", + 200L, 190L, 1, true, false), + writer, + MakeContext()); + + Assert.Equal(PgBlockingCollector.Instance.PayloadColumns.Count, writer.Values.Count); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_blocking"); + + var schedule = CollectorScheduleDefaults.All["pg_blocking"]; + Assert.Equal(1, schedule.FrequencyMinutes); + Assert.Equal(30, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } +} diff --git a/Lite.Tests/PgIoStatsCollectorDefinitionTests.cs b/Lite.Tests/PgIoStatsCollectorDefinitionTests.cs new file mode 100644 index 000000000..319c4882c --- /dev/null +++ b/Lite.Tests/PgIoStatsCollectorDefinitionTests.cs @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the pg_stat_io collector: the version floor, the PG18 column removal, the UTC conversion, and the +/// one that matters most — that NULL is preserved rather than coalesced, because on Aurora the entire write +/// side is NULL and a zero there would be a measurement nobody took. +/// +public class PgIoStatsCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext(int major = 17, ICollectorDeltaCalculator? deltas = null) + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas ?? s_deltas, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = major, + }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_io_stats", PgIoStatsCollector.Instance.Name); + Assert.Equal("pg_io_stats", PgIoStatsCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgIoStatsCollector.Instance.TargetEngine); + } + + /// pg_stat_io arrived in PostgreSQL 16; on 15 the view does not exist and the query would fail. + [Theory] + [InlineData(15, false)] + [InlineData(16, true)] + [InlineData(17, true)] + [InlineData(18, true)] + public void GatesOnThePostgres16VersionFloor(int major, bool applies) + { + Assert.Equal(applies, PgIoStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, PostgresMajorVersion = major })); + } + + /// + /// Unlike the autovacuum collector, this one RUNS on a standby. A replica's own read traffic and its + /// walreplay context are exactly what is wanted when a reader is slow, and unlike + /// pg_stat_user_tables these counters are the instance's own rather than the writer's. + /// + [Fact] + public void RunsOnAStandbyUnlikeTheAutovacuumCollector() + { + var reader = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = 17, + IsInRecovery = true, + }; + + Assert.True(PgIoStatsCollector.Instance.AppliesTo(reader)); + Assert.False(PgAutovacuumStatsCollector.Instance.AppliesTo(reader)); + } + + /// Core view, so it must not be Aurora-gated the way the wait and statement collectors are. + [Fact] + public void IsNotAuroraGated() + { + Assert.True(PgIoStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, PostgresMajorVersion = 16, IsAurora = false, + })); + Assert.False(CollectorCatalog.AppliesTo(PgIoStatsCollector.Instance, new CollectorTargetInfo())); + } + + /// Cluster-wide — a per-database fan-out here would multiply connections for identical data. + [Fact] + public void NeverRunsPerDatabase() + { + Assert.False(PgIoStatsCollector.Instance.RunsPerDatabase(MakeContext().Target)); + } + + /// + /// PG18 REMOVED op_bytes (replaced by read_bytes / write_bytes / extend_bytes). Selecting it there + /// would fail with "column does not exist" and take the whole collection down, so it is substituted — + /// and the substitution must keep the column count identical so the stored shape cannot drift. + /// + [Fact] + public void SubstitutesOpBytesOnPg18WithoutChangingShape() + { + var pg17 = PgIoStatsCollector.Instance.BuildQuery(MakeContext(17)).Text; + var pg18 = PgIoStatsCollector.Instance.BuildQuery(MakeContext(18)).Text; + + Assert.Contains(" op_bytes ", pg17, StringComparison.Ordinal); + Assert.Contains("NULL::bigint", pg18, StringComparison.Ordinal); + Assert.Equal(pg17.Split(" AS ").Length, pg18.Split(" AS ").Length); + } + + /// + /// stats_reset is `timestamp with time zone`. AT TIME ZONE 'UTC', never ::timestamp (renders in the + /// session's TimeZone) and never bare (Npgsql refuses to write a Kind=Utc DateTime to the store's + /// `timestamp without time zone` column). + /// + [Fact] + public void ConvertsStatsResetWithAnExplicitUtcZone() + { + var sql = PgIoStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("(stats_reset AT TIME ZONE 'UTC')", sql, StringComparison.Ordinal); + Assert.DoesNotContain("stats_reset::timestamp", sql, StringComparison.Ordinal); + } + + /// + /// Nothing may filter on backend_type / object / context. Aurora ADDS enum values — 17.7 showed a + /// `walreplay` context plus 'aurora cache receiver process', 'aurora wal replay process' and + /// 'slotsync worker' backend types that 16.11 did not — so any whitelist silently drops rows. + /// + [Fact] + public void FiltersNothingBecauseAuroraAddsEnumValues() + { + var sql = PgIoStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.DoesNotContain("WHERE", sql, StringComparison.Ordinal); + Assert.DoesNotContain("IN (", sql, StringComparison.Ordinal); + Assert.Contains("FROM pg_stat_io", sql, StringComparison.Ordinal); + } + + /// + /// No coalesce anywhere in the projection. PostgreSQL uses NULL for "this counter does not apply here", + /// and on Aurora the write side is NULL throughout because backends do not write data files. A + /// coalesce to 0 would turn "never measured" into "measured zero" — and a consumer averaging write + /// latency would then divide by it. + /// + [Fact] + public void NeverCoalescesACounterToZero() + { + var sql = PgIoStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.DoesNotContain("coalesce", sql, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void PayloadColumns_CountAndKeyTypes_Pinned() + { + var columns = PgIoStatsCollector.Instance.PayloadColumns; + + Assert.Equal(18, columns.Count); + Assert.Equal("backend_type", columns[0].Name); + Assert.Equal("context", columns[2].Name); + Assert.Equal("reads", columns[3].Name); + Assert.Equal(CollectorColumnType.Double, columns[4].Type); // read_time_ms + Assert.Equal(CollectorColumnType.Timestamp, columns[17].Type); // stats_reset + } + + /// + /// The Aurora row shape, end to end: reads and hits populated, and every write-side counter NULL — + /// which must survive into the row as null rather than becoming 0. + /// + [Fact] + public async Task PreservesNullWriteCountersFromAurora() + { + var reader = new FakeCollectorDataReader( + new object[] + { + "client backend", "relation", "normal", + 87L, 342.46, // reads, read_time + DBNull.Value, DBNull.Value, // writes, write_time — NULL on Aurora + DBNull.Value, DBNull.Value, // writebacks, writeback_time + 0L, 0.0, // extends, extend_time + 8192L, // op_bytes + 10_150_937_570L, 0L, 0L, // hits, evictions, reuses + DBNull.Value, DBNull.Value, // fsyncs, fsync_time — NULL on Aurora + new DateTime(2026, 5, 18, 7, 4, 22, DateTimeKind.Unspecified), + }); + + var rows = await PgIoStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var row = Assert.Single(rows); + Assert.Equal(87L, row.Reads); + Assert.Equal(10_150_937_570L, row.Hits); + + /* The whole point: null, not 0. */ + Assert.Null(row.Writes); + Assert.Null(row.WriteTimeMs); + Assert.Null(row.Writebacks); + Assert.Null(row.Fsyncs); + Assert.Null(row.FsyncTimeMs); + + /* And a real zero stays a zero — the two are distinguishable, which is the requirement. */ + Assert.Equal(0L, row.Extends); + Assert.NotNull(row.StatsReset); + } + + /// + /// The not-applicable-per-combination case, which is core PostgreSQL rather than Aurora: the + /// checkpointer performs no reads and has no hit counter at all. + /// + [Fact] + public async Task PreservesNullForCombinationsWhereACounterDoesNotApply() + { + var reader = new FakeCollectorDataReader( + new object[] + { + "checkpointer", "relation", "normal", + DBNull.Value, DBNull.Value, // reads / read_time do not apply + 0L, 0.0, DBNull.Value, DBNull.Value, + 0L, 0.0, 8192L, + DBNull.Value, // hits does not apply either + DBNull.Value, DBNull.Value, + DBNull.Value, DBNull.Value, DBNull.Value, + }); + + var rows = await PgIoStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var row = Assert.Single(rows); + Assert.Null(row.Reads); + Assert.Null(row.Hits); + Assert.Null(row.Evictions); + Assert.Null(row.StatsReset); + } + + /// + /// No stored deltas. The windowed change is computed at read time, so a "not applicable" NULL never has + /// to be given a value it does not have — which is exactly what a stored delta would force. + /// + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + + PgIoStatsCollector.Instance.WritePayload( + new PgIoStatsCollector.Row( + "client backend", "relation", "normal", 1, 1.0, null, null, null, null, + 0, 0.0, 8192, 2, 0, 0, null, null, null), + new RecordingCollectorRowWriter(), + MakeContext(deltas: deltas)); + + Assert.Empty(deltas.Calls); + } + + /// Every payload column is written, in order, including the nulls. + [Fact] + public void WritesEveryPayloadColumn() + { + var writer = new RecordingCollectorRowWriter(); + + PgIoStatsCollector.Instance.WritePayload( + new PgIoStatsCollector.Row( + "client backend", "relation", "bulkread", 5, 2.5, null, null, null, null, + null, null, 8192, 7, null, 3, null, null, null), + writer, + MakeContext()); + + Assert.Equal(PgIoStatsCollector.Instance.PayloadColumns.Count, writer.Values.Count); + Assert.Equal("client backend", writer.Values[0]); + Assert.Equal("bulkread", writer.Values[2]); + Assert.Null(writer.Values[5]); // writes + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_io_stats"); + + var schedule = CollectorScheduleDefaults.All["pg_io_stats"]; + + /* Per-minute is affordable because this is cluster-wide (one connection) and returned 25-37 rows + per snapshot on the fleet — the same order as pg_wait_stats. */ + Assert.Equal(1, schedule.FrequencyMinutes); + Assert.Equal(30, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } + +} diff --git a/Lite.Tests/PgReplicationSlotsCollectorDefinitionTests.cs b/Lite.Tests/PgReplicationSlotsCollectorDefinitionTests.cs new file mode 100644 index 000000000..9605a7d7a --- /dev/null +++ b/Lite.Tests/PgReplicationSlotsCollectorDefinitionTests.cs @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the replication-slot collector: the recovery-safe LSN reference, the computed retained-WAL +/// measure that does not depend on a column which is NULL by default, and the version gating that keeps +/// the table shape constant across a mixed-version fleet. +/// +public class PgReplicationSlotsCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext(int major = 17) + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = major, + }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_replication_slots", PgReplicationSlotsCollector.Instance.Name); + /* NOT pg_replication_slots: that is pg_catalog's view, which resolves ahead of anything in + search_path, so a store table of that name breaks CREATE INDEX loudly and every unqualified read + silently. Name and TargetTable differing is established practice (query_store -> query_store_stats). */ + Assert.Equal("pg_replication_slot_stats", PgReplicationSlotsCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgReplicationSlotsCollector.Instance.TargetEngine); + } + + [Fact] + public void AppliesToAnyPostgresTargetButNeverSqlServer() + { + Assert.True(PgReplicationSlotsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + Assert.False(CollectorCatalog.AppliesTo( + PgReplicationSlotsCollector.Instance, new CollectorTargetInfo())); + } + + /// + /// pg_current_wal_lsn() ERRORS on a standby, and Aurora readers are legitimate targets, so the LSN + /// reference has to switch on recovery state or the whole collection fails on a reader. + /// + [Fact] + public void UsesARecoverySafeLsnReference() + { + var sql = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("pg_is_in_recovery()", sql, StringComparison.Ordinal); + Assert.Contains("pg_last_wal_receive_lsn()", sql, StringComparison.Ordinal); + Assert.Contains("pg_current_wal_lsn()", sql, StringComparison.Ordinal); + } + + /// + /// Retained WAL is computed from restart_lsn rather than read from safe_wal_size, which is NULL + /// whenever max_slot_wal_keep_size is -1 — the default. Depending on that column would mean + /// reporting nothing precisely where retention is unbounded. + /// + [Fact] + public void ComputesRetainedWalRatherThanTrustingSafeWalSize() + { + var sql = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("s.restart_lsn", sql, StringComparison.Ordinal); + Assert.Contains("retained_wal_bytes", sql, StringComparison.Ordinal); + } + + /// wal_status is the single most diagnostic column and must always be selected. + [Fact] + public void SelectsWalStatusAndBothXminCounters() + { + var sql = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("s.wal_status", sql, StringComparison.Ordinal); + Assert.Contains("age(s.xmin)", sql, StringComparison.Ordinal); + Assert.Contains("age(s.catalog_xmin)", sql, StringComparison.Ordinal); + } + + /// + /// PG17 columns are read on 17 and substituted on 16, so the payload shape is identical either way. + /// A version-conditional COLUMN COUNT would change the table shape mid-fleet. + /// + [Fact] + public void GatesTheNewerColumnsByVersionWithoutChangingShape() + { + var pg17 = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext(17)).Text; + var pg16 = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext(16)).Text; + + Assert.Contains("s.inactive_since", pg17, StringComparison.Ordinal); + Assert.Contains("s.invalidation_reason", pg17, StringComparison.Ordinal); + Assert.Contains("s.conflicting", pg17, StringComparison.Ordinal); + + Assert.DoesNotContain("s.inactive_since", pg16, StringComparison.Ordinal); + Assert.DoesNotContain("s.invalidation_reason", pg16, StringComparison.Ordinal); + Assert.Contains("NULL::timestamp", pg16, StringComparison.Ordinal); + Assert.Contains("NULL::text", pg16, StringComparison.Ordinal); + + /* conflicting is 16+, so 16 reads it and only an older major would substitute. */ + Assert.Contains("s.conflicting", pg16, StringComparison.Ordinal); + + /* Same number of selected expressions on both, so the row shape cannot drift. */ + Assert.Equal( + pg17.Split(" AS ").Length, + pg16.Split(" AS ").Length); + } + + /// + /// inactive_since is `timestamp with time zone`, and selecting it bare is a runtime failure, not a + /// style issue: Npgsql maps a timestamptz read to DateTime with Kind=Utc and refuses to write that + /// into the store's `timestamp without time zone` column, so collection would break on any PG17 + /// target with a slot that has ever gone inactive — the exact servers this collector exists for. + /// `::timestamp` would be correctly typed but timezone-dependent, so neither shortcut is acceptable. + /// + [Fact] + public void ConvertsInactiveSinceWithAnExplicitUtcZone() + { + var sql = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext(17)).Text; + + Assert.Contains("(s.inactive_since AT TIME ZONE 'UTC')", sql, StringComparison.Ordinal); + Assert.DoesNotContain("s.inactive_since::timestamp", sql, StringComparison.Ordinal); + + /* The substituted branch was always correctly typed; both branches must stay `timestamp`. */ + var pg16 = PgReplicationSlotsCollector.Instance.BuildQuery(MakeContext(16)).Text; + Assert.Contains("NULL::timestamp", pg16, StringComparison.Ordinal); + } + + [Fact] + public void PayloadColumns_CountAndKeyTypes_Pinned() + { + var columns = PgReplicationSlotsCollector.Instance.PayloadColumns; + + Assert.Equal(16, columns.Count); + Assert.Equal("slot_name", columns[0].Name); + Assert.Equal("wal_status", columns[8].Name); + Assert.Equal("retained_wal_bytes", columns[10].Name); + Assert.Equal(CollectorColumnType.Timestamp, columns[13].Type); // inactive_since + Assert.Equal(CollectorColumnType.Boolean, columns[15].Type); // conflicting + } + + /// + /// No slots is normal on most servers and must yield no rows rather than an error or a placeholder. + /// + [Fact] + public async Task NoSlotsYieldsNoRows() + { + var rows = await PgReplicationSlotsCollector.Instance.ReadAsync( + new FakeCollectorDataReader(), MakeContext(), CancellationToken.None); + + Assert.Empty(rows); + } + + /// + /// The orphan shape end to end: inactive, WAL being retained because of it, and an xmin pinned — one + /// slot causing both the disk and the vacuum failure mode at once. + /// + [Fact] + public async Task ReadsAnAbandonedSlotIncludingBothFailureModes() + { + var reader = new FakeCollectorDataReader( + new object[] + { + "debezium_orphan", "logical", "pgoutput", "appdb", false, 0L, false, false, + "extended", -1L, 48_318_382_080L, 900_000L, 900_000L, + new DateTime(2026, 7, 20, 3, 0, 0, DateTimeKind.Unspecified), DBNull.Value, false, + }); + + var rows = await PgReplicationSlotsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + var slot = Assert.Single(rows); + Assert.False(slot.IsActive); + Assert.Equal("extended", slot.WalStatus); // WAL retained because of this slot + Assert.Equal(48_318_382_080L, slot.RetainedWalBytes); + Assert.Equal(-1, slot.SafeWalSizeBytes); // not-applicable sentinel, the stock default + Assert.Equal(900_000L, slot.CatalogXminAge); // and vacuum is pinned too + Assert.NotNull(slot.InactiveSince); + } + + /// Retained WAL is the size of a hole, not work done — a level, so no deltas. + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + var context = new CollectorContext + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, PostgresMajorVersion = 17 }, + ExcludedDatabases = Array.Empty(), + }; + + PgReplicationSlotsCollector.Instance.WritePayload( + new PgReplicationSlotsCollector.Row( + "s", "physical", null, null, true, 1, false, false, "reserved", -1, 0, -1, -1, null, null, false), + new RecordingCollectorRowWriter(), + context); + + Assert.Empty(deltas.Calls); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_replication_slots"); + + var schedule = CollectorScheduleDefaults.All["pg_replication_slots"]; + Assert.Equal(1, schedule.FrequencyMinutes); + Assert.Equal(90, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } +} diff --git a/Lite.Tests/PgWaitStatsCollectorDefinitionTests.cs b/Lite.Tests/PgWaitStatsCollectorDefinitionTests.cs new file mode 100644 index 000000000..67d6f3b3c --- /dev/null +++ b/Lite.Tests/PgWaitStatsCollectorDefinitionTests.cs @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the first PostgreSQL collector: the Aurora-only gate, the verified function signatures in its +/// query, the wait-type noise filter, and delta keying on the numeric event id rather than the name. +/// +public class PgWaitStatsCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext() + => new() + { + ServerId = 42, + ServerName = "aurora-writer", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = 17, + PostgresVersionNum = 170007, + IsAurora = true, + }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_wait_stats", PgWaitStatsCollector.Instance.Name); + Assert.Equal("pg_wait_stats", PgWaitStatsCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgWaitStatsCollector.Instance.TargetEngine); + Assert.Equal("collection_id", PgWaitStatsCollector.Instance.PrefixIdColumnName); + Assert.Equal("collection_time", PgWaitStatsCollector.Instance.PrefixTimeColumnName); + } + + /// + /// Aurora-only, and gated on the Aurora flag rather than a version: the wait functions are Aurora + /// built-ins, and core PostgreSQL has no cumulative wait source in ANY version, so on stock + /// Postgres there is nothing to read. + /// + [Fact] + public void AppliesOnlyToAuroraTargets() + { + Assert.True(PgWaitStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true })); + + Assert.False(PgWaitStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + } + + /// And the composed gate keeps it off SQL Server entirely, Aurora flag notwithstanding. + [Fact] + public void NeverDispatchesAtASqlServerTarget() + { + Assert.False(CollectorCatalog.AppliesTo( + PgWaitStatsCollector.Instance, new CollectorTargetInfo())); + Assert.False(CollectorCatalog.EngineMatches( + "pg_wait_stats", new CollectorTargetInfo())); + } + + [Fact] + public void PayloadColumns_OrderAndTypes_Pinned() + { + var expected = new (string Name, CollectorColumnType Type)[] + { + ("wait_type_id", CollectorColumnType.Integer), + ("wait_event_id", CollectorColumnType.BigInt), + ("wait_type", CollectorColumnType.Varchar), + ("wait_event", CollectorColumnType.Varchar), + ("waits", CollectorColumnType.BigInt), + ("wait_time_us", CollectorColumnType.BigInt), + ("delta_waits", CollectorColumnType.BigInt), + ("delta_wait_time_us", CollectorColumnType.BigInt), + }; + + var actual = PgWaitStatsCollector.Instance.PayloadColumns; + Assert.Equal(expected.Length, actual.Count); + for (var i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i].Name, actual[i].Name); + Assert.Equal(expected[i].Type, actual[i].Type); + } + } + + /// + /// The AWS reference describes aurora_stat_wait_event() as four columns including type_name. It + /// returns three, type_id first. Aliasing it wrong does not error — the join silently matches + /// nothing and every event name comes back NULL — so the correct arity is pinned here. + /// + [Fact] + public void Query_UsesTheVerifiedFunctionSignatures() + { + var sql = PgWaitStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("aurora_stat_system_waits() AS w(type_id, event_id, waits, wait_time)", sql, StringComparison.Ordinal); + Assert.Contains("aurora_stat_wait_type() AS t(type_id, type_name)", sql, StringComparison.Ordinal); + Assert.Contains("aurora_stat_wait_event() AS e(type_id, event_id, event_name)", sql, StringComparison.Ordinal); + } + + /// + /// LEFT JOIN, never NATURAL JOIN as the AWS example uses: the documented type list omits type 2, + /// Limitless adds type 12, and an inner join would silently drop any event whose type is unknown. + /// + [Fact] + public void Query_LeftJoinsTheLookupsSoUnknownTypesSurvive() + { + var sql = PgWaitStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Equal(2, sql.Split("LEFT JOIN").Length - 1); + Assert.DoesNotContain("NATURAL JOIN", sql, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("INNER JOIN", sql, StringComparison.OrdinalIgnoreCase); + } + + /// Explicit casts pin the reader's types; Npgsql throws on GetInt64 over an int4 column. + [Fact] + public void Query_CastsColumnsSoReaderTypesAreDeterministic() + { + var sql = PgWaitStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("w.type_id::int", sql, StringComparison.Ordinal); + Assert.Contains("w.event_id::bigint", sql, StringComparison.Ordinal); + Assert.Contains("w.waits::bigint", sql, StringComparison.Ordinal); + Assert.Contains("w.wait_time::bigint", sql, StringComparison.Ordinal); + } + + /// + /// Background-noise wait types are dropped. Measured on prod: Client:ClientRead alone accumulated + /// 565,758,023 seconds and every Activity event grows at ~1 second per second of uptime forever, so + /// without this filter they are 99%+ of the chart. + /// + [Fact] + public async Task ReadAsync_FiltersBackgroundNoiseWaitTypes() + { + var reader = new FakeCollectorDataReader( + new object[] { 6, 100663296L, "Client", "ClientRead", 3283144470L, 565758023440000L }, + new object[] { 5, 83886080L, "Activity", "AuroraRuntimeMain", 855576L, 8563569140000L }, + new object[] { 9, 150994944L, "Timeout", "VacuumDelay", 813456L, 4227680000L }, + new object[] { 10, 167772160L, "IO", "DataFileRead", 2065211345L, 1523253130000L }, + new object[] { 3, 50331648L, "Lock", "transactionid", 4395L, 11393070000L }); + + var rows = await PgWaitStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + Assert.Equal(2, rows.Count); + Assert.Equal(new[] { "DataFileRead", "transactionid" }, rows.Select(r => r.EventName).ToArray()); + } + + /// + /// A wait whose type did not decode is KEPT, not dropped. A NULL type_name means the lookup did not + /// know the type — which is exactly the new-Aurora-wait-type case worth seeing, not noise. + /// + [Fact] + public async Task ReadAsync_KeepsWaitsWhoseTypeDidNotDecode() + { + var reader = new FakeCollectorDataReader( + new object[] { 12, 201326607L, DBNull.Value, DBNull.Value, 10L, 5000L }); + + var rows = await PgWaitStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + Assert.Single(rows); + Assert.Null(rows[0].TypeName); + Assert.Null(rows[0].EventName); + Assert.Equal(12, rows[0].TypeId); + Assert.Equal(201326607L, rows[0].EventId); + } + + /// + /// Deltas key on the numeric event id, not the event name. Wait-event name casing differs between + /// Aurora majors — AutoVacuumMain on 16.11 versus AutovacuumMain on 17.7 — so a name-keyed delta + /// would break its own history the moment a cluster is upgraded. + /// + [Fact] + public void WritePayload_KeysDeltasOnTheStableEventId() + { + var deltas = new RecordingCollectorDeltaCalculator(); + var context = new CollectorContext + { + ServerId = 42, + ServerName = "aurora-writer", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true }, + ExcludedDatabases = Array.Empty(), + }; + + var writer = new RecordingCollectorRowWriter(); + PgWaitStatsCollector.Instance.WritePayload( + new PgWaitStatsCollector.Row(10, 167772160L, "IO", "DataFileRead", 100L, 5000L), + writer, + context); + + /* Every delta call keys on the numeric event id, never on "DataFileRead". */ + Assert.All(deltas.Calls, call => Assert.Equal("167772160", call.Key)); + + Assert.Contains(deltas.Calls, call => call.Group == "pg_wait_stats_waits"); + Assert.Contains(deltas.Calls, call => call.Group == "pg_wait_stats_time"); + + /* Same shared gap policy as wait_stats: past it, no delta rather than a spike that is + really an interval measurement. */ + Assert.All(deltas.Calls, call => Assert.Equal(CollectorDeltaCalculator.DefaultMaxGapSeconds, call.MaxGap)); + Assert.All(deltas.Calls, call => Assert.Equal(context.CollectionTime, call.Time)); + } + + /// The written payload matches the declared column count and order. + [Fact] + public void WritePayload_EmitsEveryDeclaredColumnInOrder() + { + var writer = new RecordingCollectorRowWriter(); + PgWaitStatsCollector.Instance.WritePayload( + new PgWaitStatsCollector.Row(10, 167772160L, "IO", "DataFileRead", 100L, 5000L), + writer, + MakeContext()); + + Assert.Equal(PgWaitStatsCollector.Instance.PayloadColumns.Count, writer.Values.Count); + Assert.Equal(10, writer.Values[0]); + Assert.Equal(167772160L, writer.Values[1]); + Assert.Equal("IO", writer.Values[2]); + Assert.Equal("DataFileRead", writer.Values[3]); + Assert.Equal(100L, writer.Values[4]); + Assert.Equal(5000L, writer.Values[5]); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_wait_stats"); + Assert.True(CollectorScheduleDefaults.All.ContainsKey("pg_wait_stats")); + + var schedule = CollectorScheduleDefaults.All["pg_wait_stats"]; + Assert.Equal(1, schedule.FrequencyMinutes); // same cadence as wait_stats + Assert.Equal(30, schedule.RetentionDays); // same horizon as wait_stats + Assert.True(schedule.DefaultEnabled); + } +} diff --git a/Lite.Tests/PgWraparoundStatsCollectorDefinitionTests.cs b/Lite.Tests/PgWraparoundStatsCollectorDefinitionTests.cs new file mode 100644 index 000000000..6d47f119c --- /dev/null +++ b/Lite.Tests/PgWraparoundStatsCollectorDefinitionTests.cs @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the freeze-headroom collector: it runs on any PostgreSQL target (not just Aurora), it collects +/// MultiXact age as a first-class second counter, and its percentages use the right denominators. +/// +public class PgWraparoundStatsCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext(bool isAurora = true) + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = 17, + IsAurora = isAurora, + }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_wraparound_stats", PgWraparoundStatsCollector.Instance.Name); + Assert.Equal("pg_wraparound_stats", PgWraparoundStatsCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgWraparoundStatsCollector.Instance.TargetEngine); + } + + /// + /// Unlike the wait and statement collectors, this one reads only core catalog surfaces, so it must + /// apply to stock PostgreSQL as well as Aurora. Gating it on Aurora would silently drop the single + /// most consequential PostgreSQL signal on every non-Aurora target. + /// + [Fact] + public void AppliesToAnyPostgresTargetIncludingNonAurora() + { + Assert.True(PgWraparoundStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true })); + Assert.True(PgWraparoundStatsCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + } + + /// But the engine gate still keeps it off SQL Server. + [Fact] + public void NeverDispatchesAtASqlServerTarget() + { + Assert.False(CollectorCatalog.AppliesTo( + PgWraparoundStatsCollector.Instance, new CollectorTargetInfo())); + } + + /// + /// MultiXact is a separate counter and separately fatal, and it is the one most monitoring misses. + /// Both must be in the query or the collector gives false comfort. + /// + [Fact] + public void CollectsBothTransactionIdAndMultiXactAge() + { + var sql = PgWraparoundStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("age(d.datfrozenxid)", sql, StringComparison.Ordinal); + Assert.Contains("mxid_age(d.datminmxid)", sql, StringComparison.Ordinal); + } + + /// + /// age()/mxid_age() rather than arithmetic on the raw xid: both handle the modular wrap that makes a + /// naive subtraction wrong precisely near the boundary, which is the only region that matters. + /// + [Fact] + public void UsesAgeFunctionsRatherThanRawSubtraction() + { + var sql = PgWraparoundStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.DoesNotContain("- d.datfrozenxid", sql, StringComparison.Ordinal); + Assert.DoesNotContain("txid_current()", sql, StringComparison.Ordinal); + } + + /// + /// Reads the SHARED catalog and INCLUDES templates — the opposite of the per-database fan-out, and the + /// contrast is the point. + /// The cluster-wide stop limit derives from the oldest datfrozenxid anywhere in + /// pg_database, so excluding template0/template1 would understate cluster risk by exactly the + /// amount that matters; template0 aging without ever being vacuumed is a documented route there, usually + /// after a major upgrade. This query needs no connection — pg_database is shared, and template0's + /// row reads fine despite datallowconn = false (verified on live Aurora 17.7). + /// The per-database enumeration in ITargetProvider.BuildDatabaseListPlan DOES exclude + /// templates, and must: it opens a connection per database and template0 refuses them. Two queries, two + /// correct answers — so do NOT "fix" the inconsistency by aligning them. That counterpart is pinned by + /// Darling.Tests.TargetProviderTests.PostgresDatabaseList_SkipsTemplatesAndClosedDatabases; + /// it cannot be asserted here because the provider lives in the Darling service project. + /// + [Fact] + public void ReadsTheSharedCatalogAndIncludesTemplates() + { + var sql = PgWraparoundStatsCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("FROM pg_database", sql, StringComparison.Ordinal); + Assert.DoesNotContain("NOT d.datistemplate", sql, StringComparison.Ordinal); + Assert.False(PgWraparoundStatsCollector.Instance.RunsPerDatabase(MakeContext().Target)); + + } + + [Fact] + public void PayloadColumns_OrderAndTypes_Pinned() + { + var expected = new (string Name, CollectorColumnType Type)[] + { + ("database_name", CollectorColumnType.Varchar), + ("frozen_xid_age", CollectorColumnType.BigInt), + ("min_multixid_age", CollectorColumnType.BigInt), + ("autovacuum_freeze_max_age", CollectorColumnType.BigInt), + ("autovacuum_multixact_freeze_max_age", CollectorColumnType.BigInt), + ("pct_toward_emergency_vacuum", CollectorColumnType.Double), + ("pct_toward_wraparound", CollectorColumnType.Double), + ("pct_toward_multixact_emergency", CollectorColumnType.Double), + ("pct_toward_multixact_wraparound", CollectorColumnType.Double), + ("xids_remaining", CollectorColumnType.BigInt), + ("multixids_remaining", CollectorColumnType.BigInt), + ("allows_connections", CollectorColumnType.Boolean), + }; + + var actual = PgWraparoundStatsCollector.Instance.PayloadColumns; + Assert.Equal(expected.Length, actual.Count); + for (var i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i].Name, actual[i].Name); + Assert.Equal(expected[i].Type, actual[i].Type); + } + } + + /// + /// The two percentages answer different questions against different denominators: emergency-vacuum + /// percentage is against autovacuum_freeze_max_age (200M by default), wraparound percentage against + /// the ~2^31 ceiling. Conflating them would make a routine anti-wraparound vacuum look like an + /// imminent outage, roughly a tenfold overstatement at defaults. + /// + [Fact] + public void PercentagesUseTheirOwnDenominators() + { + var writer = new RecordingCollectorRowWriter(); + PgWraparoundStatsCollector.Instance.WritePayload( + new PgWraparoundStatsCollector.Row( + DatabaseName: "app", + FrozenXidAge: 200_000_000, + MinMultiXidAge: 100_000_000, + AutovacuumFreezeMaxAge: 200_000_000, + AutovacuumMultixactFreezeMaxAge: 400_000_000, + AllowsConnections: true), + writer, + MakeContext()); + + /* At exactly autovacuum_freeze_max_age: 100% of the way to an emergency vacuum, but only + ~9.3% of the way to the wraparound ceiling. */ + Assert.Equal(100.0, (double)writer.Values[5]!, precision: 6); + Assert.Equal(200_000_000d / PgWraparoundStatsCollector.WraparoundCeiling * 100, (double)writer.Values[6]!, precision: 6); + + /* MultiXact has its own, larger default ceiling: 100M against 400M is 25%. */ + Assert.Equal(25.0, (double)writer.Values[7]!, precision: 6); + } + + /// An unreadable setting must not manufacture a percentage — or an alert. + [Fact] + public void UnknownDenominatorYieldsZeroNotInfinity() + { + var writer = new RecordingCollectorRowWriter(); + PgWraparoundStatsCollector.Instance.WritePayload( + new PgWraparoundStatsCollector.Row("app", 500, 500, 0, 0, true), + writer, + MakeContext()); + + Assert.Equal(0.0, (double)writer.Values[5]!); + Assert.Equal(0.0, (double)writer.Values[7]!); + } + + /// + /// Remaining headroom counts down to the point where writes STOP, not to the raw 2^31 ceiling. + /// PostgreSQL refuses new write transactions with roughly + /// ids still unconsumed, so counting to the ceiling overstated the runway by that margin — and disagreed + /// with the MCP tool's own 99.86%-of-space figure, which already accounts for it. This assertion changed + /// with that fix and the change IS the fix. + /// + [Fact] + public void RemainingHeadroomCountsDownToWhereWritesStop() + { + var writer = new RecordingCollectorRowWriter(); + PgWraparoundStatsCollector.Instance.WritePayload( + new PgWraparoundStatsCollector.Row("app", 1_000_000_000, 2_000_000_000, 200_000_000, 400_000_000, true), + writer, + MakeContext()); + + var stopPoint = PgWraparoundStatsCollector.WraparoundCeiling - PgWraparoundStatsCollector.StopMargin; + Assert.Equal(stopPoint - 1_000_000_000, writer.Values[9]); + Assert.Equal(stopPoint - 2_000_000_000, writer.Values[10]); + + /* And the margin is real, not zero — otherwise this test would pass against the old behaviour. */ + Assert.True(PgWraparoundStatsCollector.StopMargin > 0); + } + + /// + /// Past the stop point, headroom clamps at 0 rather than going negative. A negative "ids remaining" is + /// nonsense to render and worse to compare against a threshold. + /// + [Fact] + public void RemainingHeadroomClampsAtZeroPastTheStopPoint() + { + var writer = new RecordingCollectorRowWriter(); + PgWraparoundStatsCollector.Instance.WritePayload( + new PgWraparoundStatsCollector.Row("app", 2_147_000_000, 2_147_000_000, 200_000_000, 400_000_000, true), + writer, + MakeContext()); + + Assert.Equal(0L, writer.Values[9]); + Assert.Equal(0L, writer.Values[10]); + } + + /// Age is a distance from a wall, not accumulated work, so no deltas are taken. + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + var context = new CollectorContext + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql }, + ExcludedDatabases = Array.Empty(), + }; + + PgWraparoundStatsCollector.Instance.WritePayload( + new PgWraparoundStatsCollector.Row("app", 1, 1, 200_000_000, 400_000_000, true), + new RecordingCollectorRowWriter(), + context); + + Assert.Empty(deltas.Calls); + } + + [Fact] + public async Task ReadAsync_MapsEveryDatabaseRow() + { + var reader = new FakeCollectorDataReader( + new object[] { "app", 1_500_000_000L, 90_000_000L, 200_000_000L, 400_000_000L, true }, + new object[] { "rdsadmin", 12_345L, 0L, 200_000_000L, 400_000_000L, false }); + + var rows = await PgWraparoundStatsCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + Assert.Equal(2, rows.Count); + Assert.Equal("app", rows[0].DatabaseName); + Assert.Equal(1_500_000_000L, rows[0].FrozenXidAge); + Assert.True(rows[0].AllowsConnections); + Assert.False(rows[1].AllowsConnections); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_wraparound_stats"); + + var schedule = CollectorScheduleDefaults.All["pg_wraparound_stats"]; + Assert.Equal(5, schedule.FrequencyMinutes); + Assert.Equal(90, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } +} diff --git a/Lite.Tests/PgXminHorizonCollectorDefinitionTests.cs b/Lite.Tests/PgXminHorizonCollectorDefinitionTests.cs new file mode 100644 index 000000000..16e4cffc4 --- /dev/null +++ b/Lite.Tests/PgXminHorizonCollectorDefinitionTests.cs @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the xmin-horizon collector, whose entire value is attribution: all four causes look identical +/// from the symptom side, so the collector must name which one is holding the horizon. +/// +public class PgXminHorizonCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + private static CollectorContext MakeContext() + => new() + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = s_deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = true }, + ExcludedDatabases = Array.Empty(), + }; + + [Fact] + public void Identity_Pinned() + { + Assert.Equal("pg_xmin_horizon", PgXminHorizonCollector.Instance.Name); + Assert.Equal("pg_xmin_horizon", PgXminHorizonCollector.Instance.TargetTable); + Assert.Equal(CollectorTargetEngine.PostgreSql, PgXminHorizonCollector.Instance.TargetEngine); + } + + /// Core catalog only, so it must run on non-Aurora PostgreSQL too. + [Fact] + public void AppliesToAnyPostgresTarget() + { + Assert.True(PgXminHorizonCollector.Instance.AppliesTo( + new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql, IsAurora = false })); + Assert.False(CollectorCatalog.AppliesTo( + PgXminHorizonCollector.Instance, new CollectorTargetInfo())); + } + + /// + /// All five holder sources must be queried. Missing one does not degrade the answer, it INVERTS it: + /// the collector would report a different cause as the winner and send someone to fix the wrong thing. + /// + [Fact] + public void QueriesEveryHolderSource() + { + var sql = PgXminHorizonCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("pg_stat_activity", sql, StringComparison.Ordinal); + Assert.Contains("pg_replication_slots", sql, StringComparison.Ordinal); + Assert.Contains("pg_stat_replication", sql, StringComparison.Ordinal); + Assert.Contains("pg_prepared_xacts", sql, StringComparison.Ordinal); + + Assert.Contains("backend_xmin", sql, StringComparison.Ordinal); + /* A slot's xmin blocks row cleanup; its catalog_xmin blocks CATALOG cleanup, which is what a + logical decoding slot pins. They can differ by a lot, so both are separate sources. */ + Assert.Contains("s.xmin", sql, StringComparison.Ordinal); + Assert.Contains("s.catalog_xmin", sql, StringComparison.Ordinal); + } + + /// + /// Bounded to the oldest holder per source. pg_stat_activity can carry hundreds of backends with an + /// xmin, and storing all of them every minute would be a great many rows saying one thing. + /// + [Fact] + public void ReducesToTheOldestHolderPerSource() + { + var sql = PgXminHorizonCollector.Instance.BuildQuery(MakeContext()).Text; + + Assert.Contains("DISTINCT ON (source)", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY source, xmin_age DESC", sql, StringComparison.Ordinal); + } + + [Fact] + public void PayloadColumns_OrderAndTypes_Pinned() + { + var expected = new (string Name, CollectorColumnType Type)[] + { + ("source", CollectorColumnType.Varchar), + ("xmin_age", CollectorColumnType.BigInt), + ("holder", CollectorColumnType.Varchar), + ("detail", CollectorColumnType.Varchar), + ("is_winner", CollectorColumnType.Boolean), + }; + + var actual = PgXminHorizonCollector.Instance.PayloadColumns; + Assert.Equal(expected.Length, actual.Count); + for (var i = 0; i < expected.Length; i++) + { + Assert.Equal(expected[i].Name, actual[i].Name); + Assert.Equal(expected[i].Type, actual[i].Type); + } + } + + /// The oldest holder across all sources is the one actually setting the horizon. + [Fact] + public async Task StampsTheOldestHolderAcrossSourcesAsTheWinner() + { + var reader = new FakeCollectorDataReader( + new object[] { "session", 5_000L, "12345", "state=idle in transaction" }, + new object[] { "replication_slot", 900_000L, "debezium_slot", "active=false" }, + new object[] { "prepared_transaction", 40_000L, "gid-1", "owner=app" }); + + var rows = await PgXminHorizonCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + Assert.Equal(3, rows.Count); + var winner = Assert.Single(rows, r => r.IsWinner); + Assert.Equal("replication_slot", winner.Source); + Assert.Equal(900_000L, winner.XminAge); + } + + /// Exactly one winner even when two sources tie — a stored row must not be ambiguous. + [Fact] + public async Task StampsExactlyOneWinnerOnATie() + { + var reader = new FakeCollectorDataReader( + new object[] { "session", 700L, "1", "a" }, + new object[] { "standby_feedback", 700L, "replica-1", "state=streaming" }); + + var rows = await PgXminHorizonCollector.Instance.ReadAsync(reader, MakeContext(), CancellationToken.None); + + Assert.Single(rows, r => r.IsWinner); + } + + /// + /// No holders is the HEALTHY state. Returning zero rows must not be mistaken for a failed + /// collection, which is why nothing here throws or synthesizes a placeholder row. + /// + [Fact] + public async Task NoHoldersYieldsNoRowsRatherThanAPlaceholder() + { + var rows = await PgXminHorizonCollector.Instance.ReadAsync( + new FakeCollectorDataReader(), MakeContext(), CancellationToken.None); + + Assert.Empty(rows); + } + + /// An xmin age is a level, like freeze age — no deltas. + [Fact] + public void TakesNoDeltas() + { + var deltas = new RecordingCollectorDeltaCalculator(); + var context = new CollectorContext + { + ServerId = 42, + ServerName = "pg-target", + CollectionTime = new DateTime(2026, 8, 11, 12, 0, 0, DateTimeKind.Utc), + Deltas = deltas, + Target = new CollectorTargetInfo { Engine = CollectorTargetEngine.PostgreSql }, + ExcludedDatabases = Array.Empty(), + }; + + PgXminHorizonCollector.Instance.WritePayload( + new PgXminHorizonCollector.Row("session", 1, "1", "d", true), + new RecordingCollectorRowWriter(), + context); + + Assert.Empty(deltas.Calls); + } + + [Fact] + public void RegisteredInBothTheCatalogAndTheSchedule() + { + Assert.Contains(CollectorCatalog.All, d => d.Name == "pg_xmin_horizon"); + + var schedule = CollectorScheduleDefaults.All["pg_xmin_horizon"]; + Assert.Equal(1, schedule.FrequencyMinutes); + Assert.Equal(30, schedule.RetentionDays); + Assert.True(schedule.DefaultEnabled); + } + + /// + /// The round-2 fixes, pinned the way the three sibling collectors already pin theirs — this was the + /// one fixed collector left unpinned. Timestamptz renders go through AT TIME ZONE 'UTC', never a + /// bare ::text (which renders in the SESSION's TimeZone and agrees with UTC only while every server's + /// timezone GUC says UTC — exactly what keeps the bug invisible on this fleet); the horizon takes the + /// GREATEST of both counters so an idle-in-transaction backend (backend_xid, no snapshot) is visible; + /// and the monitor's own session can never be the reported holder. + /// + [Fact] + public void HorizonQuery_RendersUtc_SeesXidOnlyBackends_AndExcludesItself() + { + var sql = PgXminHorizonCollector.Instance.BuildQuery(MakeContext()).Text; + + /* The three timestamptz renders. */ + Assert.Contains("(a.xact_start AT TIME ZONE 'UTC')::text", sql, StringComparison.Ordinal); + Assert.Contains("(a.query_start AT TIME ZONE 'UTC')::text", sql, StringComparison.Ordinal); + Assert.Contains("(p.prepared AT TIME ZONE 'UTC')::text", sql, StringComparison.Ordinal); + Assert.DoesNotContain("a.xact_start::text", sql, StringComparison.Ordinal); + Assert.DoesNotContain("a.query_start::text", sql, StringComparison.Ordinal); + Assert.DoesNotContain("p.prepared::text", sql, StringComparison.Ordinal); + + /* Idle-in-transaction visibility: either counter qualifies a backend, and the age is the worse + of the two — an xid-only session pins the horizon exactly as hard as a snapshot-holder. */ + Assert.Contains("(a.backend_xmin IS NOT NULL OR a.backend_xid IS NOT NULL)", sql, StringComparison.Ordinal); + Assert.Contains("coalesce(age(a.backend_xid), 0)", sql, StringComparison.Ordinal); + Assert.Contains("GREATEST(", sql, StringComparison.Ordinal); + + /* Self-attribution: the collector's own connection holds a snapshot while it reads. */ + Assert.Contains("a.pid <> pg_backend_pid()", sql, StringComparison.Ordinal); + } +} diff --git a/Lite.Tests/QueryStatExtremesTests.cs b/Lite.Tests/QueryStatExtremesTests.cs new file mode 100644 index 000000000..16f40ed57 --- /dev/null +++ b/Lite.Tests/QueryStatExtremesTests.cs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Decision-table pins for the shared (#2235) — the lifetime-extremes +/// annotation both SKUs' get_top_queries_by_cpu / get_top_procedures_by_cpu serve. This SAME table is +/// pinned identically in Darling.Tests so the two SKUs cannot drift: min/max CPU and elapsed are +/// lifetime extremes for the plan's cache residency, and the note fires only on the provable case — +/// an extreme exceeding the whole window's total. +/// +public sealed class QueryStatExtremesTests +{ + /// The ordinary row: extremes inside the window's totals say nothing. + [Fact] + public void ExtremesWithinTotalsCarryNoNote() + { + Assert.Null(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 1000, maxCpu: 400, totalElapsed: 2000, maxElapsed: 900)); + } + + /// + /// Equality is NOT the provable case — a single-execution window has max == total by + /// construction, and flagging it would put the note on every one-shot query. + /// + [Fact] + public void ExactEqualityCarriesNoNote() + { + Assert.Null(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 400, maxCpu: 400, totalElapsed: 900, maxElapsed: 900)); + } + + /// + /// The field case that filed this: dbo.ClosingReportV6, window total 158,906 ms against a + /// lifetime max of 322,066 ms. The note fires and names the CPU column. + /// + [Fact] + public void CpuExtremeBeyondTheWindowTotalIsFlagged() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 158906, maxCpu: 322066, totalElapsed: 500000, maxElapsed: 400000); + + Assert.NotNull(note); + Assert.Contains("max_cpu_ms exceeds", note, System.StringComparison.Ordinal); + Assert.DoesNotContain("max_elapsed_ms", note, System.StringComparison.Ordinal); + } + + /// Elapsed alone proves it too, and the note names that column instead. + [Fact] + public void ElapsedExtremeBeyondTheWindowTotalIsFlagged() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 500000, maxCpu: 400000, totalElapsed: 158906, maxElapsed: 322066); + + Assert.NotNull(note); + Assert.Contains("max_elapsed_ms exceeds", note, System.StringComparison.Ordinal); + Assert.DoesNotContain("max_cpu_ms", note, System.StringComparison.Ordinal); + } + + /// Both provable, both named — a reader should not have to re-derive which half. + [Fact] + public void BothExtremesBeyondTotalsNameBoth() + { + var note = QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 100, maxCpu: 200, totalElapsed: 100, maxElapsed: 200); + + Assert.NotNull(note); + Assert.Contains("max_cpu_ms and max_elapsed_ms exceed", note, System.StringComparison.Ordinal); + } + + /// + /// A zero-total window with a nonzero lifetime max is the purest form of the problem — the plan + /// did nothing this window and the extreme is entirely inherited. It must flag. + /// + [Fact] + public void InheritedExtremeOnAnIdleWindowIsFlagged() + { + Assert.NotNull(QueryStatExtremes.LifetimeExtremeNote( + totalCpu: 0, maxCpu: 5, totalElapsed: 0, maxElapsed: 5)); + } +} diff --git a/Lite.Tests/QueryStatsCollectorDefinitionTests.cs b/Lite.Tests/QueryStatsCollectorDefinitionTests.cs index 341fde1d7..61c1ce5ec 100644 --- a/Lite.Tests/QueryStatsCollectorDefinitionTests.cs +++ b/Lite.Tests/QueryStatsCollectorDefinitionTests.cs @@ -165,20 +165,22 @@ public async Task ReadAsync_WritePayload_PlanCaptureOn_CapturesTrailingPlanColum var deltas = new RecordingCollectorDeltaCalculator(); var context = MakeContext(deltas: deltas, capturePlanXml: true); - /* Flag on = the SELECT carries host_object_name at ordinal 42 (#2012 stage 2) and the - trailing query_plan_xml column at ordinal 43. */ - var row44 = new object[44]; - row44[0] = "SO"; row44[1] = "0xQH"; row44[2] = "0xQPH"; - row44[3] = new DateTime(2026, 7, 2, 1, 0, 0, DateTimeKind.Utc); - row44[4] = new DateTime(2026, 7, 2, 2, 0, 0, DateTimeKind.Utc); - for (int i = 5; i < 36; i++) row44[i] = (long)i; - row44[22] = 4L; row44[23] = 8L; - row44[36] = "0xSH"; row44[37] = "0xPH"; row44[38] = "SELECT 1"; - row44[39] = 3L; row44[40] = 66; row44[41] = 512; - row44[42] = "dbo.HostProc"; - row44[43] = "captured"; - - using var reader = new FakeCollectorDataReader(row44); + /* Flag on = the SELECT carries host_object_name at ordinal 42 (#2012 stage 2), then + compile_age_seconds at 43 (#2235, inside SelectColumnsText so it is present in BOTH capture + modes), then the trailing query_plan_xml column at 44. */ + var row45 = new object[45]; + row45[0] = "SO"; row45[1] = "0xQH"; row45[2] = "0xQPH"; + row45[3] = new DateTime(2026, 7, 2, 1, 0, 0, DateTimeKind.Utc); + row45[4] = new DateTime(2026, 7, 2, 2, 0, 0, DateTimeKind.Utc); + for (int i = 5; i < 36; i++) row45[i] = (long)i; + row45[22] = 4L; row45[23] = 8L; + row45[36] = "0xSH"; row45[37] = "0xPH"; row45[38] = "SELECT 1"; + row45[39] = 3L; row45[40] = 66; row45[41] = 512; + row45[42] = "dbo.HostProc"; + row45[43] = 17; + row45[44] = "captured"; + + using var reader = new FakeCollectorDataReader(row45); var rows = await QueryStatsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); var writer = new RecordingCollectorRowWriter(); @@ -187,6 +189,12 @@ trailing query_plan_xml column at ordinal 43. */ Assert.Equal(51, writer.Values.Count); Assert.Equal("captured", writer.Values[37]); /* query_plan_xml payload slot */ Assert.Equal("dbo.HostProc", writer.Values[50]); /* host_object_name payload slot */ + + /* #2235: the compile age reaches the delta calculator and is NOT stored — 51 payload values, as + pinned above, and one age per delta'd counter. Nine, because crediting only some of them would + make one row's metrics disagree about how much work it did. */ + Assert.Equal(8, deltas.SeriesAges.Count); + Assert.All(deltas.SeriesAges, age => Assert.Equal(17, age)); } private static string Collapse(string sql) => Regex.Replace(sql, @"\s+", ""); @@ -246,17 +254,20 @@ public async Task WritePayload_PinsFullRowIdentityDeltaKey_AndIntervalCapture() var deltas = new RecordingCollectorDeltaCalculator(); var context = CollectorTestContext.Make(deltas); - var row43 = new object[43]; - row43[0] = "SO"; row43[1] = "0xQH"; row43[2] = "0xQPH"; - row43[3] = new DateTime(2026, 7, 2, 1, 0, 0, DateTimeKind.Utc); - row43[4] = new DateTime(2026, 7, 2, 2, 0, 0, DateTimeKind.Utc); - for (int i = 5; i < 36; i++) row43[i] = (long)i; - row43[22] = 4L; row43[23] = 8L; /* dop as long via GetValue */ - row43[36] = "0xSH"; row43[37] = "0xPH"; row43[38] = "SELECT 1"; - row43[39] = 3L; row43[40] = 66; row43[41] = 512; - row43[42] = "dbo.HostProc"; /* host_object_name (#2012 stage 2) */ - - using var reader = new FakeCollectorDataReader(row43); + /* 44 wide with the plan flag OFF: compile_age_seconds (#2235) lives inside SelectColumnsText, so + unlike query_plan_xml it is present in both capture modes and its ordinal never moves. */ + var row44 = new object[44]; + row44[0] = "SO"; row44[1] = "0xQH"; row44[2] = "0xQPH"; + row44[3] = new DateTime(2026, 7, 2, 1, 0, 0, DateTimeKind.Utc); + row44[4] = new DateTime(2026, 7, 2, 2, 0, 0, DateTimeKind.Utc); + for (int i = 5; i < 36; i++) row44[i] = (long)i; + row44[22] = 4L; row44[23] = 8L; /* dop as long via GetValue */ + row44[36] = "0xSH"; row44[37] = "0xPH"; row44[38] = "SELECT 1"; + row44[39] = 3L; row44[40] = 66; row44[41] = 512; + row44[42] = "dbo.HostProc"; /* host_object_name (#2012 stage 2) */ + row44[43] = 25; /* compile_age_seconds (#2235) */ + + using var reader = new FakeCollectorDataReader(row44); var rows = await QueryStatsCollector.Instance.ReadAsync(reader, context, CancellationToken.None); var writer = new RecordingCollectorRowWriter(); @@ -271,5 +282,14 @@ public async Task WritePayload_PinsFullRowIdentityDeltaKey_AndIntervalCapture() Assert.Equal( new[] { "query_stats_exec", "query_stats_worker", "query_stats_elapsed", "query_stats_reads", "query_stats_writes", "query_stats_phys_reads", "query_stats_rows", "query_stats_spills" }, deltas.Calls.Select(c => c.Group).ToArray()); + + /* #2235: plan_handle is IN that key, so a recompile presents a new key and the first sighting of + it reports 0 — which on a churning instance is most of the server's CPU, and is invisible + because the honest "unknowable" path needs the same key to reappear lower. The compile age is + what lets the calculator tell "new to us" from "new to the world", so every one of the eight + counters must receive it: crediting only some would make one row's metrics disagree about how + much work it did. */ + Assert.Equal(8, deltas.SeriesAges.Count); + Assert.All(deltas.SeriesAges, age => Assert.Equal(25, age)); } } diff --git a/Lite.Tests/QueryStoreCollectorDefinitionTests.cs b/Lite.Tests/QueryStoreCollectorDefinitionTests.cs index a24d3aece..ea0d5892a 100644 --- a/Lite.Tests/QueryStoreCollectorDefinitionTests.cs +++ b/Lite.Tests/QueryStoreCollectorDefinitionTests.cs @@ -37,7 +37,8 @@ private static CollectorContext MakeContext( object? probeResult = null, DateTime? watermark = null, DateTime? collectionTime = null, - bool capturePlanXml = false) + bool capturePlanXml = false, + bool fetchQueryTextSeparately = false) => new() { ServerId = 42, @@ -48,6 +49,7 @@ private static CollectorContext MakeContext( Watermark = watermark, EnumerationProbeResult = probeResult, CapturePlanXml = capturePlanXml, + FetchQueryTextSeparately = fetchQueryTextSeparately, }; /// @@ -192,10 +194,11 @@ ordinals on both paths. */ /* Interval-grain incremental filter since #1907, on this path too — the Azure body IS the shared body, so the WHERE→HAVING move lands here by construction rather than by a second edit. */ - Assert.Contains("HAVING\n MAX(qsrs.last_execution_time) > @cutoff_time", Lf(plan.Text), StringComparison.Ordinal); + Assert.Contains("HAVING\n MAX(qsrs.last_execution_time) > @cutoff_time", Lf(plan.Text), StringComparison.Ordinal); Assert.DoesNotContain("WHERE qsrs.last_execution_time > @cutoff_time", plan.Text, StringComparison.Ordinal); Assert.Contains("ORDER BY qsrs.last_execution_time ASC", plan.Text, StringComparison.Ordinal); - Assert.Contains("OPTION(RECOMPILE, LOOP JOIN);", plan.Text, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE);", plan.Text, StringComparison.Ordinal); + Assert.DoesNotContain("LOOP JOIN", plan.Text, StringComparison.Ordinal); var parameter = Assert.Single(plan.Parameters); Assert.Equal("@cutoff_time", parameter.Name); @@ -464,7 +467,7 @@ public void BuildPerItemQuery_CombinesTheSlicesOfOneInterval_OnTheViewsNaturalKe Anything coarser would merge work that is genuinely distinct; anything finer would leave the slices split, which is the bug. */ Assert.Contains( - "GROUP BY\n qsrs.plan_id,\n qsrs.runtime_stats_interval_id,\n qsrs.execution_type_desc,\n qsrs.replica_group_id", + "GROUP BY\n qsrs.plan_id,\n qsrs.runtime_stats_interval_id,\n qsrs.execution_type_desc,\n qsrs.replica_group_id", text, StringComparison.Ordinal); @@ -481,20 +484,137 @@ public void BuildPerItemQuery_CombinesTheSlicesOfOneInterval_OnTheViewsNaturalKe Assert.Contains("max_dop = MAX(qsrs.max_dop)", text, StringComparison.Ordinal); /* The pre-filter is a prune, not a semantic: its interval list is a superset of what the HAVING - keeps, so it can never subtract a row. Without it the aggregate runs over the database's entire - retained Query Store every cycle — measured 1203ms against 375ms on a real 212k-row store. */ + keeps, so it can never subtract a row. #2133: the ids resolve from the INTERVAL CATALOG — + hundreds of rows — never by scanning runtime_stats itself (measured 20 ms vs 426 ms for the + identical id set on the field store that wedged). */ Assert.Contains("WHERE qsrs.runtime_stats_interval_id IN", text, StringComparison.Ordinal); - Assert.Contains("WHERE f.last_execution_time > @cutoff_time", text, StringComparison.Ordinal); - - /* The row SHAPE must not move: 55 selected columns, and the TOP/ORDER BY stay OUTSIDE the - aggregate so the cap counts intervals and can never truncate one interval's slices into a + Assert.Contains("FROM sys.query_store_runtime_stats_interval AS i", text, StringComparison.Ordinal); + Assert.Contains("WHERE i.end_time > @cutoff_time", text, StringComparison.Ordinal); + Assert.DoesNotContain("FROM sys.query_store_runtime_stats AS f", text, StringComparison.Ordinal); + + /* #2133 STAGING: the aggregate lands in a temp table and the plan/query/text joins run FROM it, + so the optimizer joins with real cardinalities instead of TVF fixed guesses — the monolithic + join re-materialized a TVF per probe, a fixed ≥30s cost on an 82k-plan catalog that no + catch-up width could reduce (staged: 524 ms, same store, same window). SELECT INTO emits no + result set, so the batch still returns exactly one; the leading DROP covers Azure's pooled + direct connections (on-prem the sp_executesql scope self-cleans). */ + Assert.Contains("DROP TABLE IF EXISTS #pm_qs_slice;", text, StringComparison.Ordinal); + Assert.Contains("INTO #pm_qs_slice", text, StringComparison.Ordinal); + Assert.Contains("FROM #pm_qs_slice AS qsrs\nJOIN sys.query_store_plan AS qsp", Lf(text), StringComparison.Ordinal); + + /* The row SHAPE must not move: 55 selected columns, and the TOP/ORDER BY stay on the final + SELECT so the cap counts intervals and can never truncate one interval's slices into a partial sum. WITH TIES + ASC are the #1960 never-a-hole pair: oldest-first shipping keeps the derived watermark at the shipped boundary, and WITH TIES stops a bare TOP from splitting a group of rows tied at that boundary — the strict `> @cutoff_time` would strand the - unshipped half forever. */ + unshipped half forever. The LOOP JOIN hint must never return to this query: looping from the + temp into the TVFs is the per-probe re-materialization #2133 removed. */ Assert.Contains($"TOP ({QueryStoreCollector.MaxRowsPerDatabase}) WITH TIES", text, StringComparison.Ordinal); - Assert.Contains(") AS qsrs\nJOIN sys.query_store_plan AS qsp", text, StringComparison.Ordinal); - Assert.Contains("ORDER BY qsrs.last_execution_time ASC\nOPTION(RECOMPILE, LOOP JOIN);", text, StringComparison.Ordinal); + Assert.Contains("ORDER BY qsrs.last_execution_time ASC\nOPTION(RECOMPILE);", Lf(text), StringComparison.Ordinal); + Assert.DoesNotContain("LOOP JOIN", text, StringComparison.Ordinal); + } + + /// + /// #2150: with the flag off — which is Lite, always — the payload is UNCHANGED, text and all. + /// + /// This is the pin that makes the feature safe to add at all. Lite stores query_sql_text + /// inline in DuckDB and its grid reads it from there, so nulling that column unconditionally would + /// blind Lite. The flag exists for exactly that reason, and "off changes nothing" is the property that + /// has to be enforced rather than assumed. + /// + [Fact] + public void WithoutTheFlag_TheTextStaysInline() + { + foreach (var azure in new[] { false, true }) + { + var text = PayloadSql(MakeContext(isAzureSqlDb: azure)); + + Assert.Contains("query_sql_text = qst.query_sql_text,", text, StringComparison.Ordinal); + Assert.DoesNotContain("query_sql_text = CONVERT", text, StringComparison.Ordinal); + } + } + + /// + /// #2150: with the flag on the text column becomes a placeholder AT THE SAME ORDINAL, and nothing else + /// about the payload moves. + /// + /// The ordinal is the load-bearing part: the readers index this row by number, so a column that + /// changed position would silently shift every later field onto the wrong value. Asserted by + /// normalizing the one column out of both forms and requiring the remainder to be identical — which + /// covers "nothing else moved" for every column at once, rather than for the handful someone thought + /// to list. + /// + /// The query_store_query_text join deliberately REMAINS. It is one row per key, and the + /// 10x measurement behind this change was taken with it in place, so dropping it would be an + /// unmeasured change riding along on a measured one. + /// + [Fact] + public void WithTheFlag_TheTextIsNulledAtTheSameOrdinal() + { + var inline = PayloadSql(MakeContext()); + var nulled = PayloadSql(MakeContext(fetchQueryTextSeparately: true)); + + Assert.DoesNotContain("query_sql_text = qst.query_sql_text", nulled, StringComparison.Ordinal); + Assert.Contains("query_sql_text = CONVERT(nvarchar(1), NULL),", nulled, StringComparison.Ordinal); + /* Immediately before query_hash, exactly where the real column sat. */ + Assert.Contains("query_sql_text = CONVERT(nvarchar(1), NULL),\n query_hash", Lf(nulled), StringComparison.Ordinal); + Assert.Contains("JOIN sys.query_store_query_text AS qst", nulled, StringComparison.Ordinal); + + Assert.Equal( + inline.Replace("query_sql_text = qst.query_sql_text,", "@@TEXT@@", StringComparison.Ordinal), + nulled.Replace("query_sql_text = CONVERT(nvarchar(1), NULL),", "@@TEXT@@", StringComparison.Ordinal)); + } + + /// + /// #2150: the text fetch resumes from a query_id watermark, is cut by an exact byte budget, and + /// ships in query_id order — the ordering being what makes a budget cut a SUFFIX, so the highest + /// stored id resumes with no hole. + /// + [Fact] + public void TextFetch_ResumesFromTheWatermark_AndIsBudgetCutInQueryIdOrder() + { + var sql = QueryStoreCollector.Instance.BuildTextFetchQuery( + "SO", MakeContext(fetchQueryTextSeparately: true), watermark: 4242, + candidateTexts: QueryStoreTextState.CandidateTexts, budgetBytes: 12 * 1024 * 1024).Text; + + Assert.Contains("EXECUTE [SO].sys.sp_executesql", sql, StringComparison.Ordinal); + Assert.Contains("WHERE qsq.query_id > 4242", sql, StringComparison.Ordinal); + Assert.Contains($"TOP ({QueryStoreTextState.CandidateTexts})", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY qsq.query_id", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY b.query_id", sql, StringComparison.Ordinal); + Assert.Contains("b.running_bytes - b.text_bytes < 12582912", sql, StringComparison.Ordinal); + /* ROWS, not the RANGE default: RANGE tie-groups peers and forces a spool, and the frame has to be + per-row because the cut falls BETWEEN two statements. */ + Assert.Contains("ROWS UNBOUNDED PRECEDING", sql, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE)", sql, StringComparison.Ordinal); + /* It fetches text and nothing else — plan XML has its own fetch, with its own watermark. */ + Assert.DoesNotContain("query_plan", sql, StringComparison.Ordinal); + } + + /// + /// #2150: every input that would make the fetch ship nothing and silently stall the watermark throws + /// instead. A stalled watermark looks exactly like a quiet database, which is why these are exceptions + /// rather than no-ops — the plan fetch learned this the hard way from several directions. + /// + [Fact] + public void TextFetch_RefusesInputsThatWouldStallTheWatermark() + { + var enabled = MakeContext(fetchQueryTextSeparately: true); + + /* Issuing it while the host still ships text inline would fetch and store text nobody reads. */ + Assert.Throws(() => + QueryStoreCollector.Instance.BuildTextFetchQuery("SO", MakeContext(), 0, 5_000, 1024)); + + /* `running_bytes - text_bytes < 0` excludes even the first candidate, so the pass ships nothing. */ + Assert.Throws(() => + QueryStoreCollector.Instance.BuildTextFetchQuery("SO", enabled, 0, 5_000, 0)); + + /* TOP (0) returns no rows; a negative literal is a syntax error. */ + Assert.Throws(() => + QueryStoreCollector.Instance.BuildTextFetchQuery("SO", enabled, 0, 0, 1024)); + + Assert.Throws(() => + QueryStoreCollector.Instance.BuildTextFetchQuery("SO", null!, 0, 5_000, 1024)); } /// @@ -517,11 +637,13 @@ public void Payload_EveryAverageColumn_IsTheCountWeightedMean() { var text = PayloadSql(MakeContext(probeResult: 16)); - /* Only the aggregating derived table — the outer projection references the same names as plain - columns, which is correct there and must not be mistaken for an un-weighted aggregate. */ - var open = text.IndexOf("FROM\n(", StringComparison.Ordinal); - var close = text.IndexOf(") AS qsrs", StringComparison.Ordinal); - Assert.True(open > 0 && close > open, "could not locate the slice-aggregating derived table"); + /* Only the aggregating STAGING statement (#2133: the aggregate lands in #pm_qs_slice and the + joins run from it) — the final projection references the same names as plain columns, which + is correct there and must not be mistaken for an un-weighted aggregate. The staging SELECT + is the marker's first occurrence; the final SELECT carries TOP on the marker line. */ + var open = text.IndexOf("SELECT /* PerformanceMonitorLite */\n", StringComparison.Ordinal); + var close = text.IndexOf("INTO #pm_qs_slice", StringComparison.Ordinal); + Assert.True(open > 0 && close > open, "could not locate the slice-aggregating staging statement"); var aggregate = text[open..close]; var averages = System.Text.RegularExpressions.Regex @@ -566,18 +688,18 @@ public void BuildPerItemQuery_ReplicaGroupIdEntersTheGroupingKey_OnlyWhereItBind foreach (var probe in new object[] { 16, 17 }) { var attributed = PayloadSql(MakeContext(probeResult: probe)); - Assert.Contains("qsrs.execution_type_desc,\n qsrs.replica_group_id", attributed, StringComparison.Ordinal); + Assert.Contains("qsrs.execution_type_desc,\n qsrs.replica_group_id", attributed, StringComparison.Ordinal); } var azure = AzurePayloadSql(MakeContext(isAzureSqlDb: true, probeResult: 12)); - Assert.Contains("qsrs.execution_type_desc,\n qsrs.replica_group_id", azure, StringComparison.Ordinal); + Assert.Contains("qsrs.execution_type_desc,\n qsrs.replica_group_id", azure, StringComparison.Ordinal); /* Pre-2022 box and Managed Instance: the column must not be named anywhere, GROUP BY included. */ foreach (var probe in new object?[] { 13, 14, 15, null }) { var ungated = PayloadSql(MakeContext(probeResult: probe)); Assert.DoesNotContain("replica_group_id", ungated, StringComparison.Ordinal); - Assert.Contains("GROUP BY\n qsrs.plan_id,\n qsrs.runtime_stats_interval_id,\n qsrs.execution_type_desc\n", ungated, StringComparison.Ordinal); + Assert.Contains("GROUP BY\n qsrs.plan_id,\n qsrs.runtime_stats_interval_id,\n qsrs.execution_type_desc\n", ungated, StringComparison.Ordinal); } } @@ -604,13 +726,14 @@ public void BuildPerItemQuery_PreSql2017_GatedFamiliesLeaveTheAggregate_ButKeepT Assert.Contains("avg_log_bytes_used = NULL, min_log_bytes_used = NULL, max_log_bytes_used = NULL,", old, StringComparison.Ordinal); Assert.Contains("avg_tempdb_space_used = NULL, min_tempdb_space_used = NULL, max_tempdb_space_used = NULL,", old, StringComparison.Ordinal); - /* The inner list must end cleanly on the last ungated column when all three are absent. */ - Assert.Contains("max_rowcount = MAX(qsrs.max_rowcount)\n FROM sys.query_store_runtime_stats AS qsrs", old, StringComparison.Ordinal); + /* The staging list must end cleanly on the last ungated column when all three are absent — + #2133: the aggregate lands in #pm_qs_slice, so INTO sits between the list and FROM. */ + Assert.Contains("max_rowcount = MAX(qsrs.max_rowcount)\nINTO #pm_qs_slice\nFROM sys.query_store_runtime_stats AS qsrs", old, StringComparison.Ordinal); /* On 2017+ they are present, aggregated, and the list ends with the last gated family instead. */ var newer = PayloadSql(MakeContext(probeResult: 14)); Assert.Contains("max_rowcount = MAX(qsrs.max_rowcount),\n", newer, StringComparison.Ordinal); - Assert.Contains("max_tempdb_space_used = MAX(qsrs.max_tempdb_space_used)\n FROM sys.query_store_runtime_stats AS qsrs", newer, StringComparison.Ordinal); + Assert.Contains("max_tempdb_space_used = MAX(qsrs.max_tempdb_space_used)\nINTO #pm_qs_slice\nFROM sys.query_store_runtime_stats AS qsrs", newer, StringComparison.Ordinal); } [Fact] @@ -763,7 +886,7 @@ that would have pinned the bug. A per-slice WHERE cannot survive slice aggregati original defect with an aggregate bolted on. HAVING MAX(...) asks whether the INTERVAL saw new activity and then takes all of it. */ var normalized = plan.Text.Replace("\r\n", "\n", StringComparison.Ordinal); - Assert.Contains("HAVING\n MAX(qsrs.last_execution_time) > @cutoff_time", normalized, StringComparison.Ordinal); + Assert.Contains("HAVING\n MAX(qsrs.last_execution_time) > @cutoff_time", normalized, StringComparison.Ordinal); Assert.DoesNotContain("WHERE qsrs.last_execution_time > @cutoff_time", normalized, StringComparison.Ordinal); /* #1565: NO SQL-side self-exclusion — the old NOT LIKE was 75% of the read's elapsed time (full nvarchar(max) scan per row on a column no index can serve; field A/B: 4.3x without it), and no @@ -771,7 +894,8 @@ activity and then takes all of it. */ where the text is already materialized (pinned below). The query still CONTAINS the marker — in its own leading comment. */ Assert.DoesNotContain("NOT LIKE", plan.Text, StringComparison.Ordinal); - Assert.Contains("OPTION(RECOMPILE, LOOP JOIN);", plan.Text, StringComparison.Ordinal); + Assert.Contains("OPTION(RECOMPILE);", plan.Text, StringComparison.Ordinal); + Assert.DoesNotContain("LOOP JOIN", plan.Text, StringComparison.Ordinal); Assert.Contains("N'@cutoff_time datetime2(7)',", plan.Text, StringComparison.Ordinal); var parameter = Assert.Single(plan.Parameters); @@ -781,28 +905,23 @@ in its own leading comment. */ } [Fact] - public void BuildPerItemQuery_PlanCapture_OffEmitsNullPlaceholder_OnMirrorsDashboard() + public void BuildPerItemQuery_PlanCapture_AlwaysEmitsNullPlaceholder_RegardlessOfCapturePlanXml() { - /* Lite parity (default off): query_plan_text is the nvarchar(1) NULL placeholder, - byte-identical to the no-plan form. Darling (on): CONVERT(nvarchar(max), qsp.query_plan) - from sys.query_store_plan — install/09_collect_query_store.sql's @collect_plan path. */ + /* #2210: the runtime-stats query no longer carries plan XML at all, in EITHER capture mode — the + ROW_NUMBER-gated CASE and its watermark predicate are DELETED, not reworked. + BuildPlanFetchQuery is the only thing that reads plan XML now (it fetches plans in plan_id + order under a byte budget and is Darling-only), so CapturePlanXml gates that separate fetch + rather than this query. Lite's off path and Darling's on path are therefore byte-identical + here — there is no longer a Darling-only branch of this query to pin. */ var off = QueryStoreCollector.Instance.BuildPerItemQuery("SO", MakeContext()); - Assert.Contains("query_plan_text = CONVERT(nvarchar(1), NULL),", off.Text, StringComparison.Ordinal); - Assert.DoesNotContain("qsp.query_plan,", off.Text, StringComparison.Ordinal); - - /* #1556 plan-text dedupe (ON branch): the plan lands once per plan_id per cycle — on the newest - runtime-stats interval (rn = 1) — and NULL on the older intervals, instead of the full plan XML - repeating on every interval row. */ var on = QueryStoreCollector.Instance.BuildPerItemQuery("SO", MakeContext(capturePlanXml: true)); - Assert.Contains( - "query_plan_text = CASE WHEN ROW_NUMBER() OVER (PARTITION BY qsp.plan_id ORDER BY qsrs.last_execution_time DESC) = 1 THEN CONVERT(nvarchar(max), qsp.query_plan) ELSE CONVERT(nvarchar(max), NULL) END,", - on.Text, - StringComparison.Ordinal); - /* Scoped to query_plan_text rather than the bare placeholder: replica_role shares the same - nvarchar(1) NULL idiom on a pre-2022 target (this context's probe defaults to 13), so an - unqualified DoesNotContain would assert on an unrelated column. */ - Assert.DoesNotContain("query_plan_text = CONVERT(nvarchar(1), NULL)", on.Text, StringComparison.Ordinal); + Assert.Contains("query_plan_text = CONVERT(nvarchar(1), NULL),", off.Text, StringComparison.Ordinal); + Assert.Contains("query_plan_text = CONVERT(nvarchar(1), NULL),", on.Text, StringComparison.Ordinal); + Assert.DoesNotContain("qsp.query_plan,", off.Text, StringComparison.Ordinal); + Assert.DoesNotContain("qsp.query_plan,", on.Text, StringComparison.Ordinal); + Assert.DoesNotContain("ROW_NUMBER()", on.Text, StringComparison.Ordinal); + Assert.True(string.Equals(off.Text, on.Text, StringComparison.Ordinal), "CapturePlanXml must no longer change this query's text"); } [Fact] @@ -823,8 +942,11 @@ TOP from splitting a group of rows tied at that boundary (the strict `> @cutoff_ { Assert.Contains($"TOP ({QueryStoreCollector.MaxRowsPerDatabase}) WITH TIES", plan.Text, StringComparison.Ordinal); Assert.Contains("ORDER BY qsrs.last_execution_time ASC", plan.Text, StringComparison.Ordinal); - /* The row-bounding ORDER BY sits before the existing query hint, which the OPTION pin still checks. */ - Assert.Contains("OPTION(RECOMPILE, LOOP JOIN);", plan.Text, StringComparison.Ordinal); + /* The row-bounding ORDER BY sits before the existing query hint, which the OPTION pin still + checks. RECOMPILE only — the old LOOP JOIN hint is the #2133 pathology (per-probe TVF + re-materialization) and must never return. */ + Assert.Contains("OPTION(RECOMPILE);", plan.Text, StringComparison.Ordinal); + Assert.DoesNotContain("LOOP JOIN", plan.Text, StringComparison.Ordinal); } Assert.Equal(50_000, QueryStoreCollector.MaxRowsPerDatabase); @@ -1082,8 +1204,11 @@ live window instead of the backlog. */ Assert.Contains("EXECUTE [StackOverflow].sys.sp_executesql", plan.Text, StringComparison.Ordinal); Assert.Contains("N'@floor_time datetime2(7), @ceiling_time datetime2(7)'", plan.Text, StringComparison.Ordinal); - Assert.Contains("f.last_execution_time > @floor_time", plan.Text, StringComparison.Ordinal); - Assert.Contains("f.last_execution_time < @ceiling_time", plan.Text, StringComparison.Ordinal); + /* #2133: the pre-filter's two-sided window asks the INTERVAL CATALOG which intervals OVERLAP + (floor, ceiling) — end after the floor AND start before the ceiling — a superset the exact + HAVING below then narrows, exactly like the live path's one-sided form. */ + Assert.Contains("i.end_time > @floor_time", plan.Text, StringComparison.Ordinal); + Assert.Contains("i.start_time < @ceiling_time", plan.Text, StringComparison.Ordinal); Assert.Contains("MAX(qsrs.last_execution_time) > @floor_time", plan.Text, StringComparison.Ordinal); Assert.Contains("MAX(qsrs.last_execution_time) < @ceiling_time", plan.Text, StringComparison.Ordinal); Assert.Contains("ORDER BY qsrs.last_execution_time DESC", plan.Text, StringComparison.Ordinal); @@ -1109,8 +1234,8 @@ own catalog would be worse than a loud wrong-path error. */ var plan = QueryStoreCollector.Instance.BuildBackfillQuery(MakeContext(isAzureSqlDb: true), floor, ceiling); Assert.DoesNotContain("sp_executesql", plan.Text, StringComparison.Ordinal); - Assert.Contains("f.last_execution_time > @floor_time", plan.Text, StringComparison.Ordinal); - Assert.Contains("f.last_execution_time < @ceiling_time", plan.Text, StringComparison.Ordinal); + Assert.Contains("i.end_time > @floor_time", plan.Text, StringComparison.Ordinal); + Assert.Contains("i.start_time < @ceiling_time", plan.Text, StringComparison.Ordinal); Assert.Contains("ORDER BY qsrs.last_execution_time DESC", plan.Text, StringComparison.Ordinal); Assert.DoesNotContain("@cutoff_time", plan.Text, StringComparison.Ordinal); /* The same eligibility gate the live Azure query leads with. */ @@ -1132,7 +1257,7 @@ public void BuildBackfillPerItemQuery_LiveBodyStaysUntouched() one-sided cutoff and ASC order byte-for-byte, or phase 1's watermark-exact resume breaks in the same PR that builds on it. */ var live = QueryStoreCollector.Instance.BuildPerItemQuery("StackOverflow", MakeContext()); - Assert.Contains("f.last_execution_time > @cutoff_time", live.Text, StringComparison.Ordinal); + Assert.Contains("i.end_time > @cutoff_time", live.Text, StringComparison.Ordinal); Assert.Contains("MAX(qsrs.last_execution_time) > @cutoff_time", live.Text, StringComparison.Ordinal); Assert.Contains("ORDER BY qsrs.last_execution_time ASC", live.Text, StringComparison.Ordinal); Assert.DoesNotContain("@floor_time", live.Text, StringComparison.Ordinal); @@ -1273,4 +1398,77 @@ public async Task ReadItemAsync_WritePayload_Pins56ColumnOrder_AndTypeCoercions( Assert.Equal(new DateTime(2026, 7, 2, 10, 0, 0), writer.Values[55]); /* interval_start_time_utc, no shift applied */ Assert.Empty(s_deltas.Calls); /* incremental snapshot — no deltas */ } + + /* ---------------- #2312: the open-interval skip cycles ---------------- */ + + /// + /// The closed-only form (#2312): most cycles exclude the OPEN interval — its cumulative snapshot is + /// the whole re-read bill on a big primary (40–110 s per run measured) and every snapshot but the + /// latest is discarded by the read side's rn = 1. Closed intervals are immutable, so shipping + /// only them is final on first collection; the standing HAVING readmits a newly closed interval + /// whose content moved past our last open-snapshot, because counters only move with executions. + /// + [Fact] + public void BuildPerItemQuery_ClosedIntervalsOnly_ExcludesTheOpenInterval() + { + var context = MakeContext(probeResult: 16); + context.IncludeOpenInterval = false; + var text = PayloadSql(context); + + Assert.Contains( + "WHERE i.end_time > @cutoff_time\n AND i.end_time <= SYSUTCDATETIME()", + text, StringComparison.Ordinal); + + /* Server-evaluated exclusion — the single-parameter sp_executesql contract is untouched. */ + Assert.Single(QueryStoreCollector.Instance.BuildPerItemQuery("SO", context).Parameters); + + /* The row-level filter is byte-identical: the skip narrows the interval-id PRUNE only, never + the shipped semantics of the rows that do qualify. */ + Assert.Contains("HAVING\n MAX(qsrs.last_execution_time) > @cutoff_time", text, StringComparison.Ordinal); + } + + /// Default = today's exact form: no exclusion anywhere, so every untouched caller is byte-identical. + [Fact] + public void BuildPerItemQuery_DefaultIncludesTheOpenInterval_TodaysExactForm() + { + var text = PayloadSql(MakeContext(probeResult: 16)); + + Assert.Contains("WHERE i.end_time > @cutoff_time\n", text, StringComparison.Ordinal); + Assert.DoesNotContain("SYSUTCDATETIME", text, StringComparison.Ordinal); + } + + /// + /// The remainder pin, same discipline as the text-flag one: normalize the ONE legal difference out + /// of the closed-only form and everything else must be byte-identical to the open form — the pin + /// that catches a future edit landing in one arm only. + /// + [Fact] + public void BuildPerItemQuery_ClosedOnly_ChangesNothingButTheIntervalPrune() + { + var closedContext = MakeContext(probeResult: 16); + closedContext.IncludeOpenInterval = false; + + var open = PayloadSql(MakeContext(probeResult: 16)); + var closed = PayloadSql(closedContext); + + var normalized = closed.Replace( + "\n AND i.end_time <= SYSUTCDATETIME()", "", StringComparison.Ordinal); + Assert.NotEqual(open, closed); + Assert.Equal(open, normalized); + } + + /// The backfill window pre-dates the open interval by construction; the flag must not touch it. + [Fact] + public void BuildBackfillPerItemQuery_IgnoresTheOpenIntervalFlag() + { + var floor = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var ceiling = new DateTime(2026, 6, 2, 0, 0, 0, DateTimeKind.Utc); + + var flagged = MakeContext(probeResult: 16); + flagged.IncludeOpenInterval = false; + + Assert.Equal( + Lf(QueryStoreCollector.Instance.BuildBackfillPerItemQuery("SO", MakeContext(probeResult: 16), floor, ceiling).Text), + Lf(QueryStoreCollector.Instance.BuildBackfillPerItemQuery("SO", flagged, floor, ceiling).Text)); + } } diff --git a/Lite.Tests/QueryStoreHealthCollectorDefinitionTests.cs b/Lite.Tests/QueryStoreHealthCollectorDefinitionTests.cs new file mode 100644 index 000000000..1576e78d6 --- /dev/null +++ b/Lite.Tests/QueryStoreHealthCollectorDefinitionTests.cs @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lite.Tests.Helpers; +using PerformanceMonitor.Collectors; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the parity contract of the query_store_health definition (#2319) — database_scoped_config's +/// enumeration shape applied to sys.database_query_store_options: database-list selection +/// (AG-primary filter on-prem, plain list on Azure), the [db].sys.sp_executesql per-database query with +/// bracket escaping, the 2016+ AppliesTo gate, and the 10-column payload — including the +/// ordinal-mapped and the +/// order, the two paths most likely to silently +/// drift on a future column reorder. +/// +public sealed class QueryStoreHealthCollectorDefinitionTests +{ + private static readonly RecordingCollectorDeltaCalculator s_deltas = new(); + + [Fact] + public void EnumerationQuery_OnPrem_FiltersToAgPrimaries_AndSplicesExclusions() + { + var plan = QueryStoreHealthCollector.Instance.BuildEnumerationQuery(new CollectorContext + { + ServerId = 42, + ServerName = "test-server", + CollectionTime = DateTime.UtcNow, + Deltas = s_deltas, + ExcludedDatabases = new[] { "SO" }, + }); + + Assert.NotNull(plan); + Assert.Contains("sys.dm_hadr_database_replica_states", plan!.Text, StringComparison.Ordinal); + Assert.Contains("is_primary_replica = 1", plan.Text, StringComparison.Ordinal); + /* #1823: a least-privilege login without per-db access must be filtered out up front, the + same self-skip the sibling per-database collectors carry. */ + Assert.Contains("HAS_DBACCESS(d.name) = 1", plan.Text, StringComparison.Ordinal); + Assert.Contains("AND d.name NOT IN (@excl_db_0)", plan.Text, StringComparison.Ordinal); + Assert.Equal("SO", Assert.Single(plan.Parameters).Value); + } + + [Fact] + public void EnumerationQuery_Azure_ListsAllOnline_NoAgFilter() + { + var plan = QueryStoreHealthCollector.Instance.BuildEnumerationQuery( + CollectorTestContext.Make(s_deltas, isAzureSqlDb: true)); + + Assert.NotNull(plan); + Assert.DoesNotContain("dm_hadr_database_replica_states", plan!.Text, StringComparison.Ordinal); + Assert.Contains("state_desc = N'ONLINE'", plan.Text, StringComparison.Ordinal); + /* The asymmetry is deliberate: from master on Azure SQL DB, HAS_DBACCESS() returns 0 for + every user database, so the on-prem filter here would silently enumerate nothing. */ + Assert.DoesNotContain("HAS_DBACCESS", plan.Text, StringComparison.Ordinal); + } + + [Fact] + public void PerItemQuery_EscapesClosingBrackets_InDatabaseNames() + { + var plan = QueryStoreHealthCollector.Instance.BuildPerItemQuery("we]rd db", CollectorTestContext.Make(s_deltas)); + + Assert.Contains("EXECUTE [we]]rd db].sys.sp_executesql", plan.Text, StringComparison.Ordinal); + Assert.Contains("sys.database_query_store_options", plan.Text, StringComparison.Ordinal); + Assert.Empty(plan.Parameters); + } + + /// Query Store shipped in 2016 (v13); without the gate a pre-2016 target errors once per + /// database per hour. Same condition as QueryStoreCollector, so both SKUs skip identically. + [Theory] + [InlineData(11, false)] + [InlineData(12, false)] + [InlineData(13, true)] + [InlineData(0, true)] /* version unknown = assume newest */ + public void AppliesTo_GatesOnQueryStoresExistence(int majorVersion, bool applies) + => Assert.Equal(applies, QueryStoreHealthCollector.Instance.AppliesTo( + new CollectorTargetInfo { SqlMajorVersion = majorVersion })); + + [Fact] + public async Task ReadItemAsync_AccumulatesAcrossItems_TaggedWithDatabase_AndNullsCoalesceHonestly() + { + var rows = new List(); + var context = CollectorTestContext.Make(s_deltas); + + /* A healthy READ_WRITE database with a cap. */ + using (var reader = new FakeCollectorDataReader(new object[] + { "READ_WRITE", "READ_WRITE", 0, 512L, 1000L, "AUTO", 30L, 200L, 60L })) + { + await QueryStoreHealthCollector.Instance.ReadItemAsync("db1", reader, rows, context, CancellationToken.None); + } + + /* The cap-hit shape this collector exists for: desired READ_WRITE, actual READ_ONLY, reason + 65536 — plus DBNulls exercising every coalesce arm. */ + using (var reader = new FakeCollectorDataReader(new object[] + { "READ_ONLY", "READ_WRITE", 65536, 1000L, 1000L, DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value })) + { + await QueryStoreHealthCollector.Instance.ReadItemAsync("db2", reader, rows, context, CancellationToken.None); + } + + Assert.Equal(2, rows.Count); + + Assert.Equal("db1", rows[0].DbName); + Assert.Equal("READ_WRITE", rows[0].ActualState); + Assert.Equal("READ_WRITE", rows[0].DesiredState); + Assert.Equal(0, rows[0].ReadonlyReason); + Assert.Equal(512L, rows[0].CurrentStorageMb); + Assert.Equal(1000L, rows[0].MaxStorageMb); + Assert.Equal("AUTO", rows[0].SizeBasedCleanupMode); + Assert.Equal(30L, rows[0].StaleQueryThresholdDays); + Assert.Equal(200L, rows[0].MaxPlansPerQuery); + Assert.Equal(60L, rows[0].IntervalLengthMinutes); + + Assert.Equal("db2", rows[1].DbName); + Assert.Equal("READ_ONLY", rows[1].ActualState); + Assert.Equal("READ_WRITE", rows[1].DesiredState); + Assert.Equal(65536, rows[1].ReadonlyReason); + Assert.Null(rows[1].SizeBasedCleanupMode); + Assert.Equal(0L, rows[1].StaleQueryThresholdDays); + Assert.Equal(0L, rows[1].MaxPlansPerQuery); + Assert.Equal(0L, rows[1].IntervalLengthMinutes); + } + + [Fact] + public void PayloadColumns_MatchSchema_AndWriteOrder() + { + Assert.Equal( + new[] + { + "database_name", "actual_state", "desired_state", "readonly_reason", + "current_storage_size_mb", "max_storage_size_mb", "size_based_cleanup_mode", + "stale_query_threshold_days", "max_plans_per_query", "interval_length_minutes", + }, + QueryStoreHealthCollector.Instance.PayloadColumns.Select(c => c.Name).ToArray()); + + var writer = new RecordingCollectorRowWriter(); + QueryStoreHealthCollector.Instance.WritePayload( + new QueryStoreHealthCollector.Row + { + DbName = "db1", + ActualState = "READ_WRITE", + DesiredState = "READ_WRITE", + ReadonlyReason = 0, + CurrentStorageMb = 512L, + MaxStorageMb = 1000L, + SizeBasedCleanupMode = "AUTO", + StaleQueryThresholdDays = 30L, + MaxPlansPerQuery = 200L, + IntervalLengthMinutes = 60L, + }, + writer, CollectorTestContext.Make(s_deltas)); + + Assert.Equal( + new object?[] { "db1", "READ_WRITE", "READ_WRITE", 0, 512L, 1000L, "AUTO", 30L, 200L, 60L }, + writer.Values); + } +} diff --git a/Lite.Tests/QueryStoreReplicaSplitAnalysisTests.cs b/Lite.Tests/QueryStoreReplicaSplitAnalysisTests.cs index 76ceb6f08..8c00c15f9 100644 --- a/Lite.Tests/QueryStoreReplicaSplitAnalysisTests.cs +++ b/Lite.Tests/QueryStoreReplicaSplitAnalysisTests.cs @@ -106,7 +106,8 @@ private async Task SeedAsync( DateTime firstExecutionTime, DateTime lastExecutionTime, long avgCpuUs, - string? replicaRole) + string? replicaRole, + long? avgDurUs = null) { using var readLock = _duckDb.AcquireReadLock(); var connection = await SeedConnectionAsync(); @@ -135,8 +136,9 @@ INSERT INTO query_store_stats floor even after the dedup collapses the repeat collections down to one row. */ cmd.Parameters.Add(new DuckDBParameter { Value = 100L }); cmd.Parameters.Add(new DuckDBParameter { Value = avgCpuUs }); - /* Duration tracks CPU so GREATEST(cpu ratio, duration ratio) is unambiguous. */ - cmd.Parameters.Add(new DuckDBParameter { Value = avgCpuUs + 20_000 }); + /* Duration defaults to CPU + 20ms, so the classic seeds regress on BOTH signals and fire the + CPU-primary path (#2138); the split-signal tests pass avgDurUs to move one without the other. */ + cmd.Parameters.Add(new DuckDBParameter { Value = avgDurUs ?? avgCpuUs + 20_000 }); cmd.Parameters.Add(new DuckDBParameter { Value = planHash }); cmd.Parameters.Add(new DuckDBParameter { Value = false }); cmd.Parameters.Add(new DuckDBParameter { Value = 0L }); @@ -184,6 +186,58 @@ await SeedAsync(collectionTime, planId: 2, BadPlanHash, intervalId: 2, } } + /// + /// One plan-cache row for THE regressed query's hash ('0xREGRESSQH'), carrying — or, with a tame + /// worker-time spread, deliberately missing — the PARAMETER_SENSITIVITY detector's firing + /// signature (#2138 gap 3). Grants flat, no spills: the worker ratio is the only dial. + /// + private async Task SeedPlanCacheRowAsync(long minWorkerUs, long maxWorkerUs) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" +INSERT INTO query_stats + (collection_id, collection_time, server_id, server_name, database_name, + query_hash, query_plan_hash, creation_time, execution_count, + min_worker_time, max_worker_time, min_grant_kb, max_grant_kb, + min_spills, max_spills, query_text, delta_execution_count) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"; + cmd.Parameters.Add(new DuckDBParameter { Value = _nextId++ }); + cmd.Parameters.Add(new DuckDBParameter { Value = PeriodEnd.AddMinutes(-5) }); + cmd.Parameters.Add(new DuckDBParameter { Value = ServerId }); + cmd.Parameters.Add(new DuckDBParameter { Value = ServerName }); + cmd.Parameters.Add(new DuckDBParameter { Value = Db }); + cmd.Parameters.Add(new DuckDBParameter { Value = "0xREGRESSQH" }); + cmd.Parameters.Add(new DuckDBParameter { Value = BadPlanHash }); + cmd.Parameters.Add(new DuckDBParameter { Value = PeriodStart.AddDays(-3) }); + cmd.Parameters.Add(new DuckDBParameter { Value = 100L }); + cmd.Parameters.Add(new DuckDBParameter { Value = minWorkerUs }); + cmd.Parameters.Add(new DuckDBParameter { Value = maxWorkerUs }); + cmd.Parameters.Add(new DuckDBParameter { Value = 1_024L }); + cmd.Parameters.Add(new DuckDBParameter { Value = 1_024L }); + cmd.Parameters.Add(new DuckDBParameter { Value = 0L }); + cmd.Parameters.Add(new DuckDBParameter { Value = 0L }); + cmd.Parameters.Add(new DuckDBParameter { Value = "SELECT * FROM dbo.Orders WHERE CustomerId = @id" }); + cmd.Parameters.Add(new DuckDBParameter { Value = 50L }); + await cmd.ExecuteNonQueryAsync(); + } + + /// + /// One replica-less query, two plans, CPU and duration controlled INDEPENDENTLY — the seed shape for + /// the #2138 CPU-primary scoring pins, where which signal moved is the entire test. + /// + private async Task SeedCpuAndDurationSplitAsync( + long goodCpuUs, long goodDurUs, long badCpuUs, long badDurUs) + { + var collectionTime = PeriodEnd.AddMinutes(-10); + + await SeedAsync(collectionTime, planId: 1, GoodPlanHash, intervalId: 1, + PeriodStart.AddDays(-6), PeriodStart.AddDays(-5), goodCpuUs, replicaRole: null, goodDurUs); + await SeedAsync(collectionTime, planId: 2, BadPlanHash, intervalId: 2, + PeriodStart.AddDays(-1), PeriodEnd, badCpuUs, replicaRole: null, badDurUs); + } + private async Task CollectPlanRegressionFactAsync() { var facts = await new DuckDbFactCollector(_duckDb).CollectFactsAsync(Context()); @@ -288,4 +342,99 @@ and the regression factor with them. */ Assert.Equal(2.0, fact!.Metadata["offender_count"]); Assert.Equal(12.0, fact.Metadata["worst_regression_factor"], precision: 1); } + + [Fact] + public async Task DurationOnlyRegression_DoesNotFire_CpuIsThePrimarySignal() + { + /* #2138: duration alone is confounded by blocking, IO waits and machine contention that no plan + choice caused. CPU flat, duration 5x worse — under the old GREATEST this fired at 5.0; now the + CPU path (1x < 2) and the corroboration gate (1x < 1.25) both decline it. */ + await SeedCpuAndDurationSplitAsync( + goodCpuUs: 100_000, goodDurUs: 120_000, + badCpuUs: 100_000, badDurUs: 600_000); + + Assert.Null(await CollectPlanRegressionFactAsync()); + + /* The drill-down runs the same scoring — a row here that the fact never counted would be + incoherent in the report. */ + Assert.Empty(await CollectRegressedQueriesDrillDownAsync()); + } + + [Fact] + public async Task ExtremeDurationRegression_WithMildCpuCorroboration_FiresAtHalfTheDurationRatio() + { + /* #2138: the duration path stays open for the genuinely extreme case — 6x duration with 1.5x CPU + corroboration — but scores at HALF the duration ratio (3.0, not 6.0) so it competes honestly + with CPU-detected rows. */ + await SeedCpuAndDurationSplitAsync( + goodCpuUs: 100_000, goodDurUs: 100_000, + badCpuUs: 150_000, badDurUs: 600_000); + + var fact = await CollectPlanRegressionFactAsync(); + + Assert.NotNull(fact); + Assert.Equal(3.0, fact!.Metadata["worst_regression_factor"], precision: 1); + /* A duration-fired row reports the duration dimension. */ + Assert.Equal(2.0, fact.Metadata["regressed_dimension"]); + } + + [Fact] + public async Task CpuFiredRow_WithLargerDurationRatio_StillReportsTheCpuDimension() + { + /* Review catch on #2138: CPU has PRECEDENCE in the scoring, so cpu 2.5x with duration 10x (a + genuine CPU regression that also picked up blocking) fires the CPU branch at 2.5 — and must + be LABELED cpu. Comparing raw ratio magnitudes, correct under the old GREATEST, would call + this duration-caused; a plan-forcing bot reading the dimension would misjudge WHY. */ + await SeedCpuAndDurationSplitAsync( + goodCpuUs: 100_000, goodDurUs: 100_000, + badCpuUs: 250_000, badDurUs: 1_000_000); + + var fact = await CollectPlanRegressionFactAsync(); + + Assert.NotNull(fact); + Assert.Equal(2.5, fact!.Metadata["worst_regression_factor"], precision: 1); + Assert.Equal(1.0, fact.Metadata["regressed_dimension"]); + } + + [Fact] + public async Task BelowTheSpendFloor_ATinyQuery_DoesNotFire() + { + /* #2138: a 12x CPU ratio on a query burning 1.2 CPU-seconds across the whole 14-day window + (100 execs x 12ms) is sampling jitter, not a finding. Same 12x ratio as the NonAgServer arm — + the only difference is absolute spend, so this pins the 10-CPU-second noise floor and nothing + else. */ + await SeedCpuAndDurationSplitAsync( + goodCpuUs: 1_000, goodDurUs: 21_000, + badCpuUs: 12_000, badDurUs: 32_000); + + Assert.Null(await CollectPlanRegressionFactAsync()); + Assert.Empty(await CollectRegressedQueriesDrillDownAsync()); + } + + [Fact] + public async Task RegressedQuery_WithThePlanCachePspSignature_CarriesTheCoFiredFlag() + { + /* #2138 gap 3: the same query hash regresses in Query Store AND shows the parameter-sensitivity + signature in the plan cache (min 15ms, max 300ms — past every detector floor, ratio 20x). The + drill-down row must say so, because the force-plan remediation's caution and the future bot's + never-auto-force gate both read this flag. */ + await SeedOneReplicaAsync(role: null, BadCpuUsPrimary, offsetSeconds: 0); + await SeedPlanCacheRowAsync(minWorkerUs: 15_000, maxWorkerUs: 300_000); + + var row = Assert.Single(await CollectRegressedQueriesDrillDownAsync()); + Assert.True(row.GetProperty("parameter_sensitivity_cofired").GetBoolean()); + } + + [Fact] + public async Task RegressedQuery_BelowThePspRatio_FlagStaysFalse() + { + /* Same floors, but a 2x worker-time spread — ordinary variance, not the >= 10x signature. Pins + that the flag uses the PARAMETER_SENSITIVITY detector's own threshold, not mere presence of + the hash in the plan cache. */ + await SeedOneReplicaAsync(role: null, BadCpuUsPrimary, offsetSeconds: 0); + await SeedPlanCacheRowAsync(minWorkerUs: 150_000, maxWorkerUs: 300_000); + + var row = Assert.Single(await CollectRegressedQueriesDrillDownAsync()); + Assert.False(row.GetProperty("parameter_sensitivity_cofired").GetBoolean()); + } } diff --git a/Lite.Tests/QueryStoreServerGateTests.cs b/Lite.Tests/QueryStoreServerGateTests.cs new file mode 100644 index 000000000..ac400d781 --- /dev/null +++ b/Lite.Tests/QueryStoreServerGateTests.cs @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the #2165 gate: the per-server mutual exclusion that stops the tick's Query Store collection and the +/// #2058 backfill slice from running heavy QS text extraction against ONE server at the same time. +/// +/// The field evidence. On a 4-core multi-tenant box mid-consolidation, a 64 MB backfill slice for a +/// freshly restored database overlapped the tick's collection of a sibling database — a 12:50:58 ship against a +/// 12:51:09 tick completion — so roughly 128 MB of text extraction was in flight on the box least able to afford +/// it. The two loops collide precisely when a server is already drowning, because a big catalog arriving is what +/// triggers both. +/// +/// The two properties worth pinning are that it EXCLUDES, and that it never WAITS. The second is as +/// important as the first: these are shared fleet loops, so a blocking acquire would let one server's slice +/// stall collection for every other server, which is the #2148 wedge arriving through a lock instead of a +/// hang. +/// +public sealed class QueryStoreServerGateTests +{ + /// THE POINT: a second acquirer is refused while the first holds the gate. + [Fact] + public void ASecondAcquirerIsRefusedWhileTheFirstHoldsIt() + { + var gate = new QueryStoreServerGate(); + + using var first = gate.TryAcquire(); + + Assert.NotNull(first); + Assert.Null(gate.TryAcquire()); + Assert.True(gate.IsHeld); + } + + /// + /// Releasing hands the gate to the next caller. The loops are long-lived, so a gate that could only be taken + /// once would permanently stop one server's Query Store collection — and it would look like "that server has + /// no Query Store data" rather than like a bug. + /// + [Fact] + public void ReleasingLetsTheOtherLoopIn() + { + var gate = new QueryStoreServerGate(); + + var first = gate.TryAcquire(); + Assert.NotNull(first); + first!.Dispose(); + + Assert.False(gate.IsHeld); + using var second = gate.TryAcquire(); + Assert.NotNull(second); + } + + /// + /// It never blocks. Asserted as elapsed time against a HELD gate, because the failure this guards is a + /// blocking acquire silently replacing the try-acquire — which would still pass an exclusion-only test while + /// reintroducing the fleet stall. + /// + [Fact] + public void ARefusedAcquireReturnsImmediatelyRatherThanWaiting() + { + var gate = new QueryStoreServerGate(); + using var held = gate.TryAcquire(); + + var started = Environment.TickCount64; + for (var i = 0; i < 1_000; i++) + { + Assert.Null(gate.TryAcquire()); + } + + Assert.True(Environment.TickCount64 - started < 1_000, + "a thousand refused acquires must not block — a blocking acquire here stalls the whole sweep"); + } + + /// + /// Double-dispose must not release a gate a DIFFERENT loop has since taken. Without idempotence the sequence + /// "tick disposes twice, backfill acquires in between" leaves both running against one server — the exact + /// condition the gate exists to prevent, reached through a stray extra Dispose rather than through missing + /// exclusion. + /// + [Fact] + public void DisposingALeaseTwiceCannotReleaseSomebodyElsesHold() + { + var gate = new QueryStoreServerGate(); + + var tick = gate.TryAcquire(); + Assert.NotNull(tick); + tick!.Dispose(); + + var backfill = gate.TryAcquire(); + Assert.NotNull(backfill); + + tick.Dispose(); /* the stray second dispose */ + + Assert.True(gate.IsHeld, "the backfill's hold must survive the tick's double-dispose"); + Assert.Null(gate.TryAcquire()); + backfill!.Dispose(); + } + + /// + /// Under real contention exactly ONE holder exists at a time. The count of concurrent holders is what the + /// field bug was about — two heavy extractions at once — so it is asserted directly rather than inferred. + /// + [Fact] + public async Task UnderContentionOnlyOneHolderEverExistsAtOnce() + { + var gate = new QueryStoreServerGate(); + var concurrent = 0; + var maxObserved = 0; + var acquisitions = 0; + + await Task.WhenAll(Enumerable.Range(0, 8).Select(_ => Task.Run(() => + { + for (var i = 0; i < 2_000; i++) + { + using var lease = gate.TryAcquire(); + if (lease is null) + { + continue; + } + + Interlocked.Increment(ref acquisitions); + var now = Interlocked.Increment(ref concurrent); + var seen = Volatile.Read(ref maxObserved); + if (now > seen) + { + Interlocked.Exchange(ref maxObserved, now); + } + + Interlocked.Decrement(ref concurrent); + } + })).ToArray()); + + Assert.Equal(1, maxObserved); + Assert.True(acquisitions > 0, "the test must have actually acquired the gate"); + Assert.False(gate.IsHeld, "every lease was disposed, so the gate must be free"); + } + + /// + /// NotGated is a distinct, non-null, safely re-disposable sentinel. It is what lets a call site decide + /// "am I gated?" and "did I get the gate?" in one expression while keeping null meaning only + /// "skip" — conflating the two would silently skip collectors nobody meant to gate. + /// + [Fact] + public void NotGatedIsANonNullNoOpSentinel() + { + Assert.NotNull(QueryStoreServerGate.NotGated); + + QueryStoreServerGate.NotGated.Dispose(); + QueryStoreServerGate.NotGated.Dispose(); + + /* Still usable afterwards: it is a shared singleton every non-gated collector run disposes. */ + Assert.NotNull(QueryStoreServerGate.NotGated); + Assert.Same(QueryStoreServerGate.NotGated, QueryStoreServerGate.NotGated); + } + + /// + /// Gates are PER SERVER: one server's collection must never gate another's. The registries are keyed + /// dictionaries for this reason, and a single shared gate would serialize Query Store collection across the + /// entire fleet — a fleet-wide throughput regression dressed as a fix. + /// + [Fact] + public void GatesAreIndependentPerServer() + { + var gates = new ConcurrentDictionary(StringComparer.Ordinal); + + using var serverA = gates.GetOrAdd("server-a", static _ => new QueryStoreServerGate()).TryAcquire(); + using var serverB = gates.GetOrAdd("server-b", static _ => new QueryStoreServerGate()).TryAcquire(); + + Assert.NotNull(serverA); + Assert.NotNull(serverB); + + /* And the same key resolves the SAME gate — which is what makes the tick and the backfill exclude each + other rather than each holding a private one. */ + Assert.Null(gates.GetOrAdd("server-a", static _ => new QueryStoreServerGate()).TryAcquire()); + } + + /* ──────────────── the WIRING, pinned at the source ──────────────── + + Behavioral coverage cannot reach these: reproducing the overlap needs two live loops against one real + monitored server with a big Query Store catalog. A correct gate that one of the two loops does not take is + exactly the bug still present, and it builds and passes every other test. So both apps' call sites are + asserted textually, per the ThemeCompletenessTests idiom. */ + + private static string ReadRepoFile(string relativePath, [CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var parts = relativePath.Split('/'); + while (dir is not null && !File.Exists(Path.Combine(new[] { dir }.Concat(parts).ToArray()))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(new[] { dir! }.Concat(parts).ToArray())); + } + + /// + /// LITE: both loops resolve the gate from the SAME dictionary, and the backfill takes it OUTSIDE its + /// AbandonableStep. + /// + /// The "same dictionary" half is the one that would silently fail: two loops each holding a private + /// registry compile, pass the gate's own unit tests, and exclude nothing. The "outside the step" half matters + /// because an abandoned-but-wedged slice must keep the gate closed — the statement is still running on the + /// server, so the tick must keep yielding to it. + /// + [Fact] + public void Lite_BothLoopsShareOneGateRegistry() + { + var tick = ReadRepoFile("Lite/Services/RemoteCollectorService.QueryStore.cs"); + var backfill = ReadRepoFile("Lite/Services/RemoteCollectorService.QueryStoreBackfill.cs"); + + Assert.Contains("_queryStoreGates", tick, StringComparison.Ordinal); + Assert.Contains("_queryStoreGates", backfill, StringComparison.Ordinal); + Assert.Contains(".TryAcquire()", tick, StringComparison.Ordinal); + Assert.Contains(".TryAcquire()", backfill, StringComparison.Ordinal); + + /* Declared exactly once, so the two partials cannot drift onto separate registries. */ + Assert.Equal(1, CountOccurrences(tick + backfill, "ConcurrentDictionary")); + + /* The gate is taken before the step is even constructed. */ + var gateIndex = backfill.IndexOf(".TryAcquire()", StringComparison.Ordinal); + var stepIndex = backfill.IndexOf("_backfillSliceSteps.GetOrAdd", StringComparison.Ordinal); + Assert.True(gateIndex > 0 && gateIndex < stepIndex, + "the backfill must take the gate OUTSIDE the AbandonableStep, so a wedged slice keeps it closed"); + } + + /// + /// DARLING: the same two properties, plus that the tick gates on the collector's OWN declared name rather + /// than a string literal — renaming the collector must not silently unhook the gate. + /// + [Fact] + public void Darling_BothLoopsShareOneGateRegistry() + { + var worker = ReadRepoFile("Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs"); + + Assert.Equal(1, CountOccurrences(worker, "ConcurrentDictionary")); + Assert.Equal(2, CountOccurrences(worker, "_queryStoreGates.GetOrAdd")); + Assert.Contains("QueryStoreCollector.Instance.Name", worker, StringComparison.Ordinal); + Assert.Contains("QueryStoreServerGate.NotGated", worker, StringComparison.Ordinal); + + var gateIndex = worker.IndexOf("_queryStoreGates.GetOrAdd(runtime.ServerId", StringComparison.Ordinal); + var stepIndex = worker.IndexOf("_backfillSliceSteps.GetOrAdd", StringComparison.Ordinal); + Assert.True(gateIndex > 0 && stepIndex > 0); + } + + private static int CountOccurrences(string haystack, string needle) + { + var count = 0; + for (var i = haystack.IndexOf(needle, StringComparison.Ordinal); i >= 0; + i = haystack.IndexOf(needle, i + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Lite.Tests/QueryStoreStatePruneTests.cs b/Lite.Tests/QueryStoreStatePruneTests.cs new file mode 100644 index 000000000..c4a813325 --- /dev/null +++ b/Lite.Tests/QueryStoreStatePruneTests.cs @@ -0,0 +1,439 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using Microsoft.Extensions.Logging; +using PerformanceMonitor.Collectors; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Services; +using PerformanceMonitorLite.Tests; +using Xunit; + +namespace Lite.Tests; + +/// +/// Real-DuckDB pins for #2188 on Lite's side — retiring the per-database collector_state rows +/// query_store leaves behind for databases the server no longer has. +/// +/// Why Lite is affected at all, which the first cut of #2188 got wrong: Lite writes no +/// planwm: (it never sets CollectorContext.CapturePlanXml), but its own backfill worker writes +/// done: and hole: per database, and it only ever deletes a hole it SERVICES or expires. A +/// dropped database can do neither — its hole can never be dug and its tail can never drain — so its markers +/// were kept forever, the identical defect the issue reported against Darling's watermark. +/// +/// What has to be right is the input, not the delete. query_store's enumeration is filtered by +/// ONLINE state, AG primary-ness, exclusions, a vendor-name screen and HAS_DBACCESS, so pruning on +/// absence from it would delete LIVE state for a database that is merely parked. The prune reads +/// database_states — the unfiltered sys.databases snapshot — which is why +/// is the load-bearing test here rather +/// than the delete case. +/// +/// Real DuckDB rather than a mock, for the reason its siblings in this project give: the whole change +/// is one SQL statement, and DuckDB's dialect is where it can be wrong (starts_with, the NULL-valued +/// MAX over an empty snapshot, NOT IN against a subquery). None of that is visible above the store. +/// +public sealed class QueryStoreStatePruneTests : IClassFixture, IDisposable +{ + private const int ServerId = 21880; + private const int NeighborServerId = 21881; + + /// The snapshot's collection_time; state rows are dated relative to it. + private static readonly DateTime Newest = new(2026, 8, 11, 9, 0, 0, DateTimeKind.Unspecified); + + private static readonly DateTime BeforeNewest = Newest.AddHours(-1); + + private readonly DuckDbInitializer _duckDb; + private readonly Pruner _pruner; + private DuckDBConnection? _seedConn; + private long _nextId = 1; + + public QueryStoreStatePruneTests(SharedDuckDbFixture fixture) + { + fixture.ResetData(); + _duckDb = fixture.DuckDb; + _pruner = new Pruner(fixture.DuckDb); + } + + public void Dispose() => _seedConn?.Dispose(); + + /// Exposes the runner's protected prune and state accessors; only _duckDb is exercised. + private sealed class Pruner(DuckDbInitializer duckDb, ILogger? logger = null) + : RemoteCollectorService(duckDb, serverManager: null!, scheduleManager: null!, logger) + { + public Task PruneAsync(int serverId) => + PruneOrphanedQueryStoreDatabaseStateAsync(serverId, CancellationToken.None); + + /// The #2191 Azure arm, which prunes against the registration's own database rather than + /// against a database_states snapshot. + public Task PruneForeignAsync(int serverId, string ownDatabase) => + PruneForeignQueryStoreDatabaseStateAsync(serverId, ownDatabase, CancellationToken.None); + } + + /// + /// Captures formatted log lines so the prune's DIAGNOSTIC can be asserted, not just its effect. + /// #2205: correctness already matched Darling exactly, and the gap was forensic — Lite logged a + /// COUNT where Darling names the retired keys, so on Lite you could see that three rows went without + /// seeing WHICH databases' watermark and backfill state was retired. A count cannot be told apart from + /// a mistaken prune of the same size, which is the case an operator most needs to see. + /// + private sealed class CapturingLogger : ILogger + { + public List Lines { get; } = new(); + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) => Lines.Add(formatter(state, exception)); + } + + private static string Done(string database) => QueryStoreBackfillState.DoneKeyPrefix + database; + private static string Hole(string database) => QueryStoreBackfillState.HoleKeyPrefix + database; + + private static string EncodedHole() => QueryStoreBackfillState.EncodeHole( + new DateTime(2026, 8, 10, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2026, 8, 10, 6, 0, 0, DateTimeKind.Utc)); + + [Fact] + public async Task Prune_RetiresTheBackfillMarkersOfDroppedDatabases_AndKeepsTheRest() + { + await SeedSnapshotAsync(Newest, "Live", "App"); + + /* An OLDER snapshot still naming the dropped databases: if the prune read any snapshot but the + newest, nothing would ever be retired. */ + await SeedSnapshotAsync(Newest.AddMinutes(-15), "Live", "App", "Dropped", "AppArchive"); + + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Live"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Live"), EncodedHole()); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Dropped"), EncodedHole()); + + /* The name-shape trap: writing the anti-join with a prefix match instead of an equality would spare + AppArchive forever, because "done:App" is a prefix of "done:AppArchive". For an issue whose whole + subject is database name churn, that case belongs here. */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("AppArchive"), "2026-08-11T09:00:00Z"); + + /* Not database-keyed: the prefix filter is what protects it from a prune written as "every key of + this collector". */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping", "keep me"); + + /* Another collector, and another server whose database really was dropped here — server scoping is + the difference between pruning one server and pruning every server in the file. */ + await SeedStateAsync(ServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey, @"S:\MSSQL\Log\log_766.trc"); + await SeedStateAsync(NeighborServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"), "2026-08-11T09:00:00Z"); + + await _pruner.PruneAsync(ServerId); + + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"))); + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Dropped"))); + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("AppArchive"))); + + Assert.Equal("2026-08-11T09:00:00Z", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Live"))); + Assert.Equal(EncodedHole(), await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Live"))); + Assert.Equal("keep me", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping")); + Assert.Equal(@"S:\MSSQL\Log\log_766.trc", await ValueAsync(ServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey)); + Assert.Equal("2026-08-11T09:00:00Z", await ValueAsync(NeighborServerId, QueryStoreBackfillState.StateCollectorName, Done("Dropped"))); + + /* Idempotent — it runs on every query_store cycle, so a second pass must touch nothing. */ + await _pruner.PruneAsync(ServerId); + Assert.Equal(4L, await CountAsync(ServerId)); + } + + [Fact] + public async Task Prune_KeepsAParkedDatabase_ThatNoEnumerationWouldReturn() + { + /* The assertion this change exists for. Parked is OFFLINE, so query_store's enumeration (which + screens state_desc = ONLINE) never returns it — but it is still in sys.databases and still very + much a database. A prune keyed on the enumeration deletes its state and silently costs a full + re-drain of its history; a prune keyed on sys.databases leaves it alone. */ + await SeedSnapshotAsync(Newest, ("Live", "ONLINE"), ("Parked", "OFFLINE")); + + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Parked"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Parked"), EncodedHole()); + + await _pruner.PruneAsync(ServerId); + + Assert.Equal("2026-08-11T09:00:00Z", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Parked"))); + Assert.Equal(EncodedHole(), await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Parked"))); + } + + [Fact] + public async Task Prune_WithNoSnapshot_RetiresNothing() + { + /* The guard that stops a hygiene sweep becoming a data event. DuckDB's MAX over zero rows is NULL + and `updated_at < NULL` is NULL, so the delete matches nothing — but that is a property of the + dialect, not of the intent, which is exactly why it is pinned against a real store. + + A server with no snapshot is reachable in ordinary use: Lite never collects database_states on + Azure SQL DB, and a store whose rows have been archived out looks the same from here. */ + await SeedSnapshotAsync(Newest, NeighborServerId, ("SomeOtherServersDatabase", "ONLINE")); + + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Alpha"), "a"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Beta"), EncodedHole()); + + await _pruner.PruneAsync(ServerId); + + Assert.Equal("a", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Alpha"))); + Assert.Equal(EncodedHole(), await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Beta"))); + } + + [Fact] + public async Task Prune_LeavesStateWrittenAfterTheSnapshot() + { + /* Existing is not CURRENT. If database_states stops collecting, its newest snapshot freezes, and + every database created after that instant is missing from it while being perfectly alive — + pruning on presence alone would retire such a database's markers on every cycle forever. A + snapshot cannot judge a row written after it was taken. */ + await SeedSnapshotAsync(Newest, "OldDb"); + + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, + Done("BornAfterTheSnapshot"), "new", updatedAt: Newest.AddMinutes(30)); + + /* And the control: dropped BEFORE the snapshot froze, so its last write precedes it and it is still + prunable. Without this the test would also pass for a prune that had simply stopped working. */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, + Done("DroppedLongAgo"), "old", updatedAt: BeforeNewest); + + await _pruner.PruneAsync(ServerId); + + Assert.Equal("new", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("BornAfterTheSnapshot"))); + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("DroppedLongAgo"))); + } + + [Fact] + public async Task Prune_RunsTheWatermarkStatementToo_EvenThoughLiteWritesNone() + { + /* Lite iterates the SHARED QueryStorePerDatabaseState.PrunableKeys, which carries planwm: even + though Lite never writes it. That is deliberate: the day plan capture is enabled here, the prune + is already in place rather than being a thing somebody has to remember. Today it must simply be + harmless — one delete matching nothing — which is what this checks by proving a planted planwm: + row for a DROPPED database is retired by the same pass. */ + await SeedSnapshotAsync(Newest, "Live"); + await SeedStateAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, + QueryStorePlanXmlState.WatermarkKeyPrefix + "Dropped", "900000:1786449600"); + + await _pruner.PruneAsync(ServerId); + + Assert.Null(await ValueAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, + QueryStorePlanXmlState.WatermarkKeyPrefix + "Dropped")); + } + + /* ---------------- helpers ---------------- */ + + private async Task SeedConnectionAsync() + { + if (_seedConn is null) + { + _seedConn = _duckDb.CreateConnection(); + await _seedConn.OpenAsync(); + } + return _seedConn; + } + + private Task SeedSnapshotAsync(DateTime when, params string[] databases) + => SeedSnapshotAsync(when, ServerId, Array.ConvertAll(databases, db => (db, "ONLINE"))); + + private Task SeedSnapshotAsync(DateTime when, params (string Db, string State)[] databases) + => SeedSnapshotAsync(when, ServerId, databases); + + private async Task SeedSnapshotAsync(DateTime when, int serverId, params (string Db, string State)[] databases) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + var dbId = 1; + foreach (var (db, state) in databases) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" +INSERT INTO database_states (collection_id, collection_time, server_id, server_name, database_name, database_id, state_desc, is_in_standby) +VALUES ($1, $2, $3, $4, $5, $6, $7, false)"; + cmd.Parameters.Add(new DuckDBParameter { Value = _nextId++ }); + cmd.Parameters.Add(new DuckDBParameter { Value = when }); + cmd.Parameters.Add(new DuckDBParameter { Value = serverId }); + cmd.Parameters.Add(new DuckDBParameter { Value = "LITE-PRUNE-SRV" }); + cmd.Parameters.Add(new DuckDBParameter { Value = db }); + cmd.Parameters.Add(new DuckDBParameter { Value = dbId++ }); + cmd.Parameters.Add(new DuckDBParameter { Value = state }); + await cmd.ExecuteNonQueryAsync(); + } + } + + private async Task SeedStateAsync(int serverId, string owner, string key, string value, DateTime? updatedAt = null) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" +INSERT OR REPLACE INTO collector_state (server_id, collector_name, state_key, state_value, updated_at) +VALUES ($1, $2, $3, $4, $5)"; + cmd.Parameters.Add(new DuckDBParameter { Value = serverId }); + cmd.Parameters.Add(new DuckDBParameter { Value = owner }); + cmd.Parameters.Add(new DuckDBParameter { Value = key }); + cmd.Parameters.Add(new DuckDBParameter { Value = value }); + cmd.Parameters.Add(new DuckDBParameter { Value = updatedAt ?? BeforeNewest }); + await cmd.ExecuteNonQueryAsync(); + } + + private async Task ValueAsync(int serverId, string owner, string key) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = + "SELECT state_value FROM collector_state WHERE server_id = $1 AND collector_name = $2 AND state_key = $3"; + cmd.Parameters.Add(new DuckDBParameter { Value = serverId }); + cmd.Parameters.Add(new DuckDBParameter { Value = owner }); + cmd.Parameters.Add(new DuckDBParameter { Value = key }); + var value = await cmd.ExecuteScalarAsync(); + return value is DBNull or null ? null : (string)value; + } + + private async Task CountAsync(int serverId) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM collector_state WHERE server_id = $1"; + cmd.Parameters.Add(new DuckDBParameter { Value = serverId }); + return Convert.ToInt64(await cmd.ExecuteScalarAsync(), System.Globalization.CultureInfo.InvariantCulture); + } + + /// + /// #2205: the log line must NAME the pruned keys, matching Darling's wording field for field. + /// + /// Watched red before it went green: with the previous ExecuteNonQueryAsync + + /// counter, the message read "Pruned 2 query_store state row(s)…" and contained neither key, so both + /// key assertions failed while the row-level assertions passed — which is exactly the shape of the + /// defect. The DELETE … RETURNING it now depends on was verified against real DuckDB 1.5.5 + /// before the change, using the shipped SQL string rather than a copy. + /// + [Fact] + public async Task Prune_LogNamesTheRetiredKeys_NotJustACount() + { + var logger = new CapturingLogger(); + var pruner = new Pruner(_duckDb, logger); + + await SeedSnapshotAsync(Newest, "Live"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Live"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("DroppedOne"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("DroppedTwo"), EncodedHole()); + + await pruner.PruneAsync(ServerId); + + var line = Assert.Single(logger.Lines, l => l.Contains("pruned", StringComparison.OrdinalIgnoreCase)); + + /* The keys themselves — the whole point of the change. */ + Assert.Contains(Done("DroppedOne"), line, StringComparison.Ordinal); + Assert.Contains(Hole("DroppedTwo"), line, StringComparison.Ordinal); + + /* And it must not name a database that was NOT pruned, or the log is worse than a count. */ + Assert.DoesNotContain("Live", line, StringComparison.Ordinal); + + /* Same fields as DarlingCollectorRunner's line, so one sentence serves both SKUs. */ + Assert.Contains($"[server_id {ServerId}]", line, StringComparison.Ordinal); + Assert.Contains("2 query_store state row(s)", line, StringComparison.Ordinal); + } + /* ─────────────── #2191: the Azure arm, pruned against the registration's own database ─────────────── */ + + /// + /// The Azure prune keeps the registration's own database and retires every sibling — executed against + /// real DuckDB, so this covers the statement itself (starts_with plus + /// state_key <> prefix || own) rather than its text. + /// + /// These keys are what #2220 left behind: before that fix each Azure registration swept every + /// sibling database on the logical server and wrote a watermark for it under its OWN server_id. And + /// collector_state carries no retention, so they would persist indefinitely rather than ageing out + /// with the collected rows. + /// + [Fact] + public async Task ForeignPrune_KeepsThisRegistrationsDatabase_AndRetiresItsSiblings() + { + /* Deliberately NO database_states snapshot: an Azure server never has one, and the point of this arm + is that it does not need one. */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Payments"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Payments"), EncodedHole()); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Sibling-A"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Sibling-B"), EncodedHole()); + + /* The prefix trap, same one the on-prem test carries: an inequality written as a prefix comparison + would spare "PaymentsArchive" because "Payments" is a prefix of it. */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("PaymentsArchive"), "2026-08-11T09:00:00Z"); + + /* Not database-keyed, and another collector, and another server — all three must survive. */ + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping", "keep me"); + await SeedStateAsync(ServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey, @"S:\MSSQL\Log\log_766.trc"); + await SeedStateAsync(NeighborServerId, QueryStoreBackfillState.StateCollectorName, Done("Sibling-A"), "2026-08-11T09:00:00Z"); + + await _pruner.PruneForeignAsync(ServerId, "Payments"); + + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Sibling-A"))); + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Sibling-B"))); + Assert.Null(await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("PaymentsArchive"))); + + Assert.Equal("2026-08-11T09:00:00Z", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Payments"))); + Assert.Equal(EncodedHole(), await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Hole("Payments"))); + Assert.Equal("keep me", await ValueAsync(ServerId, QueryStoreBackfillState.StateCollectorName, "unrelated-bookkeeping")); + Assert.Equal(@"S:\MSSQL\Log\log_766.trc", await ValueAsync(ServerId, DefaultTraceEventsCollector.Instance.Name, + DefaultTraceEventsCollector.LastTraceFilePathStateKey)); + Assert.Equal("2026-08-11T09:00:00Z", await ValueAsync(NeighborServerId, QueryStoreBackfillState.StateCollectorName, Done("Sibling-A"))); + + /* Idempotent: it runs every query_store cycle, so a second pass must touch nothing. */ + await _pruner.PruneForeignAsync(ServerId, "Payments"); + Assert.Equal(4L, await CountAsync(ServerId)); + } + + /// + /// An empty own-database short-circuits BEFORE the statement runs, deleting nothing. + /// + /// This is the arm's safety property rather than an edge case. A registration naming no database is + /// a registration of the logical SERVER, whose legitimate database set is everything on it — so running + /// the single-name prune there would delete every live watermark it has. The caller gates on + /// AzureSweepScope, and the method guards again itself, because the consequence of getting it + /// wrong is silent and total. + /// + [Theory] + [InlineData("")] + public async Task ForeignPrune_WithNoOwnDatabase_RetiresNothing(string ownDatabase) + { + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Payments"), "2026-08-11T09:00:00Z"); + await SeedStateAsync(ServerId, QueryStoreBackfillState.StateCollectorName, Done("Sibling-A"), "2026-08-11T09:00:00Z"); + + await _pruner.PruneForeignAsync(ServerId, ownDatabase); + + Assert.Equal(2L, await CountAsync(ServerId)); + } + + /// + /// Every per-database prefix is pruned, not just the backfill ones — the watermark prefix included, even + /// though Lite writes none today (it never sets CapturePlanXml). Pinned for the same reason the + /// on-prem twin pins it: the shared prefix list is what stops a prefix being pruned on one SKU and + /// orphaning on the other, and a Lite-only omission would be invisible on Darling. + /// + [Fact] + public async Task ForeignPrune_CoversTheWatermarkPrefixToo() + { + var foreignWatermark = QueryStorePlanXmlState.KeyFor("Sibling-A"); + var ownWatermark = QueryStorePlanXmlState.KeyFor("Payments"); + + await SeedStateAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, foreignWatermark, "8140"); + await SeedStateAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, ownWatermark, "8150"); + + await _pruner.PruneForeignAsync(ServerId, "Payments"); + + Assert.Null(await ValueAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, foreignWatermark)); + Assert.Equal("8150", await ValueAsync(ServerId, QueryStorePlanXmlState.StateCollectorName, ownWatermark)); + } +} diff --git a/Lite.Tests/ServerIdentityEngineAndPortTests.cs b/Lite.Tests/ServerIdentityEngineAndPortTests.cs new file mode 100644 index 000000000..217655b8c --- /dev/null +++ b/Lite.Tests/ServerIdentityEngineAndPortTests.cs @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2218: the storage name carries engine and port, without re-keying anything that already exists. +/// +/// The defect. server_id was derived from host, database and read-only intent only. So a SQL +/// Server and a PostgreSQL instance on ONE host collided into a single identity and interleaved their histories, +/// and so did two PostgreSQL instances distinguished only by port — both of which #2213 made first-class +/// configuration. +/// +/// Why the new discriminators are OPTIONAL, which is the whole reason this is safe. Lite derives +/// server_id FRESH at runtime from this function, everywhere, and has no stored-id fallback: +/// RemoteCollectorService.GetServerNameForStorage hashes it on every read. So any change to what this +/// returns for an existing server re-keys it in Lite and orphans all of its collected history, silently — which +/// is the same class of harm as #2158, arrived at from the other direction. Appending nothing at the parameter +/// defaults is what keeps Lite's three-argument call byte-identical. +/// +/// These tests are therefore mostly about what did NOT change. This file lives in Lite.Tests deliberately: +/// Lite is the SKU with no stored-id safety net, so it is the one whose invariant needs guarding. +/// +public sealed class ServerIdentityEngineAndPortTests +{ + /// + /// The pre-#2218 implementation, verbatim, as the oracle. Comparing against a re-statement of the rule + /// rather than against hard-coded strings is what makes "nothing re-keyed" checkable for any input, not + /// just the handful someone thought to write down. + /// + private static string PreviousImplementation(string serverName, string? databaseName, bool readOnlyIntent) + { + var name = string.IsNullOrWhiteSpace(databaseName) ? serverName : serverName + ":" + databaseName; + return readOnlyIntent ? name + ":RO" : name; + } + + /// + /// THE INVARIANT: Lite's three-argument call is byte-identical to what it produced before, so no Lite + /// server re-keys and no Lite history is orphaned. Asserted on the derived server_id too, because + /// that — not the string — is what the stored rows are keyed by. + /// + [Theory] + [InlineData("SQLPROD01", null, false)] + [InlineData("SQLPROD01", "SalesDB", false)] + [InlineData("SQLPROD01", null, true)] + [InlineData("SQLPROD01", "SalesDB", true)] + [InlineData("host.contoso.com,1433", null, false)] + [InlineData("host,49152", "db", true)] + [InlineData("azure.database.windows.net", "AdventureWorks", false)] + public void TheThreeArgumentCallIsUnchanged_SoNothingReKeys(string host, string? database, bool readOnlyIntent) + { + var expected = PreviousImplementation(host, database, readOnlyIntent); + + Assert.Equal(expected, ServerIdHelper.BuildStorageName(host, database, readOnlyIntent)); + Assert.Equal( + ServerIdHelper.GetDeterministicHashCode(expected), + ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(host, database, readOnlyIntent))); + } + + /// + /// And a SQL Server entry that DOES pass the new arguments is also unchanged — which is what lets Darling + /// pass Engine and Port unconditionally instead of branching at the call site. Port is + /// a PostgreSQL-only field (SQL Server carries a non-default port inside the host as host,1433, so it + /// is already discriminated there), so 0 is the SQL Server case. + /// + [Theory] + [InlineData("SQLPROD01", null, false)] + [InlineData("SQLPROD01", "SalesDB", true)] + [InlineData("host.contoso.com,1433", null, false)] + public void ASqlServerEntryPassingEngineAndPortIsAlsoUnchanged(string host, string? database, bool readOnlyIntent) + { + Assert.Equal( + PreviousImplementation(host, database, readOnlyIntent), + ServerIdHelper.BuildStorageName(host, database, readOnlyIntent, "sqlserver", 0)); + } + + /// + /// An engine that is blank, SQL Server, or unrecognized appends NOTHING. The unrecognized case matters as + /// much as the others: a typo in engine must not mint a fresh identity for a server that already has + /// one, which is exactly what interpolating the raw value would do. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("sqlserver")] + [InlineData("SQLSERVER")] + [InlineData("SqlServer")] + [InlineData("mysql")] + [InlineData("postgre")] + public void AnEngineThatIsNotPostgresAppendsNothing(string? engine) + { + Assert.Equal("H:D", ServerIdHelper.BuildStorageName("H", "D", false, engine, 0)); + } + + /// THE FIX, half one: PostgreSQL and SQL Server on one host are now two identities. + [Fact] + public void PostgresAndSqlServerOnOneHostNoLongerCollide() + { + var sqlServer = ServerIdHelper.BuildStorageName("box01", null, false, "sqlserver", 0); + var postgres = ServerIdHelper.BuildStorageName("box01", null, false, "postgres", 0); + + Assert.Equal("box01", sqlServer); + Assert.Equal("box01:pg", postgres); + Assert.NotEqual( + ServerIdHelper.GetDeterministicHashCode(sqlServer), + ServerIdHelper.GetDeterministicHashCode(postgres)); + } + + /// THE FIX, half two: two PostgreSQL instances on one host, told apart by port. + [Fact] + public void TwoPostgresInstancesOnOneHostNoLongerCollide() + { + var first = ServerIdHelper.BuildStorageName("box01", null, false, "postgres", 5432); + var second = ServerIdHelper.BuildStorageName("box01", null, false, "postgres", 5433); + + Assert.NotEqual(first, second); + Assert.NotEqual( + ServerIdHelper.GetDeterministicHashCode(first), + ServerIdHelper.GetDeterministicHashCode(second)); + } + + /// + /// Every spelling of the engine folds to ONE token, so an operator writing "PostgreSQL" where a + /// colleague wrote "postgres" does not get a second identity for the same instance — a split history + /// caused by capitalisation, which nothing downstream could diagnose. + /// + [Theory] + [InlineData("postgres")] + [InlineData("PostgreSQL")] + [InlineData("Postgres")] + [InlineData("POSTGRESQL")] + [InlineData("postgresql")] + [InlineData("pg")] + [InlineData("PG")] + [InlineData(" postgres ")] + public void EverySpellingOfPostgresFoldsToOneIdentity(string engine) + { + Assert.Equal("box01:pg", ServerIdHelper.BuildStorageName("box01", null, false, engine, 0)); + } + + /// + /// The suffix order is fixed — engine, then port, then :RO. Two callers supplying the same facts + /// must not be able to produce two names, and the read-only marker stays last so the existing convention + /// (and anything that reads a name by eye) is preserved. + /// + [Theory] + [InlineData("h", "d", true, "postgres", 5433, "h:d:pg:5433:RO")] + [InlineData("h", null, true, "postgres", 0, "h:pg:RO")] + [InlineData("h", null, false, null, 5433, "h:5433")] + [InlineData("h", "d", false, "postgres", 0, "h:d:pg")] + public void TheSuffixOrderIsEngineThenPortThenReadOnly( + string host, string? database, bool readOnlyIntent, string? engine, int port, string expected) + { + Assert.Equal(expected, ServerIdHelper.BuildStorageName(host, database, readOnlyIntent, engine, port)); + } + + /// + /// A zero or negative port appends nothing. Zero is the real default (the field means "the driver's + /// default"), and a negative value is nonsense that must not become part of an identity. + /// + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void AnAbsentOrNonsensePortAppendsNothing(int port) + { + Assert.Equal("h", ServerIdHelper.BuildStorageName("h", null, false, null, port)); + } +} diff --git a/Lite.Tests/SharedCollectorDefaultsPinTests.cs b/Lite.Tests/SharedCollectorDefaultsPinTests.cs index f0557ac02..6b270cd27 100644 --- a/Lite.Tests/SharedCollectorDefaultsPinTests.cs +++ b/Lite.Tests/SharedCollectorDefaultsPinTests.cs @@ -60,7 +60,15 @@ public void CollectorScheduleDefaults_MatchScheduleManagerTable() { var liteDefaults = ScheduleManager.GetDefaultSchedules(); - Assert.Equal(liteDefaults.Count, CollectorScheduleDefaults.All.Count); + /* The SQL Server subset of the shared defaults, not all of it. The catalog is engine-mixed and Lite + has no PostgreSQL target, so its schedule table correctly does not list the PostgreSQL collectors — + a phantom row for a collector this SKU can never dispatch would show up in Lite's own schedule UI. */ + var sqlServerDefaults = CollectorCatalog.All + .Where(d => d.TargetEngine == CollectorTargetEngine.SqlServer) + .Select(d => d.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + Assert.Equal(liteDefaults.Count, sqlServerDefaults.Count); foreach (var schedule in liteDefaults) { Assert.True(CollectorScheduleDefaults.All.TryGetValue(schedule.Name, out var shared), diff --git a/Lite.Tests/SkippedMaintenanceIsReportedTests.cs b/Lite.Tests/SkippedMaintenanceIsReportedTests.cs new file mode 100644 index 000000000..5efa60ba6 --- /dev/null +++ b/Lite.Tests/SkippedMaintenanceIsReportedTests.cs @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using Xunit; + +namespace Lite.Tests; + +/// +/// #2266: a skipped database-state maintenance cycle now says so, once per transition. +/// +/// What was silent. GetDatabaseStateDeviationsAsync does its baseline seed, the #2189 heal, +/// the #2203 forget and the prune inside a block that opens the write connection with a 5-second lock +/// acquisition and, on TimeoutException, skips all of it while still running the deviation read. Skipping +/// is the right behaviour — the method's own comment makes the case that it is the only lossless option when +/// archival holds the lock — but it logged NOTHING, so a sustained window of write-lock contention meant +/// baselines quietly stopped being seeded and healed with no evidence anywhere. +/// +/// That matters more than a typical missing log line: #2189 exists because an unhealed baseline inverts +/// the alert permanently, so the failure this maintenance prevents is itself invisible. Its absence has to be +/// visible instead. +/// +/// Pinned at the source, because the alternative is a live DuckDB store plus real write-lock +/// contention from another thread, and the assertion is about which lines exist on which path rather than about +/// a computed value. The one behavioural property that is easy to break — that the READ still runs after a +/// skipped maintenance block — is pinned here too, because my first attempt at this broke exactly that. +/// +public sealed class SkippedMaintenanceIsReportedTests +{ + /// + /// Warn on the way in, Info on the way out, and both keyed off a per-server latch so a standing contention + /// window reports once rather than once per sweep. + /// + /// Warn rather than Error is deliberate and worth pinning: one skipped cycle is the expected, benign + /// outcome of colliding with archival and the next sweep re-runs everything. Logging it at Error would make + /// a routine collision look like a fault, which is the fastest way to get the line filtered — and a filtered + /// line restores the silence this closes. + /// + [Fact] + public void TheSkipIsWarnedOnce_AndTheRecoveryIsReported() + { + var source = ReadDatabaseStatesSource(); + + Assert.Contains("_lastMaintenanceSkipped", source, StringComparison.Ordinal); + Assert.Contains("AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync)", source, StringComparison.Ordinal); + Assert.Contains("AppLogger.Info(nameof(GetDatabaseStateDeviationsAsync)", source, StringComparison.Ordinal); + Assert.Contains("running again after one or more skipped", source, StringComparison.Ordinal); + + /* Not Error — see the summary. */ + Assert.DoesNotContain("AppLogger.Error(nameof(GetDatabaseStateDeviationsAsync)", source, StringComparison.Ordinal); + } + + /// + /// The message has to name the consequence, not just the event. "Skipped maintenance" means nothing to + /// whoever reads the log; "baselines are not being seeded or healed while this persists" is what tells them + /// whether to care, and naming #2189/#2203 is what lets them find out why it matters. + /// + [Fact] + public void TheWarningNamesTheConsequenceAndNotJustTheEvent() + { + var source = ReadDatabaseStatesSource(); + + Assert.Contains("Baselines are not being ", source, StringComparison.Ordinal); + Assert.Contains("#2189/#2203", source, StringComparison.Ordinal); + /* And that the read still happened, so the reader is not left wondering whether the numbers are stale. */ + Assert.Contains("deviations are still read", source, StringComparison.Ordinal); + /* And that one occurrence is not a fault, so nobody escalates the expected case. */ + Assert.Contains("Expected ", source, StringComparison.Ordinal); + } + + /// + /// THE BEHAVIOURAL PROPERTY: a skipped maintenance block must NOT skip the deviation read. + /// + /// The whole design rests on it — the method's comment says the read "still runs under its read lock + /// exactly as before", and swallowing the read instead would report every database as recovered and clear + /// the alert memory for all of them. It is one stray return away at all times, and my first cut of + /// this change added exactly that return in the catch. So: the catch sets a flag and falls through, + /// and there is no return between it and the read. + /// + [Fact] + public void ASkippedMaintenanceBlockStillRunsTheDeviationRead() + { + var source = ReadDatabaseStatesSource().Replace("\r\n", "\n"); + + var catchIndex = source.IndexOf("catch (TimeoutException)", StringComparison.Ordinal); + var readIndex = source.IndexOf("using var connection = await OpenConnectionAsync();", StringComparison.Ordinal); + + Assert.True(catchIndex > 0, "the best-effort maintenance catch must still exist"); + Assert.True(readIndex > catchIndex, "the deviation read must follow the maintenance block"); + + /* The catch records and falls through rather than returning. */ + var between = source[catchIndex..readIndex]; + Assert.Contains("maintenanceSkipped = true;", between, StringComparison.Ordinal); + Assert.DoesNotContain("return ", between); + } + + private static string ReadDatabaseStatesSource([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + var relative = Path.Combine("Lite", "Services", "LocalDataService.DatabaseStates.cs"); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return File.ReadAllText(Path.Combine(dir!, relative)); + } +} diff --git a/Lite.Tests/SpinlockStatsCollectorDefinitionTests.cs b/Lite.Tests/SpinlockStatsCollectorDefinitionTests.cs index 1a8c0c517..3c2b63b82 100644 --- a/Lite.Tests/SpinlockStatsCollectorDefinitionTests.cs +++ b/Lite.Tests/SpinlockStatsCollectorDefinitionTests.cs @@ -119,13 +119,13 @@ public void WritePayload_EmitsPayloadOrder_AndPinsDeltaGroupsKeysAndGapPolicy() new object?[] { "SOS_SUSPEND_QUEUE", 100L, 5000L, 2.5, 10L, 3L, 1000L, 50000L, 100L, 30L }, writer.Values); - /* Delta contract: group names, key = spinlock_name, host collection time, 300 s gap policy. + /* Delta contract: group names, key = spinlock_name, host collection time, the shared gap policy. No delta call for spins_per_collision — exactly four calls. */ Assert.Equal(4, deltas.Calls.Count); - Assert.Equal(("spinlock_stats_collisions", "SOS_SUSPEND_QUEUE", 100L, context.CollectionTime, 300), deltas.Calls[0]); - Assert.Equal(("spinlock_stats_spins", "SOS_SUSPEND_QUEUE", 5000L, context.CollectionTime, 300), deltas.Calls[1]); - Assert.Equal(("spinlock_stats_sleep_time", "SOS_SUSPEND_QUEUE", 10L, context.CollectionTime, 300), deltas.Calls[2]); - Assert.Equal(("spinlock_stats_backoffs", "SOS_SUSPEND_QUEUE", 3L, context.CollectionTime, 300), deltas.Calls[3]); + Assert.Equal(("spinlock_stats_collisions", "SOS_SUSPEND_QUEUE", 100L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[0]); + Assert.Equal(("spinlock_stats_spins", "SOS_SUSPEND_QUEUE", 5000L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[1]); + Assert.Equal(("spinlock_stats_sleep_time", "SOS_SUSPEND_QUEUE", 10L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[2]); + Assert.Equal(("spinlock_stats_backoffs", "SOS_SUSPEND_QUEUE", 3L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[3]); Assert.All(deltas.Calls, _ => Assert.Equal(42, deltas.LastServerId)); } } diff --git a/Lite.Tests/StructuredRemediationTests.cs b/Lite.Tests/StructuredRemediationTests.cs new file mode 100644 index 000000000..2fad9954c --- /dev/null +++ b/Lite.Tests/StructuredRemediationTests.cs @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using PerformanceMonitor.Analysis; +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Pins the #2138 machine-first remediation projection: +/// is THE policy gate a future auto-force feature consults — these tests are the "never auto-force a +/// flagged target" data contract — and is how +/// agents see it. The wire-shape pin serializes with the SAME options the MCP surfaces use, because the +/// snake_case field names exist only by attribute (McpHelpers.JsonOptions carries no naming policy) and +/// a renamed record property would silently break every agent consumer. +/// +public sealed class StructuredRemediationTests +{ + private static ForcePlanTarget Target( + bool pspCoFired = false, string? replicaRole = null, + long queryId = 123, long planId = 99) => + new( + Database: "MyDb", + QueryId: queryId, + PlanId: planId, + BestPlanHash: "0xBEST", + LatestPlanHash: "0xLATEST", + LatestCpuPerExecUs: 9000, + BestCpuPerExecUs: 1200, + RegressionFactor: 7.5, + ReplicaRole: replicaRole, + ParameterSensitivityCoFired: pspCoFired); + + private static RemediationAction Action(params ForcePlanTarget[] targets) => + new("PLAN_REGRESSION", "force", targets); + + [Fact] + public void CleanPrimaryTarget_IsEligible_WithNoBlockers() + { + /* Null replica role is every standalone/non-AG/pre-2022 server — the 99% case must be + actionable, or the verdict field is noise. */ + var target = Assert.Single( + FactRemediation.BuildStructuredRemediation(Action(Target()))!.ForcePlanTargets); + + Assert.True(target.Eligible); + Assert.Empty(target.Blockers); + Assert.Equal(7.5, target.Evidence.RegressionFactor); + } + + [Fact] + public void PspCoFiredTarget_IsIneligible_AndNamesTheBlocker() + { + /* THE contract: a flagged target is never auto-forced. The blocker is a NAMED string, not a + boolean soup — an agent (and the bot's audit log) says WHY. */ + var target = Assert.Single( + FactRemediation.BuildStructuredRemediation(Action(Target(pspCoFired: true)))!.ForcePlanTargets); + + Assert.False(target.Eligible); + Assert.Equal("parameter_sensitivity_cofired", Assert.Single(target.Blockers)); + } + + [Fact] + public void SecondaryReplicaEvidence_IsIneligible_PrimaryRoleIsNot() + { + /* #1882 as data: the statement forces on the PRIMARY, so evidence from a non-primary replica + blocks; the primary's own evidence does not. Case-insensitive like the disclosure. */ + var secondary = Assert.Single( + FactRemediation.BuildStructuredRemediation(Action(Target(replicaRole: "Secondary")))!.ForcePlanTargets); + Assert.False(secondary.Eligible); + Assert.Equal("secondary_replica_evidence", Assert.Single(secondary.Blockers)); + + var primary = Assert.Single( + FactRemediation.BuildStructuredRemediation(Action(Target(replicaRole: "Primary")))!.ForcePlanTargets); + Assert.True(primary.Eligible); + Assert.Empty(primary.Blockers); + } + + [Fact] + public void BothGates_StackAsTwoNamedBlockers() + { + var target = Assert.Single(FactRemediation.BuildStructuredRemediation( + Action(Target(pspCoFired: true, replicaRole: "Geo Secondary")))!.ForcePlanTargets); + + Assert.False(target.Eligible); + Assert.Equal( + new[] { "parameter_sensitivity_cofired", "secondary_replica_evidence" }, + target.Blockers.OrderBy(b => b, StringComparer.Ordinal)); + } + + [Fact] + public void Artifacts_AreSplitAndRunnable_AndVerifyChecksTheForceStuck() + { + var target = Assert.Single( + FactRemediation.BuildStructuredRemediation(Action(Target()))!.ForcePlanTargets); + + Assert.Contains("EXEC sys.sp_query_store_force_plan @query_id = 123, @plan_id = 99;", target.ForceSql, StringComparison.Ordinal); + Assert.Contains("EXEC sys.sp_query_store_unforce_plan @query_id = 123, @plan_id = 99;", target.UnforceSql, StringComparison.Ordinal); + + /* The verify artifact asks BOTH post-force questions: did the force stick, and what has the + per-interval cost looked like since. */ + Assert.Contains("force_failure_count", target.VerifySql, StringComparison.Ordinal); + Assert.Contains("last_force_failure_reason_desc", target.VerifySql, StringComparison.Ordinal); + Assert.Contains("sys.query_store_runtime_stats", target.VerifySql, StringComparison.Ordinal); + Assert.Contains("WHERE qsp.query_id = 123", target.VerifySql, StringComparison.Ordinal); + + /* Every artifact carries its own USE — an agent pastes them independently. */ + Assert.StartsWith("USE [MyDb];", target.ForceSql, StringComparison.Ordinal); + Assert.StartsWith("USE [MyDb];", target.UnforceSql, StringComparison.Ordinal); + Assert.StartsWith("USE [MyDb];", target.VerifySql, StringComparison.Ordinal); + } + + [Fact] + public void NullAndNonForceActions_ProjectToNull() + { + Assert.Null(FactRemediation.BuildStructuredRemediation(null)); + Assert.Null(FactRemediation.BuildStructuredRemediation( + new RemediationAction("PLAN_REGRESSION", "force", Array.Empty()))); + } + + [Fact] + public void WireShape_IsSnakeCase_UnderTheMcpSerializerOptions() + { + /* The MCP surfaces serialize with McpHelpers.JsonOptions, which has NO naming policy — the + snake_case names exist only via JsonPropertyName. This pin is what makes a record-property + rename a test failure instead of a silent agent-facing break. */ + var json = JsonSerializer.Serialize( + FactRemediation.BuildStructuredRemediation(Action(Target(pspCoFired: true))), + McpHelpers.JsonOptions); + + foreach (var field in new[] + { + "\"fact_key\"", "\"verb\"", "\"force_plan_targets\"", + "\"query_id\"", "\"plan_id\"", "\"eligible\"", "\"blockers\"", "\"evidence\"", + "\"regression_factor\"", "\"parameter_sensitivity_cofired\"", + "\"force_sql\"", "\"unforce_sql\"", "\"verify_sql\"", + }) + { + Assert.Contains(field, json, StringComparison.Ordinal); + } + } +} diff --git a/Lite.Tests/SweepPressureClassifierTests.cs b/Lite.Tests/SweepPressureClassifierTests.cs new file mode 100644 index 000000000..379b94f18 --- /dev/null +++ b/Lite.Tests/SweepPressureClassifierTests.cs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Common; +using Xunit; + +namespace Lite.Tests; + +/// +/// Decision-table pins for the shared (#2296) — the roll-up both +/// SKUs' get_collection_health serve so half-rate collection stops being visible only as a service-log +/// warning. This SAME table is pinned identically in Darling.Tests so the two SKUs cannot drift. +/// +/// The load-bearing case is the motivating measurement: prod-pos-use2-multi-01's four heavy +/// collectors averaged 22,141 + 16,590 + 13,544 + 8,437 ms against a 60s cadence — the body could not +/// fit, every relaunch was skipped (~50 warnings/hour), the server collected at half rate, and all 40 +/// collectors read HEALTHY, because from each one's own seat nothing was wrong. +/// +public sealed class SweepPressureClassifierTests +{ + private static (string, double, int) C(string name, double avgMs, int freqMin) => (name, avgMs, freqMin); + + /// The #2296 measurement verbatim: ~101% of the minute — SATURATED, not a warning-log easter egg. + [Fact] + public void TheMotivatingServerReadsSaturated() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("procedure_stats", 22_141, 1), + C("query_store", 16_590, 1), + C("plan_correction", 13_544, 1), + C("query_stats", 8_437, 1), + }); + + Assert.Equal(SweepPressureClassifier.Saturated, pressure.Verdict); + Assert.Equal(60_712, pressure.BusyMsPerMinute, 3); + Assert.True(pressure.BusyPercent > 100.0); + } + + /// An ordinary in-region profile sits far below every threshold. + [Fact] + public void AHealthyProfileReadsOk() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("wait_stats", 180, 1), + C("cpu_utilization", 95, 1), + C("query_stats", 2_400, 1), + C("database_size_stats", 1_200, 60), + }); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.True(pressure.BusyPercent < 5.0); + } + + /// + /// The band edges, both inclusive: 45,000 ms/min is exactly 75% (AT_RISK), 60,000 exactly 100% + /// (SATURATED). Inclusive because the average already smooths spikes — a body that AVERAGES the + /// boundary is over it half the time. + /// + [Fact] + public void TheBandEdgesAreInclusive() + { + Assert.Equal(SweepPressureClassifier.Ok, + SweepPressureClassifier.Compute(new[] { C("a", 44_999, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.AtRisk, + SweepPressureClassifier.Compute(new[] { C("a", 45_000, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.AtRisk, + SweepPressureClassifier.Compute(new[] { C("a", 59_999, 1) }).Verdict); + Assert.Equal(SweepPressureClassifier.Saturated, + SweepPressureClassifier.Compute(new[] { C("a", 60_000, 1) }).Verdict); + } + + /// + /// A non-recurring collector (frequency 0: on-load, unknown name) contributes nothing however long it + /// runs — it does not compete for the sweep. A zero-duration entry likewise adds nothing. + /// + [Fact] + public void OnLoadAndZeroDurationCollectorsAreExcluded() + { + var pressure = SweepPressureClassifier.Compute(new[] + { + C("database_config", 500_000, 0), + C("trace_flags", 0, 1), + C("wait_stats", 300, 1), + }); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.Equal(300, pressure.BusyMsPerMinute, 3); + } + + /// + /// Amortization is by each collector's OWN cadence: an hourly collector averaging 30s costs 500 ms of + /// every minute, not 30,000 — the mistake this pin forbids is charging slow collectors at the fast + /// cadence, which would flag every server with a heavy daily job. + /// + [Fact] + public void SlowCollectorsAreAmortizedByTheirOwnCadence() + { + var pressure = SweepPressureClassifier.Compute(new[] { C("index_object_stats", 30_000, 60) }); + + Assert.Equal(500, pressure.BusyMsPerMinute, 3); + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + } + + /// No collectors — a server before first collection — is OK with zero demand, never a verdict from nothing. + [Fact] + public void AnEmptyWindowReadsOkWithZeroDemand() + { + var pressure = SweepPressureClassifier.Compute(Array.Empty<(string, double, int)>()); + + Assert.Equal(SweepPressureClassifier.Ok, pressure.Verdict); + Assert.Equal(0, pressure.BusyMsPerMinute); + Assert.Equal(0, pressure.BusyPercent); + } +} diff --git a/Lite.Tests/WaitStatsCollectorDefinitionTests.cs b/Lite.Tests/WaitStatsCollectorDefinitionTests.cs index 6de20065e..699bfcd40 100644 --- a/Lite.Tests/WaitStatsCollectorDefinitionTests.cs +++ b/Lite.Tests/WaitStatsCollectorDefinitionTests.cs @@ -95,11 +95,11 @@ public void WritePayload_EmitsPayloadOrder_AndPinsDeltaGroupsKeysAndGapPolicy() /* Payload order: raw values then the three deltas (recording calculator returns value * 10). */ Assert.Equal(new object?[] { "PAGEIOLATCH_SH", 7L, 300L, 20L, 70L, 3000L, 200L }, writer.Values); - /* Delta contract: group names, key = wait_type, the host collection time, 300 s gap policy. */ + /* Delta contract: group names, key = wait_type, the host collection time, the shared gap policy. */ Assert.Equal(3, deltas.Calls.Count); - Assert.Equal(("wait_stats_tasks", "PAGEIOLATCH_SH", 7L, context.CollectionTime, 300), deltas.Calls[0]); - Assert.Equal(("wait_stats_time", "PAGEIOLATCH_SH", 300L, context.CollectionTime, 300), deltas.Calls[1]); - Assert.Equal(("wait_stats_signal", "PAGEIOLATCH_SH", 20L, context.CollectionTime, 300), deltas.Calls[2]); + Assert.Equal(("wait_stats_tasks", "PAGEIOLATCH_SH", 7L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[0]); + Assert.Equal(("wait_stats_time", "PAGEIOLATCH_SH", 300L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[1]); + Assert.Equal(("wait_stats_signal", "PAGEIOLATCH_SH", 20L, context.CollectionTime, CollectorDeltaCalculator.DefaultMaxGapSeconds), deltas.Calls[2]); Assert.All(deltas.Calls, _ => Assert.Equal(42, deltas.LastServerId)); } } diff --git a/Lite.Tests/WatermarkPolicyTests.cs b/Lite.Tests/WatermarkPolicyTests.cs index 92ffeb8fe..329ce2bf5 100644 --- a/Lite.Tests/WatermarkPolicyTests.cs +++ b/Lite.Tests/WatermarkPolicyTests.cs @@ -13,10 +13,11 @@ namespace Lite.Tests; /// -/// #1556: the 24h catch-up clamp boundaries. A stale query_store watermark (a service down for days) must -/// not point its cutoff days into the past — that one cycle would try to pull the whole retained backlog and -/// drive the commit-limit blowout. The clamp floors a >24h-stale watermark to now-24h; a fresh watermark -/// and a null watermark pass through untouched. +/// #1556/#2102: the catch-up clamp boundaries. A stale query_store watermark must not point its cutoff +/// far into the past — the per-database query's cost grows with window width, so a wide one-shot window +/// either blows the commit limit (#1556, days-wide) or times out every cycle and wedges the database +/// permanently (#2102, hours-wide). The clamp floors a stale watermark to now-MaxCatchup; a fresh +/// watermark and a null watermark pass through untouched. /// public sealed class WatermarkPolicyTests { @@ -32,9 +33,9 @@ public void ClampCatchup_Null_StaysNull() [Fact] public void ClampCatchup_WithinHorizon_ReturnedUnchanged() { - /* A routine restart / brief outage: the watermark is minutes-to-hours old and never clamps. */ - var oneHourAgo = Now.AddHours(-1); - Assert.Equal(oneHourAgo, WatermarkPolicy.ClampCatchup(oneHourAgo, Now)); + /* A routine restart: the watermark is minutes old and never clamps. */ + var tenMinutesAgo = Now.AddMinutes(-10); + Assert.Equal(tenMinutesAgo, WatermarkPolicy.ClampCatchup(tenMinutesAgo, Now)); var justInside = Now - WatermarkPolicy.MaxCatchup + TimeSpan.FromSeconds(1); Assert.Equal(justInside, WatermarkPolicy.ClampCatchup(justInside, Now)); @@ -43,19 +44,21 @@ public void ClampCatchup_WithinHorizon_ReturnedUnchanged() [Fact] public void ClampCatchup_ExactlyAtHorizon_NotClamped() { - /* The floor is strict (< floor clamps): a watermark exactly 24h old is at the horizon, not past it. */ + /* The floor is strict (< floor clamps): a watermark exactly MaxCatchup old is at the horizon, + not past it. */ var atHorizon = Now - WatermarkPolicy.MaxCatchup; Assert.Equal(atHorizon, WatermarkPolicy.ClampCatchup(atHorizon, Now)); } [Fact] - public void ClampCatchup_StalerThanHorizon_FlooredToNowMinus24h() + public void ClampCatchup_StalerThanHorizon_FlooredToNowMinusMaxCatchup() { - /* The field incident: a multi-day-old watermark is floored to now-24h so catch-up is bounded. */ + /* The field incidents: a stale watermark is floored to now-MaxCatchup so one cycle's window is + bounded; the skipped range is the backfill worker's job. */ var floor = Now - WatermarkPolicy.MaxCatchup; Assert.Equal(floor, WatermarkPolicy.ClampCatchup(Now.AddDays(-3), Now)); - Assert.Equal(floor, WatermarkPolicy.ClampCatchup(Now.AddHours(-30), Now)); + Assert.Equal(floor, WatermarkPolicy.ClampCatchup(Now.AddHours(-6), Now)); var justPast = Now - WatermarkPolicy.MaxCatchup - TimeSpan.FromSeconds(1); Assert.Equal(floor, WatermarkPolicy.ClampCatchup(justPast, Now)); @@ -70,10 +73,14 @@ public void ClampCatchup_FutureWatermark_ReturnedUnchanged() } [Fact] - public void MaxCatchup_IsTwentyFourHours() + public void MaxCatchup_IsOneHour_AndMatchesTheBackfillSliceSpan() { - /* Drift tripwire: the horizon is a deliberate choice (routine outages never clamp; multi-day ones - survive with a bounded, logged hole). */ - Assert.Equal(TimeSpan.FromHours(24), WatermarkPolicy.MaxCatchup); + /* Drift tripwire: one hour is the live path's one-query cost envelope — the width the fleet + proves every day under Query Store's 900s flush cadence. It was 24h until #2102 showed the + clamp sat far above the cost tipping point on big databases and never interrupted the + timeout spiral. The equality half is the design invariant: NO path, live or backfill, may + window wider than the other, or one of them re-becomes the wide-window casualty. */ + Assert.Equal(TimeSpan.FromHours(1), WatermarkPolicy.MaxCatchup); + Assert.Equal(QueryStoreBackfillState.MaxSliceSpan, WatermarkPolicy.MaxCatchup); } } diff --git a/Lite.Tests/packages.lock.json b/Lite.Tests/packages.lock.json index 411ca9446..dcc0b23bc 100644 --- a/Lite.Tests/packages.lock.json +++ b/Lite.Tests/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0-windows7.0": { "Microsoft.NET.Test.Sdk": { @@ -62,35 +62,11 @@ "Microsoft.Identity.Client.Extensions.Msal": "4.78.0" } }, - "CredentialManagement": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" - }, "DuckDB.NET.Bindings": { "type": "Transitive", "resolved": "1.5.5", "contentHash": "DrQS4YgORTdmQ7ZkPosk/Wo/f3ctlBQSmz0O23BKbIc/7Tb+hgCOgYSX5Bdy2/Oqmc3t8J9Dz8Z/BQOqCfpoww==" }, - "DuckDB.NET.Bindings.Full": { - "type": "Transitive", - "resolved": "1.5.5", - "contentHash": "QIyg93jXN+ebHtsl3UhNmoKV+UoBlHxicycGUVY8cw/kwTGB8CrrULcLr7ngUkk5pftaPoyIHycxTgshBOalaA==" - }, - "DuckDB.NET.Data": { - "type": "Transitive", - "resolved": "1.5.5", - "contentHash": "VerE5IRph9ui+E8Ljz1xNuErbuZMrYvfwxo+KLZmBngrpj3UVdYWXmr2FwvHRJWwuCpuk7PcouEbdhNnU5DYlQ==", - "dependencies": { - "Apache.Arrow": "23.0.0", - "DuckDB.NET.Bindings": "1.5.5" - } - }, - "Hardcodet.NotifyIcon.Wpf": { - "type": "Transitive", - "resolved": "2.0.1", - "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" - }, "HarfBuzzSharp": { "type": "Transitive", "resolved": "8.3.1.1", @@ -135,21 +111,6 @@ "resolved": "18.8.1", "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" }, - "Microsoft.Data.SqlClient": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", - "dependencies": { - "Microsoft.Bcl.Cryptography": "9.0.13", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", - "Microsoft.Extensions.Caching.Memory": "9.0.13", - "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", - "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" - } - }, "Microsoft.Data.SqlClient.Extensions.Abstractions": { "type": "Transitive", "resolved": "7.0.2", @@ -158,20 +119,6 @@ "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)" } }, - "Microsoft.Data.SqlClient.Extensions.Azure": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", - "dependencies": { - "Azure.Core": "1.51.1", - "Azure.Identity": "1.18.0", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Extensions.Caching.Memory": "8.0.1", - "Microsoft.Identity.Client": "4.84.2", - "Microsoft.Identity.Client.Broker": "4.84.2" - } - }, "Microsoft.Data.SqlClient.Internal.Logging": { "type": "Transitive", "resolved": "7.0.2", @@ -207,15 +154,6 @@ "Microsoft.Extensions.Primitives": "9.0.13" } }, - "Microsoft.Extensions.Configuration": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", - "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Primitives": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -263,17 +201,6 @@ "Microsoft.Extensions.Primitives": "10.0.10" } }, - "Microsoft.Extensions.Configuration.Json": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.UserSecrets": { "type": "Transitive", "resolved": "10.0.10", @@ -340,35 +267,6 @@ "resolved": "10.0.10", "contentHash": "jSOCVxEwCd4Aq925kJVz1kSO1EpX2OHYKL04qVREXkDU7Ce3pVDdHPYm+fEy8y/th2kJf/DAstRHpJAqoNWP8w==" }, - "Microsoft.Extensions.Hosting": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.Binder": "10.0.10", - "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", - "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.Configuration.Json": "10.0.10", - "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", - "Microsoft.Extensions.Diagnostics": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", - "Microsoft.Extensions.FileProviders.Physical": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging.Configuration": "10.0.10", - "Microsoft.Extensions.Logging.Console": "10.0.10", - "Microsoft.Extensions.Logging.Debug": "10.0.10", - "Microsoft.Extensions.Logging.EventLog": "10.0.10", - "Microsoft.Extensions.Logging.EventSource": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -381,24 +279,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.10" } }, - "Microsoft.Extensions.Logging": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", "resolved": "10.0.10", @@ -616,28 +496,10 @@ "resolved": "5.0.0", "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" }, - "ModelContextProtocol": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "X/KDNZDP9Zgs7YXXxxpKiDMWWPXYrs764lVgc6vEM4pCg87bYz4VaceeiKW91XDTGgUvIbmRYXgWqyNpV32xRg==", - "dependencies": { - "Microsoft.Extensions.Caching.Abstractions": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "ModelContextProtocol.Core": "[2.0.0]" - } - }, - "ModelContextProtocol.AspNetCore": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "dXrB7sBpQjUQU0UcdyFPJbOTFw7yaceD+OgAZVAeBveRzbiBlg89jEygAtcgOwd/L+O+YpM98zfwaXz067NFDQ==", - "dependencies": { - "ModelContextProtocol": "[2.0.0]" - } - }, "ModelContextProtocol.Core": { "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "piFR0HtA/2Oc1tgk96EE5Tye6qA2sg3WGRAXBhUqo/BWikdEYEs2UuqtmwLrQZJUge1nUOPgGHYsb15VIBK8iw==", + "resolved": "2.1.0", + "contentHash": "cU/urrhRxE4/iSyBIJI7QOaFqSP1FOEnwEHsct9n6t6/XluCAFD9iqnrPkBAsEYr+f/G4tVQ21U+6wN/6fQvOg==", "dependencies": { "Microsoft.Extensions.AI.Abstractions": "10.8.3", "Microsoft.Extensions.Logging.Abstractions": "10.0.10" @@ -750,17 +612,6 @@ "SkiaSharp.NativeAssets.Linux.NoDependencies": "3.119.0" } }, - "ScottPlot.WPF": { - "type": "Transitive", - "resolved": "5.1.59", - "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", - "dependencies": { - "OpenTK": "4.9.4", - "OpenTK.GLWpfControl": "4.3.3", - "ScottPlot": "5.1.59", - "SkiaSharp.Views.WPF": "3.119.0" - } - }, "SkiaSharp": { "type": "Transitive", "resolved": "3.119.0", @@ -843,11 +694,6 @@ "resolved": "10.0.1", "contentHash": "BZC4mhdL569AXV56ep9YO6ShjhxFXGP7SwVX0Bc/e0dJPWnS6aBEXZJXqh64RVx8HquqWHkJUINBydLRQ1yq0g==" }, - "Velopack": { - "type": "Transitive", - "resolved": "1.2.0", - "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" - }, "xunit.analyzers": { "type": "Transitive", "resolved": "1.27.0", @@ -932,7 +778,7 @@ "dependencies": { "CredentialManagement": "[1.0.2, )", "Microsoft.Extensions.Logging.Abstractions": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )" + "ModelContextProtocol": "[2.1.0, )" } }, "performancemonitor.notifications": { @@ -968,8 +814,8 @@ "Microsoft.Data.SqlClient.Extensions.Azure": "[7.0.2, )", "Microsoft.Extensions.Hosting": "[10.0.10, )", "Microsoft.Extensions.Logging": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )", - "ModelContextProtocol.AspNetCore": "[2.0.0, )", + "ModelContextProtocol": "[2.1.0, )", + "ModelContextProtocol.AspNetCore": "[2.1.0, )", "PerformanceMonitor.Alerting": "[1.0.0, )", "PerformanceMonitor.Analysis": "[1.0.0, )", "PerformanceMonitor.Collectors": "[1.0.0, )", @@ -980,6 +826,175 @@ "ScottPlot.WPF": "[5.1.59, )", "Velopack": "[1.2.0, )" } + }, + "CredentialManagement": { + "type": "CentralTransitive", + "requested": "[1.0.2, )", + "resolved": "1.0.2", + "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" + }, + "DuckDB.NET.Bindings.Full": { + "type": "CentralTransitive", + "requested": "[1.5.5, )", + "resolved": "1.5.5", + "contentHash": "QIyg93jXN+ebHtsl3UhNmoKV+UoBlHxicycGUVY8cw/kwTGB8CrrULcLr7ngUkk5pftaPoyIHycxTgshBOalaA==" + }, + "DuckDB.NET.Data": { + "type": "CentralTransitive", + "requested": "[1.5.5, )", + "resolved": "1.5.5", + "contentHash": "VerE5IRph9ui+E8Ljz1xNuErbuZMrYvfwxo+KLZmBngrpj3UVdYWXmr2FwvHRJWwuCpuk7PcouEbdhNnU5DYlQ==", + "dependencies": { + "Apache.Arrow": "23.0.0", + "DuckDB.NET.Bindings": "1.5.5" + } + }, + "Hardcodet.NotifyIcon.Wpf": { + "type": "CentralTransitive", + "requested": "[2.0.1, )", + "resolved": "2.0.1", + "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" + }, + "Microsoft.Data.SqlClient": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "9.0.13", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", + "Microsoft.Extensions.Caching.Memory": "9.0.13", + "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", + "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" + } + }, + "Microsoft.Data.SqlClient.Extensions.Azure": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", + "dependencies": { + "Azure.Core": "1.51.1", + "Azure.Identity": "1.18.0", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Extensions.Caching.Memory": "8.0.1", + "Microsoft.Identity.Client": "4.84.2", + "Microsoft.Identity.Client.Broker": "4.84.2" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" + } + }, + "Microsoft.Extensions.Hosting": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.Binder": "10.0.10", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.Configuration.Json": "10.0.10", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Diagnostics": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", + "Microsoft.Extensions.FileProviders.Physical": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging.Configuration": "10.0.10", + "Microsoft.Extensions.Logging.Console": "10.0.10", + "Microsoft.Extensions.Logging.Debug": "10.0.10", + "Microsoft.Extensions.Logging.EventLog": "10.0.10", + "Microsoft.Extensions.Logging.EventSource": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" + } + }, + "ModelContextProtocol": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "Oa4rU7EL9C2qyFjQj1dx+ysGMzfWDRpM8RRaUMmLGs5vPvfJ9xyz4ZtyF4ychY+Nx1b/auGCqIQLqSz/IpPkKA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "ModelContextProtocol.Core": "[2.1.0]" + } + }, + "ModelContextProtocol.AspNetCore": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "yhJ8bBXIgrX0mAgRYRgzcbH3bLdv3MDSkG52utRW9EAAtQrPw/g7Q/T6EurxKV+L+Zefv8VVUYcNbXEOd9GgfA==", + "dependencies": { + "ModelContextProtocol": "[2.1.0]" + } + }, + "ScottPlot.WPF": { + "type": "CentralTransitive", + "requested": "[5.1.59, )", + "resolved": "5.1.59", + "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", + "dependencies": { + "OpenTK": "4.9.4", + "OpenTK.GLWpfControl": "4.3.3", + "ScottPlot": "5.1.59", + "SkiaSharp.Views.WPF": "3.119.0" + } + }, + "Velopack": { + "type": "CentralTransitive", + "requested": "[1.2.0, )", + "resolved": "1.2.0", + "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" } } } diff --git a/Lite/Analysis/DrillDownCollector.Queries.cs b/Lite/Analysis/DrillDownCollector.Queries.cs index ac7d0a22e..566655dcc 100644 --- a/Lite/Analysis/DrillDownCollector.Queries.cs +++ b/Lite/Analysis/DrillDownCollector.Queries.cs @@ -287,7 +287,45 @@ private async Task CollectRegressedQueries(AnalysisFinding finding, AnalysisCont using var cmd = connection.CreateCommand(); cmd.CommandText = @" -WITH deduped AS +WITH psp_signature AS +( + -- #2138 gap 3: the PARAMETER_SENSITIVITY detector's EXACT firing signature (same floors, same + -- ratio, same analysis window) reduced to the (database, query_hash) set it would report. Using + -- the detector's own thresholds is what keeps the flag honest: a query flagged here IS one the + -- detector counts when it fires, never a looser lookalike. Grant/spill divergence stay metadata + -- on the PSP side — they do not fire the detector alone, so they do not fire this flag alone. + SELECT DISTINCT + database_name, + query_hash + FROM + ( + SELECT + database_name, + query_hash, + query_plan_hash, + execution_count, + creation_time, + min_worker_time, + max_worker_time, + ROW_NUMBER() OVER + ( + PARTITION BY database_name, query_hash, query_plan_hash + ORDER BY collection_time DESC + ) AS rn + FROM v_query_stats + WHERE server_id = $1 + AND collection_time >= $3 + AND collection_time <= $4 + AND delta_execution_count > 0 + ) AS latest_cache + WHERE rn = 1 + AND min_worker_time >= 10000 + AND max_worker_time >= 250000 + AND execution_count >= 20 + AND creation_time <= $3 + AND max_worker_time::DOUBLE PRECISION / NULLIF(min_worker_time, 0) >= 10 +), +deduped AS ( -- #1850: replica_role is part of the interval's identity, not a passenger. -- sys.query_store_runtime_stats is keyed by (plan_id, interval, execution_type, replica_group), and @@ -305,6 +343,7 @@ WITH deduped AS plan_id, replica_role, query_plan_hash, + query_hash, execution_count, avg_cpu_time_us, avg_duration_us, @@ -332,6 +371,7 @@ plan_agg AS plan_id, replica_role, any_value(query_plan_hash) AS query_plan_hash, + any_value(query_hash) AS query_hash, any_value(query_text) AS query_text, SUM(execution_count) AS execs, SUM(avg_cpu_time_us * execution_count)::DOUBLE PRECISION / NULLIF(SUM(execution_count), 0) AS cpu_per_exec, @@ -352,6 +392,7 @@ plan_dedup AS replica_role, query_plan_hash, MAX(plan_id) AS plan_id, + any_value(query_hash) AS query_hash, any_value(query_text) AS query_text, SUM(execs) AS execs, SUM(cpu_per_exec * execs) / NULLIF(SUM(execs), 0) AS cpu_per_exec, @@ -388,12 +429,28 @@ compared AS b.cpu_per_exec AS best_cpu, b.dur_per_exec AS best_dur, l.query_text, - GREATEST + -- #2138: the SAME CPU-primary scoring as the PLAN_REGRESSION fact (DuckDbFactCollector.QueryPerf.cs, + -- where the rationale lives). The drill-down must agree with the fact that displays it: under the + -- old GREATEST a duration-only regression could appear here that the fact never counted. + CASE + WHEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 2 + THEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) + WHEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) >= 4 + AND l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 1.25 + THEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) / 2 + END AS regression_factor, + l.replica_role, + l.execs * l.cpu_per_exec AS latest_total_cpu_us, + -- #2138 gap 3: does this regressed query ALSO carry the parameter-sensitivity signature in the + -- plan cache? Keyed on (database, query_hash) — the hash bridges Query Store and the cache. + -- Steers the force-plan remediation's caution text; the future bot never auto-forces on true. + EXISTS ( - l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0), - l.dur_per_exec / NULLIF(b.dur_per_exec, 0) - ) AS regression_factor, - l.replica_role + SELECT 1 + FROM psp_signature AS p + WHERE p.database_name = l.database_name + AND p.query_hash = l.query_hash + ) AS parameter_sensitivity_cofired FROM ranked AS l JOIN ranked AS b ON b.database_name = l.database_name @@ -415,14 +472,21 @@ AND l.query_plan_hash <> b.query_plan_hash best_dur, regression_factor, LEFT(query_text, 500) AS query_text, - replica_role + replica_role, + parameter_sensitivity_cofired FROM compared WHERE regression_factor >= 2 +AND latest_total_cpu_us >= 10000000 ORDER BY regression_factor DESC LIMIT 5"; cmd.Parameters.Add(new DuckDBParameter { Value = context.ServerId }); cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeStart.AddDays(-14) }); + /* $3/$4: the STANDARD analysis window for the psp_signature CTE — deliberately not the 14-day + comparison window above, so the flag matches what the PARAMETER_SENSITIVITY detector itself + would report for this run. */ + cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeStart }); + cmd.Parameters.Add(new DuckDBParameter { Value = context.TimeRangeEnd }); var items = new List(); using var reader = await cmd.ExecuteReaderAsync(); @@ -445,8 +509,11 @@ ORDER BY regression_factor DESC standalone/non-AG/pre-2022 server, which is the overwhelming majority; it is only populated on an AG primary with Query Store for secondary replicas enabled, where two rows for the same query are now legitimately distinct rather than one silently dropped. - Last in the row so the existing reader ordinals are untouched. */ - replica_role = reader.IsDBNull(11) ? "" : reader.GetString(11) + Appended after the older columns so the existing reader ordinals are untouched. */ + replica_role = reader.IsDBNull(11) ? "" : reader.GetString(11), + /* #2138 gap 3: the plan-cache PSP signature co-fired for this query's hash. Steers the + force-plan caution text; the future bot never auto-forces a flagged target. */ + parameter_sensitivity_cofired = !reader.IsDBNull(12) && Convert.ToBoolean(reader.GetValue(12)) }); } diff --git a/Lite/Analysis/DuckDbFactCollector.QueryPerf.cs b/Lite/Analysis/DuckDbFactCollector.QueryPerf.cs index 07690d46b..2ac62f305 100644 --- a/Lite/Analysis/DuckDbFactCollector.QueryPerf.cs +++ b/Lite/Analysis/DuckDbFactCollector.QueryPerf.cs @@ -315,11 +315,22 @@ compared AS l.force_failure_count AS force_failure_count, b.cpu_per_exec AS best_cpu, b.dur_per_exec AS best_dur, - GREATEST - ( - l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0), - l.dur_per_exec / NULLIF(b.dur_per_exec, 0) - ) AS regression_factor + -- #2138: CPU is the PRIMARY signal — duration alone is confounded by blocking, IO waits, and + -- machine contention that no plan choice caused, so it must not fire a plan-regression verdict + -- by itself. A CPU regression scores at its own ratio; a duration-dominant one fires only when + -- EXTREME (>= 4x) AND corroborated by at least mild CPU worsening (>= 1.25x), scored at half + -- the duration ratio so it competes honestly with CPU-detected rows. NULL when neither path + -- fires — the >= 2 gate below drops it. + CASE + WHEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 2 + THEN l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) + WHEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) >= 4 + AND l.cpu_per_exec / NULLIF(b.cpu_per_exec, 0) >= 1.25 + THEN l.dur_per_exec / NULLIF(b.dur_per_exec, 0) / 2 + END AS regression_factor, + -- The resource-expenditure half of the importance gate (#2138): total CPU the LATEST plan burned + -- over the window. The exec-count floor above only counts; this weighs. + l.execs * l.cpu_per_exec AS latest_total_cpu_us FROM ranked AS l JOIN ranked AS b ON b.database_name = l.database_name @@ -340,6 +351,10 @@ AND l.query_plan_hash <> b.query_plan_hash regression_factor FROM compared WHERE regression_factor >= 2 +-- 10 CPU-seconds across the window: a NOISE floor, not an importance ranking — it exists to exclude +-- near-zero-cost queries whose ratios are all sampling jitter; magnitude ranking stays with +-- regression_factor and the scorer. +AND latest_total_cpu_us >= 10000000 ORDER BY regression_factor DESC LIMIT 20"; @@ -363,18 +378,18 @@ ORDER BY regression_factor DESC { worstQueryId = reader.IsDBNull(0) ? 0L : ToInt64(reader.GetValue(0)); var latestCpu = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); - var latestDur = reader.IsDBNull(2) ? 0.0 : Convert.ToDouble(reader.GetValue(2)); worstLatestForced = (!reader.IsDBNull(3) && Convert.ToBoolean(reader.GetValue(3))) ? 1 : 0; worstForceFailures = reader.IsDBNull(4) ? 0L : ToInt64(reader.GetValue(4)); var bestCpu = reader.IsDBNull(5) ? 0.0 : Convert.ToDouble(reader.GetValue(5)); - var bestDur = reader.IsDBNull(6) ? 0.0 : Convert.ToDouble(reader.GetValue(6)); worstFactor = reader.IsDBNull(7) ? 0.0 : Convert.ToDouble(reader.GetValue(7)); worstLatestCpu = latestCpu; worstBestCpu = bestCpu; + // Which CASE branch fired, not which raw ratio is larger (review catch on #2138): + // CPU has PRECEDENCE in the scoring, so a row with cpu 2.5x and duration 10x is a + // CPU-detected regression at 2.5 — comparing magnitudes would mislabel it duration. var cpuRatio = bestCpu > 0 ? latestCpu / bestCpu : 0.0; - var durRatio = bestDur > 0 ? latestDur / bestDur : 0.0; - worstDimension = cpuRatio >= durRatio ? 1 : 2; // 1 = cpu, 2 = duration + worstDimension = cpuRatio >= 2 ? 1 : 2; // 1 = cpu, 2 = duration } offenderCount++; } diff --git a/Lite/App.xaml.cs b/Lite/App.xaml.cs index f2e71cf14..512f0a133 100644 --- a/Lite/App.xaml.cs +++ b/Lite/App.xaml.cs @@ -16,6 +16,7 @@ using System.Threading; using System.Threading.Tasks; using System.Windows; +using System.Windows.Interop; using PerformanceMonitor.Notifications; using System.Windows.Threading; using PerformanceMonitorLite.Services; @@ -150,6 +151,8 @@ being handed back the stale in-memory version. The coordinator owns the mutex + public static bool AlertLowDiskEnabled { get; set; } = true; public static int AlertLowDiskThresholdPercent { get; set; } = 10; // Alert when a volume's free space < X% (0 disables this check) public static int AlertLowDiskThresholdGb { get; set; } = 5; // Alert when a volume's free space < X GB (0 disables this check) + public static int AlertDiskCriticalFreePercent { get; set; } = 3; // #2107: at/below this % free the low-disk alert grades CRITICAL (#1136 tier) + public static int AlertDiskCriticalFreeGb { get; set; } = 2; // #2107: at/below this many GB free is CRITICAL on any volume (OR-ed with the %) public static bool AlertPvsEnabled { get; set; } = true; // #1984 ADR persistent version store pressure public static int AlertPvsThresholdPercent { get; set; } = 40; // Alert when an ADR database's PVS >= X% of its data files (0 disables this check) public static int AlertPvsFloorGb { get; set; } = 1; // AND-qualifier: the PVS must also be >= X GB (0 removes the floor) @@ -194,6 +197,15 @@ private static string GetDefaultCsvSeparator() return System.Globalization.CultureInfo.CurrentCulture.NumberFormat.NumberDecimalSeparator == "," ? ";" : ","; } + /* Collection settings */ + /* #2167: the Query Store history backfill (#2058) — fills gaps the live path never takes (a + first-contact tail, an outage hole, a freshly restored database's imported catalog) in bounded + background slices. Default ON. Turn it off when a heavy catch-up is costing the monitored server + more than the history is worth; live collection is unaffected and re-enabling resumes exactly + where the watermarks left off, so nothing is lost by pausing it. Darling's equivalent is a store + column (V58) because a headless service has no window to click. */ + public static bool QueryStoreBackfillEnabled { get; set; } = true; + /* System tray settings */ public static bool MinimizeToTray { get; set; } = true; @@ -448,11 +460,82 @@ land in the file a few statements later. */ DispatcherUnhandledException += OnDispatcherUnhandledException; TaskScheduler.UnobservedTaskException += OnUnobservedTaskException; + /* Entra MFA needs a parent window handle for the WAM broker, or interactive auth fails with + 0xwindow_handle_required instead of prompting (#2184). Registered once, before any window + exists, because SqlAuthenticationProvider installs process-wide: the Add/Edit dialog's Test + Connection and every collector connection are covered without per-site wiring. The handle + itself is resolved lazily per prompt, so registering this early is safe. */ + Services.EntraInteractiveAuth.Register(ActiveWindowHandle); + // Create and show main window (StartupUri removed for Velopack custom Main) _mainWindow = new MainWindow(); _mainWindow.Show(); } + /// + /// The window that should own an Entra MFA prompt, resolved at the moment MSAL asks (#2184). + /// + /// Prefers whichever window is currently active over the main window, because a connection is + /// usually triggered from the Add/Edit Server dialog — parenting the account picker to the main + /// window behind it would let the picker appear behind the dialog the user is looking at. Falls back + /// to the main window, then to , which MSAL treats the same as no handle: + /// the prompt fails rather than the app crashing, which is the right way round for an auth path. + /// + /// Resolved per call rather than captured once: a window's HWND does not exist until the + /// window has been sourced, and the right parent is whichever window is in front now, not the one + /// that existed at startup. + /// + /// Marshaled to the UI thread: MSAL invokes this from whatever thread SqlClient's token + /// acquisition runs on — collector worker threads included — and WPF enforces dispatcher affinity + /// on properties, so an off-thread read would throw rather than merely race. + /// The blocking Invoke is safe here because no UI-thread path blocks on a SQL connection open + /// (opens are async throughout; the UI stays pumping). If the dispatcher cannot deliver anyway + /// (shutdown timing), Zero degrades to MSAL's normal no-handle failure instead of throwing from + /// inside the auth callback. + /// + private static IntPtr ActiveWindowHandle() + { + try + { + var dispatcher = Current?.Dispatcher; + if (dispatcher is null) + return IntPtr.Zero; + + return dispatcher.CheckAccess() + ? ActiveWindowHandleOnUIThread() + : dispatcher.Invoke(ActiveWindowHandleOnUIThread); + } + catch (Exception ex) + { + /* Zero reproduces the original #2184 symptom (0xwindow_handle_required), so a throw here + must leave a trace - a silent fallback would be this bug's own shape one layer down. The + log call is guarded because this can fire during shutdown, after the dispatcher and + logger are gone, and logging must never be the thing that breaks auth. */ + try { AppLogger.Warn("App", $"Entra parent-window handle resolution failed; WAM will see no handle: {ex.Message}"); } catch { /* nothing left to log to */ } + return IntPtr.Zero; + } + } + + private static IntPtr ActiveWindowHandleOnUIThread() + { + var app = Current; + if (app is null) + return IntPtr.Zero; + + Window? active = null; + foreach (Window window in app.Windows) + { + if (window.IsActive) + { + active = window; + break; + } + } + + var owner = active ?? app.MainWindow; + return owner is null ? IntPtr.Zero : new WindowInteropHelper(owner).Handle; + } + /// /// Invoked on 's background thread when a second launch asks us /// to surface the window. Marshals to the UI thread and restores via WPF's Show() path (#1050). @@ -650,6 +733,9 @@ cannot drive a nonsense threshold in either app. */ if (root.TryGetProperty("alert_low_disk_enabled", out v)) AlertLowDiskEnabled = v.GetBoolean(); if (root.TryGetProperty("alert_low_disk_threshold_percent", out v)) AlertLowDiskThresholdPercent = (int)Math.Clamp(v.GetInt64(), 0, 100); if (root.TryGetProperty("alert_low_disk_threshold_gb", out v)) AlertLowDiskThresholdGb = (int)Math.Max(0, v.GetInt64()); + /* #2107: the CRITICAL tier floors, clamped like the WARNING thresholds above. */ + if (root.TryGetProperty("alert_disk_critical_free_percent", out v)) AlertDiskCriticalFreePercent = Math.Clamp(v.GetInt32(), 0, 100); + if (root.TryGetProperty("alert_disk_critical_free_gb", out v)) AlertDiskCriticalFreeGb = (int)Math.Max(0, v.GetInt64()); if (root.TryGetProperty("alert_pvs_enabled", out v)) AlertPvsEnabled = v.GetBoolean(); if (root.TryGetProperty("alert_pvs_threshold_percent", out v)) AlertPvsThresholdPercent = (int)Math.Clamp(v.GetInt64(), 0, 100); if (root.TryGetProperty("alert_pvs_floor_gb", out v)) AlertPvsFloorGb = (int)Math.Max(0, v.GetInt64()); @@ -774,6 +860,7 @@ only the enable flag and EU-region toggle are plain prefs. */ if (root.TryGetProperty("smtp_recipients", out v)) SmtpRecipients = v.GetString() ?? ""; if (root.TryGetProperty("analysis_enabled", out v)) AnalysisEnabled = v.GetBoolean(); + if (root.TryGetProperty("query_store_backfill_enabled", out v)) QueryStoreBackfillEnabled = v.GetBoolean(); if (root.TryGetProperty("analysis_notifications_enabled", out v)) AnalysisNotificationsEnabled = v.GetBoolean(); if (root.TryGetProperty("analysis_interval_minutes", out v)) AnalysisIntervalMinutes = (int)Math.Clamp(v.GetInt64(), 5, 360); if (root.TryGetProperty("analysis_notify_severity", out v)) AnalysisNotifySeverity = Math.Clamp(v.GetDouble(), 0.0, 2.0); diff --git a/Lite/Controls/FinOpsTab.xaml.cs b/Lite/Controls/FinOpsTab.xaml.cs index 5d0abc7cd..cf877f236 100644 --- a/Lite/Controls/FinOpsTab.xaml.cs +++ b/Lite/Controls/FinOpsTab.xaml.cs @@ -37,6 +37,11 @@ public partial class FinOpsTab : UserControl private readonly Dictionary _filterManagers = new(); + /* #2306: suppresses ServerSelector_SelectionChanged while RefreshServerList reselects the SAME + logical server through a new instance — without it, any Manage Servers edit would clear filters + for a server switch that never happened. Darling's FinOps tab carries the same flag. */ + private bool _populatingServers; + private DataGridFilterManager? _dbResourcesFilterMgr; private DataGridFilterManager? _storageGrowthFilterMgr; private DataGridFilterManager? _dbSizesFilterMgr; @@ -83,18 +88,33 @@ public void RefreshServerList() var previousSelection = ServerSelector.SelectedItem as ServerConnection; var servers = _serverManager.GetAllServers(); - ServerSelector.ItemsSource = servers; - if (previousSelection != null) + if (previousSelection != null + && servers.FirstOrDefault(s => s.Id == previousSelection.Id) is { } match) { - var match = servers.FirstOrDefault(s => s.Id == previousSelection.Id); - if (match != null) + /* #2306 review catch (Darling's _populatingServers, ported): the same logical server + reselected through a NEW instance (ServerManager replaces edited entries, and + ComboBox compares by reference) still raises SelectionChanged. Without the guard, a + tag or favorite edit in Manage Servers would wipe active column filters for a server + switch that never happened. Nothing changed for this tab, so the handler — clear, + drill reset, reload — is suppressed entirely. */ + _populatingServers = true; + try { + ServerSelector.ItemsSource = servers; ServerSelector.SelectedItem = match; - return; } + finally + { + _populatingServers = false; + } + + return; } + /* The previous selection is gone (or never existed): the selection genuinely moves, so the + assignments below fire the handler on purpose — a real switch clears filters and reloads. */ + ServerSelector.ItemsSource = servers; if (servers.Count > 0) ServerSelector.SelectedIndex = 0; } @@ -370,9 +390,13 @@ private void UpdateUtilizationSummary(UtilizationEfficiencyRow? data) { "RIGHT_SIZED" => $"CPU is moderately loaded (avg {data.AvgCpuPct:N1}%, p95 {data.P95CpuPct:N1}%) and memory is well-utilized (buffer pool uses {bpPct:N0}% of physical RAM). No action needed.", "OVER_PROVISIONED" => $"CPU is lightly loaded (avg {data.AvgCpuPct:N1}%, max {data.MaxCpuPct}%) and buffer pool uses only {bpPct:N0}% of physical RAM. This server may have more resources than it needs.", - "UNDER_PROVISIONED" => data.P95CpuPct > 85 - ? $"CPU p95 is {data.P95CpuPct:N1}% (threshold: 85%). This server may need more CPU capacity." - : $"Buffer pool uses {bpPct:N0}% of physical RAM and memory ratio is {data.MemoryRatio:N2} (threshold: 0.95). Memory pressure is high.", + /* The reason comes from the same place as the verdict. This branch used to read + "P95CpuPct > 85 ? CPU : memory ratio is {x} (threshold: 0.95)", so a server flagged for grant + pressure or worker saturation would have been explained as a memory ratio that no longer + decides anything, citing a threshold the code does not check (#2246). */ + "UNDER_PROVISIONED" => ProvisioningVerdict.UnderProvisionedReason( + data.P95CpuPct, data.MaxGrantWaiters, data.GrantTimeouts, data.ForcedGrants, + data.MaxWorkersCount, data.CurrentWorkersCount), _ => "" }; @@ -805,7 +829,19 @@ private async System.Threading.Tasks.Task LoadMemoryGrantEfficiencyAsync(int ser private async void ServerSelector_SelectionChanged(object sender, SelectionChangedEventArgs e) { + if (_populatingServers) return; // same-server list repopulation, not a switch — see RefreshServerList + ResetStorageDrill(); // a new server invalidates any open object/index drill + + /* #2306: column filters belong to the previous server too — same mechanism as Darling's FinOps + tab: a filter set against server A silently zeroes server B's grid while the count indicators + (computed from the unfiltered list) stay full, and Refresh cannot clear it. Cleared via the + map every FinOps manager registers into, so a new grid inherits this without a second edit. */ + foreach (var manager in _filterManagers.Values) + { + manager.ClearFilters(); + } + await LoadPerServerDataAsync(); } diff --git a/Lite/Controls/JobHistoryTab.xaml.cs b/Lite/Controls/JobHistoryTab.xaml.cs index caff16155..82f58a51c 100644 --- a/Lite/Controls/JobHistoryTab.xaml.cs +++ b/Lite/Controls/JobHistoryTab.xaml.cs @@ -30,6 +30,7 @@ namespace PerformanceMonitorLite.Controls; public partial class JobHistoryTab : UserControl { private LocalDataService? _dataService; + private Func>? _displayNames; private DataGridFilterManager? _filterManager; private Popup? _filterPopup; private ColumnFilterPopup? _filterPopupContent; @@ -43,10 +44,17 @@ public JobHistoryTab() _staleDataTimer.Tick += StaleDataTimer_Tick; } - /// Initializes the control with required dependencies. - public void Initialize(LocalDataService dataService) + /// + /// Initializes the control with required dependencies. snapshots + /// server_id → operator display name from the CONFIG layer (#2126's Lite half): Lite's display-name + /// concept lives on ServerConnection, not in DuckDB (the stored servers.display_name + /// column is unpopulated), so the shell supplies the mapping the same way the Overview tab passes + /// DisplayNameWithIntent into GetServerSummaryAsync. + /// + public void Initialize(LocalDataService dataService, Func>? displayNames = null) { _dataService = dataService; + _displayNames = displayNames; _filterManager = new DataGridFilterManager(JobHistoryDataGrid); _staleDataTimer.Start(); } @@ -68,6 +76,20 @@ private async System.Threading.Tasks.Task LoadJobsAsync() var all = await System.Threading.Tasks.Task.Run(() => _dataService.GetJobHistoryAsync(hoursBack, 2000, serverId)); + /* #2126: rows carry the raw collected server name; swap in the operator's alias where the + config layer knows one, so the Server column and filter speak the same names as every + other tab. A server no longer in config keeps its raw name (the durable-record case). */ + if (_displayNames?.Invoke() is { Count: > 0 } names) + { + foreach (var row in all) + { + if (names.TryGetValue(row.ServerId, out var alias) && !string.IsNullOrEmpty(alias)) + { + row.ServerName = alias; + } + } + } + /* Populate the Server / Category combos from the full (pre status/category) result, then apply Status + Category client-side — those must NOT go into the reader's window (they'd skew the per-job long-running / last-success baselines, which are computed over every run in the diff --git a/Lite/Controls/ServerTab.Filters.cs b/Lite/Controls/ServerTab.Filters.cs index e3fe77ef9..ed5a6e8a9 100644 --- a/Lite/Controls/ServerTab.Filters.cs +++ b/Lite/Controls/ServerTab.Filters.cs @@ -36,6 +36,7 @@ private void InitializeFilterManagers() _serverConfigFilterMgr = new DataGridFilterManager(ServerConfigGrid); _databaseConfigFilterMgr = new DataGridFilterManager(DatabaseConfigGrid); _dbScopedConfigFilterMgr = new DataGridFilterManager(DatabaseScopedConfigGrid); + _queryStoreHealthFilterMgr = new DataGridFilterManager(QueryStoreHealthGrid); _automaticTuningFilterMgr = new DataGridFilterManager(AutomaticTuningGrid); _traceFlagsFilterMgr = new DataGridFilterManager(TraceFlagsGrid); _collectionHealthFilterMgr = new DataGridFilterManager(CollectionHealthGrid); @@ -69,6 +70,7 @@ private void InitializeFilterManagers() _filterManagers[ServerConfigGrid] = _serverConfigFilterMgr; _filterManagers[DatabaseConfigGrid] = _databaseConfigFilterMgr; _filterManagers[DatabaseScopedConfigGrid] = _dbScopedConfigFilterMgr; + _filterManagers[QueryStoreHealthGrid] = _queryStoreHealthFilterMgr; _filterManagers[AutomaticTuningGrid] = _automaticTuningFilterMgr; _filterManagers[TraceFlagsGrid] = _traceFlagsFilterMgr; _filterManagers[CollectionHealthGrid] = _collectionHealthFilterMgr; diff --git a/Lite/Controls/ServerTab.Refresh.cs b/Lite/Controls/ServerTab.Refresh.cs index 2badd4efa..039e7ea99 100644 --- a/Lite/Controls/ServerTab.Refresh.cs +++ b/Lite/Controls/ServerTab.Refresh.cs @@ -695,14 +695,16 @@ private async System.Threading.Tasks.Task RefreshConfigurationAsync(int hoursBac var serverConfigTask = Helpers.MethodProfiler.TimeAsync("Config.ServerConfig", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestServerConfigAsync(_serverId)))); var databaseConfigTask = Helpers.MethodProfiler.TimeAsync("Config.DatabaseConfig", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestDatabaseConfigAsync(_serverId, SelectedDatabaseFilter)))); var databaseScopedConfigTask = Helpers.MethodProfiler.TimeAsync("Config.DatabaseScopedConfig", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestDatabaseScopedConfigAsync(_serverId, SelectedDatabaseFilter)))); + var queryStoreHealthTask = Helpers.MethodProfiler.TimeAsync("Config.QueryStoreHealth", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestQueryStoreHealthAsync(_serverId, SelectedDatabaseFilter)))); var automaticTuningTask = Helpers.MethodProfiler.TimeAsync("Config.AutomaticTuning", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestAutomaticTuningAsync(_serverId, SelectedDatabaseFilter)))); var traceFlagsTask = Helpers.MethodProfiler.TimeAsync("Config.TraceFlags", () => Task.Run(() => SafeQueryAsync(() => _dataService.GetLatestTraceFlagsAsync(_serverId)))); - await System.Threading.Tasks.Task.WhenAll(serverConfigTask, databaseConfigTask, databaseScopedConfigTask, automaticTuningTask, traceFlagsTask); + await System.Threading.Tasks.Task.WhenAll(serverConfigTask, databaseConfigTask, databaseScopedConfigTask, queryStoreHealthTask, automaticTuningTask, traceFlagsTask); _serverConfigFilterMgr!.UpdateData(serverConfigTask.Result); _databaseConfigFilterMgr!.UpdateData(databaseConfigTask.Result); _dbScopedConfigFilterMgr!.UpdateData(databaseScopedConfigTask.Result); + _queryStoreHealthFilterMgr!.UpdateData(queryStoreHealthTask.Result); _automaticTuningFilterMgr!.UpdateData(automaticTuningTask.Result); _traceFlagsFilterMgr!.UpdateData(traceFlagsTask.Result); } diff --git a/Lite/Controls/ServerTab.TimeRange.cs b/Lite/Controls/ServerTab.TimeRange.cs index a6ed132f2..879390376 100644 --- a/Lite/Controls/ServerTab.TimeRange.cs +++ b/Lite/Controls/ServerTab.TimeRange.cs @@ -272,6 +272,16 @@ private async void TimeRangeCombo_SelectionChanged(object sender, SelectionChang FromDatePicker.SelectedDate = DateTime.Today.AddDays(-1); ToDatePicker.SelectedDate = DateTime.Today; } + + if (!isCustom) + { + /* #2154: a DatePicker's calendar dropdown is a POPUP, which lives outside the visual + tree's visibility — collapsing the picker does not close an already-open dropdown, + so backing out of Custom Range without picking a date left an orphaned floating + calendar on screen. Close them explicitly alongside the collapse. */ + FromDatePicker.IsDropDownOpen = false; + ToDatePicker.IsDropDownOpen = false; + } } if (!isCustom) diff --git a/Lite/Controls/ServerTab.xaml b/Lite/Controls/ServerTab.xaml index 933ba9ab8..77fc2adf8 100644 --- a/Lite/Controls/ServerTab.xaml +++ b/Lite/Controls/ServerTab.xaml @@ -1034,9 +1034,15 @@ + private async Task RunQueryStoreBackfillIfDueAsync(CancellationToken stoppingToken) { + /* #2167: the off switch, checked BEFORE the due-time stamp so a disabled backfill does not quietly + consume its own schedule — flipping it back on runs on the next due tick rather than waiting out + an interval that elapsed while it was off. Read live from the setting (not captured), so the + Settings window takes effect without restarting Lite, matching Darling's store-reload behavior. */ + if (!App.QueryStoreBackfillEnabled) + { + if (_queryStoreBackfillWasEnabled) + { + _queryStoreBackfillWasEnabled = false; + _logger?.LogInformation("Query Store backfill disabled in settings — idling; in-flight slices finish and no new ones start"); + } + + return; + } + + if (!_queryStoreBackfillWasEnabled) + { + _queryStoreBackfillWasEnabled = true; + _logger?.LogInformation("Query Store backfill re-enabled in settings — resuming from the stored watermarks"); + } + if (DateTime.UtcNow - _lastQueryStoreBackfill < QueryStoreBackfillInterval) { return; } _lastQueryStoreBackfill = DateTime.UtcNow; + + /* #2148: the hang protection lives INSIDE the tick, per server (see + RemoteCollectorService.RunQueryStoreBackfillTickAsync) — a wedged slice is abandoned and + quarantines only ITS server, so the tick itself is bounded by construction. A tick-level + deadline here was the first cut and was wrong twice over (review catch, round 2): it stalled + every server's backfill behind one wedge, and it would false-trip as fleet size grows + because a shared deadline sized for one slice was applied to the sum of all of them. */ try { await _collectorService.RunQueryStoreBackfillTickAsync(stoppingToken); } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + /* Shutdown — quiet. */ + } catch (Exception ex) { _logger?.LogError(ex, "Query Store backfill tick failed"); } } + /// One log vocabulary for every abandonable ladder step (#2148): abandonment and + /// still-wedged skips are ERRORS — the deadlines are generous multiples of healthy behavior, so + /// either one is a defect signal, and this line is the difference between a diagnosable field + /// report and "the charts just stopped". + private void LogStepOutcome(AbandonableStepResult result, string stepName, TimeSpan deadline) + { + switch (result.Outcome) + { + case AbandonableStepOutcome.Faulted: + _logger?.LogError(result.Exception, "{Step} failed", stepName); + break; + case AbandonableStepOutcome.Abandoned: + _logger?.LogError( + "{Step} exceeded {Deadline}s and was ABANDONED — collection continues; the step is " + + "quarantined until the wedged task ends. This is a defect signal: please report it " + + "with this log file (#2148).", + stepName, (int)deadline.TotalSeconds); + break; + case AbandonableStepOutcome.SkippedStillRunning: + _logger?.LogError( + "{Step} skipped — a previously-abandoned run is still wedged (#2148).", stepName); + break; + /* Completed and Cancelled are the quiet outcomes. */ + } + } + private async Task RunArchivalIfDueAsync() { if (_archiveService == null) diff --git a/Lite/Services/DuckDbAlertHistoryStore.cs b/Lite/Services/DuckDbAlertHistoryStore.cs index 8cc45aa25..b5952e4b2 100644 --- a/Lite/Services/DuckDbAlertHistoryStore.cs +++ b/Lite/Services/DuckDbAlertHistoryStore.cs @@ -367,6 +367,140 @@ INSERT OR REPLACE INTO config_edge_trigger_watermarks (server_id, metric_name, w } } + /// + /// Loads the per-fingerprint occurrence accounting for one server/metric (#2216) — + /// (dedup_key, total, observed window count, incident start, last observed) per row. + /// + /// Returns an EMPTY list on failure rather than the rows it managed to read. A partial read is the + /// worst outcome available: the fingerprints that made it keep accumulating while the ones that did not + /// silently restart mid-incident, so one alert reports some totals continuing and others reset. Empty is + /// at least uniform — every fingerprint reads as new and its total equals the window count, which is + /// the pre-#2216 information. + /// + public async Task> + LoadIncidentOccurrencesAsync(int serverId, string metricName) + { + var result = new List<(string, long, int, DateTime, DateTime)>(); + try + { + var duckDb = _duckDb; + if (duckDb == null) + { + var dbPath = App.DatabasePath; + if (string.IsNullOrEmpty(dbPath)) return result; + duckDb = new DuckDbInitializer(dbPath); + } + + using var readLock = duckDb.AcquireReadLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(); + + using var command = connection.CreateCommand(); + command.CommandText = @" +SELECT dedup_key, total_occurrences, observed_window_count, incident_started_at, last_observed_at +FROM config_incident_occurrences +WHERE server_id = $1 +AND metric_name = $2"; + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = metricName }); + + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + result.Add(( + reader.GetString(0), + Convert.ToInt64(reader.GetValue(1)), + Convert.ToInt32(reader.GetValue(2)), + Convert.ToDateTime(reader.GetValue(3)), + Convert.ToDateTime(reader.GetValue(4)))); + } + } + catch (Exception ex) + { + AppLogger.Error("Alerts", $"Could not load incident occurrences ({metricName}): {ex.Message}"); + return new List<(string, long, int, DateTime, DateTime)>(); + } + return result; + } + + /// + /// REPLACES the persisted occurrence set for one server/metric (#2216): the rows passed in become the + /// metric's complete state, and anything the store held for it that is not in the list is removed. An + /// empty list therefore clears the metric, which is how the falling edge is recorded — there is no + /// separate clear method to forget to call. + /// + /// Delete-then-insert inside ONE transaction rather than an INSERT OR REPLACE per row, because + /// absence carries meaning: a fingerprint with no events left in the window has a FINISHED incident, and + /// leaving its row behind would make that fingerprint's next incident read as a continuation of the old + /// one — an undercount reported under a stale start time. Upserting only the live rows cannot express + /// that, and splitting the delete from the insert would let a crash between them zero live counters. + /// + /// Called once per delivered alert (cooldown-gated by construction), so it is a low-frequency + /// write like the watermarks above. + /// + public async Task SaveIncidentOccurrencesAsync( + int serverId, + string metricName, + IReadOnlyList<(string DedupKey, long TotalOccurrences, int ObservedWindowCount, DateTime IncidentStartedUtc, DateTime LastObservedUtc)> states) + { + if (states == null) return; + + try + { + var duckDb = _duckDb; + if (duckDb == null) + { + var dbPath = App.DatabasePath; + if (string.IsNullOrEmpty(dbPath)) return; + duckDb = new DuckDbInitializer(dbPath); + } + + using var writeLock = duckDb.AcquireWriteLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(); + using var transaction = connection.BeginTransaction(); + + using (var prune = connection.CreateCommand()) + { + prune.Transaction = transaction; + prune.CommandText = @" +DELETE FROM config_incident_occurrences +WHERE server_id = $1 +AND metric_name = $2"; + prune.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + prune.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = metricName }); + await prune.ExecuteNonQueryAsync(); + } + + foreach (var state in states) + { + using var insert = connection.CreateCommand(); + insert.Transaction = transaction; + insert.CommandText = @" +INSERT INTO config_incident_occurrences + (server_id, metric_name, dedup_key, total_occurrences, observed_window_count, incident_started_at, last_observed_at) +VALUES ($1, $2, $3, $4, $5, $6, $7)"; + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = metricName }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = state.DedupKey }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = state.TotalOccurrences }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = state.ObservedWindowCount }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = state.IncidentStartedUtc }); + insert.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = state.LastObservedUtc }); + await insert.ExecuteNonQueryAsync(); + } + + transaction.Commit(); + } + catch (Exception ex) + { + /* A dropped write costs accuracy on the NEXT delivery's total — the fingerprint reads as new and + restarts, with a fresh start time saying so — never a missed or duplicated alert, which the + gate has already decided by this point. */ + AppLogger.Error("Alerts", $"Could not persist incident occurrences ({metricName}): {ex.Message}"); + } + } + /* The failed-Agent-job watermark shares the edge-trigger table but is time-based, not a count: it holds the newest already-alerted failure's server-local run time (stored in watermark_time, not the INTEGER watermark column). One reserved metric_name row per server. */ @@ -456,4 +590,94 @@ INSERT OR REPLACE INTO config_edge_trigger_watermarks (server_id, metric_name, w AppLogger.Error("Alerts", $"Could not persist failed-job watermark: {ex.Message}"); } } + + /// + /// #2203: stamps the alerted state onto the database's row in config_database_state_expected, the + /// table that already holds this alert's per-database config. The Lite half of #2166's edge trigger. + /// + /// UPDATE, never upsert — the constraint #2166 established on the Darling side. An INSERT would have + /// to supply expected_state (NOT NULL) and the only value on hand is the state being alerted ON, so + /// a database first observed SUSPECT would get SUSPECT as its accepted baseline, stop deviating, read as + /// recovered while still corrupt, and never alert again. The seed deliberately refuses to baseline the + /// integrity states for exactly that reason; this must not do it behind the seed's back. + /// + public async Task SaveDatabaseStateAlertedAsync(int serverId, string databaseName, string effectiveState) + { + try + { + var duckDb = _duckDb; + if (duckDb == null) + { + var dbPath = App.DatabasePath; + if (string.IsNullOrEmpty(dbPath)) return; + duckDb = new DuckDbInitializer(dbPath); + } + + using var writeLock = duckDb.AcquireWriteLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(); + + using var command = connection.CreateCommand(); + command.CommandText = @" +UPDATE config_database_state_expected +SET last_alerted_state = $3, + last_alerted_at = $4 +WHERE server_id = $1 +AND database_name = $2"; + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = databaseName }); + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = effectiveState }); + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = DateTime.UtcNow }); + + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + AppLogger.Error("Alerts", $"Could not record the alerted database state for {databaseName}: {ex.Message}"); + } + } + + /// + /// #2203: forgets the alerted state when a database returns to its expected one. The falling edge of an + /// edge trigger — without it each database announces once and then never again, so a second parking of + /// the same database in the same state is silently swallowed. + /// + /// A failed clear costs a MISSED alert on the next episode rather than a duplicate, which makes it + /// the more consequential of this pair's two failures — hence logged with the database named, and hence + /// the store-derived sweep in LocalDataService.GetDatabaseStateDeviationsAsync that heals the + /// memory independently of whether this call ever ran. + /// + public async Task ClearDatabaseStateAlertedAsync(int serverId, string databaseName) + { + try + { + var duckDb = _duckDb; + if (duckDb == null) + { + var dbPath = App.DatabasePath; + if (string.IsNullOrEmpty(dbPath)) return; + duckDb = new DuckDbInitializer(dbPath); + } + + using var writeLock = duckDb.AcquireWriteLock(); + using var connection = duckDb.CreateConnection(); + await connection.OpenAsync(); + + using var command = connection.CreateCommand(); + command.CommandText = @" +UPDATE config_database_state_expected +SET last_alerted_state = NULL, + last_alerted_at = NULL +WHERE server_id = $1 +AND database_name = $2"; + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + command.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = databaseName }); + + await command.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + AppLogger.Error("Alerts", $"Could not clear the alerted database state for {databaseName}: {ex.Message}"); + } + } } diff --git a/Lite/Services/EntraInteractiveAuth.cs b/Lite/Services/EntraInteractiveAuth.cs new file mode 100644 index 000000000..673d82331 --- /dev/null +++ b/Lite/Services/EntraInteractiveAuth.cs @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Microsoft.Data.SqlClient; + +namespace PerformanceMonitorLite.Services; + +/// +/// Makes Authentication=ActiveDirectoryInteractive (Microsoft Entra MFA) work by giving MSAL a +/// parent window handle (#2184). +/// +/// Why this has to exist: current Microsoft.Data.SqlClient (7.0.2 here) routes interactive +/// Entra auth through the Windows WAM broker, and WAM requires the calling application to supply +/// the HWND that will own its account picker. An application that never supplies one does not get a +/// prompt — it gets 0xwindow_handle_required / "A window handle must be configured" and the +/// connection fails outright. Lite set the authentication mode in +/// but never registered a provider, so Entra +/// MFA was broken for every user on a current build, not for some particular tenant. Old builds worked +/// because they predate WAM being SqlClient's default and fell back to a browser. +/// +/// Registration is process-wide. installs +/// against the authentication METHOD, not a connection, so one call at startup covers every +/// SqlConnection the process opens — the Add/Edit dialog's Test Connection and every collector +/// loop — without threading a handle through call sites that could each forget it. +/// +/// Same seam as Performance Studio's fix +/// (PerformanceStudio#426), +/// which was verified against a real Entra-MFA tenant by the reporter of #2184; only the handle source +/// differs (WPF's WindowInteropHelper here vs Avalonia's platform handle there). +/// +public static class EntraInteractiveAuth +{ + private static readonly object Gate = new(); + private static bool _registered; + + /// + /// Registers the interactive-auth provider, resolving the parent window through + /// at the moment MSAL asks for it. + /// + /// The handle is fetched per prompt rather than captured once, deliberately: a window's HWND + /// does not exist until the window is sourced, and the right parent is whichever window is actually + /// in front when a connection fires — usually the Add/Edit Server dialog, not the main window + /// behind it. + /// + /// + /// Returns the owning window handle, or when no window is available. + /// Zero degrades to MSAL's normal no-handle failure rather than a crash. + /// + /// True if this call registered the provider; false if already registered. + public static bool Register(Func parentWindowHandleProvider) + { + ArgumentNullException.ThrowIfNull(parentWindowHandleProvider); + + lock (Gate) + { + if (_registered) + return false; + + var provider = new ActiveDirectoryAuthenticationProvider(); + + /* Func rather than Func is MSAL's shape — it takes an Android Activity on + mobile and an HWND on Windows. Boxing the IntPtr is the intended usage. */ + provider.SetParentActivityOrWindowFunc(() => parentWindowHandleProvider()); + + SqlAuthenticationProvider.SetProvider(SqlAuthenticationMethod.ActiveDirectoryInteractive, provider); + + _registered = true; + return true; + } + } + + /* Test-only escape hatch for the one-way registration flag. Registration is deliberately + irreversible in production — SqlAuthenticationProvider offers no unregister — so a test that + needs to observe registration order resets the flag rather than the provider. The provider may + stay registered with SqlClient; tests open no interactive connections, so that is inert. */ + internal static void ResetRegistrationForTests() + { + lock (Gate) + { + _registered = false; + } + } +} diff --git a/Lite/Services/LiteAlertReadAdapter.cs b/Lite/Services/LiteAlertReadAdapter.cs index dd9e282f5..3df7f8a9a 100644 --- a/Lite/Services/LiteAlertReadAdapter.cs +++ b/Lite/Services/LiteAlertReadAdapter.cs @@ -189,6 +189,16 @@ public async Task> GetDatabaseStatesAsync( return await Task.Run(() => _dataService.GetDatabaseStateDeviationsAsync(serverId), cancellationToken); } + /// + /// #2157. Off the WPF dispatcher via Task.Run for the same reason as its siblings here (#1202). + /// + public async Task> GetForcePlanFailuresAsync( + string serverKey, CancellationToken cancellationToken = default) + { + var serverId = ParseServerKey(serverKey); + return await Task.Run(() => _dataService.GetForcePlanFailuresAsync(serverId), cancellationToken); + } + private int ResolveRunningJobsCadence(int serverId) => ResolveCadence(_runningJobsCadenceMinutes, serverId, "running_jobs"); diff --git a/Lite/Services/LiteAlertStateStore.cs b/Lite/Services/LiteAlertStateStore.cs index b8e65fd56..67d877925 100644 --- a/Lite/Services/LiteAlertStateStore.cs +++ b/Lite/Services/LiteAlertStateStore.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; using PerformanceMonitor.Alerting; @@ -93,6 +94,91 @@ public Task SaveFailedJobWatermarkAsync(string serverKey, DateTime watermark) return Task.Run(() => _store.SaveFailedJobWatermarkAsync(serverId, watermark)); } + /// + /// #2203: records the state a database was just alerted about, so the next evaluation can tell a NEW + /// deviation from the one it already reported. This is the Lite half of #2166 — until it existed, + /// alreadyAnnounced was always false here and a database parked OFFLINE for a month alerted every + /// cooldown forever, which is the complaint #2166 was filed about. + /// + /// UPDATE, never upsert, for the reason #2166 established on the Darling side: an INSERT would have + /// to invent expected_state (NOT NULL), and the only value available is the state being alerted ON + /// — so a database first observed SUSPECT would get SUSPECT written as its accepted baseline, stop + /// deviating, be read as recovered while still corrupt, and never alert again. Nothing is lost by + /// skipping the no-row case: a database with no baseline was first observed in an integrity state, and + /// those are never edge-suppressed, so this memory is never consulted for them. + /// + public Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState) + { + var serverId = ParseServerKey(serverKey); + return Task.Run(() => _store.SaveDatabaseStateAlertedAsync(serverId, databaseName, effectiveState)); + } + + /// + /// #2203: forgets what recorded, on the falling edge. Without + /// it the memory is permanent and each database can only ever announce once: park a database OFFLINE, + /// restore it, park it again weeks later, and the stale memory swallows the second parking — the repeat + /// soft-delete workflow this alert exists for. + /// + /// This is the IMMEDIATE path only. It runs off the engine's in-memory active set, which empties on + /// restart, so it cannot be the whole answer — the store-derived clear in + /// LocalDataService.GetDatabaseStateDeviationsAsync is what owns the invariant across restarts. + /// Both exist for the same reason they do in Darling: a recovery inside one process should not wait for + /// the next cycle's sweep. + /// + public Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName) + { + var serverId = ParseServerKey(serverKey); + return Task.Run(() => _store.ClearDatabaseStateAlertedAsync(serverId, databaseName)); + } + + /// + /// #2216: loads the per-fingerprint occurrence accounting for one server/metric. Wrapped in + /// Task.Run like every other method here — DuckDB.NET's I/O is synchronous under its async + /// facade and the engine runs on the WPF dispatcher, so an unwrapped call is a UI hitch (#1202). + /// + public Task> LoadIncidentOccurrencesAsync( + string serverKey, string metricName) + { + var serverId = ParseServerKey(serverKey); + return Task.Run(async () => + { + var rows = await _store.LoadIncidentOccurrencesAsync(serverId, metricName); + var states = new Dictionary(rows.Count, StringComparer.Ordinal); + foreach (var row in rows) + { + states[row.DedupKey] = new IncidentOccurrenceState( + row.TotalOccurrences, row.ObservedWindowCount, row.IncidentStartedUtc, row.LastObservedUtc); + } + return (IReadOnlyDictionary)states; + }); + } + + /// + /// #2216: replaces the persisted occurrence set for one server/metric — see the store method for why + /// the contract is replace-the-set rather than upsert-each-row, and why an empty set is the falling edge + /// rather than a no-op. + /// + public Task SaveIncidentOccurrencesAsync( + string serverKey, string metricName, IReadOnlyDictionary states) + { + var serverId = ParseServerKey(serverKey); + var rows = new List<(string, long, int, DateTime, DateTime)>(states?.Count ?? 0); + if (states is not null) + { + foreach (var entry in states) + { + rows.Add(( + entry.Key, + entry.Value.TotalOccurrences, + entry.Value.ObservedWindowCount, + entry.Value.IncidentStartedUtc, + entry.Value.LastObservedUtc)); + } + } + + return Task.Run(() => _store.SaveIncidentOccurrencesAsync(serverId, metricName, rows)); + } + private static int ParseServerKey(string serverKey) => int.Parse(serverKey, CultureInfo.InvariantCulture); } diff --git a/Lite/Services/LocalDataService.CollectionHealth.cs b/Lite/Services/LocalDataService.CollectionHealth.cs index bf9e45e4c..33460fd5f 100644 --- a/Lite/Services/LocalDataService.CollectionHealth.cs +++ b/Lite/Services/LocalDataService.CollectionHealth.cs @@ -358,8 +358,10 @@ public class CollectorHealthRow /// The collector's default cadence from the shared /// (0 for an on-load or unknown collector — both fall to the floor thresholds). The banding uses the - /// shipped default, not the per-install ScheduleManager override, so all three surfaces stay in parity. - private int FrequencyMinutes => + /// shipped default, not the per-install ScheduleManager override, so all three surfaces stay in parity. + /// Internal since #2296: the tool's sweep-pressure roll-up amortizes each collector's average + /// duration by this same cadence, so both readers of it share one resolution. + internal int FrequencyMinutes => CollectorScheduleDefaults.All.TryGetValue(CollectorName, out var schedule) ? schedule.FrequencyMinutes : 0; public string HealthStatus => CollectorHealthClassifier.Classify( diff --git a/Lite/Services/LocalDataService.Config.cs b/Lite/Services/LocalDataService.Config.cs index 872494283..f5ff976a2 100644 --- a/Lite/Services/LocalDataService.Config.cs +++ b/Lite/Services/LocalDataService.Config.cs @@ -115,6 +115,48 @@ FROM v_database_config return items; } + + /// + /// Gets the latest per-database Query Store health snapshot (#2319 — the Query Store grid). + /// + public async Task> GetLatestQueryStoreHealthAsync(int serverId, IReadOnlyList? databaseNames = null) + { + using var connection = await OpenConnectionAsync(); + using var command = connection.CreateCommand(); + var dbClause = BuildDbInClause(databaseNames, "database_name", 2, out var dbValues); + command.CommandText = @" +SELECT database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes +FROM v_query_store_health +WHERE server_id = $1 +AND capture_time = (SELECT MAX(capture_time) FROM v_query_store_health WHERE server_id = $1)" + dbClause + @" +ORDER BY database_name"; + + command.Parameters.Add(new DuckDBParameter { Value = serverId }); + foreach (var db in dbValues) + command.Parameters.Add(new DuckDBParameter { Value = db }); + + var items = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + items.Add(new QueryStoreHealthRow + { + DatabaseName = reader.GetString(0), + ActualState = reader.IsDBNull(1) ? "" : reader.GetString(1), + DesiredState = reader.IsDBNull(2) ? "" : reader.GetString(2), + ReadonlyReason = reader.IsDBNull(3) ? 0 : Convert.ToInt32(reader.GetValue(3)), + CurrentStorageMb = reader.IsDBNull(4) ? 0L : Convert.ToInt64(reader.GetValue(4)), + MaxStorageMb = reader.IsDBNull(5) ? 0L : Convert.ToInt64(reader.GetValue(5)), + SizeBasedCleanupMode = reader.IsDBNull(6) ? "" : reader.GetString(6), + StaleQueryThresholdDays = reader.IsDBNull(7) ? 0L : Convert.ToInt64(reader.GetValue(7)), + MaxPlansPerQuery = reader.IsDBNull(8) ? 0L : Convert.ToInt64(reader.GetValue(8)), + IntervalLengthMinutes = reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9)), + }); + } + + return items; + } + /// /// Gets the latest database-scoped configuration snapshot. /// @@ -247,6 +289,41 @@ public class DatabaseConfigRow public string OptimizedLockingDisplay => IsOptimizedLockingOn ? "Yes" : "No"; } + +/// +/// One database's Query Store health row (#2319) — the latest collected +/// sys.database_query_store_options snapshot. folds the classic silent +/// failure into one glanceable cell: actual and desired agreeing shows one state; disagreeing shows +/// both, because desired READ_WRITE with actual READ_ONLY is precisely the condition this collector +/// exists to surface. decodes the bitmask values an operator +/// actually meets; unknown bits fall back to the raw number rather than guessing. +/// +public class QueryStoreHealthRow +{ + public string DatabaseName { get; set; } = ""; + public string ActualState { get; set; } = ""; + public string DesiredState { get; set; } = ""; + public int ReadonlyReason { get; set; } + public long CurrentStorageMb { get; set; } + public long MaxStorageMb { get; set; } + public string SizeBasedCleanupMode { get; set; } = ""; + public long StaleQueryThresholdDays { get; set; } + public long MaxPlansPerQuery { get; set; } + public long IntervalLengthMinutes { get; set; } + + public string StateDisplay => + string.Equals(ActualState, DesiredState, StringComparison.OrdinalIgnoreCase) + ? ActualState + : $"{ActualState} (wanted {DesiredState})"; + + /// Percent of the storage cap in use; blank when the cap is 0 (unlimited/unknown). + public string PercentOfCapDisplay => + MaxStorageMb > 0 ? $"{100.0 * CurrentStorageMb / MaxStorageMb:F0}%" : ""; + + /// The shared bit-by-bit decode — one label table for every surface that shows this value. + public string ReadonlyReasonDisplay => PerformanceMonitor.Common.QueryStoreReadonlyReason.Decode(ReadonlyReason); +} + public class DatabaseScopedConfigRow { public string DatabaseName { get; set; } = ""; diff --git a/Lite/Services/LocalDataService.Cpu.cs b/Lite/Services/LocalDataService.Cpu.cs index 50e7be8ae..1e4fff366 100644 --- a/Lite/Services/LocalDataService.Cpu.cs +++ b/Lite/Services/LocalDataService.Cpu.cs @@ -56,8 +56,51 @@ FROM v_cpu_utilization_stats return items; } + + /// + /// The attributed-CPU denominator's pieces (#2320): sample count, coverage bounds, and average SQL + /// CPU% over the window. Windowed on collection_time (UTC) — the SAME bounds the top-queries and + /// top-procedures rankings use — so numerator and denominator share collection gaps; sample_time's + /// server-local skew is irrelevant to an average. Takes the window EXPLICITLY (not hours_back) so + /// the caller can hand the identical bounds to CpuAttribution.Compute — review catch: three + /// independently-sampled UtcNow calls backing one disclosure is drift by construction. + /// + public async Task GetCpuWindowAggregateAsync(int serverId, DateTime startUtc, DateTime endUtc) + { + using var connection = await OpenConnectionAsync(); + using var command = connection.CreateCommand(); + + command.CommandText = @" +SELECT + COUNT(*), + MIN(collection_time), + MAX(collection_time), + AVG(CAST(sqlserver_cpu_utilization AS DOUBLE)) +FROM v_cpu_utilization_stats +WHERE server_id = $1 +AND collection_time >= $2 +AND collection_time <= $3"; + + command.Parameters.Add(new DuckDBParameter { Value = serverId }); + command.Parameters.Add(new DuckDBParameter { Value = startUtc }); + command.Parameters.Add(new DuckDBParameter { Value = endUtc }); + + using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + return new CpuWindowAggregateRow(0, null, null, null); + } + + return new CpuWindowAggregateRow( + reader.IsDBNull(0) ? 0 : Convert.ToInt32(reader.GetValue(0)), + reader.IsDBNull(1) ? null : reader.GetDateTime(1), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? null : reader.GetDouble(3)); + } } +public sealed record CpuWindowAggregateRow(int SampleCount, DateTime? FirstSample, DateTime? LastSample, double? AvgSqlCpuPercent); + public class CpuUtilizationRow { public DateTime SampleTime { get; set; } diff --git a/Lite/Services/LocalDataService.DatabaseStates.cs b/Lite/Services/LocalDataService.DatabaseStates.cs index 3b57c3eb7..b1ecf085c 100644 --- a/Lite/Services/LocalDataService.DatabaseStates.cs +++ b/Lite/Services/LocalDataService.DatabaseStates.cs @@ -15,6 +15,22 @@ namespace PerformanceMonitorLite.Services; public partial class LocalDataService { + /// + /// Whether the last deviation sweep for a server SKIPPED its maintenance block (#2266). Transition-logged + /// from this, so a sustained contention window reports once rather than once per sweep. + /// + /// Why this needs recording at all. The maintenance block below — the baseline seed, the #2189 + /// heal, the #2203 forget and the prune — is best-effort: it opens the write connection with a 5-second lock + /// acquisition and, on TimeoutException, skips everything and lets the deviation read run anyway. + /// Skipping is the right call and the block comment there argues it well, but it was completely SILENT, so a + /// sustained window of write-lock contention meant baselines quietly stopped being seeded and healed with no + /// evidence anywhere. That matters because #2189 exists precisely because an unhealed baseline inverts the + /// alert permanently — the failure it prevents is invisible, so its absence has to be visible. + /// + /// Per server, because the lock is process-wide but the consequence is not: one server's skipped heal + /// says nothing about another's. Never pruned — one bool per server ever swept. + /// + private readonly System.Collections.Concurrent.ConcurrentDictionary _lastMaintenanceSkipped = new(); /* Effective state = STANDBY for a read-only log-shipping secondary (is_in_standby), else the raw state_desc. A standby secondary reports state_desc = ONLINE with is_in_standby = 1 and flips through RESTORING on every log restore; collapsing it to a single stable STANDBY token means it baselines as @@ -26,19 +42,42 @@ public partial class LocalDataService /// database-state alert. Fires only when the deviation is present in the TWO most recent collections /// (so a restart's RECOVERY_PENDING / RECOVERING transients — and a standby secondary's per-restore /// RESTORING flicker — don't page unless the condition actually sticks). First AUTO-SEEDS a baseline - /// (the effective state) for any database in the newest snapshot that has none — EXCEPT a critical - /// effective state (SUSPECT / RECOVERY_PENDING / EMERGENCY), which is left pending so onboarding a - /// server mid-outage doesn't learn the bad state as expected. Also tidies auto-baselines for databases + /// (the effective state) for any database in the newest snapshot that has none — EXCEPT an integrity or + /// transient effective state (), which is left + /// pending so onboarding a server mid-outage or mid-restore doesn't learn that state as expected. Then + /// HEALS an auto-baseline that nonetheless records one of those states — written by an older build, or + /// by re-baselining a database by hand while it was mid-something — to ONLINE once the database reaches + /// ONLINE, so it stops deviating by being healthy (#2189); a user override, and an OFFLINE or STANDBY + /// baseline, are never touched. Then FORGETS the recorded alerted-state of any database now back at its + /// expected state (#2203), so a second episode can announce. Also tidies auto-baselines for databases /// that have dropped off the newest snapshot (user overrides are preserved). The base table always /// holds the newest snapshots (archival only moves older rows to parquet), so it is queried directly. /// public async Task> GetDatabaseStateDeviationsAsync(int serverId) { - using var connection = await OpenConnectionAsync(); + /* #2208: the four statements below INSERT, UPDATE and DELETE, so they run under the WRITE lock — which + is what its own contract asks for ("operations that must not race with archival or compaction"). They + used the READ lock, which was wrong twice over: the writes could interleave with archival, and holding + a read lock across four statements starves writers, because a ReaderWriterLockSlim writer waits for + every reader to drain and OpenWriteConnectionAsync gives up after 5 seconds. That is how this + surfaced — an unrelated server-tags test timed out acquiring the write lock while this method held the + read lock, on a static lock shared by the whole process. + + BEST-EFFORT, and deliberately separate from the read below. If archival is mid-flight the maintenance + is skipped for this cycle and the deviation read still runs under its read lock exactly as before: a + cycle without seeding is a cycle where a brand-new database has no baseline yet, which the no-baseline + arm already handles. The alternative — letting the timeout escape — would either crash the sweep or, + if swallowed into an empty result, read as "every database recovered" and clear the alert memory for + all of them. Skipping maintenance is the only failure mode here that loses nothing. */ + var maintenanceSkipped = false; + try + { + using var maintenance = await OpenWriteConnectionAsync(); - /* Seed missing baselines from the latest snapshot (insert-if-absent; effective state; non-critical - only — a critical first observation stays pending and alerts via the no-baseline arm below). */ - using (var seed = connection.CreateCommand()) + /* Seed missing baselines from the latest snapshot (insert-if-absent; effective state). An integrity + or transient state is never learned: a critical first observation stays pending and alerts via the + no-baseline arm below, and a transient one stays pending SILENTLY until it settles. */ + using (var seed = maintenance.CreateCommand()) { seed.CommandText = $@" INSERT INTO config_database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at) @@ -47,7 +86,7 @@ FROM database_states ds WHERE ds.server_id = $1 AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) AND ds.state_desc IS NOT NULL -AND {EffectiveStateSql} NOT IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY') +AND {EffectiveStateSql} NOT IN ({DatabaseStateTokens.NeverBaselinedSqlList}) AND NOT EXISTS ( SELECT 1 FROM config_database_state_expected e WHERE e.server_id = $1 AND e.database_name = ds.database_name @@ -56,9 +95,81 @@ SELECT 1 FROM config_database_state_expected e await seed.ExecuteNonQueryAsync(); } + /* #2189: re-learn an ILLEGITIMATE inferred baseline as ONLINE once the database reaches ONLINE — the + seed's own rule applied after the fact. An expectation recording a state the seed would refuse to + learn is not a baseline, it is a snapshot of a database mid-something, and left alone it inverts + the alert permanently. The widened seed above cannot fix that on its own because it only governs + rows that do not exist yet; this heals the ones already written, by the old seed or by "reset to + current" pressed during a restore or an outage (that path records whatever it sees, no filter). + + Two gates. is_user_override = false: an operator who declared an expected state meant it, and a + database parked at expected OFFLINE must still alert when it comes back ONLINE. And the state list + is NOT "anything that is not ONLINE" — OFFLINE and STANDBY are steady states worth learning, and + leaving one is real news: a STANDBY secondary that turns up ONLINE has stopped being a secondary + (somebody recovered it, log shipping is broken), and an auto-OFFLINE database brought up for an + hour and re-parked would come back deviating forever. Both are this bug's own shape. + + The EFFECTIVE state is what is matched, never state_desc — a standby secondary reports + state_desc = 'ONLINE' with is_in_standby set, so matching the raw column would re-baseline every + log-shipping secondary to ONLINE and then alert it forever for being STANDBY. Uncorrelated + IN (...), like the prune below. */ + using (var heal = maintenance.CreateCommand()) + { + heal.CommandText = $@" +UPDATE config_database_state_expected +SET expected_state = 'ONLINE', + updated_at = now()::TIMESTAMP +WHERE server_id = $1 +AND is_user_override = false +AND expected_state IN ({DatabaseStateTokens.NeverBaselinedSqlList}) +AND database_name IN ( + SELECT ds.database_name + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND {EffectiveStateSql} = 'ONLINE' +)"; + heal.Parameters.Add(new DuckDBParameter { Value = serverId }); + await heal.ExecuteNonQueryAsync(); + } + + /* #2203: forget the announced-state for any database the store now shows back AT its expected state, so + this cycle judges against a healed memory. Darling learned why this must be store-derived rather than + engine-derived (#2166): the engine also clears on the falling edge it witnesses, but that path is + reachable only through an in-memory active set that empties on every restart, so a restart landing + between an alert and the recovery left the memory sticky FOREVER and silently swallowed the next + episode. Asking the store cannot have that gap. One sample at expected is enough where the deviation + rule needs two: clearing can only cause an extra alert, never a missed one, and a flap cannot exploit + it because a flap never survives the two-sample test to alert at all. + + Placed AFTER the #2189 heal, not before the seed, because the heal REWRITES expected_state: a database + whose illegitimate RESTORING baseline was just healed to ONLINE is, from this statement's point of + view, a database that has arrived back at its expected state, and its memory is about a deviation that + no longer exists. Running first would leave that dead memory for a cycle. Ordering against the seed is + immaterial either way — a row the seed just inserted has a NULL memory, which this skips. */ + using (var clearRecovered = maintenance.CreateCommand()) + { + clearRecovered.CommandText = $@" +UPDATE config_database_state_expected AS e +SET last_alerted_state = NULL, + last_alerted_at = NULL +WHERE e.server_id = $1 +AND e.last_alerted_state IS NOT NULL +AND (e.expected_state = '{DatabaseStateTokens.Ignore}' + OR EXISTS ( + SELECT 1 FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND ds.database_name = e.database_name + AND {EffectiveStateSql} = e.expected_state + ))"; + clearRecovered.Parameters.Add(new DuckDBParameter { Value = serverId }); + await clearRecovered.ExecuteNonQueryAsync(); + } + /* Tidy auto-baselines for databases no longer in the newest snapshot (dropped/renamed). User overrides are kept — an operator's intent shouldn't vanish because a database is briefly gone. */ - using (var prune = connection.CreateCommand()) + using (var prune = maintenance.CreateCommand()) { prune.CommandText = @" DELETE FROM config_database_state_expected @@ -72,7 +183,40 @@ SELECT database_name FROM database_states prune.Parameters.Add(new DuckDBParameter { Value = serverId }); await prune.ExecuteNonQueryAsync(); } + } + catch (TimeoutException) + { + /* Archival or compaction holds the write lock. Skip this cycle's maintenance and read anyway — + see the block comment at the top of the method for why skipping is the only lossless option. */ + maintenanceSkipped = true; + } + + /* #2266: report the skip, on the TRANSITION. Warn rather than Error, because a single skipped cycle is + the expected benign outcome of colliding with archival and the next sweep re-runs everything; it is a + SUSTAINED run of them that means baselines have stopped being seeded and healed. One line when it + starts and one when it recovers, rather than a line per sweep that would read as noise and be + filtered — which is how the silence would effectively return. */ + if (maintenanceSkipped) + { + if (!_lastMaintenanceSkipped.TryGetValue(serverId, out var wasSkipped) || !wasSkipped) + { + _lastMaintenanceSkipped[serverId] = true; + AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync), + $"server {serverId}: skipped this cycle's database-state maintenance — could not acquire the " + + "store write lock within 5s (archival or compaction holds it). Baselines are not being " + + "seeded or healed while this persists (#2189/#2203); deviations are still read. Expected " + + "occasionally — if it repeats, the write lock is contended."); + } + } + else if (_lastMaintenanceSkipped.TryGetValue(serverId, out var hadSkipped) && hadSkipped) + { + _lastMaintenanceSkipped[serverId] = false; + AppLogger.Info(nameof(GetDatabaseStateDeviationsAsync), + $"server {serverId}: database-state maintenance is running again after one or more skipped " + + "cycles (#2266)."); + } + using var connection = await OpenConnectionAsync(); using var command = connection.CreateCommand(); command.CommandText = $@" WITH newest AS ( @@ -92,7 +236,7 @@ previous AS ( FROM database_states ds WHERE ds.server_id = $1 AND ds.collection_time = (SELECT t FROM prev) ) -SELECT l.database_name, l.eff, COALESCE(e.expected_state, '') +SELECT l.database_name, l.eff, COALESCE(e.expected_state, ''), COALESCE(e.last_alerted_state, '') FROM latest l JOIN previous p ON p.database_name = l.database_name @@ -100,8 +244,8 @@ LEFT JOIN config_database_state_expected e ON e.server_id = $1 AND e.database_name = l.database_name WHERE (e.expected_state IS NULL - AND l.eff IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY') - AND p.eff IN ('SUSPECT', 'RECOVERY_PENDING', 'EMERGENCY')) + AND l.eff IN ({DatabaseStateTokens.CriticalSqlList}) + AND p.eff IN ({DatabaseStateTokens.CriticalSqlList})) OR (e.expected_state IS NOT NULL AND e.expected_state <> '(ignore)' AND l.eff IS DISTINCT FROM e.expected_state AND p.eff IS DISTINCT FROM e.expected_state) @@ -116,7 +260,10 @@ AND l.eff IS DISTINCT FROM e.expected_state { DatabaseName = reader.IsDBNull(0) ? "" : reader.GetString(0), StateDesc = reader.IsDBNull(1) ? "" : reader.GetString(1), - ExpectedState = reader.IsDBNull(2) ? "" : reader.GetString(2) + ExpectedState = reader.IsDBNull(2) ? "" : reader.GetString(2), + /* #2203: what the engine last TOLD an operator about, so alreadyAnnounced can be true in + Lite the way it already is in Darling. Empty means never announced. */ + LastAlertedState = reader.IsDBNull(3) ? "" : reader.GetString(3) }); } @@ -178,7 +325,11 @@ LEFT JOIN config_database_state_expected e /// public async Task SetDatabaseStateExpectedAsync(int serverId, string databaseName, string expectedState) { - using var connection = await OpenConnectionAsync(); + /* #2208: an upsert, so the WRITE lock. Unlike the deviation read's maintenance prologue this one does + NOT swallow a timeout: it is a user action, and TimeoutException's message ("try again in a few + moments") is written to be shown. Silently dropping an operator's declared expected state would be + the worst outcome available here. */ + using var connection = await OpenWriteConnectionAsync(); using var command = connection.CreateCommand(); /* now()::TIMESTAMP, not a bare current_timestamp: DuckDB resolves a bare current_timestamp against the table's columns (and errors) inside a VALUES row and an ON CONFLICT DO UPDATE SET, so the @@ -201,7 +352,9 @@ ON CONFLICT (server_id, database_name) /// public async Task ResetDatabaseStateExpectedToCurrentAsync(int serverId, string databaseName) { - using var connection = await OpenConnectionAsync(); + /* #2208: an upsert, so the WRITE lock, and the timeout surfaces for the same reason as the override + write above — this is the operator pressing a button. */ + using var connection = await OpenWriteConnectionAsync(); using var command = connection.CreateCommand(); command.CommandText = $@" INSERT INTO config_database_state_expected (server_id, database_name, expected_state, is_user_override, updated_at) diff --git a/Lite/Services/LocalDataService.FinOps.ServerProperties.cs b/Lite/Services/LocalDataService.FinOps.ServerProperties.cs index c0a03e813..b50b675a8 100644 --- a/Lite/Services/LocalDataService.FinOps.ServerProperties.cs +++ b/Lite/Services/LocalDataService.FinOps.ServerProperties.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using DuckDB.NET.Data; using Microsoft.Data.SqlClient; +using PerformanceMonitor.Common; namespace PerformanceMonitorLite.Services; @@ -207,9 +208,12 @@ FROM v_cpu_utilization_stats WHERE server_id = $1 AND collection_time >= $2 ), +/* Only the worker counts are consumed now: memory_ratio used to feed this read's own CASE, and that + CASE was the #2246 bug. The verdict comes from ProvisioningVerdict, so the division would be dead. */ mem_latest AS ( SELECT - CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0) AS memory_ratio + max_workers_count, + current_workers_count FROM v_memory_stats WHERE server_id = $1 AND (server_id, collection_time) IN ( @@ -219,6 +223,19 @@ FROM v_memory_stats GROUP BY server_id ) ), +/* Same workspace-memory pressure signals as the drill-down read, so the INVENTORY GRID cannot classify a + server by a rule the drill-down no longer uses (#2246 — this grid is the screen the field report was + looking at). */ +grants AS ( + SELECT + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 +), storage_totals AS ( SELECT SUM(total_size_mb) / 1024.0 AS total_storage_gb @@ -257,18 +274,20 @@ AND delta_execution_count > 0 c.avg_cpu_pct, st.total_storage_gb, id.idle_db_count, - CASE - WHEN c.avg_cpu_pct < 15 AND c.max_cpu_pct < 40 AND COALESCE(m.memory_ratio, 0) < 0.5 - THEN 'OVER_PROVISIONED' - WHEN c.p95_cpu_pct > 85 OR COALESCE(m.memory_ratio, 0) > 0.95 - THEN 'UNDER_PROVISIONED' - ELSE 'RIGHT_SIZED' - END AS provisioning_status + c.max_cpu_pct, + c.p95_cpu_pct, + COALESCE(m.max_workers_count, 0), + COALESCE(m.current_workers_count, 0), + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0) FROM (SELECT 1) AS anchor LEFT JOIN cpu_24h c ON true LEFT JOIN mem_latest m ON true LEFT JOIN storage_totals st ON true -LEFT JOIN idle_dbs id ON true"; +LEFT JOIN idle_dbs id ON true +LEFT JOIN grants g ON true"; command.Parameters.Add(new DuckDBParameter { Value = serverId }); command.Parameters.Add(new DuckDBParameter { Value = cpuCutoff }); @@ -277,11 +296,25 @@ LEFT JOIN storage_totals st ON true using var reader = await command.ExecuteReaderAsync(); if (await reader.ReadAsync()) { + /* The verdict is computed HERE rather than as a SQL CASE, so this grid and the drill-down cannot + disagree — they now call the same predicate. The old inline CASE was copies 5 and 6 of the + ratio bug, on the screen the field report was actually looking at (#2246). */ + var status = ProvisioningVerdict.Evaluate( + avgCpuPercent: reader.IsDBNull(0) ? 0m : Convert.ToDecimal(reader.GetValue(0)), + maxCpuPercent: reader.IsDBNull(3) ? 0m : Convert.ToDecimal(reader.GetValue(3)), + p95CpuPercent: reader.IsDBNull(4) ? 0m : Convert.ToDecimal(reader.GetValue(4)), + maxGrantWaiters: reader.IsDBNull(7) ? 0L : ToInt64(reader.GetValue(7)), + grantTimeouts: reader.IsDBNull(8) ? 0L : ToInt64(reader.GetValue(8)), + forcedGrants: reader.IsDBNull(9) ? 0L : ToInt64(reader.GetValue(9)), + grantUtilizationPercent: reader.IsDBNull(10) ? 0m : Convert.ToDecimal(reader.GetValue(10)), + maxWorkers: reader.IsDBNull(5) ? 0 : Convert.ToInt32(reader.GetValue(5)), + currentWorkers: reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6))); + return ( reader.IsDBNull(0) ? null : Convert.ToDecimal(reader.GetValue(0)), reader.IsDBNull(1) ? null : Convert.ToDecimal(reader.GetValue(1)), reader.IsDBNull(2) ? null : Convert.ToInt32(reader.GetValue(2)), - reader.IsDBNull(3) ? null : reader.GetString(3) + status ); } diff --git a/Lite/Services/LocalDataService.FinOps.Utilization.cs b/Lite/Services/LocalDataService.FinOps.Utilization.cs index 030c38d86..312c180ab 100644 --- a/Lite/Services/LocalDataService.FinOps.Utilization.cs +++ b/Lite/Services/LocalDataService.FinOps.Utilization.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using DuckDB.NET.Data; +using PerformanceMonitor.Common; namespace PerformanceMonitorLite.Services; @@ -57,6 +58,20 @@ FROM v_server_properties WHERE server_id = $1 ORDER BY collection_time DESC LIMIT 1 +), +/* Workspace-memory pressure, which is what being short of memory actually looks like: a query asked the + resource semaphore for a grant and did not simply get it. Counts of events, so no threshold to tune + (#2246). The utilization peak rides along to stop a CPU-quiet server that is straining its semaphore + from being called idle. Same CTE as Darling's read, against the same column names. */ +grants AS ( + SELECT + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 ) SELECT c.avg_cpu_pct, @@ -70,10 +85,15 @@ LIMIT 1 m.memory_ratio, m.max_workers_count, m.current_workers_count, - s.cpu_count + s.cpu_count, + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0) FROM cpu_stats c CROSS JOIN mem_latest m -LEFT JOIN server_info s ON true"; +LEFT JOIN server_info s ON true +LEFT JOIN grants g ON true"; command.Parameters.Add(new DuckDBParameter { Value = serverId }); command.Parameters.Add(new DuckDBParameter { Value = cutoff }); @@ -86,11 +106,20 @@ CROSS JOIN mem_latest m var p95Cpu = reader.IsDBNull(2) ? 0m : Convert.ToDecimal(reader.GetValue(2)); var memRatio = reader.IsDBNull(8) ? 0m : Convert.ToDecimal(reader.GetValue(8)); - var status = "RIGHT_SIZED"; - if (avgCpu < 15 && maxCpu < 40 && memRatio < 0.5m) - status = "OVER_PROVISIONED"; - else if (p95Cpu > 85 || memRatio > 0.95m) - status = "UNDER_PROVISIONED"; + var maxWorkers = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)); + var currentWorkers = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)); + + /* memory_ratio is still SELECTed and still displayed — it is a real fact about the instance — but it + is no longer part of the verdict: Total over Target Server Memory converges at 1.0 on any warmed + server, so it reported every server as under-provisioned (#2246). */ + var status = ProvisioningVerdict.Evaluate( + avgCpu, maxCpu, p95Cpu, + maxGrantWaiters: reader.IsDBNull(12) ? 0L : ToInt64(reader.GetValue(12)), + grantTimeouts: reader.IsDBNull(13) ? 0L : ToInt64(reader.GetValue(13)), + forcedGrants: reader.IsDBNull(14) ? 0L : ToInt64(reader.GetValue(14)), + grantUtilizationPercent: reader.IsDBNull(15) ? 0m : Convert.ToDecimal(reader.GetValue(15)), + maxWorkers: maxWorkers, + currentWorkers: currentWorkers); return new UtilizationEfficiencyRow { @@ -104,8 +133,12 @@ CROSS JOIN mem_latest m BufferPoolMb = reader.IsDBNull(7) ? 0 : Convert.ToInt32(reader.GetValue(7)), MemoryRatio = memRatio, ProvisioningStatus = status, - MaxWorkersCount = reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)), - CurrentWorkersCount = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)), + MaxGrantWaiters = reader.IsDBNull(12) ? 0L : ToInt64(reader.GetValue(12)), + GrantTimeouts = reader.IsDBNull(13) ? 0L : ToInt64(reader.GetValue(13)), + ForcedGrants = reader.IsDBNull(14) ? 0L : ToInt64(reader.GetValue(14)), + GrantUtilizationPct = reader.IsDBNull(15) ? 0m : Convert.ToDecimal(reader.GetValue(15)), + MaxWorkersCount = maxWorkers, + CurrentWorkersCount = currentWorkers, CpuCount = reader.IsDBNull(11) ? 0 : Convert.ToInt32(reader.GetValue(11)) }; } @@ -135,20 +168,43 @@ GROUP BY CAST(collection_time AS DATE) daily_mem AS ( SELECT CAST(collection_time AS DATE) AS day, - AVG(CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0)) AS avg_memory_ratio + AVG(CAST(total_server_memory_mb AS DECIMAL(10,2)) / NULLIF(target_server_memory_mb, 0)) AS avg_memory_ratio, + MAX(max_workers_count) AS max_workers_count, + MAX(current_workers_count) AS current_workers_count FROM v_memory_stats WHERE server_id = $1 AND collection_time >= $2 GROUP BY CAST(collection_time AS DATE) +), +/* Same pressure signals as the point-in-time read, per day, so a day cannot be classified by a rule the + current verdict does not use (#2246). */ +daily_grants AS ( + SELECT + CAST(collection_time AS DATE) AS day, + MAX(waiter_count) AS max_grant_waiters, + SUM(COALESCE(timeout_error_count_delta, 0)) AS grant_timeouts, + SUM(COALESCE(forced_grant_count_delta, 0)) AS forced_grants, + MAX(100.0 * granted_memory_mb / NULLIF(target_memory_mb, 0)) AS grant_utilization_pct + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 + GROUP BY CAST(collection_time AS DATE) ) SELECT c.day, c.avg_cpu_pct, c.max_cpu_pct, c.p95_cpu_pct, - COALESCE(m.avg_memory_ratio, 0) + COALESCE(m.avg_memory_ratio, 0), + COALESCE(g.max_grant_waiters, 0), + COALESCE(g.grant_timeouts, 0), + COALESCE(g.forced_grants, 0), + COALESCE(g.grant_utilization_pct, 0), + COALESCE(m.max_workers_count, 0), + COALESCE(m.current_workers_count, 0) FROM daily_cpu c LEFT JOIN daily_mem m ON m.day = c.day +LEFT JOIN daily_grants g ON g.day = c.day ORDER BY c.day"; command.Parameters.Add(new DuckDBParameter { Value = serverId }); @@ -163,11 +219,14 @@ FROM daily_cpu c var p95Cpu = reader.IsDBNull(3) ? 0m : Convert.ToDecimal(reader.GetValue(3)); var memRatio = reader.IsDBNull(4) ? 0m : Convert.ToDecimal(reader.GetValue(4)); - var status = "RIGHT_SIZED"; - if (avgCpu < 15 && maxCpu < 40 && memRatio < 0.5m) - status = "OVER_PROVISIONED"; - else if (p95Cpu > 85 || memRatio > 0.95m) - status = "UNDER_PROVISIONED"; + var status = ProvisioningVerdict.Evaluate( + avgCpu, maxCpu, p95Cpu, + maxGrantWaiters: reader.IsDBNull(5) ? 0L : ToInt64(reader.GetValue(5)), + grantTimeouts: reader.IsDBNull(6) ? 0L : ToInt64(reader.GetValue(6)), + forcedGrants: reader.IsDBNull(7) ? 0L : ToInt64(reader.GetValue(7)), + grantUtilizationPercent: reader.IsDBNull(8) ? 0m : Convert.ToDecimal(reader.GetValue(8)), + maxWorkers: reader.IsDBNull(9) ? 0 : Convert.ToInt32(reader.GetValue(9)), + currentWorkers: reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10))); items.Add(new ProvisioningTrendRow { diff --git a/Lite/Services/LocalDataService.FinOps.cs b/Lite/Services/LocalDataService.FinOps.cs index 4b47a1a72..3eccc8904 100644 --- a/Lite/Services/LocalDataService.FinOps.cs +++ b/Lite/Services/LocalDataService.FinOps.cs @@ -77,6 +77,21 @@ public class UtilizationEfficiencyRow public int PhysicalMemoryMb { get; set; } public int BufferPoolMb { get; set; } public decimal MemoryRatio { get; set; } + + /// Peak resource-semaphore waiters over the window. Any waiter at all means a query asked for + /// workspace memory and did not simply get it — the signal the verdict uses in place of the ratio that + /// pinned at 1.0 (#2246). + public long MaxGrantWaiters { get; set; } + + /// Grant timeouts accrued over the window (delta, not cumulative). + public long GrantTimeouts { get; set; } + + /// Grants forced through below what was requested, over the window. + public long ForcedGrants { get; set; } + + /// Peak granted-over-target workspace memory, as a percentage. Fleet max is 18.8%. + public decimal GrantUtilizationPct { get; set; } + public int MaxWorkersCount { get; set; } public int CurrentWorkersCount { get; set; } public int CpuCount { get; set; } diff --git a/Lite/Services/LocalDataService.ForcePlanFailures.cs b/Lite/Services/LocalDataService.ForcePlanFailures.cs new file mode 100644 index 000000000..44e573807 --- /dev/null +++ b/Lite/Services/LocalDataService.ForcePlanFailures.cs @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using PerformanceMonitor.Alerting; + +namespace PerformanceMonitorLite.Services; + +/// +/// The DuckDB read behind the forced-plan-failure alert (#2157) — Darling's Postgres query ported, kept +/// deliberately shape-for-shape with it so the two apps can never disagree about what counts as a new +/// failure. +/// +public sealed partial class LocalDataService +{ + /// + /// Forced plans whose failure counter ROSE between the two most recent collections that carried the + /// plan. $1 server_id. + /// + /// query_store_stats holds one row per plan PER INTERVAL per collection and the forcing columns + /// are plan-level attributes repeated across them, so each (plan, collection_time) collapses to one + /// value with MAX before any comparison. The two-hour window bounds the scan: a plan not collected + /// inside it is not failing right now, and Query Store's 900s flush cadence means an active plan + /// appears several times within it. + /// + /// The > is what makes this a delta read — equal counters are silence, and a LOWER + /// counter (an unforce/re-force reset) is silence rather than a negative delta. + /// + public const string ForcePlanFailuresSql = @" +WITH per_collection AS ( + SELECT + qs.database_name, + qs.query_id, + qs.plan_id, + qs.collection_time, + MAX(COALESCE(qs.force_failure_count, 0)) AS failures, + MAX(CASE WHEN qs.is_forced_plan THEN 1 ELSE 0 END) AS forced, + MAX(COALESCE(qs.plan_forcing_type, '')) AS forcing_type, + MAX(COALESCE(qs.last_force_failure_reason, '')) AS reason + FROM v_query_store_stats AS qs + WHERE qs.server_id = $1 + AND qs.collection_time > now() - INTERVAL '2 hours' + GROUP BY qs.database_name, qs.query_id, qs.plan_id, qs.collection_time +), +ranked AS ( + SELECT + pc.*, + ROW_NUMBER() OVER (PARTITION BY pc.database_name, pc.query_id, pc.plan_id ORDER BY pc.collection_time DESC) AS rn + FROM per_collection AS pc +) +SELECT + n.database_name, + n.query_id, + n.plan_id, + n.forcing_type, + n.reason, + n.failures - p.failures AS failure_delta, + n.failures AS total_failures +FROM ranked AS n +JOIN ranked AS p + ON p.database_name = n.database_name + AND p.query_id = n.query_id + AND p.plan_id = n.plan_id + AND p.rn = 2 +WHERE n.rn = 1 +AND n.forced = 1 +AND n.failures > p.failures +ORDER BY n.database_name, n.query_id, n.plan_id"; + + /// Runs for one server. + public async Task> GetForcePlanFailuresAsync(int serverId) + { + using var connection = await OpenConnectionAsync(); + using var command = connection.CreateCommand(); + command.CommandText = ForcePlanFailuresSql; + command.Parameters.Add(new DuckDBParameter { Value = serverId }); + + var items = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + items.Add(new ForcePlanFailureInfo + { + DatabaseName = reader.IsDBNull(0) ? "" : reader.GetString(0), + QueryId = reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + PlanId = reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + ForcingType = reader.IsDBNull(3) ? "" : reader.GetString(3), + FailureReason = reader.IsDBNull(4) ? "" : reader.GetString(4), + FailureDelta = reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + TotalFailures = reader.IsDBNull(6) ? 0 : reader.GetInt64(6) + }); + } + + return items; + } +} diff --git a/Lite/Services/LocalDataService.Perfmon.cs b/Lite/Services/LocalDataService.Perfmon.cs index 58b02e8e3..8da8a29e0 100644 --- a/Lite/Services/LocalDataService.Perfmon.cs +++ b/Lite/Services/LocalDataService.Perfmon.cs @@ -96,7 +96,8 @@ public async Task> GetPerfmonTrendAsync(int serverId, st SELECT collection_time, SUM(cntr_value) AS cntr_value, - SUM(delta_cntr_value) AS delta_cntr_value + SUM(delta_cntr_value) AS delta_cntr_value, + MAX(sample_interval_seconds) AS sample_interval_seconds FROM v_perfmon_stats WHERE server_id = $1 AND counter_name = $2 @@ -118,7 +119,8 @@ GROUP BY collection_time { CollectionTime = reader.GetDateTime(0), Value = reader.IsDBNull(1) ? 0 : reader.GetInt64(1), - DeltaValue = reader.IsDBNull(2) ? 0 : reader.GetInt64(2) + DeltaValue = reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + SampleIntervalSeconds = reader.IsDBNull(3) ? 0 : Convert.ToInt64(reader.GetValue(3)) }); } @@ -146,7 +148,8 @@ public async Task>> GetPerfmonTrendsB counter_name, collection_time, SUM(cntr_value) AS cntr_value, - SUM(delta_cntr_value) AS delta_cntr_value + SUM(delta_cntr_value) AS delta_cntr_value, + MAX(sample_interval_seconds) AS sample_interval_seconds FROM v_perfmon_stats WHERE server_id = $1 AND collection_time >= $2 @@ -174,7 +177,8 @@ AND counter_name IN ({nameParams}) { CollectionTime = reader.GetDateTime(1), Value = reader.IsDBNull(2) ? 0 : reader.GetInt64(2), - DeltaValue = reader.IsDBNull(3) ? 0 : reader.GetInt64(3) + DeltaValue = reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + SampleIntervalSeconds = reader.IsDBNull(4) ? 0 : Convert.ToInt64(reader.GetValue(4)) }); } @@ -196,4 +200,11 @@ public class PerfmonTrendPoint public DateTime CollectionTime { get; set; } public long Value { get; set; } public long DeltaValue { get; set; } + + /// The wall-clock seconds covers, and the only thing that makes a + /// zero delta readable: the collector reports 0 in exactly the cases where no delta was knowable + /// (first sighting, counter reset, gap past the policy), so (0, 0) is "unknown" while (0, n) is + /// "genuinely idle" (#2234). MAX, never SUM, across a counter's instance rows — it is one measured + /// sweep gap repeated per instance, and Transactions/sec carries a median of 12 of them. + public long SampleIntervalSeconds { get; set; } } diff --git a/Lite/Services/LocalDataService.cs b/Lite/Services/LocalDataService.cs index 41c87e512..1924b7744 100644 --- a/Lite/Services/LocalDataService.cs +++ b/Lite/Services/LocalDataService.cs @@ -76,7 +76,7 @@ internal async Task OpenWriteConnectionAsync() /// /// Safely converts a DuckDB value to double, handling BigInteger from SUM aggregations. /// - protected static double ToDouble(object value) + private static double ToDouble(object value) { if (value is BigInteger bi) return (double)bi; @@ -86,7 +86,7 @@ protected static double ToDouble(object value) /// /// Safely converts a DuckDB value to long, handling BigInteger from SUM/COUNT aggregations. /// - protected static long ToInt64(object value) + private static long ToInt64(object value) { if (value is BigInteger bi) return (long)bi; @@ -98,7 +98,7 @@ protected static long ToInt64(object value) /// Returns UTC time for collection_time queries (most tables store collection_time in UTC). /// When fromDate/toDate are provided, they should already be in UTC. /// - protected static (DateTime startTime, DateTime endTime) GetTimeRange(int hoursBack, DateTime? fromDate, DateTime? toDate) + private static (DateTime startTime, DateTime endTime) GetTimeRange(int hoursBack, DateTime? fromDate, DateTime? toDate) { if (fromDate.HasValue && toDate.HasValue) { @@ -115,7 +115,7 @@ protected static (DateTime startTime, DateTime endTime) GetTimeRange(int hoursBa /// /// Gets the time range in server local time (for tables like cpu_utilization_stats.sample_time). /// - protected static (DateTime startTime, DateTime endTime) GetTimeRangeServerLocal(int hoursBack, DateTime? fromDate, DateTime? toDate) + private static (DateTime startTime, DateTime endTime) GetTimeRangeServerLocal(int hoursBack, DateTime? fromDate, DateTime? toDate) { var serverNow = DateTime.UtcNow.AddMinutes(ServerTimeHelper.UtcOffsetMinutes); @@ -132,7 +132,7 @@ protected static (DateTime startTime, DateTime endTime) GetTimeRangeServerLocal( /// Starts query timing for performance logging. Use with 'using' statement. /// Only logs queries that exceed the slow query threshold (default 500ms). /// - protected static Helpers.QueryExecutionContext TimeQuery(string context, string sql) + private static Helpers.QueryExecutionContext TimeQuery(string context, string sql) { return Helpers.QueryLogger.StartQuery(context, sql, source: "DuckDB"); } diff --git a/Lite/Services/QueryStoreSliceRepairService.cs b/Lite/Services/QueryStoreSliceRepairService.cs index 4d1526feb..70d5c8112 100644 --- a/Lite/Services/QueryStoreSliceRepairService.cs +++ b/Lite/Services/QueryStoreSliceRepairService.cs @@ -573,9 +573,6 @@ union_by_name view keeps reading the un-rewritten file with the #1907 read-side return file.RowsRemoved; } - /// - /// Drops DuckDB's cached view of every external file, by toggling the cache off and back on. - /// /// /// Promotes a rewritten file over the original, and makes the store able to READ it — the two are one /// operation, which is why they live in one helper rather than being two things a caller must remember. diff --git a/Lite/Services/RemoteCollectorService.DefinitionRunner.cs b/Lite/Services/RemoteCollectorService.DefinitionRunner.cs index ab3241d3c..a8ab74417 100644 --- a/Lite/Services/RemoteCollectorService.DefinitionRunner.cs +++ b/Lite/Services/RemoteCollectorService.DefinitionRunner.cs @@ -56,7 +56,7 @@ private async Task RunCollectorDefinitionAsync( /* Some collectors don't exist on some targets (e.g. ring buffers on Azure SQL DB) — skip the cycle entirely, matching the original hand-rolled collectors. */ - if (!definition.AppliesTo(target)) + if (!CollectorCatalog.AppliesTo(definition, target)) { return 0; } @@ -90,6 +90,66 @@ declared keys (every other collector) means no query runs. */ ? null : await GetCollectorStateAsync(serverId, definition.Name, cancellationToken); + /* #2312: the open-interval refresh stamps, HOST-owned under their own state owner — the same + pattern as Darling's plan/text watermarks: the definition cannot declare these keys (one per + DATABASE, only known at runtime). Read unconditionally for query_store and merged into the + same flat State; a store predating this owner has no rows, and absent keys read as "include + the open interval", which is today's behavior exactly. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var openIntervalState = await GetCollectorStateAsync( + serverId, QueryStoreOpenIntervalState.StateCollectorName, cancellationToken); + + if (openIntervalState is { Count: > 0 }) + { + var merged = new Dictionary(StringComparer.Ordinal); + if (collectorState is not null) + { + foreach (var entry in collectorState) + { + merged[entry.Key] = entry.Value; + } + } + + foreach (var entry in openIntervalState) + { + merged[entry.Key] = entry.Value; + } + + collectorState = merged; + } + } + + /* #2188: retire the per-database state rows of databases that no longer exist. Lite's backfill + worker writes done: and hole: per database and only ever deletes a hole it SERVICES or expires, + so a dropped database's markers were kept forever — the same defect as Darling's watermark rows, + in Lite's own collector_state. Same trigger and same placement as Darling's, before the state + load, so the two hosts cannot drift on when they prune. + + Gated on the SAME AppliesTo that decides whether database_states is collected at all: on Azure + SQL DB there is no snapshot by design, so this would otherwise be a guaranteed no-op every cycle + and #2191's boundary would be emergent rather than stated. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal) + && DatabaseStateCollector.Instance.AppliesTo(target)) + { + await PruneOrphanedQueryStoreDatabaseStateAsync(serverId, cancellationToken); + } + else if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal) + && target.IsAzureSqlDb) + { + /* #2191's boundary, now crossable. Azure SQL DB has no database_states snapshot by design, which + is why the arm above states it as a no-op — but after #2220 a registration that names a database + sweeps only that database, so its one legitimate key is the connection string's own catalog. A + registration naming NO database is still skipped: it is a registration of the logical SERVER, + and a single-name prune there would delete every live watermark it has. */ + var ownDatabase = new SqlConnectionStringBuilder( + _serverManager.CredentialResolver.GetConnectionString(server)).InitialCatalog; + if (AzureSweepScope.OwnDatabaseOrEmpty(ownDatabase).Count > 0) + { + await PruneForeignQueryStoreDatabaseStateAsync(serverId, ownDatabase, cancellationToken); + } + } + var context = new CollectorContext { ServerId = serverId, @@ -149,6 +209,20 @@ watermark would let one busy database's newer event silence another database's o { cancellationToken.ThrowIfCancellationRequested(); attempted++; + + /* #2150: THE path the field report is on — Azure SQL DB collects query_store per database + here, not through the enumerated driver, so the wall-clock ceiling has to be applied on + both. Null for every collector that declares none, in which case dbToken IS + cancellationToken and this loop is byte-for-byte what it was. Darling's twin is the same + shape in DarlingCollectorRunner. */ + using var dbBudget = EnumeratedCollectorDriver.StartItemBudget( + definition.PerItemWallClockBudget, cancellationToken); + var dbToken = dbBudget?.Token ?? cancellationToken; + + /* #2312: this database's open-interval stamp, staged at decision time and landed only + after its read and flush succeed — per iteration, so a fault cannot leak a stamp + into a sibling database's landing. Mirrors Darling. */ + string? stagedOpenIntervalStamp = null; try { /* The authoritative database_name for XE rows read on this path — see @@ -166,9 +240,66 @@ where flooring a stale watermark would WRONGLY truncate legitimate catch-up branch on Azure SQL DB (#1836) and does need the bound, so it applies WatermarkPolicy.ClampCatchup inside its own cutoff computation: the clamp travels with the collector that needs it instead of with the path. */ + /* dbToken throughout this branch (#2150 review catch): the interface contract says the + budget covers "the watermark refresh, the command, and the whole drain", and the + enumerated path's perItemWatermark delegate already honours that. Leaving these three + store round-trips on cancellationToken made THIS loop — the one the field report is + actually on — the only place the promise was not kept, and a store that has stopped + answering is exactly the stall the budget exists to bound. Safe for the hole records + specifically: a budget expiry abandons the whole pass, so the watermark does not + advance, the clamp is re-derived next cycle, and the hole is re-recorded (merged wider + with any already pending) rather than lost. */ context.Watermark = await GetLastCollectedTimeForDatabaseAsync( serverId, definition.TargetTable, definition.WatermarkColumn!, - definition.PerDatabaseWatermarkColumn!, databaseName, cancellationToken); + definition.PerDatabaseWatermarkColumn!, databaseName, dbToken); + + /* #2111 adaptive shrink, Azure arm — tighten BEFORE BuildQuery: the + definition's own clamp only floors OLDER watermarks, so a tighter one + passes through untouched; the skipped range rides the backfill hole. */ + var azureFailures = ConsecutiveQueryStoreItemFailures(serverId, databaseName); + if (azureFailures > 0 + && string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var adaptiveSpan = QueryStoreBackfillState.AdaptiveSpan(WatermarkPolicy.MaxCatchup, azureFailures); + var tighterFloor = collectionTime - adaptiveSpan; + if (context.Watermark is DateTime azureRaw) + { + if (azureRaw < tighterFloor) + { + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive catch-up shrink: {Failures} consecutive failed cycles — window narrowed to {Minutes:F0}m; the skipped range rides the backfill hole.", + server.DisplayName, databaseName, azureFailures, adaptiveSpan.TotalMinutes); + await RecordQueryStoreBackfillHoleAsync(serverId, databaseName, azureRaw, tighterFloor, dbToken); + context.Watermark = tighterFloor; + } + } + else + { + /* Never-succeeded database: tighten the first-run fallback too (the + review catch); no hole — pre-watermark history is the tail's job. */ + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive first-contact shrink: {Failures} consecutive failed cycles — first-run window narrowed to {Minutes:F0}m.", + server.DisplayName, databaseName, azureFailures, adaptiveSpan.TotalMinutes); + context.Watermark = tighterFloor; + } + } + + /* #2312, Azure arm: same per-database open-interval decision as the enumerated + delegate, BEFORE BuildQuery bakes the predicate. Staged into the local, landed + only in the post-flush success block below — a per-database fault this loop + tolerates must re-include next cycle, not spend the refresh window. Mirrors + Darling. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var includeOpen = QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + context.State, databaseName, collectionTime); + context.IncludeOpenInterval = includeOpen; + if (includeOpen) + { + stagedOpenIntervalStamp = QueryStoreOpenIntervalState.Format(collectionTime); + } + } + dbPlan = definition.BuildQuery(context); /* The definition clamped its own cutoff — surface the same WARNING the @@ -190,18 +321,21 @@ backfill state they have no worker for. */ && string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal) && WatermarkPolicy.ClampCatchup(context.Watermark, collectionTime) is DateTime azureClampedFloor) { - await RecordQueryStoreBackfillHoleAsync(serverId, databaseName, context.Watermark.Value, azureClampedFloor, cancellationToken); + await RecordQueryStoreBackfillHoleAsync(serverId, databaseName, context.Watermark.Value, azureClampedFloor, dbToken); } } } var sqlSlice = Stopwatch.StartNew(); List batch; - using (var dbConnection = await OpenAzureDatabaseConnectionAsync(server, databaseName, cancellationToken)) + /* dbToken, not cancellationToken (#2150): connect, execute and drain are the phases the + budget bounds. The FLUSH below deliberately stays on cancellationToken — abandoning a + write already in flight would trade a slow cycle for a partially-written one. */ + using (var dbConnection = await OpenAzureDatabaseConnectionAsync(server, databaseName, dbToken)) using (var dbCommand = CreateCollectorCommand(dbPlan, dbConnection, commandTimeout)) - using (var dbReader = await dbCommand.ExecuteReaderAsync(cancellationToken)) + using (var dbReader = await dbCommand.ExecuteReaderAsync(dbToken)) { - batch = await definition.ReadAsync(dbReader, context, cancellationToken); + batch = await definition.ReadAsync(dbReader, context, dbToken); /* #1875: the payload path's probe-failure contract, on the path that used to ignore it. blocked_process_report is the declaring collector that also runs per @@ -212,7 +346,7 @@ set and the loop simply never advanced the reader to it — the rows were built if (definition.EmitsProbeFailures) { cycleProbeFailures.Add( - await EnumeratedCollectorDriver.ReadPayloadProbeFailuresAsync(dbReader, cancellationToken)); + await EnumeratedCollectorDriver.ReadPayloadProbeFailuresAsync(dbReader, dbToken)); } } sqlMs += sqlSlice.ElapsedMilliseconds; @@ -233,6 +367,18 @@ database whose cycle was cut at the bound would look like a clean collection. shipped boundary rather than dropping it — this log is how a long catch-up stays observable. Read after the flush, as on the other path: the context signal stays this database's until the next read resets it. */ + /* #2111: success resets the adaptive-shrink count on the Azure arm too. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemSucceeded(serverId, databaseName); + + /* #2312: read and flush both landed — the staged open-interval stamp may too. */ + if (stagedOpenIntervalStamp is not null) + { + context.PendingState[QueryStoreOpenIntervalState.KeyFor(databaseName)] = stagedOpenIntervalStamp; + } + } + var capHit = definition.PerItemRowCountWarnThreshold is int cap && batch.Count >= cap; if (capHit || context.PerItemTextBudgetExceeded) { @@ -244,12 +390,62 @@ signal stays this database's until the next read resets it. */ context.PerItemShippedBoundary?.ToString("o") ?? "n/a"); } } + catch (OutOfMemoryException) + { + /* AHEAD of the budget arm, because ItemBudgetExpired classifies on the TOKENS and never + looks at the exception type (review catch). Without this, an OOM thrown while the + budget's timer had already fired — materializing a large batch, or inside the store + write — would be caught by that arm and logged as a routine per-database timeout, + silently breaking the invariant the generic catch below states outright. The shared + EnumeratedCollectorDriver already orders it this way; these two loops did not. */ + throw; + } + catch (Exception ex) when (EnumeratedCollectorDriver.ItemBudgetExpired(dbBudget, cancellationToken)) + { + /* #2150: this database ran out of wall clock. Counted as a per-database failure so the + cycle moves on — one database must not be able to starve the rest, which is the harm + the field report describes, and it bites hardest on Lite because its live collectors + run strictly one after another. Ahead of the generic catch because a cancelled command + does not reliably arrive as an OperationCanceledException, so that filter cannot be + trusted to claim it; the token check is what keeps a real shutdown out of this arm. */ + _ = ex; + var budgetFailure = EnumeratedCollectorDriver.ItemBudgetException( + definition.PerItemWallClockBudget!.Value); + failed++; + firstFailure ??= budgetFailure; + + /* Same #2111 stamp the generic arm makes, and it MATTERS more here: this is what turns + the bound from a cut that repeats forever into one that converges. The consecutive + count narrows this database's next catch-up window, so a database that cannot finish + in the budget keeps halving until it can. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(serverId, databaseName); + } + + /* WARNING, not Debug, unlike the routine per-database skip beside it: an offline + database is ordinary and this is a collector that could not finish its work. */ + _logger?.LogWarning( + "{Collector} on '{Server}' database [{Database}] {Message}", + definition.Name, server.DisplayName, databaseName, budgetFailure.Message); + } catch (Exception ex) when (ex is not OperationCanceledException and not OutOfMemoryException) { /* OOM is filtered OUT of this per-database skip and propagates: it is fatal, not a routine one-database miss. */ failed++; firstFailure ??= ex; + + /* #2111: the yield-to-live stamp + adaptive-shrink count for the Azure SQL DB + arm — query_store reaches THIS per-database loop there, not the enumeration + path's onItemError, and without the stamp the backfill worker would never + yield on an Azure target (the review catch on #2112). Same query_store-only + guard as the hole recording above. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(serverId, databaseName); + } + _logger?.LogDebug("Skipping database '{Database}' for {Collector}: {Error}", databaseName, definition.Name, ex.Message); } } @@ -347,6 +543,12 @@ that row distinguishable from a healthy collector whose databases were just quie using var duckConnection = _duckDb.CreateConnection(); await duckConnection.OpenAsync(cancellationToken); + /* #2312: open-interval stamps STAGED at decision time (perItemWatermark, below), landed + into PendingState only from onItemComplete — after the item's read AND flush succeeded. + A per-item fault the driver tolerates must re-include next cycle, not spend the + 15-minute refresh window on a cycle that captured nothing. Mirrors Darling. */ + var stagedOpenIntervalStamps = new Dictionary(StringComparer.Ordinal); + var driverResult = await EnumeratedCollectorDriver.RunAsync( items, /* Per-database watermark refresh + the 24h catch-up clamp, computed INSIDE the loop — @@ -376,7 +578,58 @@ hole already pending. Name-guarded like the Azure site. */ await RecordQueryStoreBackfillHoleAsync(serverId, item, raw.Value, clamped.Value, ct); } } + + /* #2111 adaptive shrink — see Darling's twin; the skipped range rides the + same hole records the clamp writes, deferred to the trickle, never + dropped. Success resets the count via onItemComplete. */ + var failures = ConsecutiveQueryStoreItemFailures(serverId, item); + if (failures > 0 + && string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var span = QueryStoreBackfillState.AdaptiveSpan(WatermarkPolicy.MaxCatchup, failures); + var tighterFloor = collectionTime - span; + if (clamped is DateTime current) + { + if (current < tighterFloor) + { + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive catch-up shrink: {Failures} consecutive failed cycles — window narrowed to {Minutes:F0}m; the skipped range rides the backfill hole.", + server.DisplayName, item, failures, span.TotalMinutes); + await RecordQueryStoreBackfillHoleAsync(serverId, item, current, tighterFloor, ct); + clamped = tighterFloor; + } + } + else + { + /* Never-succeeded database (null watermark): tighten the 60-minute + first-run fallback the same way — the review catch; see Darling's + twin. No hole: pre-watermark history is the tail's job. */ + _logger?.LogWarning( + "query_store on '{Server}' database [{Database}] adaptive first-contact shrink: {Failures} consecutive failed cycles — first-run window narrowed to {Minutes:F0}m.", + server.DisplayName, item, failures, span.TotalMinutes); + clamped = tighterFloor; + } + } + context.Watermark = clamped; + + /* #2312: decide per database whether this cycle reads the OPEN interval. The + stamp is only STAGED here — it lands in PendingState from onItemComplete, + after this item's read and flush actually succeeded, so a per-item fault + (which this driver swallows by design) re-includes next time instead of + spending the refresh window on a cycle that captured nothing. Mirrors + Darling. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + var includeOpen = QueryStoreOpenIntervalState.ShouldIncludeOpenInterval( + context.State, item, collectionTime); + context.IncludeOpenInterval = includeOpen; + if (includeOpen) + { + stagedOpenIntervalStamps[QueryStoreOpenIntervalState.KeyFor(item)] = + QueryStoreOpenIntervalState.Format(collectionTime); + } + } }, readItem: async (item, ct) => { @@ -389,6 +642,21 @@ hole already pending. Name-guarded like the Azure site. */ writeBatch: (batch, ct) => Task.FromResult(WriteBatch(duckConnection, definition, batch, serverId, context.ServerName, collectionTime, context)), onItemComplete: (item, batchCount, itemSqlMs, itemStorageMs) => { + /* #2111: a completed item resets the adaptive-shrink count — recovery returns + the member to the full catch-up width on its next cycle. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemSucceeded(serverId, item); + + /* #2312: NOW the open-interval stamp may land — this hook only fires after + the item's read and flush both succeeded. Remove, not read: a stamp left + staged (read faulted) must not leak into a later run's landing. */ + if (stagedOpenIntervalStamps.Remove(QueryStoreOpenIntervalState.KeyFor(item), out var landedStamp)) + { + context.PendingState[QueryStoreOpenIntervalState.KeyFor(item)] = landedStamp; + } + } + /* Per-DATABASE line for non-empty batches (#1565): the per-server summary blends every database into one number, hiding a single busy database's burst behind quiet siblings. Quiet databases (0 rows) stay silent. */ @@ -410,9 +678,22 @@ quiet siblings. Quiet databases (0 rows) stay silent. */ } }, onItemError: (item, ex) => + { + /* #2111: stamp the yield-to-live signal (any database's live failure vouches + for the whole replica being contended) + the per-database adaptive-shrink + count. */ + if (string.Equals(definition.Name, QueryStoreCollector.Instance.Name, StringComparison.Ordinal)) + { + OnQueryStoreItemFailed(serverId, item); + } + _logger?.LogWarning("Failed to collect {Collector} from [{Database}] on '{Server}': {Message}", - definition.Name, item, server.DisplayName, ex.Message), - cancellationToken); + definition.Name, item, server.DisplayName, ex.Message); + }, + cancellationToken, + /* #2150: the per-database wall-clock ceiling. Null for every collector but + query_store, so this argument leaves every other cycle untouched. */ + perItemBudget: definition.PerItemWallClockBudget); rowsWritten = driverResult.Rows; sqlMs += driverResult.SqlMs; @@ -482,7 +763,32 @@ exactly what they were. */ path. Outside the storage-phase timer: this is host bookkeeping, not collected data. */ if (context.PendingState.Count > 0) { - await SaveCollectorStateAsync(serverId, definition.Name, context.PendingState, cancellationToken); + /* #2312: the open-interval stamps belong to their OWN state owner, not the definition's name + — a row written under "query_store" would load back (nothing reads that owner here) but the + shared prune set pairs qsowm: with query_store_open_interval, and a prefix pruned under the + wrong owner deletes nothing. Split by prefix on the way out, like Darling's runner. */ + var openIntervalKeys = context.PendingState + .Where(entry => entry.Key.StartsWith(QueryStoreOpenIntervalState.WatermarkKeyPrefix, StringComparison.Ordinal)) + .ToDictionary(entry => entry.Key, entry => entry.Value, StringComparer.Ordinal); + + if (openIntervalKeys.Count > 0) + { + var others = context.PendingState + .Where(entry => !openIntervalKeys.ContainsKey(entry.Key)) + .ToDictionary(entry => entry.Key, entry => entry.Value, StringComparer.Ordinal); + + await SaveCollectorStateAsync( + serverId, QueryStoreOpenIntervalState.StateCollectorName, openIntervalKeys, cancellationToken); + + if (others.Count > 0) + { + await SaveCollectorStateAsync(serverId, definition.Name, others, cancellationToken); + } + } + else + { + await SaveCollectorStateAsync(serverId, definition.Name, context.PendingState, cancellationToken); + } } telemetry.SqlMs = sqlMs; diff --git a/Lite/Services/RemoteCollectorService.QueryStore.cs b/Lite/Services/RemoteCollectorService.QueryStore.cs index ab695a39a..bb84a475e 100644 --- a/Lite/Services/RemoteCollectorService.QueryStore.cs +++ b/Lite/Services/RemoteCollectorService.QueryStore.cs @@ -8,7 +8,9 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; using PerformanceMonitorLite.Models; namespace PerformanceMonitorLite.Services; @@ -21,6 +23,31 @@ public partial class RemoteCollectorService /// 2017+/2022+ column gates, the last_execution_time incremental watermark, and the /// per-database sp_executesql query live there — the cross-SKU parity contract). /// - private Task CollectQueryStoreAsync(ServerConnection server, CancellationToken cancellationToken) - => RunCollectorDefinitionAsync(QueryStoreCollector.Instance, server, cancellationToken); + private async Task CollectQueryStoreAsync(ServerConnection server, CancellationToken cancellationToken) + { + /* #2165: never run beside this server's own Query Store BACKFILL slice. The two loops are + independent and used to overlap freely — measured as two heavy QS text extractions in flight at + once on a 4-core box, because a large catalog arriving is what triggers both. Gated inside the + collector's entry point rather than at the tick's dispatch switch so every caller is covered, + including an on-demand collection. + + Zero-wait by construction (see QueryStoreServerGate): a blocking acquire here would let one + server's slice stall the whole sweep, which is the #2148 wedge wearing a lock. Skipping costs + nothing durable because this collector's window is a watermark (#1960) — the next tick resumes + from the same boundary. */ + using var gate = _queryStoreGates + .GetOrAdd(server.Id, static _ => new QueryStoreServerGate()) + .TryAcquire(); + + if (gate is null) + { + _logger?.LogInformation( + "query_store collection on '{Server}' skipped this tick — its Query Store backfill slice is " + + "mid-flight (#2165). Resumes next tick from the same watermark; no rows are lost.", + server.DisplayName); + return 0; + } + + return await RunCollectorDefinitionAsync(QueryStoreCollector.Instance, server, cancellationToken); + } } diff --git a/Lite/Services/RemoteCollectorService.QueryStoreBackfill.cs b/Lite/Services/RemoteCollectorService.QueryStoreBackfill.cs index e5e87cb96..e9076cfc4 100644 --- a/Lite/Services/RemoteCollectorService.QueryStoreBackfill.cs +++ b/Lite/Services/RemoteCollectorService.QueryStoreBackfill.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Threading; @@ -14,6 +15,7 @@ using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; using PerformanceMonitorLite.Models; namespace PerformanceMonitorLite.Services; @@ -42,11 +44,33 @@ collection paths. /// retention default. private const int BackfillFallbackRetentionDays = 30; + /// #2148: per-server abandonment guards, the Darling loop's exact shape — keyed by server + /// id so one wedged server never blocks its neighbors, never pruned (one small object per server + /// ever monitored). + private readonly ConcurrentDictionary _backfillSliceSteps = new(StringComparer.Ordinal); + + /// + /// #2165: per-server gates shared by the tick's Query Store collection and this backfill slice, so the two + /// never run heavy QS text extraction against one server at the same time. Keyed like + /// and likewise never pruned — one tiny object per server. + /// + /// Both loops must resolve the SAME gate instance per server, which is why there is one dictionary + /// rather than one per loop. Pinned by a test for exactly that reason. + /// + private readonly ConcurrentDictionary _queryStoreGates = new(StringComparer.Ordinal); + + /// #2148: the hard ceiling ONE server's slice may hold the tick — a healthy slice is one + /// 30s-capped statement plus DuckDB writes, so this is a defect signal, never jitter. Per SERVER + /// deliberately (review catch, round 2): a shared tick-level deadline would both stall every + /// server's backfill behind one wedge AND false-trip as fleet size grows. + private static readonly TimeSpan BackfillSliceDeadline = TimeSpan.FromSeconds(180); + /// /// Runs AT MOST one backfill slice per enabled server: the first database found with a pending /// hole or an undrained first-contact tail gets one byte-budgeted slice; everything else waits - /// for a later tick. Per-server failures log and skip — one unreachable server never stalls the - /// sweep. Called from CollectionBackgroundService on its own due-cadence. + /// for a later tick. Per-server failures log and skip, and per-server WEDGES are abandoned and + /// quarantined (#2148) — one stuck server never stalls the sweep in either failure mode. Called + /// from CollectionBackgroundService on its own due-cadence. /// public async Task RunQueryStoreBackfillTickAsync(CancellationToken cancellationToken) { @@ -57,22 +81,105 @@ public async Task RunQueryStoreBackfillTickAsync(CancellationToken cancellationT return; } - try - { - await RunQueryStoreBackfillSliceAsync(server, cancellationToken); - } - catch (OperationCanceledException) + /* #2165: the other half of the gate. Taken OUTSIDE the AbandonableStep on purpose — an + abandoned-but-still-wedged slice keeps the gate closed, which is right, because the statement + is genuinely still running on the monitored server and the tick must keep yielding to it. */ + var gate = _queryStoreGates + .GetOrAdd(server.Id, static _ => new QueryStoreServerGate()) + .TryAcquire(); + + if (gate is null) { - return; + _logger?.LogInformation( + "query_store backfill slice on '{Server}' deferred — the tick's Query Store collection is running (#2165)", + server.DisplayName); + continue; } - catch (Exception ex) + + using var backfillGate = gate; + + var step = _backfillSliceSteps.GetOrAdd(server.Id, static _ => new AbandonableStep()); + var result = await step.RunAsync( + () => RunQueryStoreBackfillSliceAsync(server, cancellationToken), + BackfillSliceDeadline, + onLateFault: ex => _logger?.LogError(ex, + "query_store backfill slice on '{Server}' faulted AFTER being abandoned — this is the wedge's own exception (#2148)", + server.DisplayName), + cancellationToken: cancellationToken); + + switch (result.Outcome) { - _logger?.LogWarning("query_store backfill slice on '{Server}' failed: {Message}", - server.DisplayName, ex.Message); + case AbandonableStepOutcome.Cancelled: + return; + case AbandonableStepOutcome.Faulted when result.Exception is OperationCanceledException: + return; + case AbandonableStepOutcome.Faulted: + _logger?.LogWarning("query_store backfill slice on '{Server}' failed: {Message}", + server.DisplayName, result.Exception!.Message); + break; + case AbandonableStepOutcome.Abandoned: + _logger?.LogError( + "query_store backfill slice on '{Server}' exceeded {Deadline}s and was ABANDONED — " + + "other servers' backfill continues; this server is quarantined until the wedged task " + + "ends. Defect signal: report with this log (#2148).", + server.DisplayName, (int)BackfillSliceDeadline.TotalSeconds); + break; + case AbandonableStepOutcome.SkippedStillRunning: + _logger?.LogError( + "query_store backfill slice on '{Server}' skipped — a previously-abandoned slice is still wedged (#2148).", + server.DisplayName); + break; } } } + /// + /// When a server's live query_store collection last failed a per-database item — the yield-to- + /// live signal (#2111), stamped by the definition runner's item-error path and judged by + /// . In-memory on purpose — a restart + /// forgetting the stamps just means one backfill slice races one live cycle once. + /// + private readonly ConcurrentDictionary _lastQueryStoreItemFailureUtc = new(); + + /// Consecutive live query_store failures per (server, database) — the adaptive-shrink + /// signal (#2111 promoted); see Darling's twin for the semantics. Reset on the database's next + /// successful item. + private readonly ConcurrentDictionary<(int ServerId, string Database), int> _consecutiveQueryStoreItemFailures = new(); + + private int ConsecutiveQueryStoreItemFailures(int serverId, string database) + => _consecutiveQueryStoreItemFailures.TryGetValue((serverId, database), out var count) ? count : 0; + + private void OnQueryStoreItemFailed(int serverId, string database) + { + _lastQueryStoreItemFailureUtc[serverId] = DateTime.UtcNow; + _consecutiveQueryStoreItemFailures.AddOrUpdate((serverId, database), 1, static (_, current) => current + 1); + } + + private void OnQueryStoreItemSucceeded(int serverId, string database) + => _consecutiveQueryStoreItemFailures.TryRemove((serverId, database), out _); + + /// Consecutive failed backfill slices per server — the shrink signal's backfill half; + /// any completed slice resets it. + private readonly ConcurrentDictionary _consecutiveSliceFailures = new(); + + /// Runs one slice with the failure accounting wrapped around it — the caller's outer + /// catch still logs the throw exactly as before. + private async Task RunCountedBackfillSliceAsync( + ServerConnection server, int serverId, CollectorTargetInfo target, string databaseName, + DateTime floorUtc, DateTime ceilingUtc, bool isHole, CancellationToken cancellationToken) + { + try + { + await RunBackfillSliceAsync(server, serverId, target, databaseName, floorUtc, ceilingUtc, isHole, cancellationToken); + _consecutiveSliceFailures.TryRemove(serverId, out _); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _consecutiveSliceFailures.AddOrUpdate(serverId, 1, static (_, current) => current + 1); + throw; + } + } + /// One server's scan-and-slice — the twin of Darling's RunServerSliceAsync, on Lite's /// plumbing (DuckDB reads, ServerConnection credentials, the shared appender write). internal async Task RunQueryStoreBackfillSliceAsync(ServerConnection server, CancellationToken cancellationToken) @@ -93,6 +200,20 @@ internal async Task RunQueryStoreBackfillSliceAsync(ServerConnection serve } var serverId = GetDeterministicHashCode(GetServerNameForStorage(server)); + + /* #2111 yield-to-live: a backfill slice scans the same QS internal tables the live sweep + reads — when the live path is failing on this server, running a slice anyway is the + contention that keeps it failing. Skip the server this tick; the hole waits, live + recovers, backfill resumes. Same policy, same window as Darling's worker. */ + if (QueryStoreBackfillState.ShouldYieldToLive( + _lastQueryStoreItemFailureUtc.TryGetValue(serverId, out var lastLiveFailure) ? lastLiveFailure : null, + DateTime.UtcNow)) + { + _logger?.LogDebug( + "query_store backfill on '{Server}': yielding to the live path (recent live query_store failure)", + server.DisplayName); + return false; + } var state = await GetCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorName, cancellationToken); var databases = await GetBackfillCandidateDatabasesAsync(serverId, cancellationToken); @@ -114,7 +235,7 @@ internal async Task RunQueryStoreBackfillSliceAsync(ServerConnection serve } var holeFloor = holeFrom > floorLimit ? holeFrom : floorLimit; - await RunBackfillSliceAsync(server, serverId, target, databaseName, holeFloor, holeTo, isHole: true, cancellationToken); + await RunCountedBackfillSliceAsync(server, serverId, target, databaseName, holeFloor, holeTo, isHole: true, cancellationToken); return true; } @@ -142,7 +263,7 @@ await SaveCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorNa continue; } - await RunBackfillSliceAsync(server, serverId, target, databaseName, floorLimit, storedFloor.Value, isHole: false, cancellationToken); + await RunCountedBackfillSliceAsync(server, serverId, target, databaseName, floorLimit, storedFloor.Value, isHole: false, cancellationToken); return true; } @@ -163,6 +284,17 @@ private async Task RunBackfillSliceAsync( ServerConnection server, int serverId, CollectorTargetInfo target, string databaseName, DateTime floorUtc, DateTime ceilingUtc, bool isHole, CancellationToken cancellationToken) { + /* #2102: one slice queries at most the top MaxSliceSpan of the remaining range. The byte + budget bounds what SHIPS, not what the query aggregates and sorts — an unchunked wide + window on a big database times out at the command timeout every tick and the range never + drains, the same row-cap-is-not-a-cost-cap flaw that wedged the live path. */ + /* #2111 adaptive shrink: after consecutive failed slices this server digs in narrower + chunks until one fits its command timeout; a completed slice resets to full width. */ + var sliceSpan = QueryStoreBackfillState.AdaptiveSpan( + QueryStoreBackfillState.MaxSliceSpan, + _consecutiveSliceFailures.TryGetValue(serverId, out var recentFailures) ? recentFailures : 0); + var sliceFloor = QueryStoreBackfillState.BoundSliceFloor(floorUtc, ceilingUtc, sliceSpan); + var definition = QueryStoreCollector.Instance; var context = new CollectorContext { @@ -183,7 +315,7 @@ private async Task RunBackfillSliceAsync( /* Azure arm: the window travels as command parameters on a per-database connection — same contract as Darling's, same shared BuildBackfillQuery. */ context.CurrentDatabaseName = databaseName; - var azurePlan = definition.BuildBackfillQuery(context, floorUtc, ceilingUtc); + var azurePlan = definition.BuildBackfillQuery(context, sliceFloor, ceilingUtc); using var dbConnection = await OpenAzureDatabaseConnectionAsync(server, databaseName, cancellationToken); using var dbCommand = new SqlCommand(azurePlan.Text, dbConnection) { CommandTimeout = timeout }; AddCollectorParameters(dbCommand, azurePlan); @@ -215,7 +347,7 @@ private async Task RunBackfillSliceAsync( } } - var plan = definition.BuildBackfillPerItemQuery(databaseName, context, floorUtc, ceilingUtc); + var plan = definition.BuildBackfillPerItemQuery(databaseName, context, sliceFloor, ceilingUtc); using var command = new SqlCommand(plan.Text, sqlConnection) { CommandTimeout = timeout }; AddCollectorParameters(command, plan); using var reader = await command.ExecuteReaderAsync(cancellationToken); @@ -224,6 +356,32 @@ private async Task RunBackfillSliceAsync( if (rows.Count == 0) { + if (sliceFloor > floorUtc) + { + /* Only this CHUNK is quiet — the range below it is unexplored, so this is an + advance, not a terminal verdict (#2102). The persisted hole ceiling shrinks past + the quiet chunk; a derived-boundary tail converts its remainder to a hole record, + because MIN over stored rows cannot walk through quiet space (an empty chunk + ships nothing, so the derived ceiling would re-ask the same chunk forever). The + tail marks done in the same breath — the hole owns the rest of the dig, and the + scan services holes first. */ + var advance = new Dictionary(StringComparer.Ordinal) + { + [QueryStoreBackfillState.HoleKeyPrefix + databaseName] = QueryStoreBackfillState.EncodeHole(floorUtc, sliceFloor) + }; + if (!isHole) + { + advance[QueryStoreBackfillState.DoneKeyPrefix + databaseName] = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + } + + await SaveCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorName, advance, cancellationToken); + + _logger?.LogInformation( + "query_store backfill on '{Server}' [{Database}]: quiet chunk {Floor:o}..{Ceiling:o}, continuing below ({Range}).", + server.DisplayName, databaseName, sliceFloor, ceilingUtc, isHole ? "hole" : "tail"); + return; + } + if (isHole) { await DeleteCollectorStateKeyAsync(serverId, QueryStoreBackfillState.StateCollectorName, QueryStoreBackfillState.HoleKeyPrefix + databaseName, cancellationToken); @@ -255,7 +413,11 @@ await SaveCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorNa var boundary = context.PerItemShippedBoundary; if (isHole) { - if (boundary is null || boundary <= floorUtc) + /* A chunked slice's rows all sit at or above its own chunk floor, so a missing shipped + boundary falls back to the chunk floor rather than deleting (#2102) — deletion under + a bounded window would orphan the unexplored range below it. */ + var shippedTo = boundary ?? sliceFloor; + if (shippedTo <= floorUtc) { await DeleteCollectorStateKeyAsync(serverId, QueryStoreBackfillState.StateCollectorName, QueryStoreBackfillState.HoleKeyPrefix + databaseName, cancellationToken); } @@ -264,7 +426,7 @@ await SaveCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorNa await SaveCollectorStateAsync(serverId, QueryStoreBackfillState.StateCollectorName, new Dictionary(StringComparer.Ordinal) { - [QueryStoreBackfillState.HoleKeyPrefix + databaseName] = QueryStoreBackfillState.EncodeHole(floorUtc, boundary.Value) + [QueryStoreBackfillState.HoleKeyPrefix + databaseName] = QueryStoreBackfillState.EncodeHole(floorUtc, shippedTo) }, cancellationToken); } } @@ -375,6 +537,177 @@ protected async Task DeleteCollectorStateKeyAsync( } } + /// + /// Retires one collector's per-database collector_state rows for databases the server no longer + /// has (#2188) — the DuckDB twin of DarlingCollectorRunner.PruneOrphanedDatabaseStateKeysSql, + /// same guards in DuckDB's dialect. $1 server_id, $2 collector_name, $3 the key + /// prefix, which is also what reconstructs each live database's key for the anti-join. + /// + /// The existence list is database_states, not query_store's enumeration. The + /// enumeration is heavily filtered — ONLINE only, AG primaries only, the excluded-database filter, the + /// vendor-name screen, HAS_DBACCESS, and a per-database probe that can fail — so a database + /// missing from one cycle's items is far more often offline or unprobeable than dropped, and pruning on + /// that absence would delete LIVE state on exactly the servers that keep databases parked. database_states + /// is an unfiltered SELECT ... FROM sys.databases, which answers the only question asked here. + /// Lite already reads the newest snapshot this same way in + /// LocalDataService.GetDatabaseStateDeviationsAsync, which prunes auto-baselines for dropped + /// databases — this is that established idiom applied to collector_state. + /// + /// The snapshot must be NEWER than the row it judges (updated_at < newest). + /// Existing is not current: if database_states stops collecting, its newest snapshot freezes, and every + /// database created after that instant is missing from it while being perfectly alive — presence alone + /// would prune such a database's state on EVERY tick forever. A snapshot cannot judge a row written + /// after it was taken. Both stamps are the service clock's UTC (collectionTime and + /// both read DateTime.UtcNow). This also subsumes the + /// empty-snapshot case for free: < against a NULL MAX is NULL, so a server that has never + /// collected database_states prunes nothing rather than everything. + /// + /// Which keys comes from the SHARED , + /// deliberately including planwm: that Lite never writes: running one no-op delete is what + /// guarantees that enabling plan capture here later cannot quietly create an orphan class this forgot + /// about. Best-effort like its siblings — a failed prune leaves the rows and the next tick retries. + /// + private const string PruneOrphanedDatabaseStateKeysSql = @" +DELETE FROM collector_state +WHERE server_id = $1 +AND collector_name = $2 +AND starts_with(state_key, $3) +AND updated_at < (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) +AND NOT EXISTS + ( + SELECT 1 + FROM database_states ds + WHERE ds.server_id = $1 + AND ds.collection_time = (SELECT MAX(collection_time) FROM database_states WHERE server_id = $1) + AND collector_state.state_key = $3 || ds.database_name + ) +RETURNING state_key"; + + /// + /// The Azure SQL DB variant (#2191) — the DuckDB twin of + /// DarlingCollectorRunner.PruneForeignDatabaseStateKeysSql. Prunes every per-database state key + /// that is not the ONE database this registration names. + /// + /// No snapshot and no freshness guard, and that is the difference rather than an omission: the + /// on-prem statement guards because a SNAPSHOT can be empty or stale, while the single legitimate name + /// here comes from the connection string's own catalog, which is current by construction. #2191 asked for + /// "an authoritative unfiltered sys.databases read from master, used only on the success path"; #2220 + /// removed the need for any master read on this path, which is what makes it reachable now. + /// + private const string PruneForeignDatabaseStateKeysSql = @" +DELETE FROM collector_state +WHERE server_id = $1 +AND collector_name = $2 +AND starts_with(state_key, $3) +AND state_key <> $3 || $4 +RETURNING state_key"; + + /// + /// Runs for every shared owner/prefix pair, once per + /// query_store cycle for one server — the same trigger and the same placement as Darling's, so the two + /// cannot drift on WHEN they prune either. + /// + protected async Task PruneOrphanedQueryStoreDatabaseStateAsync(int serverId, CancellationToken cancellationToken) + { + try + { + using var conn = _duckDb.CreateConnection(); + await conn.OpenAsync(cancellationToken); + var pruned = new List(); + + foreach (var (owner, prefix) in QueryStorePerDatabaseState.PrunableKeys) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = PruneOrphanedDatabaseStateKeysSql; + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = owner }); + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = prefix }); + + /* RETURNING and a reader, not a rows-affected count (#2205). The only symptom of a WRONG + delete here is a silent refetch, so a bare number leaves nothing to diagnose it with — on + Lite you could see that three rows went without seeing WHICH databases' watermark and + backfill state was retired. Correctness already matched Darling exactly (same anti-join, + same freshness guard, pinned by the #2195 tests); this closes the FORENSICS gap. */ + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + pruned.Add(reader.GetString(0)); + } + } + + if (pruned.Count > 0) + { + /* Same shape and same fields as DarlingCollectorRunner's, so an operator reading either + SKU's log sees the same sentence. */ + _logger?.LogInformation( + "[server_id {ServerId}] pruned {Count} query_store state row(s) for database(s) no longer on the server: {Keys}", + serverId, pruned.Count, string.Join(", ", pruned)); + } + } + catch (Exception ex) + { + _logger?.LogDebug(ex, "Pruning orphaned query_store database state failed; next cycle retries"); + } + } + + /// + /// The Azure SQL DB arm of the #2188 prune (#2191) — Darling's twin is + /// PruneForeignQueryStoreDatabaseStateAsync, same trigger, same placement, same shared + /// set, so the two cannot drift on which prefixes + /// get pruned or when. + /// + /// What it deletes today is mostly #2220's residue: before that fix each Azure registration swept + /// every sibling database on the logical server and wrote a watermark for each under its own server_id. + /// collector_state carries no retention, so unlike the collected rows those orphans would persist + /// indefinitely rather than ageing out. + /// + /// The registration's own database. Callers must only reach here when it is + /// non-empty — a registration naming none is a registration of the logical SERVER, and a single-name + /// prune there would delete every live watermark it legitimately has. + protected async Task PruneForeignQueryStoreDatabaseStateAsync( + int serverId, string ownDatabase, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(ownDatabase)) + { + return; + } + + try + { + using var conn = _duckDb.CreateConnection(); + await conn.OpenAsync(cancellationToken); + var pruned = new List(); + + foreach (var (owner, prefix) in QueryStorePerDatabaseState.PrunableKeys) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = PruneForeignDatabaseStateKeysSql; + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = serverId }); + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = owner }); + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = prefix }); + cmd.Parameters.Add(new DuckDB.NET.Data.DuckDBParameter { Value = ownDatabase }); + + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + pruned.Add(reader.GetString(0)); + } + } + + if (pruned.Count > 0) + { + /* Same sentence as Darling's twin, so an operator reading either SKU's log sees one wording. */ + _logger?.LogInformation( + "[server_id {ServerId}] pruned {Count} query_store state row(s) belonging to databases other than this registration's [{Database}]: {Keys}", + serverId, pruned.Count, ownDatabase, string.Join(", ", pruned)); + } + } + catch (Exception ex) + { + _logger?.LogDebug(ex, "Pruning foreign query_store database state failed; next cycle retries"); + } + } + /// Records a clamp-opened Query Store hole for the backfill worker (#2058), under the /// WORKER's collector_state name — merged wider with any pending hole so a repeat outage cannot /// overwrite an unserviced one. Best-effort: a lost record is a lost backfill opportunity, diff --git a/Lite/Services/RemoteCollectorService.ServerConfig.cs b/Lite/Services/RemoteCollectorService.ServerConfig.cs index 739df3af3..2be668699 100644 --- a/Lite/Services/RemoteCollectorService.ServerConfig.cs +++ b/Lite/Services/RemoteCollectorService.ServerConfig.cs @@ -41,6 +41,16 @@ private Task CollectDatabaseConfigAsync(ServerConnection server, Cancellati private Task CollectDatabaseScopedConfigAsync(ServerConnection server, CancellationToken cancellationToken) => RunCollectorDefinitionAsync(DatabaseScopedConfigCollector.Instance, server, cancellationToken); + /// + /// Collects per-database Query Store health via the shared + /// definition (#2319 — the database enumeration with the AG-primary filter and the + /// [db].sys.sp_executesql per-database loop live there, the cross-SKU parity contract). Hourly, not + /// on-load: actual_state and the storage numbers change by themselves, and the cap-hit transition + /// to READ_ONLY is the point of collecting this. + /// + private Task CollectQueryStoreHealthAsync(ServerConnection server, CancellationToken cancellationToken) + => RunCollectorDefinitionAsync(QueryStoreHealthCollector.Instance, server, cancellationToken); + /// /// Collects active trace flags via the shared definition. /// Wrapped in a permission-tolerant catch — DBCC may be denied — so a failure degrades to diff --git a/Lite/Services/RemoteCollectorService.cs b/Lite/Services/RemoteCollectorService.cs index 12f3ca49b..13bfac501 100644 --- a/Lite/Services/RemoteCollectorService.cs +++ b/Lite/Services/RemoteCollectorService.cs @@ -485,7 +485,9 @@ blocked_process_report is the live example — and the catches all fall through try { /* Target-gate collectors through the shared AppliesTo — the single authoritative gate surface - both SKUs consult. Darling's collector runner calls definition.AppliesTo(target) directly; + both SKUs consult. Darling's collector runner calls CollectorCatalog.AppliesTo(definition, target) + — the COMPOSED overload, which also requires the definition's TargetEngine to match, so a + PostgreSQL definition is never handed a SQL Server target or vice versa; here it drives Lite's clean pre-dispatch SKIPPED log (a genuine skip with no collection_log row, vs. the SUCCESS/0-rows a gated collector would otherwise record). The gate CONDITION lives ONLY in each definition's AppliesTo override — never re-encoded in the host — so Lite @@ -567,6 +569,7 @@ failed first attempt self-heals instead of staying broken until a manual "blocked_process_report" => await CollectBlockedProcessReportsAsync(server, cancellationToken), "long_query_completions" => await CollectLongQueryCompletionsAsync(server, cancellationToken), "database_scoped_config" => await CollectDatabaseScopedConfigAsync(server, cancellationToken), + "query_store_health" => await CollectQueryStoreHealthAsync(server, cancellationToken), "trace_flags" => await CollectTraceFlagsAsync(server, cancellationToken), "running_jobs" => await CollectRunningJobsAsync(server, cancellationToken), "database_size_stats" => await CollectDatabaseSizeStatsAsync(server, cancellationToken), @@ -813,15 +816,24 @@ internal void NoteServerOnline(ServerConnection server) } /// - /// Enumerates online databases on an Azure SQL DB logical server. - /// HAS_DBACCESS() returns false for user databases from master on Azure SQL DB, - /// so we skip that filter — inaccessible databases should be handled by callers via try/catch. + /// The databases one Azure SQL DB registration's per-database sweep covers. /// - /// On Azure SQL DB, logins are sometimes granted access only to a specific user database and - /// not to master (e.g. Microsoft Dynamics 365 FO). In that case, master enumeration fails with - /// an access/login error; we fall back to returning the connection's initial catalog as a - /// single-database list, and throttle re-probes of master so we don't retry it every cycle. - /// See issue #857. + /// A registration that names a database sweeps that database, and nothing else (#2220) — + /// the common case, since server_id hashes host[:database][:RO] and registering each + /// database separately is how you get separate identities. That path returns immediately and never + /// touches master. It also covers #857's own case better than #857 did: a login with access to one + /// user database but not to master has a named database, so it no longer probes master, fails, and falls + /// back — it simply never probes. + /// + /// Only a registration naming NO database — or naming master, where a catalog-less Azure + /// connection lands — is a registration of the logical SERVER, and only that one enumerates. + /// HAS_DBACCESS() returns false for user databases from master on Azure SQL DB, so that filter is + /// skipped and inaccessible databases are handled by callers via try/catch. The re-probe throttle is + /// deliberately NOT consulted on that path; see the comment at the call site. + /// + /// It read master unconditionally before #2220, sweeping every online database on the logical + /// server into whichever registration ran the sweep — N registrations of N databases meant N² collection + /// with every registration's history contaminated by its siblings'. /// protected async Task> GetAzureDatabaseListAsync(ServerConnection server, CancellationToken cancellationToken) { @@ -829,17 +841,36 @@ protected async Task> GetAzureDatabaseListAsync(ServerConnection se var baseConnStr = _serverManager.CredentialResolver.GetConnectionString(server); var targetDb = new SqlConnectionStringBuilder(baseConnStr).InitialCatalog; - /* Skip the throttle when there is nothing to fall back TO. With no target database the fallback - can only throw, and an error lands in collection_log either way — so honouring the throttle - here would buy one saved round-trip at the cost of 15 minutes of guaranteed failure with no - attempt to recover. Probe master instead: it might work now. */ - var hasFallback = SingleDbOrEmpty(targetDb).Count > 0; - - if (hasFallback && IsMasterProbeThrottled(serverId)) + /* #2220: a registration that NAMES a database is a registration OF that database, so its sweep + covers exactly that one and never touches master. Before this, EVERY database-scoped collector + enumerated master and swept every online database on the logical server, storing all of it under + the one server_id of whichever registration ran the sweep — N registrations of N databases on one + server meant N² collection with every registration's history contaminated by its siblings'. + + This also subsumes the #857 case it looks like it bypasses, and improves on it: a login granted + access to one user database but not to master HAS a named database, so it now returns here without + probing master at all, rather than probing, failing, forming a verdict and falling back. Master is + reached only by a registration that names no database — the logical-server registration, which has + nothing else to enumerate from. */ + var ownDatabase = AzureSweepScope.OwnDatabaseOrEmpty(targetDb); + if (ownDatabase.Count > 0) { - return FallbackDatabaseList(server, targetDb, reason: "master previously inaccessible", quiet: true); + return ownDatabase; } + /* NO throttle check here, and that is deliberate rather than an omission — restoring what the + `hasFallback &&` guard used to achieve. This branch is reached ONLY when the registration names no + database, so there is nothing to fall back TO: honouring the throttle would return + FallbackDatabaseList, which throws immediately without probing, and would keep throwing for the + whole recheck interval while never attempting the one thing that could recover. Probing master + every cycle is the cheaper failure. (Review caught me reintroducing exactly this: I read + `hasFallback &&` as a redundant condition when it was there to DISABLE the throttle.) + + The throttle machinery itself is left alone. It is tested behaviour from #857/#1506, and it is now + unreachable in production for a different reason than this one: its whole purpose was to stop + re-probing master for a registration that HAS a fallback, and such a registration no longer probes + master at all. Retiring it is its own change, with those tests. */ + var connStr = new SqlConnectionStringBuilder(baseConnStr) { ConnectTimeout = ConnectionTimeoutSeconds, @@ -1022,12 +1053,9 @@ string Escape(string s) => forNestedDynamicSql return $"AND {columnExpression} NOT IN ({string.Join(", ", quoted)})"; } - private static List SingleDbOrEmpty(string? targetDb) - { - if (string.IsNullOrEmpty(targetDb) || string.Equals(targetDb, "master", StringComparison.OrdinalIgnoreCase)) - return new List(); - return new List { targetDb }; - } + /* #2220: delegates to the shared rule — see AzureSweepScope for why this is not duplicated per host. */ + private static List SingleDbOrEmpty(string? targetDb) => + AzureSweepScope.OwnDatabaseOrEmpty(targetDb); /// /// Whether master enumeration failed in a way that means database-scoped collectors should fall back diff --git a/Lite/Services/ScheduleManager.cs b/Lite/Services/ScheduleManager.cs index 7f9ca65f2..b8e953456 100644 --- a/Lite/Services/ScheduleManager.cs +++ b/Lite/Services/ScheduleManager.cs @@ -652,7 +652,8 @@ internal static List GetDefaultSchedules() new() { Name = "ag_replica_states", Enabled = true, FrequencyMinutes = 1, RetentionDays = 30, Description = "Availability Group replica health (role, operational/connected state, recovery and synchronization health) from sys.dm_hadr_availability_replica_states; zero rows on a server with no AGs (not collected on Azure SQL DB)" }, new() { Name = "ag_database_replica_states", Enabled = true, FrequencyMinutes = 1, RetentionDays = 30, Description = "Availability Group per-database replica health (synchronization state, send/redo queue sizes and rates, secondary lag) from sys.dm_hadr_database_replica_states; zero rows on a server with no AGs (not collected on Azure SQL DB)" }, new() { Name = "plan_correction", Enabled = true, FrequencyMinutes = 5, RetentionDays = 30, Description = "Automatic plan correction: per-database FORCE_LAST_GOOD_PLAN enablement from sys.database_automatic_tuning_options plus the engine's live recommendation set from sys.dm_db_tuning_recommendations, with the regressed query's text resolved through Query Store (SQL Server 2017+ and Azure; Enterprise/Developer edition)" }, - new() { Name = "pvs_stats", Enabled = true, FrequencyMinutes = 60, RetentionDays = 90, Description = "Accelerated Database Recovery persistent version store size and cleanup state per database from sys.dm_tran_persistent_version_store_stats, with the aborted-transaction count and the skipped-page counters that say why cleanup is not reclaiming; SQL Server 2019+ only, always collected on Azure SQL DB (ADR is always on there)" } + new() { Name = "pvs_stats", Enabled = true, FrequencyMinutes = 60, RetentionDays = 90, Description = "Accelerated Database Recovery persistent version store size and cleanup state per database from sys.dm_tran_persistent_version_store_stats, with the aborted-transaction count and the skipped-page counters that say why cleanup is not reclaiming; SQL Server 2019+ only, always collected on Azure SQL DB (ADR is always on there)" }, + new() { Name = "query_store_health", Enabled = true, FrequencyMinutes = 60, RetentionDays = 30, Description = "Per-database Query Store health from sys.database_query_store_options: actual vs desired state (the cap-hit READ_ONLY transition and its readonly_reason), current vs max storage, cleanup mode and thresholds, and the runtime-stats interval length; one row per database, OFF recorded explicitly" } }; } diff --git a/Lite/Windows/SettingsWindow.xaml b/Lite/Windows/SettingsWindow.xaml index b0e90b83b..d25d5b5be 100644 --- a/Lite/Windows/SettingsWindow.xaml +++ b/Lite/Windows/SettingsWindow.xaml @@ -42,6 +42,7 @@ + @@ -58,6 +59,16 @@ + /// + /// #2216: optional hook applied to the grouped incidents BEFORE they are rendered, so a caller that + /// keeps per-fingerprint history (the engine, via ) can + /// attach each incident's monotonic total. It has to run here rather than on the finished context + /// because the renderer projects the incidents into detail items in the same pass — decorating + /// afterwards would leave the rendered facts describing the undecorated values. Null for callers with + /// no such history, which is every path that built this context before #2216. + /// public static AlertContext? BuildBlockingContext( - string serverName, IReadOnlyList? events, IReadOnlyList excludedDatabases) + string serverName, IReadOnlyList? events, IReadOnlyList excludedDatabases, + Func, IReadOnlyList>? decorateIncidents = null) { if (events == null || events.Count == 0) return null; - IReadOnlyList filtered = events; - if (excludedDatabases is { Count: > 0 }) - { - filtered = events - .Where(e => string.IsNullOrEmpty(e.DatabaseName) || - !excludedDatabases.Any(ex => - string.Equals(ex, e.DatabaseName, StringComparison.OrdinalIgnoreCase))) - .ToList(); - if (filtered.Count == 0) return null; - } + var filtered = FilterBlocking(events, excludedDatabases); + if (filtered.Count == 0) return null; - /* #1140/#1141: collapse samples of the same chain into one group (true occurrence count - + wait range) instead of listing it once per sample, and attach the dedup fingerprint. - Identity is the resolved contentious object (collected server-side, §5.3), falling back - to database + literal-stripped query pair only when the object did not resolve. */ - var groups = BlockingIncidentGrouper.Group( - serverName, - filtered.Select(e => new BlockingIncidentGrouper.BlockedEvent( - e.DatabaseName, e.ContentiousObject, e.BlockedSqlText, e.BlockingSqlText, e.WaitTimeMs, e.LockMode))); + var groups = GroupBlocking(serverName, filtered); const int maxGroups = 10; var shown = groups.Take(maxGroups).ToList(); @@ -107,36 +101,57 @@ to database + literal-stripped query pair only when the object did not resolve. context.AttachmentFileName = "blocked_process_report.xml"; } - AlertIncidentRenderer.Apply(context, shown.Select(g => g.Incident).ToList()); + AlertIncidentRenderer.Apply(context, Decorate(shown.Select(g => g.Incident).ToList(), decorateIncidents)); return context.Details.Count == 0 ? null : context; } /// - /// The deadlock-alert context from the store's deadlock rows. Body verbatim from Lite's - /// pre-slice-B BuildDeadlockContextAsync minus the fetch: deadlocks whose processes ALL - /// ran in excluded databases are dropped (); the first 3 render - /// as "Deadlock Victim" items; the first graph XML becomes the attachment; ALL deadlocks in the - /// window feed the #1140 involved-object fingerprint grouping. Null when nothing survives. + /// The deadlock-alert context from the store's deadlock rows. Deadlocks whose processes ALL ran in + /// excluded databases are dropped (); the first graph XML becomes + /// the attachment; ALL deadlocks in the window feed the #1140 involved-object fingerprint grouping. + /// Null when nothing survives. + /// + /// #2108 reshaped what displays: each fingerprint incident is now a SELF-CONTAINED unit — its + /// own Database (#2109), Victim SQL, Processes, Dedup Key, Involved Objects, Occurrences — rendered + /// via with the forensic fields INCLUDED, and the old + /// standalone "Deadlock Victim" items are kept only for deadlocks the fingerprint cannot see + /// (no parseable objects). Before, the victim fields and the fingerprint metadata lived in separate + /// items — on a multi-incident card there was no way to tell which victim belonged to which + /// fingerprint, and the two lists even disagreed on membership (victims = first 3 raw events, + /// incidents = all fingerprints). /// + /// + /// #2216: see — the same pre-render hook, for the same reason. This + /// builder renders each incident itself rather than through + /// (#2108's self-contained cards), so the hook has to sit + /// ahead of that loop too. + /// public static AlertContext? BuildDeadlockContext( - string serverName, IReadOnlyList? deadlocks, IReadOnlyList excludedDatabases) + string serverName, IReadOnlyList? deadlocks, IReadOnlyList excludedDatabases, + Func, IReadOnlyList>? decorateIncidents = null) { if (deadlocks == null || deadlocks.Count == 0) return null; - IReadOnlyList filtered = deadlocks; - if (excludedDatabases is { Count: > 0 }) + var filtered = FilterDeadlocks(deadlocks, excludedDatabases); + if (filtered.Count == 0) return null; + + var context = new AlertContext(); + var firstGraph = filtered.FirstOrDefault(d => d.HasDeadlockXml)?.DeadlockGraphXml; + if (!string.IsNullOrEmpty(firstGraph)) { - filtered = deadlocks - .Where(d => !IsDeadlockExcluded(d, excludedDatabases)) - .ToList(); - if (filtered.Count == 0) return null; + context.AttachmentXml = firstGraph; + context.AttachmentFileName = "deadlock_graph.xml"; } - var context = new AlertContext(); - var firstGraph = (string?)null; + /* One parse pass per deadlock: the fingerprint's object set and the discrete Database fact's + database set (#2109) both come off the graph. */ + var parsed = ParseDeadlocks(filtered); - foreach (var d in filtered.Take(3)) + /* Deadlocks the fingerprint cannot see (no parseable objects) would vanish entirely under the + incident-only rendering, so they keep the standalone victim item — the #1140 rule that "the + builder still displays them", now scoped to exactly the events that need it. */ + foreach (var p in parsed.Where(p => p.Objects.Count == 0).Take(3)) { var item = new AlertDetailItem { @@ -144,40 +159,166 @@ to database + literal-stripped query pair only when the object did not resolve. Fields = new() }; - if (!string.IsNullOrEmpty(d.VictimSqlText)) - item.Fields.Add(("Victim SQL", TruncateText(d.VictimSqlText))); - if (!string.IsNullOrEmpty(d.ProcessSummary)) - item.Fields.Add(("Processes", d.ProcessSummary)); + if (p.Databases.Count > 0) + item.Fields.Add(("Database", string.Join(", ", p.Databases))); + if (!string.IsNullOrEmpty(p.Row.VictimSqlText)) + item.Fields.Add(("Victim SQL", TruncateText(p.Row.VictimSqlText))); + if (!string.IsNullOrEmpty(p.Row.ProcessSummary)) + item.Fields.Add(("Processes", p.Row.ProcessSummary)); context.Details.Add(item); - if (firstGraph == null && d.HasDeadlockXml) - firstGraph = d.DeadlockGraphXml; } - if (!string.IsNullOrEmpty(firstGraph)) + /* #1140: fingerprint each deadlock by its sorted involved-object set, across ALL deadlocks in + the window, grouped so recurrences over the same objects collapse to one incident with a + count. Each incident renders self-contained (#2108): heading + its representative's forensic + fields + the dedup metadata, one item per incident. */ + var groups = GroupParsedDeadlocks(serverName, parsed); + var incidents = Decorate(groups.Select(g => g.Incident).ToList(), decorateIncidents); + if (incidents.Count > 0) { - context.AttachmentXml = firstGraph; - context.AttachmentFileName = "deadlock_graph.xml"; + context.Incidents = new List(incidents); + for (int n = 0; n < incidents.Count; n++) + { + var heading = incidents.Count == 1 ? "Deadlock" : $"Deadlock {n + 1} of {incidents.Count}"; + context.Details.Add(AlertIncidentRenderer.BuildItem(incidents[n], heading, includeDetailFields: true)); + } + } + + return context; + } + + /// + /// #2216: the fingerprinted incidents for a set of blocked-process rows — the SAME grouping + /// renders, exposed so the alert engine can observe them on every + /// sweep rather than only on the sweeps that deliver an alert. + /// + /// It has to be the same grouping, not a parallel implementation: the engine's occurrence state is + /// keyed by fingerprint, so a filter or identity rule that drifted between the counting path and the + /// rendering path would silently key them differently and every delivered incident would look like a + /// first contact. Both paths share FilterBlocking and GroupBlocking for that reason. + /// + /// Uncapped, unlike the rendered list. The render cap is a display budget; a fingerprint outside + /// the top 10 still has a live incident, and dropping it from the observation would reset its total the + /// next time it surfaced. + /// + public static IReadOnlyList BlockingIncidents( + string serverName, IReadOnlyList? events, IReadOnlyList excludedDatabases) + { + if (events == null || events.Count == 0) return Array.Empty(); + + var filtered = FilterBlocking(events, excludedDatabases); + if (filtered.Count == 0) return Array.Empty(); + + return GroupBlocking(serverName, filtered).Select(g => g.Incident).ToList(); + } + + /// + /// #2216: the deadlock twin of — same grouping + /// uses, for the same reason. + /// + public static IReadOnlyList DeadlockIncidents( + string serverName, IReadOnlyList? deadlocks, IReadOnlyList excludedDatabases) + { + if (deadlocks == null || deadlocks.Count == 0) return Array.Empty(); + + var filtered = FilterDeadlocks(deadlocks, excludedDatabases); + if (filtered.Count == 0) return Array.Empty(); + + return GroupDeadlocks(serverName, filtered).Select(g => g.Incident).ToList(); + } + + /* Excluded databases drop their rows; rows with no database always pass. Shared by the render path and + #2216's observation path so the two can never disagree about which rows exist. */ + private static IReadOnlyList FilterBlocking( + IReadOnlyList events, IReadOnlyList excludedDatabases) + { + if (excludedDatabases is not { Count: > 0 }) + { + return events; } - /* #1140: fingerprint each deadlock by its sorted involved-object set (parsed from the - graph), across ALL deadlocks in the window — not just the 3 displayed — grouped so - recurrences over the same objects collapse to one incident with a count. */ - var groups = DeadlockIncidentGrouper.Group( + return events + .Where(e => string.IsNullOrEmpty(e.DatabaseName) || + !excludedDatabases.Any(ex => + string.Equals(ex, e.DatabaseName, StringComparison.OrdinalIgnoreCase))) + .ToList(); + } + + /* A deadlock whose processes ALL ran in excluded databases is dropped. Shared, as above. */ + private static IReadOnlyList FilterDeadlocks( + IReadOnlyList deadlocks, IReadOnlyList excludedDatabases) + { + if (excludedDatabases is not { Count: > 0 }) + { + return deadlocks; + } + + return deadlocks.Where(d => !IsDeadlockExcluded(d, excludedDatabases)).ToList(); + } + + /* #1140/#1141: collapse samples of the same chain into one group (true occurrence count + wait range) + instead of listing it once per sample, and attach the dedup fingerprint. Identity is the resolved + contentious object (collected server-side, §5.3), falling back to database + literal-stripped query + pair only when the object did not resolve. */ + private static List GroupBlocking( + string serverName, IReadOnlyList filtered) => + BlockingIncidentGrouper.Group( serverName, - filtered.Select(d => new DeadlockIncidentGrouper.DeadlockEvent( - DeadlockObjectExtractor.FromGraphXml(d.DeadlockGraphXml), - DeadlockDetailFields(d.VictimSqlText, d.ProcessSummary)))); - AlertIncidentRenderer.Apply(context, groups.Select(g => g.Incident).ToList()); + filtered.Select(e => new BlockingIncidentGrouper.BlockedEvent( + e.DatabaseName, e.ContentiousObject, e.BlockedSqlText, e.BlockingSqlText, e.WaitTimeMs, e.LockMode))); - return context; + /* The graph parse, shared by the render path and #2216's observation path. Both the fingerprint's object + set and the #2109 Database fact come off the same pass, so parsing once per deadlock is the point. */ + private static List<(DeadlockAlertRow Row, IReadOnlyList Objects, IReadOnlyList Databases)> + ParseDeadlocks(IReadOnlyList filtered) => + filtered + .Select(d => (Row: d, + Objects: DeadlockObjectExtractor.FromGraphXml(d.DeadlockGraphXml), + Databases: DeadlockObjectExtractor.DatabasesFromGraphXml(d.DeadlockGraphXml))) + .ToList(); + + /* #1140: fingerprint each deadlock by its sorted involved-object set, across ALL deadlocks in the window, + grouped so recurrences over the same objects collapse to one incident with a count. */ + private static List GroupParsedDeadlocks( + string serverName, + List<(DeadlockAlertRow Row, IReadOnlyList Objects, IReadOnlyList Databases)> parsed) => + DeadlockIncidentGrouper.Group( + serverName, + parsed.Select(p => new DeadlockIncidentGrouper.DeadlockEvent( + p.Objects, + DeadlockDetailFields(p.Databases, p.Row.VictimSqlText, p.Row.ProcessSummary)))); + + private static List GroupDeadlocks( + string serverName, IReadOnlyList filtered) => + GroupParsedDeadlocks(serverName, ParseDeadlocks(filtered)); + + /* #2216: runs the caller's incident decorator, with the no-decorator and no-incident cases short- + circuited. A decorator that returned a different NUMBER of incidents would silently change what the + alert renders — dropped incidents, or a "+N more" trailer that no longer matches the items below it — + so a mismatched result is discarded in favour of the originals. The accumulator's contract is + same-order-same-count; this makes a breach of it inert rather than invisible. */ + private static IReadOnlyList Decorate( + List incidents, + Func, IReadOnlyList>? decorate) + { + if (decorate is null || incidents.Count == 0) + { + return incidents; + } + + var decorated = decorate(incidents); + return decorated is not null && decorated.Count == incidents.Count ? decorated : incidents; } - /* #1141: forensic detail carried on a deadlock incident so per-event cards keep the victim SQL - + process summary (Summary mode shows them via the builder's own items). */ - private static List? DeadlockDetailFields(string? victimSql, string? processes) + /* #1141/#2109: forensic detail carried on a deadlock incident — the representative event's + databases, victim SQL, and process summary. Since #2108 these render on the incident's own + summary item too, not just per-event cards. */ + private static List? DeadlockDetailFields( + IReadOnlyList databases, string? victimSql, string? processes) { var f = new List(); + if (databases.Count > 0) f.Add(new AlertIncidentField("Database", string.Join(", ", databases))); if (!string.IsNullOrWhiteSpace(victimSql)) f.Add(new AlertIncidentField("Victim SQL", TruncateText(victimSql))); if (!string.IsNullOrWhiteSpace(processes)) f.Add(new AlertIncidentField("Processes", processes!)); return f.Count > 0 ? f : null; @@ -345,6 +486,9 @@ public static string FormatPvsThreshold(double thresholdPercent, double floorGb) { var fields = new List<(string, string)> { + /* #2109: the database as a discrete fact, not only in the heading — downstream + automation routes on the fact name, and headings are display prose. */ + ("Database", d.DatabaseName), ("PVS Size (off-row)", $"{d.PvsGb:F1} GB"), ("Database Data Size", $"{d.DatabaseDataSizeMb / 1024.0:F1} GB"), ("Aborted Transactions", d.CurrentAbortedTransactionCount.ToString()), diff --git a/PerformanceMonitor.Alerting/AlertEngine.cs b/PerformanceMonitor.Alerting/AlertEngine.cs index c63940d12..2e9975fea 100644 --- a/PerformanceMonitor.Alerting/AlertEngine.cs +++ b/PerformanceMonitor.Alerting/AlertEngine.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -66,6 +67,19 @@ shared so Lite's existing config_edge_trigger_watermarks rows seed this engine u public const string BlockingWatermarkMetric = "Blocking Detected"; public const string DeadlockWatermarkMetric = "Deadlocks Detected"; + /// + /// The rolling window both count gates read, in hours (#1091's "in the last hour"). Named because + /// #2216's occurrence accumulator has to agree with it: its staleness horizon is what stops a row + /// stranded by a crash from being trusted on the same fingerprint's NEXT incident, and the only value + /// that makes that judgement correct is the window itself — inside the window a persisted row is + /// describing the very events the gauge is still counting, outside it the row cannot be. Two literals + /// that must match are two literals that will eventually not. + /// + public const int RollingCountWindowHours = 1; + + /* #2216: rows untouched for longer than the read window are treated as absent by the accumulator. */ + private static readonly TimeSpan OccurrenceStaleAfter = TimeSpan.FromHours(RollingCountWindowHours); + private readonly IAlertEngineSettings _settings; private readonly IAlertReadAdapter _readAdapter; private readonly IAlertStateStore _stateStore; @@ -137,7 +151,22 @@ persisted through IAlertStateStore on change (#1145 parity). */ is keyed per database (serverKey + "|" + dbName) so each database throttles independently, and an entry is removed when its database recovers. In-memory only, like the other family state. */ private readonly ConcurrentDictionary> _activeDatabaseStateAlerts = new(); - private readonly ConcurrentDictionary _lastDatabaseStateAlert = new(); + + /* #2166: keyed per (server, database, STATE) as a tuple rather than a delimited string. Per-state + because a chosen state now goes quiet indefinitely, so letting one state's clock rate-limit a + transition to a DIFFERENT state is a silence rather than a delay — and the state it would silence is + SUSPECT. Structural rather than concatenated because clearing a database's clocks means matching on + two of the three parts, and a string key makes that a prefix match: SQL Server permits '|' in a + database name, so `Foo|Bar` would collide with `Foo` under any delimiter a sysname can contain. */ + private readonly ConcurrentDictionary<(string Server, string Database, string State), DateTime> _lastDatabaseStateAlert = new(); + + /* #2157: per-PLAN active set and cooldowns. The alerting unit is one forced plan, not one server — + two plans failing on the same database are independent conditions that resolve independently. + Keyed by the internal plan key but VALUED with the plan's identity, because the resolution has to + name the plan in an operator-readable way: a bare key set left the recovery message reading + 'forceplan:Sales:11:22 no longer failing to force' in every email and webhook (review catch). */ + private readonly ConcurrentDictionary> _activeForcePlanAlerts = new(); + private readonly ConcurrentDictionary _lastForcePlanAlert = new(); /// Live threshold surface — read every sweep, never cached. /// The collected alert feeds (slice B seam). @@ -236,6 +265,7 @@ private async Task EvaluateCoreAsync(AlertServerSnapshot snaps await CheckAnomalousJobsAsync(key, serverName, now, alertCooldown, suppressed, ct); bool failedJobConditionPresent = await CheckFailedJobsAsync(snapshot, key, serverName, now, alertCooldown, suppressed, ct); await CheckDatabaseStateAsync(key, serverName, now, alertCooldown, suppressed, ct); + await CheckForcePlanFailuresAsync(key, serverName, now, alertCooldown, suppressed, ct); return new AlertSweepResult(true, lowDiskConditionPresent, failedJobConditionPresent); } @@ -361,7 +391,7 @@ private async Task CheckBlockingAsync( { /* ONE fetch serves the rolling count, the excluded-database recount (:118-133), and the fired alert's context (:172) — see class remarks adaptation (1). */ - blockingRows = await _readAdapter.GetRecentBlockedProcessReportsAsync(key, hoursBack: 1, ct); + blockingRows = await _readAdapter.GetRecentBlockedProcessReportsAsync(key, hoursBack: RollingCountWindowHours, ct); /* Lite's overview count semantics (LocalDataService.Overview.cs:74-77): prefer the XE blocked-process-report count; fall back to the DMV snapshot count when the XE @@ -410,6 +440,20 @@ the merged count IS the DMV count. */ bool wasBlockingActive = _activeBlockingAlert.TryGetValue(key, out var wasBlocking) && wasBlocking; /* :152 */ _activeBlockingAlert[key] = blockingDecision.Active; /* :153 */ + /* #2216: observe THIS sweep's fingerprints, whether or not an alert is delivered. Outside the Fire + branch deliberately — see ObserveOccurrencesAsync: counting only at delivery time lets an event + that ages out during a cooldown mask an arrival, and the total undercounts by exactly the number + of events the window retired while nobody was looking. Skipped when the gate is disabled or the + fetch failed (blockingRows null), because there is no observation to make. */ + var blockingOccurrences = default(OccurrenceTotals); + if (blockingRows is not null) + { + blockingOccurrences = await ObserveOccurrencesAsync( + key, BlockingWatermarkMetric, + AlertContextBuilders.BlockingIncidents(serverName, blockingRows, _settings.ExcludedDatabases), + now); + } + if (blockingDecision.Fire) /* :155 */ { var muteCtx = new AlertMuteContext { ServerName = serverName, MetricName = "Blocking Detected" }; /* :157 */ @@ -418,7 +462,8 @@ the merged count IS the DMV count. */ /* :172-173 — Lite's BuildBlockingContextAsync refetches the same rows; the engine reuses this sweep's fetch (identical query/window). */ - var blockingContext = AlertContextBuilders.BuildBlockingContext(serverName, blockingRows, _settings.ExcludedDatabases); + var blockingContext = AlertContextBuilders.BuildBlockingContext( + serverName, blockingRows, _settings.ExcludedDatabases, blockingOccurrences.Decorate); var detailText = AlertContextBuilders.ContextToDetailText(blockingContext); /* :175-183 — SendDetectedAlertAsync's #1141/#1236 delivery-mode fan-out is an @@ -436,6 +481,17 @@ await FireAsync(new AlertOutcome( } else if (!blockingDecision.Active && wasBlockingActive) /* :185 */ { + /* #2216: the incident is over, so its per-fingerprint counters are too — the next incident's + total should start from 1 with a start time that says so. When this sweep OBSERVED (rows + fetched), the observation above already recorded that: an empty window yields an empty state + set, which the replace-the-set contract writes as a delete. Rows are null only when the gate + is DISABLED — a fetch failure returns before reaching here — and turning the alert off should + still drop the counters rather than leave them for the staleness horizon. */ + if (blockingRows is null) + { + await ClearOccurrencesAsync(key, BlockingWatermarkMetric); + } + if (!suppressed && _settings.BlockingEnabled) /* :187 */ { await NotifyResolutionAsync(new AlertResolution( @@ -454,6 +510,124 @@ incident content is worse than skipping the sweep (state untouched, same as ever await CheckBlockingWaitAsync(key, serverName, now, alertCooldown, suppressed, blockingRows, ct); } + /* ---------------- per-fingerprint occurrence counters (#2216) ---------------- */ + + /// + /// Observes one sweep's incidents for a metric: loads the persisted per-fingerprint state, accumulates + /// this sweep's window counts into it, persists when there is something to write, and returns the totals + /// for the fired alert to attach. + /// + /// Called on EVERY sweep that successfully fetched rows — NOT only the sweeps that deliver. That is + /// the whole reason the accumulator keeps a mark separate from + /// 's: observing only at delivery time makes the two marks advance at + /// the same cadence, and then every event that ages out of the window during a cooldown masks an arrival + /// and the total silently undercounts. A sweep's grouping is UNCAPPED for the same reason the observation + /// is unconditional — the render path's top-N cap is a display budget, and a fingerprint outside it still + /// has a live incident whose state must not be dropped. + /// + /// Failure-isolated at both ends: a store that cannot answer yields an empty map, which the + /// accumulator treats as first contact — every total equals its window count, exactly the pre-#2216 + /// information. An alert that is already firing must never be lost to bookkeeping. + /// + private async Task ObserveOccurrencesAsync( + string key, string metricName, IReadOnlyList incidents, DateTime now) + { + IReadOnlyDictionary persisted; + try + { + persisted = await _stateStore.LoadIncidentOccurrencesAsync(key, metricName) + ?? EmptyOccurrenceStates; + } + catch (Exception ex) + { + _logger?.LogWarning("Could not load incident occurrences for {Metric}: {Message}", metricName, ex.Message); + persisted = EmptyOccurrenceStates; + } + + var result = IncidentOccurrenceAccumulator.Accumulate(incidents, persisted, now, OccurrenceStaleAfter); + + if (result.Changed) + { + await SaveOccurrencesAsync(key, metricName, result.States); + } + + return new OccurrenceTotals(result.States); + } + + /// + /// Records the falling edge: the metric has no incidents left, so its counters are cleared and the next + /// incident starts from 1 with a fresh start time. An empty set IS the clear — see + /// . + /// + private Task ClearOccurrencesAsync(string key, string metricName) => + SaveOccurrencesAsync(key, metricName, EmptyOccurrenceStates); + + private async Task SaveOccurrencesAsync( + string key, string metricName, IReadOnlyDictionary states) + { + try + { + await _stateStore.SaveIncidentOccurrencesAsync(key, metricName, states); + } + catch (Exception ex) + { + /* A dropped write costs accuracy on the next delivery's total — that fingerprint reads as new + and restarts, with a start time saying so — never a missed or duplicated alert. */ + _logger?.LogWarning("Could not persist incident occurrences for {Metric}: {Message}", metricName, ex.Message); + } + } + + private static readonly IReadOnlyDictionary EmptyOccurrenceStates = + new Dictionary(StringComparer.Ordinal); + + /// + /// This sweep's per-fingerprint totals, ready for the fired alert's incidents to pick up. + /// + /// The accounting is already DONE by the time this exists — is a pure + /// lookup, not a second accumulation. That split is what keeps the arithmetic honest: the counting + /// happens once per sweep against the store, and the render path merely reads it. An earlier shape had + /// the builder's decorator do the accumulating, which meant it only ran on the sweeps that delivered an + /// alert and only for the incidents that survived the render cap. + /// + private readonly struct OccurrenceTotals + { + private readonly IReadOnlyDictionary _states; + + internal OccurrenceTotals(IReadOnlyDictionary states) => + _states = states; + + /// + /// The builder's pre-render hook. Attaches each incident's total; an incident with no state (a blank + /// fingerprint, or the vanishingly unlikely case of the render path grouping to a key the sweep's + /// grouping did not produce) is passed through carrying null, which reads as "no total available" + /// rather than a fabricated zero. + /// + internal IReadOnlyList Decorate(IReadOnlyList incidents) + { + if (_states is null || _states.Count == 0) + { + return incidents; + } + + var decorated = new List(incidents.Count); + foreach (var incident in incidents) + { + decorated.Add( + incident is not null + && !string.IsNullOrEmpty(incident.DedupKey) + && _states.TryGetValue(incident.DedupKey, out var state) + ? incident with + { + TotalOccurrences = state.TotalOccurrences, + IncidentStartedUtc = state.IncidentStartedUtc, + } + : incident!); + } + + return decorated; + } + } + /* ---------------- blocking wait time (#1839) ---------------- */ /// @@ -560,7 +734,7 @@ private async Task CheckDeadlocksAsync( { /* ONE fetch serves the rolling count, the excluded-database recount (:198-211), and the fired alert's context (:249) — class remarks adaptation (1). */ - deadlockRows = await _readAdapter.GetRecentDeadlocksAsync(key, hoursBack: 1, ct); + deadlockRows = await _readAdapter.GetRecentDeadlocksAsync(key, hoursBack: RollingCountWindowHours, ct); effectiveDeadlockCount = deadlockRows.Count; /* :198-205 — recount excluding deadlocks whose processes ALL ran in excluded @@ -599,6 +773,16 @@ private async Task CheckDeadlocksAsync( bool wasDeadlockActive = _activeDeadlockAlert.TryGetValue(key, out var wasDeadlock) && wasDeadlock; /* :229 */ _activeDeadlockAlert[key] = deadlockDecision.Active; /* :230 */ + /* #2216: observe every sweep — see the blocking twin above for why this cannot sit inside Fire. */ + var deadlockOccurrences = default(OccurrenceTotals); + if (deadlockRows is not null) + { + deadlockOccurrences = await ObserveOccurrencesAsync( + key, DeadlockWatermarkMetric, + AlertContextBuilders.DeadlockIncidents(serverName, deadlockRows, _settings.ExcludedDatabases), + now); + } + if (deadlockDecision.Fire) /* :232 */ { var muteCtx = new AlertMuteContext { ServerName = serverName, MetricName = "Deadlocks Detected" }; /* :234 */ @@ -606,7 +790,8 @@ private async Task CheckDeadlocksAsync( _lastDeadlockAlert[key] = now; /* :236 */ /* :249-250 — context from this sweep's fetch. */ - var deadlockContext = AlertContextBuilders.BuildDeadlockContext(serverName, deadlockRows, _settings.ExcludedDatabases); + var deadlockContext = AlertContextBuilders.BuildDeadlockContext( + serverName, deadlockRows, _settings.ExcludedDatabases, deadlockOccurrences.Decorate); var detailText = AlertContextBuilders.ContextToDetailText(deadlockContext); /* :252-260 — ShortMessage = the toast body of :244. Numerics carried explicitly (#1830): @@ -622,6 +807,13 @@ await FireAsync(new AlertOutcome( } else if (!deadlockDecision.Active && wasDeadlockActive) /* :262 */ { + /* #2216: the falling edge — see the blocking twin above for why this is only the disabled-gate + case; an observed empty window already cleared itself. */ + if (deadlockRows is null) + { + await ClearOccurrencesAsync(key, DeadlockWatermarkMetric); + } + if (!suppressed && _settings.DeadlockEnabled) /* :264 */ { await NotifyResolutionAsync(new AlertResolution( @@ -880,7 +1072,8 @@ private async Task CheckLowDiskAsync( var lowDiskContext = AlertContextBuilders.BuildVolumeFreeSpaceContext(serverName, breached); /* :515 */ /* :516-522 — #1136: grade WARNING normally, CRITICAL when critically low. */ - if (lowDiskContext is not null && LowDiskAlertGate.IsCriticallyLow(worst.FreePercent, worst.FreeGb)) + if (lowDiskContext is not null && LowDiskAlertGate.IsCriticallyLow( + worst.FreePercent, worst.FreeGb, _settings.DiskCriticalFreePercent, _settings.DiskCriticalFreeGb)) { lowDiskContext.SeverityOverride = AlertSeverityLevel.Critical; } @@ -1239,8 +1432,27 @@ other alerts' treatment of that user-facing list. */ foreach (var (dbName, db) in current) { active.Add(dbName); - var cooldownKey = DatabaseStateCooldownKey(key, dbName); - if (!suppressed && CooldownElapsed(_lastDatabaseStateAlert, cooldownKey, now, alertCooldown)) + /* Keyed per database AND per STATE (#2166). It used to be per database, which was survivable when + every deviation re-fired every cooldown: a transition suppressed by the previous state's + cooldown re-announced on the next tick anyway. Now that a chosen state goes quiet + indefinitely, that suppression would be permanent for the length of a cooldown window — so a + database going OFFLINE and then SUSPECT inside one window could have its SUSPECT transition + swallowed, which is precisely the integrity case this alert must never go quiet about. Each + state now rate-limits itself and cannot borrow another's clock. */ + var cooldownKey = (Server: key, Database: dbName, State: db.StateDesc); + /* #2166: for the states an operator usually CHOSE (a parked OFFLINE, a secondary flickering + RESTORING), repetition is noise — alert on the transition and stay quiet until the state + changes. Compared against the PERSISTED last-alerted state, so a service restart cannot + re-announce every parked database. The integrity states skip this entirely: nobody parks a + database in SUSPECT, so their repetition is the signal and the cooldown still governs. + + A host that does not persist the memory (Lite today) reports empty here, every deviation + reads as new, and behavior is exactly as it was before this change. */ + var alreadyAnnounced = + DatabaseStateTokens.RepeatsAreNoise(db.StateDesc) + && string.Equals(db.LastAlertedState, db.StateDesc, StringComparison.OrdinalIgnoreCase); + + if (!suppressed && !alreadyAnnounced && CooldownElapsed(_lastDatabaseStateAlert, cooldownKey, now, alertCooldown)) { var severity = DatabaseStateTokens.SeverityFor(db.StateDesc); var stateText = DatabaseStateTokens.Humanize(db.StateDesc); @@ -1265,14 +1477,46 @@ as a first-observation alert rather than "expected UNKNOWN". */ ? $"{dbName} first observed {stateText} (no baseline yet)" : $"{dbName} changed to {stateText} (expected {expectedText})"; + /* #2109: the same fields the prose carries, as discrete facts — this alert fired with + Context: null, which left the database name reachable only by parsing the title. */ + var stateContext = new AlertContext(); + stateContext.Details.Add(new AlertDetailItem + { + Heading = dbName, + Fields = new() + { + ("Database", dbName), + ("Current State", stateText), + ("Expected State", expectedText) + } + }); + await FireAsync(new AlertOutcome( key, serverName, DatabaseStateTokens.MetricName, $"{dbName}: {stateText}", expectedText, - Context: null, DetailText: detailText, + Context: stateContext, DetailText: detailText, NumericCurrentValue: null, NumericThresholdValue: null, Muted: isMuted, Severity: severity, ShortMessage: shortMessage), ct); + + /* Stamped AFTER delivery so a failed fire is retried next cycle rather than silenced, and + written for every state rather than only the edge-triggered ones, so that reclassifying a + state later has correct history to work from. + + NOT stamped when MUTED, which is the one place this memory and the cooldown beside it must + disagree. The cooldown is rate limiting and applies whether or not anyone was told; this + memory means "the operator has been told about this state", and under a mute they have not. + Stamping it anyway made a mute permanent: the four edge-triggered states gate all future + firing on this value, so muting a parked database, then REMOVING the mute, left + LastAlertedState equal to the current state forever and the alert never returned — the + operator's mute silently became irreversible for as long as the state held. Skipping the + stamp costs a repeat inside the mute (invisible by definition, and exactly the pre-#2166 + cooldown behavior) and keeps unmuting meaningful. */ + if (!isMuted) + { + await _stateStore.SaveDatabaseStateAlertedAsync(key, dbName, db.StateDesc); + } } } @@ -1283,10 +1527,42 @@ drop their cooldown. Guarded by the master enable (we're past the early return), if (active.Count > 0) { var recovered = active.Where(d => !current.ContainsKey(d)).ToList(); + + /* Every recovered database's clocks are dropped in ONE pass over the cooldown map, not one pass + each (#2166). The key is per-state, so a single removal per database would leave its other + states' stamps behind to rate-limit a future episode against a cooldown that started before the + recovery — but the map holds every server's entries, so scanning it per database made the sweep + O(recovered x everything tracked) where the old string key was an O(1) remove. Hoisting it back + to one scan keeps the correctness and drops a factor. Matching on two tuple parts rather than a + string prefix is what keeps a database named 'Foo|Bar' from being swept when 'Foo' recovers. */ + if (recovered.Count > 0) + { + /* ORDINAL, like `current` and `active` above and for the same reason: per-database keys here + must be case-SENSITIVE to match the stores' case-sensitive expected-state joins. A + case-insensitive set would let recovering `Foo` clear `foo`'s per-state stamps on a + case-sensitive collation where both exist — resetting the only quiet mechanism an integrity + state has, which is the same collision class the tuple key just removed for '|'. */ + var recoveredSet = new HashSet(recovered, StringComparer.Ordinal); + foreach (var stamped in _lastDatabaseStateAlert.Keys) + { + if (string.Equals(stamped.Server, key, StringComparison.Ordinal) + && recoveredSet.Contains(stamped.Database)) + { + _lastDatabaseStateAlert.TryRemove(stamped, out _); + } + } + } + foreach (var dbName in recovered) { active.Remove(dbName); - _lastDatabaseStateAlert.TryRemove(DatabaseStateCooldownKey(key, dbName), out _); + + /* #2166 falling edge: forget the announced state as well as the in-memory cooldown, or the + edge only ever triggers once per database. Cleared even when suppressed — suppression + governs whether operators are TOLD about a transition, never whether the engine keeps + accurate state, and leaving a stale memory behind would swallow the next real episode. */ + await _stateStore.ClearDatabaseStateAlertedAsync(key, dbName); + if (!suppressed) { await NotifyResolutionAsync(new AlertResolution( @@ -1299,18 +1575,156 @@ await NotifyResolutionAsync(new AlertResolution( } /// - /// Per-database cooldown key. The serverKey is always a digit-only int (see the adapters' - /// ParseServerKey), so the first '|' unambiguously ends it regardless of what the database name - /// contains — no collision between e.g. (server 1, db "23") and (server 12, db "3"). + /// Forced Query Store plans the engine is currently failing to reproduce (#2157). The adapter returns + /// only plans whose force_failure_count ROSE since the previous collection, so every row here is + /// a live failure rather than accumulated history — see + /// for why a level would be wrong. + /// + /// Why it deserves an alert at all: when a force fails, the query keeps running on whatever plan + /// the optimizer picks. Nothing else in the product witnesses that — the operator's mitigation is + /// silently not in effect, and the only trace is a counter climbing inside Query Store. + /// + /// Standing condition with per-plan resolution, mirroring the database-state family: while a plan + /// keeps failing it re-fires on the cooldown, and when it stops appearing it announces a recovery. /// - private static string DatabaseStateCooldownKey(string serverKey, string dbName) => - serverKey + "|" + dbName; + private async Task CheckForcePlanFailuresAsync( + string key, string serverName, DateTime now, TimeSpan alertCooldown, bool suppressed, CancellationToken ct) + { + if (!_settings.ForcePlanFailureEnabled) + { + return; + } + + List failures; + try + { + failures = await _readAdapter.GetForcePlanFailuresAsync(key, ct); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + /* Log-and-skip, like every other collected read: never resolve an active plan on a failed + fetch (that would fabricate a recovery), and never fire on absent evidence. */ + _logger?.LogError("Failed to check forced-plan failures for {Server}: {Message}", serverName, ex.Message); + return; + } + + var excluded = _settings.ExcludedDatabases; + + /* Per-PLAN keys are ORDINAL for the same reason the database-state family's are: the stores compare + database names case-sensitively, so a plan must not key differently here than it does there. The + excluded-databases list stays case-insensitive, matching how every alert treats that user list. */ + var current = new Dictionary(StringComparer.Ordinal); + foreach (var failure in failures) + { + if (string.IsNullOrWhiteSpace(failure.DatabaseName) || failure.PlanId <= 0) + { + continue; + } + + if (excluded.Count > 0 && excluded.Any(e => string.Equals(e, failure.DatabaseName, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + current[ForcePlanTokens.PlanKey(failure.DatabaseName, failure.QueryId, failure.PlanId)] = failure; + } + + var active = _activeForcePlanAlerts.GetOrAdd(key, _ => new Dictionary(StringComparer.Ordinal)); + + foreach (var (planKey, failure) in current) + { + active[planKey] = failure; + var cooldownKey = key + "|" + planKey; + if (!suppressed && CooldownElapsed(_lastForcePlanAlert, cooldownKey, now, alertCooldown)) + { + var reasonText = ForcePlanTokens.HumanizeReason(failure.FailureReason); + var forcingText = string.IsNullOrWhiteSpace(failure.ForcingType) ? "unknown" : failure.ForcingType.Trim(); + var muteCtx = new AlertMuteContext + { + ServerName = serverName, + MetricName = ForcePlanTokens.MetricName, + DatabaseName = failure.DatabaseName + }; + bool isMuted = _isAlertMuted(muteCtx); + _lastForcePlanAlert[cooldownKey] = now; /* stamped even when muted, like the others */ + + var detailText = + $" Database: {failure.DatabaseName}\n" + + $" Query / Plan: {failure.QueryId} / {failure.PlanId}\n" + + $" Forcing: {forcingText}\n" + + $" Reason: {reasonText}\n" + + $" New failures since last collection: {failure.FailureDelta} (total {failure.TotalFailures})\n" + + " The query is running on the optimizer's plan, not the forced one."; + + /* #2109 discipline: the same facts the prose carries, as discrete fields, so a consumer + never has to parse the title to learn which plan this is about. */ + var context = new AlertContext(); + context.Details.Add(new AlertDetailItem + { + Heading = $"{failure.DatabaseName} query {failure.QueryId} plan {failure.PlanId}", + Fields = new() + { + ("Database", failure.DatabaseName), + ("Query ID", failure.QueryId.ToString(CultureInfo.InvariantCulture)), + ("Plan ID", failure.PlanId.ToString(CultureInfo.InvariantCulture)), + ("Forcing Type", forcingText), + ("Failure Reason", reasonText), + ("New Failures", failure.FailureDelta.ToString(CultureInfo.InvariantCulture)), + ("Total Failures", failure.TotalFailures.ToString(CultureInfo.InvariantCulture)) + } + }); + + await FireAsync(new AlertOutcome( + key, serverName, ForcePlanTokens.MetricName, + $"{failure.DatabaseName}: plan {failure.PlanId} failing to force ({reasonText})", + reasonText, + Context: context, DetailText: detailText, + NumericCurrentValue: failure.FailureDelta, NumericThresholdValue: null, + Muted: isMuted, Severity: ForcePlanTokens.SeverityFor(failure), + ShortMessage: $"{failure.DatabaseName} plan {failure.PlanId} failed to force {failure.FailureDelta}x ({reasonText})"), ct); + } + } + + /* Plans that were alerting and no longer are: the counter stopped rising, because the force was + removed, the plan became reproducible again, or the query stopped running. All three mean "no + longer failing", which is what the recovery says — deliberately not claiming it was fixed. */ + if (active.Count > 0) + { + var recovered = active.Where(p => !current.ContainsKey(p.Key)).ToList(); + foreach (var (planKey, lastSeen) in recovered) + { + active.Remove(planKey); + _lastForcePlanAlert.TryRemove(key + "|" + planKey, out _); + if (!suppressed) + { + /* Named from the identity we stored when it fired, never from the internal key: an + operator reads this in a toast, an email and a history row. */ + await NotifyResolutionAsync(new AlertResolution( + key, serverName, ForcePlanTokens.MetricName, + "Forced Plan Failing Resolved", + $"{serverName}: {lastSeen.DatabaseName} query {lastSeen.QueryId} plan {lastSeen.PlanId} no longer failing to force"), ct); + } + } + } + } /* ---------------- helpers ---------------- */ - /// Lite's per-check cooldown test: no prior fire, or the cooldown has elapsed. - private static bool CooldownElapsed( - ConcurrentDictionary lastFired, string key, DateTime now, TimeSpan cooldown) => + /// + /// Lite's per-check cooldown test: no prior fire, or the cooldown has elapsed. + /// + /// Generic in the KEY type only (#2166) so a family whose cooldown is scoped by more than one thing + /// can key it structurally instead of concatenating a string. Every existing caller is string-keyed and + /// infers unchanged; the database-state family keys by (server, database, state), where a string key + /// would need a delimiter no sysname can contain — and SQL Server permits |. + /// + private static bool CooldownElapsed( + ConcurrentDictionary lastFired, TKey key, DateTime now, TimeSpan cooldown) + where TKey : notnull => !lastFired.TryGetValue(key, out var last) || now - last >= cooldown; /// diff --git a/PerformanceMonitor.Alerting/DatabaseStateInfo.cs b/PerformanceMonitor.Alerting/DatabaseStateInfo.cs index 5a53cd33d..1682afcd9 100644 --- a/PerformanceMonitor.Alerting/DatabaseStateInfo.cs +++ b/PerformanceMonitor.Alerting/DatabaseStateInfo.cs @@ -30,4 +30,16 @@ public sealed class DatabaseStateInfo /// (e.g. "ONLINE → OFFLINE"). /// public string ExpectedState { get; set; } = ""; + + /// + /// The effective state this database was LAST alerted about (#2166), or empty when it has never been + /// alerted. Persisted per (server, database) so it survives a service restart — without that, an + /// edge-triggered alert would re-fire every deliberately-parked database on every restart, which is + /// worse than the cooldown-repeat it replaces. + /// + /// The engine compares this against for the states where repetition is + /// noise rather than signal (a chosen OFFLINE, a flickering RESTORING). The integrity states are + /// never chosen, so they keep re-firing on the cooldown and ignore this field. + /// + public string LastAlertedState { get; set; } = ""; } diff --git a/PerformanceMonitor.Alerting/DatabaseStateTokens.cs b/PerformanceMonitor.Alerting/DatabaseStateTokens.cs index 3a229613b..a6736b528 100644 --- a/PerformanceMonitor.Alerting/DatabaseStateTokens.cs +++ b/PerformanceMonitor.Alerting/DatabaseStateTokens.cs @@ -48,6 +48,39 @@ public static class DatabaseStateTokens /// The metric name every database-state alert fires under (the AlertSeverity map key). public const string MetricName = "Database State"; + /// + /// The integrity-failure states as a SQL IN-list literal, for the arm of both stores' deviation reads + /// that alerts a database with no baseline yet. Composed from the constants above so the + /// one-spelling-each rule this class exists for reaches the SQL too, where a typo would not fail to + /// compile — it would silently never match. + /// + /// This list is deliberately NARROWER than and must stay that + /// way. It answers "which states are bad enough to page about with no baseline to compare against", + /// where the other answers "which states must never be LEARNED as normal". A transient state belongs + /// only in the second: a database sitting in RESTORING has nothing wrong with it, so widening this list + /// to match would page for every restore in progress. + /// + public const string CriticalSqlList = "'" + Suspect + "', '" + RecoveryPending + "', '" + Emergency + "'"; + + /// + /// The effective states a first observation must never be baselined from, as a SQL IN-list literal — + /// the integrity states plus the transient operational ones (#2189). A database in one of these is + /// mid-something, not in a steady state anybody would choose as "expected", and learning one inverts + /// the alert permanently: a database observed mid-restore learned RESTORING as its baseline and then + /// deviated by being HEALTHY, forever (636 alerts in 24 hours from 5 databases on one fleet). + /// + /// A database in one of these states simply stays pending — no row — until it settles into + /// something the seed will learn. That keeps a NORECOVERY log-shipping secondary permanently quiet + /// (pending plus a non-critical state alerts nothing) while still covering it for the integrity states + /// through the pending arm; an operator who wants deviation coverage for one sets the expected state + /// explicitly, which is an override and therefore honoured over anything inferred. + /// + /// Note STANDBY is absent on purpose. It is synthetic and stable by construction — the whole + /// reason exists is to give a log-shipping secondary one steady token instead of + /// the RESTORING flicker underneath it — so it is exactly the kind of state worth learning. + /// + public const string NeverBaselinedSqlList = CriticalSqlList + ", '" + Restoring + "', '" + Recovering + "'"; + /// /// The fixed severity for a given state_desc: CRITICAL for the integrity-failure states /// (SUSPECT / RECOVERY_PENDING / EMERGENCY), WARNING for everything else. Case-insensitive. @@ -59,6 +92,27 @@ public static AlertSeverityLevel SeverityFor(string? stateDesc) => _ => AlertSeverityLevel.Warning }; + /// + /// Whether repetition carries information for this state (#2166). + /// + /// OFFLINE and RESTORING are usually states somebody CHOSE — routine maintenance, a soft-delete + /// park, a log-shipping secondary flickering through restores. Re-firing for weeks tells the operator + /// nothing they did not already know, and the reporter's case (#2166) generated hundreds of identical + /// alerts for one intended action. Those get edge semantics: alert on the transition, then stay quiet + /// until the state changes. + /// + /// The integrity states are never chosen — nobody parks a database in SUSPECT — so continued + /// repetition IS the signal, and they keep re-firing on the cooldown. The engine already applies this + /// discipline to Server Unreachable/Restored; this extends it to the database states that behave the + /// same way. + /// + public static bool RepeatsAreNoise(string? stateDesc) => + (stateDesc?.Trim().ToUpperInvariant()) switch + { + Offline or Restoring or Recovering or Standby => true, + _ => false + }; + /// A human-friendly rendering of a state token (RECOVERY_PENDING → "RECOVERY PENDING"). public static string Humanize(string? stateDesc) => string.IsNullOrWhiteSpace(stateDesc) ? "UNKNOWN" : stateDesc.Trim().Replace('_', ' '); diff --git a/PerformanceMonitor.Alerting/ForcePlanFailureInfo.cs b/PerformanceMonitor.Alerting/ForcePlanFailureInfo.cs new file mode 100644 index 000000000..7690a0d77 --- /dev/null +++ b/PerformanceMonitor.Alerting/ForcePlanFailureInfo.cs @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +namespace PerformanceMonitor.Alerting; + +/// +/// A forced Query Store plan whose force_failure_count ROSE between the two most recent +/// collections — the unit the forced-plan-failure alert fires on (#2157). The read adapter returns only +/// risen rows, exactly as the database-state adapter returns only deviating ones, so the engine never +/// has to decide what "new" means. +/// +/// Why the delta and not the level. force_failure_count is cumulative and it travels +/// with the database: restore a database somewhere else and its Query Store arrives carrying every +/// historical failure. An alert on the level would therefore fire forever about failures that happened +/// on a machine the operator may no longer own. A rise, by contrast, means the engine is failing to +/// reproduce that plan RIGHT NOW — which is the actionable event, because the query silently falls back +/// to the optimizer's plan and nothing else in the product witnesses it. +/// +/// A counter that DROPS (an unforce/re-force cycle resets it) is not a failure and must not alert; +/// the adapter treats that as a silent re-arm. +/// +public sealed class ForcePlanFailureInfo +{ + /// The user database the forced plan lives in. + public string DatabaseName { get; set; } = ""; + + /// Query Store query_id — half of the identity an operator needs to find the plan. + public long QueryId { get; set; } + + /// Query Store plan_id — the forced plan itself. + public long PlanId { get; set; } + + /// + /// plan_forcing_type_desc: MANUAL (a human or a tool forced it) or AUTO (Automatic Plan + /// Correction did). Both matter and are reported: a failing MANUAL force is somebody's mitigation + /// silently not working, while a failing AUTO force is the engine's own correction not applying. + /// + public string ForcingType { get; set; } = ""; + + /// + /// last_force_failure_reason_desc — the engine's own words for why the plan could not be + /// reproduced (NO_PLAN, NO_INDEX, INVALID_STARTING_JOIN_ORDER, …). Carried into the alert body + /// because it is the difference between "the index it needs is gone" and "the plan is unusable". + /// + public string FailureReason { get; set; } = ""; + + /// How much the counter rose between the two samples — how many failures are NEW. + public long FailureDelta { get; set; } + + /// The cumulative count as of the newer sample, for context on whether this is chronic. + public long TotalFailures { get; set; } +} diff --git a/PerformanceMonitor.Alerting/ForcePlanTokens.cs b/PerformanceMonitor.Alerting/ForcePlanTokens.cs new file mode 100644 index 000000000..8f132b39c --- /dev/null +++ b/PerformanceMonitor.Alerting/ForcePlanTokens.cs @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Alerting; + +/// +/// The spellings and grading the forced-plan-failure alert reasons about (#2157), fixed here so the +/// engine, both apps' stores, and the mute-rule surface reference one string each — the same discipline +/// exists for. +/// +public static class ForcePlanTokens +{ + /// The metric name every forced-plan-failure alert fires under (the AlertSeverity map key). + public const string MetricName = "Forced Plan Failing"; + + /// + /// Per-object alert key prefix. The alerting unit is one FORCED PLAN, not one server: two plans + /// failing on the same database are two independent conditions that resolve independently, so the + /// cooldown and active-set keys have to carry the plan identity. + /// + public const string KeyPrefix = "forceplan:"; + + /// The forcing types Query Store reports, both of which this alert covers. + public const string Manual = "MANUAL"; + public const string Auto = "AUTO"; + + /// + /// Query Store's sentinel for "no failure recorded" in last_force_failure_reason_desc. Treated + /// as no reason rather than a reason named NONE — a row can carry it while the counter still rose, and + /// rendering "Reason: NONE" would read as though the engine declined to say. + /// + public const string NoFailureReason = "NONE"; + + /// + /// Severity for a failure rise. Deliberately WARNING for every rise, with no Critical tier: + /// + /// A failing force is not an outage — the query still runs, on the optimizer's plan — so the + /// honest grading is "somebody's mitigation is silently not working", which is a warning. A Critical + /// tier would need evidence about which reasons or rates actually correlate with harm, and inventing + /// thresholds without that evidence is how alert streams become noise nobody reads. If field data + /// later shows a class that IS urgent (a MANUAL force failing on a hot query, say), grade it then and + /// say what the data was. + /// + public static AlertSeverityLevel SeverityFor(ForcePlanFailureInfo _) => AlertSeverityLevel.Warning; + + /// A human-friendly rendering of a failure reason (NO_INDEX → "NO INDEX"), or "unspecified" + /// when Query Store recorded none. + public static string HumanizeReason(string? reason) + { + var trimmed = reason?.Trim(); + if (string.IsNullOrEmpty(trimmed) + || string.Equals(trimmed, NoFailureReason, System.StringComparison.OrdinalIgnoreCase)) + { + return "unspecified"; + } + + return trimmed.Replace('_', ' '); + } + + /// + /// The per-plan identity used in alert keys and mute contexts. Includes the database because + /// query_id/plan_id are only unique within one database's Query Store. + /// + public static string PlanKey(string databaseName, long queryId, long planId) => + $"{KeyPrefix}{databaseName}:{queryId}:{planId}"; +} diff --git a/PerformanceMonitor.Alerting/IAlertEngineSettings.cs b/PerformanceMonitor.Alerting/IAlertEngineSettings.cs index 893643d5b..dc083ea2e 100644 --- a/PerformanceMonitor.Alerting/IAlertEngineSettings.cs +++ b/PerformanceMonitor.Alerting/IAlertEngineSettings.cs @@ -75,6 +75,14 @@ public interface IAlertEngineSettings bool PvsEnabled { get; } bool DatabaseStateEnabled { get; } + /// + /// The forced-plan-failure alert (#2157). Default ON in both apps: it fires only on a counter that ROSE + /// since the previous collection, so a quiet fleet is silent by construction rather than by threshold — + /// which is what makes it safe to enable without asking. Deliberately NOT a store column yet; if + /// operators need per-deployment control it becomes one, with the full migration ladder that implies. + /// + bool ForcePlanFailureEnabled { get; } + /* Thresholds. */ /// Fire when the selected CPU metric (see ) is at/above this %. @@ -135,6 +143,33 @@ public interface IAlertEngineSettings /// Fire when a volume's free space is below this many GB (0 disables the GB dimension). int LowDiskThresholdGb { get; } + /// + /// The low-disk CRITICAL severity tier's percent floor (#1136/#2107): free space at/below this + /// % grades the Volume Free Space alert CRITICAL instead of WARNING. Was a compile-time 3.0 in + /// LowDiskAlertGate; both apps now pass their configured value. + /// + int DiskCriticalFreePercent { get; } + + /// The critical tier's GB floor — at/below this many GB free is CRITICAL on any + /// volume, OR-ed with the percent floor exactly as before (#1136/#2107). + int DiskCriticalFreeGb { get; } + + /// + /// The store/self-monitoring warning percent (#2107, Darling's self-alerts): the monitor's own + /// store volume warns below this % free. Lite has no headless store volume to self-monitor and + /// returns the shipped default — on the engine surface anyway so the two apps' settings + /// objects stay one shape (the PVS-knob precedent). + /// + int SelfDiskFreeWarnPercent { get; } + + /// How long collection may go quiet before Collection Stopped / Agent Not Running + /// fire (#2107; was a compile-time 30 minutes). Lite returns the default. + int CollectionStaleMinutes { get; } + + /// The Collection Stopped fast path — this many consecutive failures with zero + /// successes fires without waiting out the staleness window (#2107). Lite returns the default. + int CollectionFailureThreshold { get; } + /// /// Fire when an ADR database's persistent version store reaches this % of the database's data /// files (#1984). Percent rather than absolute size because a shipped absolute guess is diff --git a/PerformanceMonitor.Alerting/IAlertReadAdapter.cs b/PerformanceMonitor.Alerting/IAlertReadAdapter.cs index 247a5179d..2e4b754ef 100644 --- a/PerformanceMonitor.Alerting/IAlertReadAdapter.cs +++ b/PerformanceMonitor.Alerting/IAlertReadAdapter.cs @@ -156,12 +156,19 @@ Task GetAnomalousJobsAsync( /// returns the rows where current != expected in both samples and expected is not the /// sentinel, each carrying both the current and expected state. /// - /// CONTRACT — the read also AUTO-SEEDS and PRUNES: any database in the latest snapshot with no + /// CONTRACT — the read also AUTO-SEEDS, HEALS and PRUNES: any database in the latest snapshot with no /// expected-state row yet gets its current effective state recorded as the first-observation baseline, - /// EXCEPT a critical effective state (SUSPECT / RECOVERY_PENDING / EMERGENCY), which is left pending so - /// it alerts rather than learning the bad state as expected. Auto-baselines for databases that have - /// dropped off the newest snapshot are pruned (user overrides preserved). Seeding is idempotent - /// (insert-if-absent) and never overwrites a user override or an existing baseline. + /// EXCEPT the states in — the integrity ones + /// (which stay pending so they alert rather than learning the bad state as expected) and the transient + /// ones (RESTORING / RECOVERING, which stay pending SILENTLY until the database settles, so onboarding + /// mid-restore cannot learn a state nobody chose). An AUTO-seeded baseline that nonetheless records one + /// of those states — written by an older build, or by re-baselining a database by hand while it was + /// mid-something — is HEALED to ONLINE once the database's effective state reaches ONLINE, since such a + /// row is not a baseline anyone chose and would otherwise make the database deviate by being healthy + /// (#2189). The heal never touches a user override, and never touches an OFFLINE or STANDBY baseline: + /// those are steady states, and departing one is a real deviation that must still fire. Auto-baselines + /// for databases that have dropped off the newest snapshot are pruned (user overrides preserved). + /// Seeding is idempotent (insert-if-absent) and never overwrites a user override or an existing baseline. /// /// /// Empty when the store has no snapshot for this server. Unlike the anomalous-jobs read this is @@ -172,4 +179,24 @@ Task GetAnomalousJobsAsync( /// Task> GetDatabaseStatesAsync( string serverKey, CancellationToken cancellationToken = default); + + /// + /// Forced Query Store plans whose force_failure_count ROSE between the two most recent + /// collections that carried the plan (#2157) — i.e. the engine is failing to reproduce that plan now. + /// + /// The store computes the delta, exactly as returns only + /// deviating rows: the engine must never see a level. The counter is cumulative AND travels with a + /// restored database, so a level-based read would alert forever about failures that happened on + /// hardware the operator may no longer own. + /// + /// A counter that DROPPED is an unforce/re-force cycle, not a failure, and is omitted (silent + /// re-arm). A plan seen for the FIRST time carries no previous sample and is therefore omitted too — + /// one cycle of delay, deliberately, because "new" is unknowable from a single observation. + /// + /// Empty when the store has fewer than two samples for every forced plan. Not freshness-gated, + /// for the same reason as database state: a failing force is a standing condition, so a stale snapshot + /// keeps it active rather than fabricating a recovery. + /// + Task> GetForcePlanFailuresAsync( + string serverKey, CancellationToken cancellationToken = default); } diff --git a/PerformanceMonitor.Alerting/IAlertStateStore.cs b/PerformanceMonitor.Alerting/IAlertStateStore.cs index 3357f8fa0..13046e364 100644 --- a/PerformanceMonitor.Alerting/IAlertStateStore.cs +++ b/PerformanceMonitor.Alerting/IAlertStateStore.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace PerformanceMonitor.Alerting; @@ -65,4 +66,63 @@ public interface IAlertStateStore /// only — when a failed-job alert actually fires. /// Task SaveFailedJobWatermarkAsync(string serverKey, DateTime watermark); + + /// + /// Records the effective state a database was just alerted about (#2166), so the next evaluation can + /// tell a NEW deviation from the same one it already reported. Called on-fire only, which makes it a + /// low-frequency write like the watermarks above. + /// + /// Persistence is the requirement, not an optimization: the case this exists for is a database + /// deliberately parked OFFLINE for weeks. In-memory edge state would re-fire every parked database on + /// every service restart, which is worse than the cooldown-repeat being replaced. + /// + /// A host that does not persist it may no-op; the engine then sees no memory, every deviation + /// reads as new, and behavior is exactly the pre-#2166 cooldown-repeat. That is the intended fallback + /// rather than a broken state. + /// + Task SaveDatabaseStateAlertedAsync(string serverKey, string databaseName, string effectiveState); + + /// + /// Forgets what recorded, called when a database returns to + /// its expected state (#2166). The falling edge of an edge trigger: without it the memory is permanent + /// and each database can only ever announce once, so a second parking of the same database in the same + /// state — the repeat soft-delete workflow this alert is FOR — would be silently swallowed. + /// + /// Same no-op fallback as its sibling for hosts that do not persist: no memory to clear means + /// nothing to go stale. + /// + Task ClearDatabaseStateAlertedAsync(string serverKey, string databaseName); + + /// + /// Loads the per-fingerprint occurrence accounting for one server/metric (#2216), keyed by #1140 dedup + /// fingerprint — the state needs to turn the groupers' + /// window gauge into a monotonic total. Returns an empty map when nothing is persisted. + /// + /// Persistence is the requirement, not an optimization: the value of the counter is that it + /// survives the throttling the consumer is subject to, and a counter reset by every service restart + /// would be a second gauge wearing a total's name. A host that cannot persist may return an empty map + /// and no-op the save — the accumulator then reports the total as equal to the window count, which is + /// exactly the pre-#2216 information rather than a wrong number. + /// + Task> LoadIncidentOccurrencesAsync( + string serverKey, string metricName); + + /// + /// REPLACES the persisted occurrence set for one server/metric with (#2216): + /// rows in the map are upserted, and rows the store holds for this (server, metric) that are NOT in the + /// map are deleted. An empty map therefore clears the metric — which is how the falling edge is + /// recorded, so there is no separate clear method to forget to call. + /// + /// Replace-the-set rather than upsert-each-row because absence carries meaning: a fingerprint + /// with no events left in the window has a FINISHED incident, and leaving its row behind would make the + /// next incident on that fingerprint read as a continuation of the old one — an undercount reported + /// with a stale start time. The accumulator's staleness horizon is the backstop for rows a crash + /// stranded before any of this could run; it is not a substitute for deleting them here. + /// + /// Called once per delivered alert (cooldown-gated by construction), so it is a low-frequency + /// write like the watermarks above. Implementations should make the upsert-and-delete atomic: a + /// partially-applied set can strand a row that the staleness horizon then has to catch. + /// + Task SaveIncidentOccurrencesAsync( + string serverKey, string metricName, IReadOnlyDictionary states); } diff --git a/PerformanceMonitor.Alerting/IPostgresAlertReadAdapter.cs b/PerformanceMonitor.Alerting/IPostgresAlertReadAdapter.cs new file mode 100644 index 000000000..af6aa59b7 --- /dev/null +++ b/PerformanceMonitor.Alerting/IPostgresAlertReadAdapter.cs @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Alerting; + +/// +/// The PostgreSQL-only alert reads, kept OFF on purpose. +/// Extending the shared adapter would have forced Lite — which has no PostgreSQL target and no +/// PostgreSQL collectors — to implement three methods that can only ever return empty, leaving permanent +/// dead code in a shipping SKU to satisfy a contract it has no stake in. A separate adapter consulted only +/// for PostgreSQL targets mirrors what collection already does: CollectorCatalog.AppliesTo gates by +/// engine rather than having every definition claim every target. +/// Each method returns the CURRENT state of one Tier 0 outage predictor, already reduced to the +/// worst row per subject. Reduction belongs on the read side because the store queries can do it in one +/// pass, and because the evaluator should decide severity, not shape data. +/// +public interface IPostgresAlertReadAdapter +{ + /// + /// Freeze headroom per database — the worst (oldest) of XID and MultiXact age. + /// The most consequential condition PostgreSQL has: exhaust transaction IDs and the server stops + /// accepting writes entirely, and the remedy is hours of vacuuming, so the alert has to fire while + /// there is still time to act rather than at the cliff. + /// + Task> GetWraparoundRiskAsync( + int serverId, CancellationToken cancellationToken = default); + + /// + /// What is currently holding back the xmin horizon, and for how long it has been doing so. + /// Persistence is part of the read rather than the threshold because it is what separates a + /// chronic holder from a query that ran long, and only the first is worth waking someone for. + /// + Task GetXminHorizonAsync( + int serverId, CancellationToken cancellationToken = default); + + /// + /// Replication slots that are retaining WAL, with whether the pile is still growing. + /// Growth is the difference between a consumer that is behind and a volume filling in front of + /// you. Per-instance by nature — slots live on the writer. + /// + Task> GetReplicationSlotRiskAsync( + int serverId, CancellationToken cancellationToken = default); +} diff --git a/PerformanceMonitor.Alerting/IncidentOccurrenceAccumulator.cs b/PerformanceMonitor.Alerting/IncidentOccurrenceAccumulator.cs new file mode 100644 index 000000000..f4c623677 --- /dev/null +++ b/PerformanceMonitor.Alerting/IncidentOccurrenceAccumulator.cs @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Alerting; + +/// +/// One fingerprint's occurrence accounting as it is persisted between observations (#2216). +/// +/// +/// Occurrences of this fingerprint accumulated over the life of the incident. Monotonic until the +/// incident ends. +/// +/// +/// The window count this accumulator has ALREADY counted. Its own high-water mark, deliberately +/// separate from RollingCountAlertGate's watermark: that one advances only when an alert +/// fires (so an event arriving during a cooldown is still reported later), while this one advances +/// on every observation (so the same event is never counted twice). Sharing one watermark between +/// the two jobs would break whichever job lost the argument. +/// +/// +/// When this fingerprint's incident was first observed. Constant for the incident's life; a change +/// is how a consumer distinguishes a new incident from a continuing one. +/// +/// +/// When this row was last touched. Not display data — it is what makes a row's staleness decidable +/// (see ), so a row abandoned by a crash +/// cannot silently suppress the next incident on the same fingerprint. +/// +public readonly record struct IncidentOccurrenceState( + long TotalOccurrences, + int ObservedWindowCount, + DateTime IncidentStartedUtc, + DateTime LastObservedUtc); + +/// +/// Turns the per-fingerprint window gauge on into the +/// monotonic per-fingerprint total on (#2216). +/// +/// +/// The reported problem: OccurrenceCount counts events inside the groupers' rolling read +/// window, so it rises as events arrive and falls as they age out. A consumer that only ever sees +/// throttled deliveries — one per cooldown, per #1154's per-fingerprint cooldown — cannot recover +/// from a sequence of gauge readings how many events actually happened, because a reading of 3 +/// followed by a reading of 3 is indistinguishable from nothing-happened and +/// three-happened-while-three-aged-out. +/// +/// +/// +/// The accumulation is the gauge's RISE above what has already been counted, per fingerprint: +/// total += window - min(observedWindow, window). The inner min is the decay — as +/// events age out the gauge falls, and the already-counted mark has to fall with it or a later rise +/// back to the same level would read as no new events. This is the same decay +/// RollingCountAlertGate applies to its own watermark, for the same reason, and it is why the +/// two marks cannot be one mark. +/// +/// +/// +/// EXACTNESS BOUND, stated plainly because a counter that quietly undercounts is worse than a gauge +/// that is honestly a gauge: this is a lower bound on occurrences, exact whenever the read window +/// outlives the gap between OBSERVATIONS. The precise loss is one occurrence per event that ages out +/// of the window between two observations — a retirement and an arrival cancel in the gauge, so the +/// arrival is invisible. +/// +/// +/// +/// That is why the caller must observe on every SWEEP, not on every delivery. With a one-hour window +/// and a sweep measured in seconds, an event can only be missed if it arrives and ages out inside a +/// single sweep interval, which cannot happen for a window this long — so per-sweep observation is +/// exact in practice. Observing only at delivery time is NOT: any event the window retires during a +/// cooldown masks an arrival, so a long incident under sustained load undercounts by roughly the +/// number of events that aged out while the cooldown was suppressing delivery. That was the shipped +/// behavior in the first cut of #2216 and it made this class's own contract false; the review of +/// PR #2221 caught it. +/// +/// +/// +/// What no arrangement of a window gauge can recover is an occurrence that both arrives AND ages out +/// between two observations. Counting those would take an event-identity watermark at the collector, +/// which is a different feature. +/// +/// +/// +/// FIRST CONTACT counts the whole window. With no usable persisted state the accumulator cannot know +/// which of the events already in the window it would have counted before, so the total starts at +/// the window count and IncidentStartedUtc is stamped now. That is the same first-read +/// behavior the gauge has always had, and it is why the incident timestamp travels with the total: a +/// consumer seeing the total restart can check whether the incident restarted with it. +/// +/// +/// +/// STALENESS is a correctness rule, not housekeeping. State is written when an alert is delivered and +/// deleted when the condition clears, but a host that dies while an incident is active leaves a row +/// behind with no clearing sweep to remove it. If that row were trusted when the same fingerprint +/// recurred weeks later, its high observed-mark would decay to the new window count, the recurrence +/// would read as nothing new, and the total plus the incident timestamp would both describe an +/// incident that ended long ago — an undercount reported with a confident timestamp, which is worse +/// than no total at all. So a row untouched for longer than staleAfter is treated as absent. +/// The horizon belongs to the caller because only the caller knows its read window: any value at or +/// above the window works, since a row inside the window is describing the same events the gauge is. +/// +/// +/// +/// Pure function: no clock, no I/O, no store. The caller supplies nowUtc and persists +/// — matching , so both are testable +/// without a host. +/// +/// +public static class IncidentOccurrenceAccumulator +{ + /// + /// The input incidents with and + /// filled in. Same order, same count. + /// + /// + /// The state to persist for this (server, metric) — keyed by dedup fingerprint. This is the + /// COMPLETE set for the metric: a fingerprint the caller previously persisted and which is + /// absent here has no events left in the window, so its incident has ended and its row must be + /// removed rather than left to go stale. The store contract is therefore replace-the-set, not + /// upsert-each-row. + /// + /// + /// True when the caller should write. The accumulator observes every SWEEP, not only the sweeps that + /// deliver an alert (that is what makes the arithmetic exact), so a naive "always write" would put a + /// store round trip on every metric of every server on every sweep — on a 52-server fleet at a 30-second + /// cadence, thousands of writes an hour to record nothing. + /// + /// So this is true when something SUBSTANTIVE moved (a total, a mark, a fingerprint appearing or + /// disappearing) — and additionally on a HEARTBEAT, when the oldest surviving row has not been touched + /// for half the staleness horizon. The heartbeat is not an optimization detail: staleness is judged from + /// , so an incident whose gauge sits perfectly flat + /// for longer than the horizon would be judged stale and reset ITSELF. Half the horizon means a live + /// incident is always refreshed at least once before it could expire, at two writes per horizon rather + /// than one per sweep. + /// + public readonly record struct Result( + IReadOnlyList Incidents, + IReadOnlyDictionary States, + bool Changed); + + /// + /// Accumulates one observation of a metric's incidents. Returns the incidents carrying their + /// totals plus the full state set to persist. + /// + /// + /// This observation's incidents for ONE (server, metric), each carrying its window count in + /// . Null or empty means the window is empty: the + /// result clears the metric's state. + /// + /// + /// State loaded for the same (server, metric), or null when the host does not persist occurrence + /// state. A null/empty map degrades to "every delivery is a fresh incident", which reports the + /// total as equal to the window count — the pre-#2216 information, never a false zero. + /// + /// + /// Observation time. Stamped as IncidentStartedUtc for new incidents, as + /// LastObservedUtc for all of them, and used as the staleness reference. + /// + /// + /// How long a persisted row may go untouched before it is treated as absent (see class remarks). + /// Pass the groupers' read window or longer. or negative disables + /// the rule, trusting every row however old — available for tests that want the unguarded + /// behavior, not for hosts. + /// + public static Result Accumulate( + IReadOnlyList? incidents, + IReadOnlyDictionary? persisted, + DateTime nowUtc, + TimeSpan staleAfter) + { + int persistedCount = persisted?.Count ?? 0; + + if (incidents is not { Count: > 0 }) + { + /* Nothing in the window. Any persisted row is a finished incident, so the caller has a + write to do (a delete) exactly when it had state. */ + return new Result( + Array.Empty(), + new Dictionary(StringComparer.Ordinal), + Changed: persistedCount > 0); + } + + var states = new Dictionary(incidents.Count, StringComparer.Ordinal); + var accumulated = new List(incidents.Count); + bool substantive = false; + bool heartbeatDue = false; + + foreach (var incident in incidents) + { + /* A blank fingerprint cannot be keyed (AlertFingerprint returns null rather than an + empty key, and IncidentCooldown filters blanks defensively for the same reason). Pass + it through untouched instead of inventing a key that would pool unrelated incidents + under one total. */ + if (incident is null || string.IsNullOrEmpty(incident.DedupKey)) + { + accumulated.Add(incident!); + continue; + } + + /* Negative cannot arrive from the groupers, but clamping here means a bad count can only + fail to advance the total, never walk it backwards — the one property a monotonic + counter has to keep. */ + int window = incident.OccurrenceCount > 0 ? incident.OccurrenceCount : 0; + + /* The working set is consulted BEFORE the persisted one: a fingerprint repeated within a + single observation then accumulates against the mark its first appearance just set, + instead of counting the same window twice against a stale mark. */ + bool usable = states.TryGetValue(incident.DedupKey, out var prior); + if (!usable && persisted is not null && persisted.TryGetValue(incident.DedupKey, out prior)) + { + usable = !IsStale(prior, nowUtc, staleAfter); + } + + long total; + int observed; + DateTime started; + + if (usable) + { + /* Decay first (see class remarks), then count only the rise above the decayed mark. */ + observed = prior.ObservedWindowCount < window ? prior.ObservedWindowCount : window; + total = prior.TotalOccurrences + (window - observed); + observed = window; + started = prior.IncidentStartedUtc; + } + else + { + total = window; + observed = window; + started = nowUtc; + } + + accumulated.Add(incident with + { + TotalOccurrences = total, + IncidentStartedUtc = started, + }); + + if (!substantive) + { + /* A fingerprint the store has never seen, or one whose numbers moved. The touch timestamp is + deliberately NOT part of this comparison — it moves on every sweep, and treating that as a + change is exactly the always-write behavior the heartbeat exists to avoid. */ + substantive = !usable + || total != prior.TotalOccurrences + || observed != prior.ObservedWindowCount + || started != prior.IncidentStartedUtc; + } + + if (!heartbeatDue && usable && staleAfter > TimeSpan.Zero + && nowUtc - prior.LastObservedUtc >= HeartbeatFraction(staleAfter)) + { + heartbeatDue = true; + } + + states[incident.DedupKey] = new IncidentOccurrenceState(total, observed, started, nowUtc); + } + + /* A fingerprint that was persisted and is no longer in the window is a finished incident — its + removal is substantive even when every surviving fingerprint held steady. */ + if (!substantive && persistedCount != states.Count) + { + substantive = true; + } + + /* No keyable incidents at all: a write only if there is persisted state to clear. */ + if (states.Count == 0) + { + return new Result(accumulated, states, Changed: persistedCount > 0); + } + + return new Result(accumulated, states, Changed: substantive || heartbeatDue); + } + + /* Half the staleness horizon. A live incident is refreshed at least once before it could expire, and a + flat gauge costs two writes per horizon instead of one per sweep. Anything closer to the horizon risks + a sweep landing after expiry; anything much shorter gives the writes back. */ + private static TimeSpan HeartbeatFraction(TimeSpan staleAfter) => + TimeSpan.FromTicks(staleAfter.Ticks / 2); + + /* A row untouched for longer than the horizon is describing events that have left the window, so + it cannot inform this observation. Rows stamped in the FUTURE are trusted rather than + discarded: that is a clock going backwards (NTP correction, host migration), and treating a + live incident as stale would reset a total and its start time on a machine whose clock simply + stepped. A zero or negative horizon disables the rule. */ + private static bool IsStale(IncidentOccurrenceState state, DateTime nowUtc, TimeSpan staleAfter) + { + if (staleAfter <= TimeSpan.Zero) + { + return false; + } + + return nowUtc - state.LastObservedUtc > staleAfter; + } +} diff --git a/PerformanceMonitor.Alerting/PostgresAlertEvaluator.cs b/PerformanceMonitor.Alerting/PostgresAlertEvaluator.cs new file mode 100644 index 000000000..6551bae8b --- /dev/null +++ b/PerformanceMonitor.Alerting/PostgresAlertEvaluator.cs @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitor.Alerting; + +/// +/// Turns the three Tier 0 PostgreSQL outage predictors into fired alerts. +/// Deliberately NOT folded into . That class is 1500 lines shared with Lite +/// and is what SQL Server monitoring alerts through today; adding engine-specific branches inside it would +/// put the highest-blast-radius file on this branch in the path of every PostgreSQL change. This is a pure +/// function of (rows, settings) instead — no I/O, no state — so it is exhaustively testable and the host +/// keeps ownership of delivery and dedup. +/// Thresholds are constants here, on purpose, for this first cut. Every one is derived from +/// PostgreSQL's own mechanics rather than picked — see each constant — so there is no obvious knob a user +/// would set differently, and the product's stated position is to add configuration when it is genuinely +/// needed rather than speculatively (the same reasoning as having no collection-schedule settings). Making +/// them configurable means new columns on config_alert_settings, a migration, and Settings-window +/// work, and that is worth doing once someone wants a different number, not before. +/// +public static class PostgresAlertEvaluator +{ + /* Wraparound. The wall is 2 billion transactions, but the number that matters first is the server's + OWN autovacuum_freeze_max_age: at that age autovacuum force-starts a wraparound-prevention vacuum + whether or not a table is otherwise due, so crossing it means the server has begun defending itself. + Warning fires at 90% of it — before the forced vacuum, while a planned one is still an option. + Critical fires at 2x it, which on a stock 200-million setting is 400 million: comfortably clear of + the 2-billion stop, but far enough past the engine's own line to mean its defence is not keeping up. + Both are ratios of a setting the row carries, so a cluster tuned to 1.5 billion gets thresholds + scaled to its own configuration rather than to a constant that would never fire for it. */ + public const double WraparoundWarningFractionOfFreezeMaxAge = 0.9; + public const double WraparoundCriticalMultipleOfFreezeMaxAge = 2.0; + + /// + /// The 32-bit comparison space both counters age within — the same denominator the collector stores its + /// percentages against, so the alert and pct_toward_wraparound can never disagree. + /// + public const long WraparoundCeiling = 2_147_483_648L; + + /// + /// The absolute Critical arm, as a fraction of : PostgreSQL's own + /// vacuum_failsafe_age (1.6B by default) is ~74.5% of the space, and past it the engine abandons + /// cost limits and skips index cleanup to catch up. Matching the ladder + /// DarlingMcpPgWraparoundTools already classifies against. + /// This exists because the RELATIVE arm alone leaves Critical unreachable on exactly the clusters + /// most at risk: criticalAt = 2 x setting exceeds the 2^31 wall once the setting passes ~1.07B, and + /// the setting is tunable to 2B. A tuned cluster would have warned and then never escalated. + /// + public const double WraparoundCriticalFractionOfCeiling = 0.745; + + /* xmin horizon. 50 million transactions of held-back horizon is roughly where bloat becomes visible + rather than theoretical on a busy database. The persistence gate is what makes it actionable: a + holder seen in a majority of the window's observations is chronic, while one seen once is a query + that ran long, and only the first is worth waking anyone for. */ + public const long XminAgeWarningThreshold = 50_000_000; + public const double XminPersistenceFraction = 0.5; + + /* Replication slots. No byte threshold for the terminal states — `lost` and `unreserved` are failures + that have already happened, at any size. For a slot merely retaining WAL, 10 GB is the point where + an unbounded pile stops being noise on any volume worth monitoring; growth is what escalates it, + since max_slot_wal_keep_size defaults to -1 and nothing will stop it. */ + public const long SlotRetainedWalWarningBytes = 10L * 1024 * 1024 * 1024; + + /// Metric names, kept as constants because mute rules and history filtering match on them. + public const string WraparoundMetric = "PostgreSQL Wraparound Risk"; + public const string XminHorizonMetric = "PostgreSQL Vacuum Horizon Blocked"; + public const string SlotRetentionMetric = "PostgreSQL Replication Slot Retention"; + + /// + /// One evaluated finding, ready for the host to turn into an . Kept separate + /// from AlertOutcome so this library stays free of the host's mute/dedup concerns. + /// + /// Mute-rule and history key. + /// Graded per condition. + /// The specific database / holder / slot, for the host's dedup fingerprint. + /// Human-readable current value. + /// Human-readable threshold breached. + /// One-line body, server-name prefix excluded, matching the engine's contract. + /// For history charting. + /// Numeric twin. + public sealed record Finding( + string MetricName, + AlertSeverityLevel Severity, + string Subject, + string CurrentValue, + string ThresholdValue, + string ShortMessage, + double? NumericCurrentValue, + double? NumericThresholdValue); + + /// + /// Evaluates every predictor. Returns findings worst-first so a host that caps delivery keeps the ones + /// that matter. An empty list is the healthy case and must not be confused with "not evaluated". + /// + public static List Evaluate( + IReadOnlyList? wraparound, + PostgresXminHorizonAlertInfo? xmin, + IReadOnlyList? slots) + { + var findings = new List(); + + if (wraparound is not null) + { + foreach (var db in wraparound) + { + var finding = EvaluateWraparound(db); + if (finding is not null) + { + findings.Add(finding); + } + } + } + + var xminFinding = EvaluateXmin(xmin); + if (xminFinding is not null) + { + findings.Add(xminFinding); + } + + if (slots is not null) + { + foreach (var slot in slots) + { + var finding = EvaluateSlot(slot); + if (finding is not null) + { + findings.Add(finding); + } + } + } + + findings.Sort((a, b) => b.Severity.CompareTo(a.Severity)); + return findings; + } + + public static Finding? EvaluateWraparound(PostgresWraparoundAlertInfo db) + { + ArgumentNullException.ThrowIfNull(db); + + /* A non-positive setting would make every derived threshold zero and fire on every database forever, + so a missing or nonsensical value means "cannot judge" rather than "everything is critical". Judged + per counter now: a server can have a sane autovacuum_freeze_max_age and a broken multixact one. */ + var xidJudgeable = db.AutovacuumFreezeMaxAge > 0; + var multiJudgeable = db.AutovacuumMultixactFreezeMaxAge > 0; + if (!xidJudgeable && !multiJudgeable) + { + return null; + } + + /* Each counter against ITS OWN governing setting. Defaults differ by 2x (200M vs 400M), so grading + MultiXact age against the XID setting warned 2.2x premature. */ + var xid = xidJudgeable ? Grade(db.XidAge, db.AutovacuumFreezeMaxAge) : null; + var multi = multiJudgeable ? Grade(db.MultiXactAge, db.AutovacuumMultixactFreezeMaxAge) : null; + + /* The worse breach wins, and "worse" is the relative position, not the raw age — the only comparison + that means anything when the denominators differ. Severity first so a Critical MultiXact cannot be + hidden behind a merely-warning XID that happens to have a bigger number. */ + var multiWins = multi is not null + && (xid is null + || multi.Value.Severity > xid.Value.Severity + || (multi.Value.Severity == xid.Value.Severity + && db.MultiXactFractionOfSetting > db.XidFractionOfSetting)); + + var winner = multiWins ? multi : xid; + if (winner is null) + { + return null; + } + + var counter = multiWins ? "MultiXact" : "XID"; + var setting = multiWins ? db.AutovacuumMultixactFreezeMaxAge : db.AutovacuumFreezeMaxAge; + var settingName = multiWins ? "autovacuum_multixact_freeze_max_age" : "autovacuum_freeze_max_age"; + var age = multiWins ? db.MultiXactAge : db.XidAge; + var (severity, breached, viaCeiling) = winner.Value; + var critical = severity == AlertSeverityLevel.Critical; + var pctOfCeiling = 100.0 * age / WraparoundCeiling; + + return new Finding( + WraparoundMetric, + severity, + db.DatabaseName, + $"{counter} age {age:N0} in [{db.DatabaseName}]", + viaCeiling + ? $"{breached:N0} ({WraparoundCriticalFractionOfCeiling * 100:0.#}% of the 2^31 wraparound space)" + : $"{breached:N0} ({(critical ? "2x" : "90% of")} {settingName} {setting:N0})", + critical + ? $"[{db.DatabaseName}] {counter} age {age:N0} is {pctOfCeiling:0.#}% of the way to the " + + $"wraparound wall" + + (viaCeiling + ? " — past vacuum_failsafe_age, where PostgreSQL abandons its cost limits and skips " + + "index cleanup trying to catch up." + : $" and past twice {settingName} ({setting:N0}) — autovacuum's own wraparound defence " + + "is not keeping up.") + + " At the wall the server stops accepting writes, and the remedy is hours of vacuuming, so act now." + : $"[{db.DatabaseName}] {counter} age {age:N0} is approaching {settingName} ({setting:N0}), " + + "where autovacuum will force a wraparound-prevention vacuum. Vacuum on your schedule now " + + "rather than on its schedule later.", + age, + breached); + } + + /// + /// Grades one counter against its own setting, then OR-s in the absolute arm. Returns the severity, the + /// threshold actually breached, and whether it was the absolute one — so the message can say which. + /// + private static (AlertSeverityLevel Severity, long Breached, bool ViaCeiling)? Grade(long age, long setting) + { + var warnAt = (long)(setting * WraparoundWarningFractionOfFreezeMaxAge); + var criticalAt = (long)(setting * WraparoundCriticalMultipleOfFreezeMaxAge); + var ceilingCriticalAt = (long)(WraparoundCeiling * WraparoundCriticalFractionOfCeiling); + + /* The absolute arm is checked FIRST and independently: on a cluster tuned past ~1.07B the relative + criticalAt sits beyond the wall and can never be reached, which is precisely where an alert is + needed most. */ + if (age >= ceilingCriticalAt) + { + return (AlertSeverityLevel.Critical, ceilingCriticalAt, true); + } + + if (age >= criticalAt) + { + return (AlertSeverityLevel.Critical, criticalAt, false); + } + + return age >= warnAt ? (AlertSeverityLevel.Warning, warnAt, false) : null; + } + + public static Finding? EvaluateXmin(PostgresXminHorizonAlertInfo? xmin) + { + if (xmin is null || xmin.XminAge < XminAgeWarningThreshold) + { + return null; + } + + /* The persistence gate. Without it this fires on any long-running report, which is how an alert + earns a mute rule instead of a response. */ + var persistent = xmin.ObservationsTotal > 0 + && (double)xmin.ObservationsHeld / xmin.ObservationsTotal >= XminPersistenceFraction; + + if (!persistent) + { + return null; + } + + var subject = string.IsNullOrWhiteSpace(xmin.Identifier) + ? xmin.Source + : $"{xmin.Source}:{xmin.Identifier}"; + + return new Finding( + XminHorizonMetric, + AlertSeverityLevel.Warning, + subject, + $"{xmin.XminAge:N0} transactions held by {subject}", + $"{XminAgeWarningThreshold:N0} transactions, held in at least " + + $"{XminPersistenceFraction:P0} of observations", + $"Vacuum is reclaiming nothing cluster-wide: {subject} is holding the xmin horizon " + + $"{xmin.XminAge:N0} transactions back, in {xmin.ObservationsHeld} of " + + $"{xmin.ObservationsTotal} observations. {RemedyFor(xmin.Source)}" + + (string.IsNullOrWhiteSpace(xmin.Detail) ? string.Empty : $" ({xmin.Detail})"), + xmin.XminAge, + XminAgeWarningThreshold); + } + + /// + /// The four causes look identical by symptom and need completely different fixes, so the alert carries + /// the fix rather than making the reader go and find out which one it is. + /// + public static string RemedyFor(string? source) => source switch + { + "session" => "A backend is idle in transaction — end it, and look at why the application left it open.", + "replication_slot" => "An inactive replication slot is pinning it — if its consumer is gone, the slot must be dropped.", + "replication_slot_catalog" => "A logical slot's catalog_xmin is pinning it — its consumer is not confirming; check the CDC/logical pipeline.", + "standby_feedback" => "A standby's hot_standby_feedback is pinning it — a long query on the replica, or the feedback setting itself.", + "prepared_transaction" => "An orphaned prepared transaction is pinning it — COMMIT PREPARED or ROLLBACK PREPARED it.", + _ => "Unrecognized holder — read pg_stat_activity, pg_replication_slots and pg_prepared_xacts directly.", + }; + + public static Finding? EvaluateSlot(PostgresSlotAlertInfo slot) + { + var terminal = slot.WalStatus is "lost" or "unreserved"; + var growing = slot.RetainedWalGrowthBytes > 0; + var overBytes = slot.RetainedWalBytes >= SlotRetainedWalWarningBytes; + + if (!terminal && !overBytes) + { + return null; + } + + /* Grading. A terminal state has already failed. An inactive slot over the byte line and still + growing is the disk-fill emergency — unbounded by default, so nothing will stop it. Anything + else over the line is a warning. */ + var severity = terminal || (!slot.IsActive && growing && overBytes) + ? AlertSeverityLevel.Critical + : AlertSeverityLevel.Warning; + + var gb = slot.RetainedWalBytes / 1024.0 / 1024.0 / 1024.0; + var growthGb = slot.RetainedWalGrowthBytes / 1024.0 / 1024.0 / 1024.0; + + var body = slot.WalStatus switch + { + "lost" => $"Slot [{slot.SlotName}] is LOST — the WAL its consumer needs is gone and the slot " + + "cannot resume. It has to be recreated, and its consumer resynchronised.", + "unreserved" => $"Slot [{slot.SlotName}] is UNRESERVED — required WAL has already been removed. " + + "Its consumer will fail on next connect.", + _ when severity == AlertSeverityLevel.Critical => + $"Slot [{slot.SlotName}] is inactive and still accumulating: {gb:N1} GB retained, " + + $"up {growthGb:N1} GB. WAL retention is unbounded by default " + + "(max_slot_wal_keep_size = -1), so this fills the volume and stops the server. If the " + + "consumer is gone, drop the slot.", + _ => $"Slot [{slot.SlotName}] is retaining {gb:N1} GB of WAL" + + (growing ? $" and growing ({growthGb:N1} GB this window)" : " (not growing)") + + $", status {slot.WalStatus ?? "unknown"}, " + + (slot.IsActive ? "consumer active." : "consumer INACTIVE."), + }; + + return new Finding( + SlotRetentionMetric, + severity, + slot.SlotName, + terminal + ? $"wal_status = {slot.WalStatus} on [{slot.SlotName}]" + : string.Create(CultureInfo.InvariantCulture, $"{gb:N1} GB retained by [{slot.SlotName}]"), + terminal + ? "any (wal_status lost/unreserved is a failure that has already happened)" + : string.Create(CultureInfo.InvariantCulture, + $"{SlotRetainedWalWarningBytes / 1024.0 / 1024.0 / 1024.0:N0} GB retained"), + body, + slot.RetainedWalBytes, + terminal ? null : SlotRetainedWalWarningBytes); + } +} diff --git a/PerformanceMonitor.Alerting/PostgresAlertInfo.cs b/PerformanceMonitor.Alerting/PostgresAlertInfo.cs new file mode 100644 index 000000000..8d37edc72 --- /dev/null +++ b/PerformanceMonitor.Alerting/PostgresAlertInfo.cs @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; + +namespace PerformanceMonitor.Alerting; + +/// +/// Freeze headroom for one database. travels with the row +/// because the threshold that matters is the SERVER's own setting, not a constant: a cluster tuned to +/// 1.5 billion is in a very different place at an age of 400 million than a stock one at 200 million. +/// +/// The database this headroom belongs to. +/// Age of the oldest unfrozen XID (datfrozenxid). +/// Age of the oldest unfrozen MultiXact (datminmxid). +/// The server's autovacuum_freeze_max_age — the age at which +/// autovacuum force-starts a wraparound-prevention vacuum whether or not the table is otherwise due. +public sealed record PostgresWraparoundAlertInfo( + string DatabaseName, + long XidAge, + long MultiXactAge, + long AutovacuumFreezeMaxAge, + long AutovacuumMultixactFreezeMaxAge) +{ + /* WorstAge/WorstCounter used to pick by raw age and the evaluator graded the winner against + autovacuum_freeze_max_age whichever counter it was. That is wrong: the two counters have DIFFERENT + governing settings — 200,000,000 versus 400,000,000 by default — so a MultiXact age was graded against + a threshold half the size of its own and warned 2.2x premature, while the alert body printed + "autovacuum_freeze_max_age N" next to a WorstCounter saying MultiXact. Each counter is now graded + against its own setting and the worse RELATIVE breach wins, which is the only comparison that means + anything when the denominators differ. */ + + /// How far this database's XID age has gone toward its own freeze threshold, as a fraction. + public double XidFractionOfSetting => + AutovacuumFreezeMaxAge > 0 ? (double)XidAge / AutovacuumFreezeMaxAge : 0; + + /// The same for MultiXacts, against autovacuum_multixact_freeze_max_age. + public double MultiXactFractionOfSetting => + AutovacuumMultixactFreezeMaxAge > 0 ? (double)MultiXactAge / AutovacuumMultixactFreezeMaxAge : 0; + + /// Which counter is in the worse position RELATIVE to its own governing setting. + public bool MultiXactIsWorse => MultiXactFractionOfSetting > XidFractionOfSetting; + + /// The age of whichever counter is relatively worse. + public long WorstAge => MultiXactIsWorse ? MultiXactAge : XidAge; + + /// Which counter that is, so the message names the right remedy. + public string WorstCounter => MultiXactIsWorse ? "MultiXact" : "XID"; + + /// The setting that governs the relatively-worse counter — the one the message must quote. + public long WorstSetting => MultiXactIsWorse ? AutovacuumMultixactFreezeMaxAge : AutovacuumFreezeMaxAge; + + /// The name of that setting, so the body cannot contradict itself. + public string WorstSettingName => + MultiXactIsWorse ? "autovacuum_multixact_freeze_max_age" : "autovacuum_freeze_max_age"; +} + +/// +/// The current xmin-horizon holder, with how persistent it has been. +/// +/// Which of the four causes wins — session, replication_slot, +/// replication_slot_catalog, standby_feedback or prepared_transaction. They are indistinguishable by +/// symptom and need completely different fixes, so the alert must name it. +/// The specific holder (pid, slot name, gid, replica). +/// How far behind the horizon this holder is holding, in transactions. +/// How many collections in the window showed this source winning — the +/// chronic-versus-transient discriminator. +/// Collections in the window, so a caller can read the ratio. +/// Free-text state the collector captured (e.g. "state=idle in transaction"). +public sealed record PostgresXminHorizonAlertInfo( + string Source, + string? Identifier, + long XminAge, + int ObservationsHeld, + int ObservationsTotal, + string? Detail); + +/// +/// One replication slot's retention risk. +/// +/// The slot. +/// reserved / extended / unreserved / lost — the single most diagnostic column. +/// Whether anything is currently consuming it. +/// WAL held because of this slot. +/// Change across the window. Growth is what turns a large figure into +/// an emergency; a flat figure is a consumer that is behind but keeping pace. +/// When it went quiet, when the server reports it (PG17+). +public sealed record PostgresSlotAlertInfo( + string SlotName, + string? WalStatus, + bool IsActive, + long RetainedWalBytes, + long RetainedWalGrowthBytes, + DateTime? InactiveSince); diff --git a/PerformanceMonitor.Analysis/AnalysisContext.cs b/PerformanceMonitor.Analysis/AnalysisContext.cs index 405bed480..f285a9218 100644 --- a/PerformanceMonitor.Analysis/AnalysisContext.cs +++ b/PerformanceMonitor.Analysis/AnalysisContext.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; namespace PerformanceMonitor.Analysis; @@ -12,6 +13,15 @@ public class AnalysisContext public DateTime TimeRangeStart { get; set; } public DateTime TimeRangeEnd { get; set; } + /// + /// The host's stopping token, observed by the pass's store reads (#2299). Default + /// — a caller that does not plumb one (Lite, the + /// fact-inspection paths) keeps the prior behavior exactly, because every shutdown + /// classification requires this token to be SIGNALLED. Carried on the context rather than + /// on thirty method signatures because the context already reaches every pipeline stage. + /// + public CancellationToken CancellationToken { get; set; } + /// /// The monitored SERVER's UTC offset (SYSDATETIME − SYSUTCDATETIME), captured once at /// analysis start. / are in the diff --git a/PerformanceMonitor.Analysis/FactRemediation.cs b/PerformanceMonitor.Analysis/FactRemediation.cs index 5c889d7e2..41781cb8a 100644 --- a/PerformanceMonitor.Analysis/FactRemediation.cs +++ b/PerformanceMonitor.Analysis/FactRemediation.cs @@ -771,7 +771,8 @@ where the query sits in the list. */ LatestCpuPerExecUs: GetDouble(row, "latest_cpu_per_exec_us"), BestCpuPerExecUs: GetDouble(row, "best_cpu_per_exec_us"), RegressionFactor: GetDouble(row, "regression_factor"), - ReplicaRole: string.IsNullOrEmpty(replicaRole) ? null : replicaRole); + ReplicaRole: string.IsNullOrEmpty(replicaRole) ? null : replicaRole, + ParameterSensitivityCoFired: GetBool(row, "parameter_sensitivity_cofired")); } foreach (var key in order) @@ -816,6 +817,7 @@ where the query sits in the list. */ if (!string.IsNullOrEmpty(target.ReplicaRole)) sb.AppendLine($"-- measured on replica: {target.ReplicaRole}"); AppendSecondaryReplicaDisclosure(sb, target); + AppendParameterSensitivityCaution(sb, target); sb.AppendLine($"USE {QuoteName(target.Database)};"); sb.AppendLine($"EXEC sys.sp_query_store_force_plan @query_id = {target.QueryId}, @plan_id = {target.PlanId};"); sb.AppendLine(); @@ -895,6 +897,135 @@ private static void AppendSecondaryReplicaDisclosure(StringBuilder sb, ForcePlan sb.AppendLine("--"); } + /// + /// The #2138 gap-3 caution, emitted only for a target whose query ALSO carried the + /// PARAMETER_SENSITIVITY detector's plan-cache signature in the same analysis window (the + /// regressed_queries drill-down computes the flag with the detector's own thresholds, so this text + /// can never appear without the detector's evidence). Forcing under parameter sensitivity is the + /// one case where the recommendation itself can become the regression: the "best" and "regressed" + /// plans may each be right for DIFFERENT parameter populations, and pinning the cheap one hands the + /// other population the wrong plan permanently — quietly, because a forced plan no longer + /// recompiles away. So the gentler levers are named first, and the future auto-force bot treats + /// this flag as a hard gate: a flagged target is never auto-forced, it gets an investigate verdict. + /// Emits NOTHING when the flag is false, which keeps the render-stability golden meaningful — the + /// unflagged rendering is still the one it pins (the #1882 replica-disclosure discipline). + /// + private static void AppendParameterSensitivityCaution(StringBuilder sb, ForcePlanTarget target) + { + if (!target.ParameterSensitivityCoFired) + return; + + sb.AppendLine("--"); + sb.AppendLine("-- CAUTION: this query also shows the parameter-sensitivity signature in the plan cache"); + sb.AppendLine("-- (one cached plan whose per-execution cost varies >= 10x across parameter values). The"); + sb.AppendLine("-- regressed plan and the best plan may each be right for DIFFERENT parameter values, and"); + sb.AppendLine("-- forcing pins one shape for all of them -- the population that preferred the other plan"); + sb.AppendLine("-- inherits the wrong one permanently. Before forcing, consider the gentler levers first:"); + sb.AppendLine("-- update statistics on the tables involved and watch whether the plan settles, or on"); + sb.AppendLine("-- SQL Server 2022+ evaluate PSP optimization / a Query Store hint instead of a hard force."); + sb.AppendLine("-- If you do force, re-check the per-parameter cost spread afterwards, not just the average."); + } + + /// + /// THE force-plan policy gate (#2138): the named reasons this target must not be force-planned + /// without a human. Fills on the MCP surfaces + /// today, and is the function the Phase 1+ auto-force bot consults before acting — one + /// implementation, so what agents inspect is what the bot enforces. Deliberately built ONLY from + /// fields the persisted target carries; a gate that re-derives evidence at judgment time can + /// disagree with the evidence the finding displayed. + /// + /// parameter_sensitivity_cofired — the query's plan-cache history shows the PSP + /// signature (#2140); forcing pins ONE shape for every parameter value, so the "best" plan may be + /// the wrong plan for the population that preferred the other one. + /// secondary_replica_evidence — the regression was measured on a non-primary replica, + /// but the statement forces on the PRIMARY (#1882's disclosure, as data): acting on it changes the + /// primary's write workload on the strength of what a read-only replica did. + /// + /// + public static IReadOnlyList ForcePlanBlockers(ForcePlanTarget target) + { + if (target is null) + { + return Array.Empty(); + } + + var blockers = new List(); + if (target.ParameterSensitivityCoFired) + { + blockers.Add("parameter_sensitivity_cofired"); + } + + if (IsNonPrimaryReplicaRow(target.ReplicaRole)) + { + blockers.Add("secondary_replica_evidence"); + } + + return blockers; + } + + /// + /// The machine-first remediation projection (#2138) — see for + /// why it exists and why it is built at read time rather than persisted. Null when the action is + /// null or carries no force-plan targets (other verbs can gain shapes when a consumer needs them). + /// + public static StructuredRemediation? BuildStructuredRemediation(RemediationAction? action) + { + if (action?.Targets is not { Count: > 0 } targets) + { + return null; + } + + var structured = new List(targets.Count); + foreach (var t in targets) + { + var blockers = ForcePlanBlockers(t); + structured.Add(new StructuredForcePlanTarget( + t.Database, + t.QueryId, + t.PlanId, + t.LatestPlanHash, + t.BestPlanHash, + string.IsNullOrEmpty(t.ReplicaRole) ? null : t.ReplicaRole, + Eligible: blockers.Count == 0, + blockers, + new StructuredForcePlanEvidence( + t.RegressionFactor, + t.LatestCpuPerExecUs, + t.BestCpuPerExecUs, + t.ParameterSensitivityCoFired), + ForceSql: $"USE {QuoteName(t.Database)};{Environment.NewLine}" + + $"EXEC sys.sp_query_store_force_plan @query_id = {t.QueryId}, @plan_id = {t.PlanId};", + UnforceSql: $"USE {QuoteName(t.Database)};{Environment.NewLine}" + + $"EXEC sys.sp_query_store_unforce_plan @query_id = {t.QueryId}, @plan_id = {t.PlanId};", + VerifySql: BuildForcePlanVerifySql(t))); + } + + return new StructuredRemediation(action.FactKey, action.Action, structured); + } + + /// + /// The post-force verification an agent (or the future bot's self-review window) runs: did the force + /// STICK (is_forced_plan, force_failure_count, and the failure reason when it did not), + /// and what has the per-interval cost looked like SINCE — the same two questions the #2141 arc's + /// "re-check the spread, not just the average" advice asks, as runnable statements. + /// + private static string BuildForcePlanVerifySql(ForcePlanTarget t) + { + var nl = Environment.NewLine; + return + $"USE {QuoteName(t.Database)};{nl}" + + $"SELECT qsp.plan_id, qsp.is_forced_plan, qsp.force_failure_count, qsp.last_force_failure_reason_desc{nl}" + + $"FROM sys.query_store_plan AS qsp{nl}" + + $"WHERE qsp.query_id = {t.QueryId};{nl}" + + $"{nl}" + + $"SELECT TOP (24) rs.plan_id, rs.runtime_stats_interval_id, rs.count_executions, rs.avg_cpu_time, rs.avg_duration, rs.max_cpu_time{nl}" + + $"FROM sys.query_store_runtime_stats AS rs{nl}" + + $"JOIN sys.query_store_plan AS qsp{nl}" + + $" ON qsp.plan_id = rs.plan_id{nl}" + + $"WHERE qsp.query_id = {t.QueryId}{nl}" + + $"ORDER BY rs.runtime_stats_interval_id DESC;"; + } + /// /// The back-out counterpart of , emitted for any target /// the server attributed to a replica at all — including the primary's own rows, because the trap it @@ -1306,8 +1437,17 @@ public static string BuildModifyFileStatement(string database, string logicalFil "-- defaults there when omitted). Scope it with @replica_group_id to target that replica." + nl : string.Empty; + // #2138 gap 3: two lines, same discipline as the replica disclosure — this surface is + // PASTED, so the parameter-sensitivity warning matters here at least as much as in the + // preview, and it matters that it stays short enough to survive the paste. + var pspCaution = t.ParameterSensitivityCoFired + ? "-- CAUTION: parameter-sensitive (plan-cache cost varies >= 10x across parameter values)." + nl + + "-- Forcing pins ONE shape for every value; consider stats updates first (see the preview)." + nl + : string.Empty; + blocks.Add( disclosure + + pspCaution + $"USE {QuoteName(t.Database)};" + nl + $"EXEC sys.sp_query_store_force_plan @query_id = {t.QueryId}, @plan_id = {t.PlanId};"); } diff --git a/PerformanceMonitor.Analysis/RemediationAction.cs b/PerformanceMonitor.Analysis/RemediationAction.cs index 857ef1ce5..0150a44fc 100644 --- a/PerformanceMonitor.Analysis/RemediationAction.cs +++ b/PerformanceMonitor.Analysis/RemediationAction.cs @@ -276,4 +276,13 @@ public sealed record ForcePlanTarget( an execution input — see FactRemediation.ExtractPlanRegressionTargets for why naming a replica is not the same as being able to TARGET one. Appended last with a default so the positional construction sites and the persisted-action JSON round-trip stay source- and wire-compatible. */ - string? ReplicaRole = null); + string? ReplicaRole = null, + + /* #2138 gap 3: this query's hash ALSO carried the PARAMETER_SENSITIVITY detector's plan-cache + signature in the same analysis window (computed with the detector's own thresholds inside the + regressed_queries drill-down, so the flag can never disagree with the fact). Steers the caution + block in the force-plan preview — forcing the "best" plan on a parameter-sensitive query pins + one shape for ALL parameter values — and is the standing gate for the future auto-force bot: + a flagged target is never auto-forced, it gets an investigate verdict. Display + disclosure + only. Appended with a default for the same wire-compatibility reasons as ReplicaRole. */ + bool ParameterSensitivityCoFired = false); diff --git a/PerformanceMonitor.Analysis/StructuredRemediation.cs b/PerformanceMonitor.Analysis/StructuredRemediation.cs new file mode 100644 index 000000000..c0c0aade8 --- /dev/null +++ b/PerformanceMonitor.Analysis/StructuredRemediation.cs @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace PerformanceMonitor.Analysis; + +/// +/// The machine-first projection of a remediation (#2138): the same decisions the prose caution and +/// disclosure blocks carry, as NAMED FIELDS, because MCP consumers read these findings more than people +/// do — an agent should never have to regex a T-SQL comment to learn what a gate decided. Built at READ +/// time from the persisted s, deliberately never persisted itself: there is +/// no DTO mirror to forget (the #2140 review catch) and exactly one source of truth. Serialized property +/// names are snake_case by attribute because the MCP surfaces carry no naming policy — sibling fields are +/// snake_case only by their anonymous-object spellings. +/// Currently force-plan only (the #2138 arc); other verbs return null from the builder and can gain +/// their own shapes when an agent consumer needs them. +/// +public sealed record StructuredRemediation( + [property: JsonPropertyName("fact_key")] string FactKey, + [property: JsonPropertyName("verb")] string Verb, + [property: JsonPropertyName("force_plan_targets")] IReadOnlyList ForcePlanTargets); + +/// +/// One force-plan target, machine-first. The verdict half ( / +/// ) is the future auto-force bot's policy surface: it is filled by +/// FactRemediation.ForcePlanBlockers, the SAME function Phase 1+ will consult before acting — so +/// what agents inspect today is what the bot enforces tomorrow, and "never auto-force a flagged target" +/// is a testable data contract rather than a promise in comments. The artifacts are split +/// ( / / ) because agents compose +/// steps; a single blob makes them parse it apart. All three are ADVISORY — the read-only MCP surfaces +/// never execute anything. +/// +public sealed record StructuredForcePlanTarget( + [property: JsonPropertyName("database")] string Database, + [property: JsonPropertyName("query_id")] long QueryId, + [property: JsonPropertyName("plan_id")] long PlanId, + [property: JsonPropertyName("latest_plan_hash")] string? LatestPlanHash, + [property: JsonPropertyName("best_plan_hash")] string? BestPlanHash, + [property: JsonPropertyName("replica_role")] string? ReplicaRole, + [property: JsonPropertyName("eligible")] bool Eligible, + [property: JsonPropertyName("blockers")] IReadOnlyList Blockers, + [property: JsonPropertyName("evidence")] StructuredForcePlanEvidence Evidence, + [property: JsonPropertyName("force_sql")] string ForceSql, + [property: JsonPropertyName("unforce_sql")] string UnforceSql, + [property: JsonPropertyName("verify_sql")] string VerifySql); + +/// +/// The numbers behind a target's verdict — only what the persisted target actually carries, nothing +/// re-derived or estimated at read time. +/// +public sealed record StructuredForcePlanEvidence( + [property: JsonPropertyName("regression_factor")] double RegressionFactor, + [property: JsonPropertyName("latest_cpu_per_exec_us")] double LatestCpuPerExecUs, + [property: JsonPropertyName("best_cpu_per_exec_us")] double BestCpuPerExecUs, + [property: JsonPropertyName("parameter_sensitivity_cofired")] bool ParameterSensitivityCoFired); diff --git a/PerformanceMonitor.Collectors/AzureSweepScope.cs b/PerformanceMonitor.Collectors/AzureSweepScope.cs new file mode 100644 index 000000000..d12a36626 --- /dev/null +++ b/PerformanceMonitor.Collectors/AzureSweepScope.cs @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; + +namespace PerformanceMonitor.Collectors; + +/// +/// Which databases one Azure SQL DB registration's per-database sweep covers (#2220). +/// +/// The defect this decides. A database-scoped collector on Azure SQL DB used to enumerate +/// master unconditionally and sweep EVERY online database on the logical server, storing all of it +/// under the one server_id of whichever registration ran the sweep. On a logical server holding N +/// separately-registered databases that is N registrations × N databases: N² collection, and every +/// registration's stored history contaminated with its siblings'. The field report is byte-identical +/// deadlock graphs and the same top query appearing under six unrelated identities, which is exactly what +/// reading one database's rows six times looks like. +/// +/// Why it happened rather than being an oversight. Two parts of the product hold incompatible +/// ideas of what an Azure SQL DB registration IS, and both are deliberate. The enumeration assumes one +/// registration = one LOGICAL SERVER, which is the shape #857 was written for. Identity assumes one +/// registration = one DATABASE: server_id is the hash of host[:database][:RO], so registering +/// two databases on one server is the supported way to get two identities — and the Azure query_store path +/// (#1836) needs a per-database connection anyway. Nothing reconciled the two, so the second shape silently +/// behaved like the first, N times over. +/// +/// The rule. A registration that NAMES a database is a registration OF that database, and its +/// sweep covers exactly that one. Only a registration that names none — or names master, which on +/// Azure SQL DB is where a catalog-less connection lands — is a registration of the logical SERVER, and only +/// that one enumerates. +/// +/// Shared rather than duplicated per host on purpose: both runners had their own private copy of this +/// predicate, and a scoping rule that disagrees between Lite and Darling is the same class of bug as the one +/// being fixed. One implementation, pinned by both test suites. +/// +public static class AzureSweepScope +{ + /// + /// The single database this registration names, or an EMPTY list when it names none. + /// + /// Empty is not "no databases" — it means "this registration is of the logical server, so the + /// caller must enumerate". The two are told apart by the count, which is why this returns a list rather + /// than a nullable string: the caller's next step is a list either way. + /// + /// master counts as naming none. A connection string with no Initial Catalog lands in + /// master on Azure SQL DB, so treating it as a named database would scope every catalog-less + /// registration to a database that holds none of the user's data. + /// + public static List OwnDatabaseOrEmpty(string? initialCatalog) + { + if (string.IsNullOrEmpty(initialCatalog) + || string.Equals(initialCatalog, "master", StringComparison.OrdinalIgnoreCase)) + { + return new List(); + } + + return new List { initialCatalog }; + } +} diff --git a/PerformanceMonitor.Collectors/CollectorCatalog.cs b/PerformanceMonitor.Collectors/CollectorCatalog.cs index c113c7629..acccba731 100644 --- a/PerformanceMonitor.Collectors/CollectorCatalog.cs +++ b/PerformanceMonitor.Collectors/CollectorCatalog.cs @@ -39,6 +39,7 @@ public static class CollectorCatalog DatabaseStateCollector.Instance, TraceFlagsCollector.Instance, DatabaseScopedConfigCollector.Instance, + QueryStoreHealthCollector.Instance, SessionStatsCollector.Instance, SessionSummaryStatsCollector.Instance, WaitingTasksCollector.Instance, @@ -62,6 +63,18 @@ public static class CollectorCatalog AgDatabaseReplicaStatesCollector.Instance, PlanCorrectionCollector.Instance, PvsStatsCollector.Instance, + /* PostgreSQL definitions. They live in the same catalog as the T-SQL ones on purpose: the + schema generator walks this list to create tables, and one store can hold data from both + engines, so splitting the catalog per engine would fragment DDL generation. Dispatch is + kept honest by the engine gate in AppliesTo(definition, target). */ + PgWaitStatsCollector.Instance, + PgStatementStatsCollector.Instance, + PgWraparoundStatsCollector.Instance, + PgXminHorizonCollector.Instance, + PgReplicationSlotsCollector.Instance, + PgAutovacuumStatsCollector.Instance, + PgIoStatsCollector.Instance, + PgBlockingCollector.Instance, }; /// Name → definition, for the by-name target-gate lookup. Built once from . @@ -78,7 +91,35 @@ public static class CollectorCatalog /// gated) so a typo surfaces as the dispatch switch's "Unknown collector" rather than a silent skip. /// public static bool AppliesTo(string collectorName, CollectorTargetInfo target) => - s_byName.TryGetValue(collectorName, out var definition) ? definition.AppliesTo(target) : true; + s_byName.TryGetValue(collectorName, out var definition) ? AppliesTo(definition, target) : true; + + /// + /// The full dispatch gate: a definition runs only when its + /// matches the target's + /// AND its own + /// gate passes. Both runners call this rather + /// than AppliesTo directly, so a definition written in one engine's dialect can never be + /// sent to the other — the individual definitions stay free to reason only about the hosting + /// flavour and version floors within their own engine. + /// Both defaults are , so this is behaviour- + /// identical to the previous direct AppliesTo call for every definition and target that + /// exists today. + /// + public static bool AppliesTo(ICollectorSchemaInfo definition, CollectorTargetInfo target) => + definition.TargetEngine == target.Engine && definition.AppliesTo(target); + + /// + /// The engine half of the gate alone, by name — for callers that want to drop a foreign-dialect + /// collector BEFORE dispatch rather than let it run and report zero rows. Darling's sweep uses + /// this so a wrong-engine collector produces no collection_log row at all: a gated run + /// would otherwise be recorded as SUCCESS, and with two engines in one catalog that would mean a + /// fake success per foreign collector per cycle on every server. + /// An unknown name returns true (not filtered), matching + /// , so a typo still surfaces as the dispatch + /// switch's "unknown collector" rather than a silent disappearance. + /// + public static bool EngineMatches(string collectorName, CollectorTargetInfo target) => + !s_byName.TryGetValue(collectorName, out var definition) || definition.TargetEngine == target.Engine; /// /// True when 's query carries the deliberate short diff --git a/PerformanceMonitor.Collectors/CollectorContext.cs b/PerformanceMonitor.Collectors/CollectorContext.cs index 0a505d5a8..7f0194ff1 100644 --- a/PerformanceMonitor.Collectors/CollectorContext.cs +++ b/PerformanceMonitor.Collectors/CollectorContext.cs @@ -125,6 +125,41 @@ public sealed class CollectorContext /// public bool CapturePlanXml { get; init; } + /// + /// When true, the query_store payload leaves query_sql_text NULL and the host is responsible + /// for resolving statement text through instead + /// (#2150). Default false, which keeps the text inline exactly as it ships today. + /// + /// Why the text has to leave that projection. The payload selects it inside a + /// TOP ... WITH TIES ... ORDER BY last_execution_time, and a Top-N Sort carries every output + /// column through the sort while reading ALL of its input before emitting a row — so choosing the rows + /// to ship materialized nvarchar(max) text for the entire qualifying set. Measured on a + /// purpose-built Azure SQL DB store with #2210's plan XML already gone, the ONE column being the only + /// difference: time-to-first-row 4.67s against 0.45s at 1,505 rows, 5.02s against 0.57s at 4,037. + /// Neither knob bounds it — TOP (500) measured the same as TOP (50000), and wall time was + /// flat from a 4 MB to a 256 MB client budget, because the server finishes before the client sees a + /// byte. + /// + /// Why this is a flag and not simply removed, which is the part worth being careful about: + /// Lite stores that text inline in DuckDB and its grid reads it from there, so nulling the column + /// unconditionally would blind Lite. Gated, the emitted column keeps its ORDINAL either way — same + /// shape, one value — which is the pattern the version-gated columns in this collector already use, and + /// it means a host that has not built text storage keeps working unchanged. + /// + public bool FetchQueryTextSeparately { get; init; } + + /// + /// Whether this cycle's query_store payload includes the OPEN Query Store interval (#2312). Default + /// true — today's behavior for every caller that never touches it. When false, the interval + /// pre-filter adds i.end_time <= SYSUTCDATETIME(), shipping only CLOSED intervals — which + /// are immutable and therefore final on first collection, while the open interval's cumulative + /// snapshot is the whole re-read bill on a big primary (40–110 s per run measured). Set PER ITEM by + /// the hosts' per-database watermark delegates from + /// , like ; + /// mutable (not init) for exactly that reason. Only the query_store payload reads it. + /// + public bool IncludeOpenInterval { get; set; } = true; + /// /// When true (the default — today's behavior), the default_trace_events collector records /// Object:Created/Altered/Deleted schema-change (DDL) events; when false its @@ -145,6 +180,23 @@ public sealed class CollectorContext /// public IReadOnlyList? PerfmonCounterOverride { get; init; } + /// + /// Host override for the per-item text byte budget (#2164), in BYTES. Null keeps the definition's + /// own — which is what Lite passes, + /// so its behavior is unchanged. Darling supplies this from the store's operator knob. + /// + /// Why a knob at all: the budget's job is bounding memory, and the compile-time 64 MB was + /// sized for a same-region client. Over a cross-region link the same 64 MB is a MINUTE of the + /// monitored server holding one query open draining to the client (ASYNC_NETWORK_IO), which is + /// tenant-visible on small hardware — the 2026-08-10 field case. A smaller budget costs catch-up + /// latency, never data, because #1960's boundary-group completion makes every cut resumable. + /// + /// Composes multiplicatively with the host's fleet sweep width — peak transient is + /// approximately (concurrent sweeps) × (this budget) — which is why both are operator knobs on + /// the same store rung and why their UI hints name each other (#2170). + /// + public int? TextByteBudgetOverride { get; init; } + /// /// Result of the definition's enumeration probe (see /// ICollectorDefinition.BuildEnumerationProbe), set by the host between enumeration @@ -167,6 +219,60 @@ public sealed class CollectorContext /// public bool PerItemTextBudgetExceeded { get; set; } + /// + /// Milliseconds spent waiting for ExecuteReaderAsync to return for the item just read (#2164), + /// set by the host around the open. Splits a batch's server time into the part the client cannot + /// influence and the part it can: + /// + /// For a multi-statement batch like query_store's staged shape, ADO.NET returns the reader only + /// when the first ROWSET is available — so this number spans every preceding non-rowset statement (the + /// SELECT … INTO #pm_qs_slice aggregate) plus the final select's time-to-first-row. The + /// remaining time, drain, is row streaming the client's byte budget and read loop actually govern. + /// + /// Why it exists: cutting the byte budget 64 MB → 12 MB on a production server moved bytes 5x and + /// the batch clock ~0%, which said the dominant term is upstream of shipping — but the single blended + /// sql: number could not prove WHICH statement, so any next fix would have been a guess. Zero + /// when the host does not measure it (Lite today), so a zero must never be read as "instant". + /// + public long PerItemOpenMs { get; set; } + + /// + /// Milliseconds the item's watermark refresh took (#2164), set by the host when it runs one. This is NOT + /// server think-time or streaming — for query_store it is a STORE read (and on the catch-up/adaptive + /// path a store write too), yet the driver's sql: stopwatch starts before it. Measured so it can + /// be subtracted rather than silently inflating drain, which would corrupt the one number this + /// instrumentation exists to make trustworthy. Zero when the host runs no per-item watermark. + /// + public long PerItemWatermarkMs { get; set; } + + /// + /// Milliseconds the item's separate plan-XML fetch took (#2312 investigation), set by the host that + /// runs one (Darling; Lite has no separate fetch and leaves it zero). The fetch is INSIDE the driver's + /// per-item sql: stopwatch but is neither open nor drain — it is its own query against + /// sys.query_store_plan, and on a database with a huge Query Store catalog it can dominate the + /// whole item (ayr-01: a 0-row closed-only cycle still cost 298s, and the blended number could not say + /// where). Measured so drain stops absorbing it, exactly the #2164 argument one seam further down. + /// + public long PerItemPlanFetchMs { get; set; } + + /// + /// Milliseconds the item's separate statement-text fetch took (#2150's fetch, split out for the #2312 + /// investigation) — same contract as : inside sql:, not drain, + /// zero when the host runs no separate fetch. + /// + public long PerItemTextFetchMs { get; set; } + + /// + /// The item's row-STREAMING time: the driver's blended per-item total minus the phases that are not + /// streaming (, , + /// , ). Lives here rather than at + /// the log site so the subtraction has exactly one definition and a test can pin the shipped arithmetic + /// instead of a copy of it. Clamped at zero: the phases are measured on separate stopwatches, so tiny + /// skew must never surface as negative drain. + /// + public long DrainMsFrom(long itemSqlMs) => + Math.Max(0, itemSqlMs - PerItemOpenMs - PerItemWatermarkMs - PerItemPlanFetchMs - PerItemTextFetchMs); + /// /// Cumulative text bytes the budgeted read actually materialized for the item just read (#1960), /// reset and written alongside . Read by the host purely diff --git a/PerformanceMonitor.Collectors/CollectorDefinitionBase.cs b/PerformanceMonitor.Collectors/CollectorDefinitionBase.cs index 4719fd9bc..985abced4 100644 --- a/PerformanceMonitor.Collectors/CollectorDefinitionBase.cs +++ b/PerformanceMonitor.Collectors/CollectorDefinitionBase.cs @@ -6,6 +6,7 @@ * Licensed under the MIT License. See LICENSE file in the project root for full license information. */ +using System; using System.Collections.Generic; using System.Data.Common; using System.Threading; @@ -47,6 +48,17 @@ public abstract class CollectorDefinitionBase : ICollectorDefinition public virtual int? PerItemTextByteBudget => null; + public virtual TimeSpan? PerItemWallClockBudget => null; + + public virtual CollectorTargetEngine TargetEngine => CollectorTargetEngine.SqlServer; + + /// + /// Target-shape gating WITHIN this definition's engine — hosting flavour, version floors, msdb + /// reach. Overrides do not need to consider : the engine check is + /// composed on top by , + /// which is what the runners call, so a T-SQL definition can never reach a non-SQL-Server target + /// even though this returns true by default. + /// public virtual bool AppliesTo(CollectorTargetInfo target) => true; public virtual bool YieldsOnLockTimeout => false; diff --git a/PerformanceMonitor.Collectors/CollectorDeltaCalculator.cs b/PerformanceMonitor.Collectors/CollectorDeltaCalculator.cs index c490cb144..e5630f234 100644 --- a/PerformanceMonitor.Collectors/CollectorDeltaCalculator.cs +++ b/PerformanceMonitor.Collectors/CollectorDeltaCalculator.cs @@ -21,22 +21,39 @@ namespace PerformanceMonitor.Collectors; /// public class CollectorDeltaCalculator : ICollectorDeltaCalculator { + /// + /// The gap past which a cached baseline is treated as too stale to subtract from, shared by every + /// delta call site in this assembly so the policy cannot drift collector by collector. + /// + /// One hour, chosen from measurement rather than intuition. The previous value — 300 s, + /// hard-coded at all 41 call sites — sat almost exactly on the fleet's median sweep gap, so it + /// fired during ordinary operation instead of after the restarts it was written for. Measured over + /// 99,717 consecutive perfmon gaps across 52 production servers and 7 days: p50 299 s, + /// p90 580 s, p99 830 s, p99.9 1,190 s, max 2,514 s. The share of ordinary gaps each candidate + /// rejects: 300 s → 50.0%, 600 s → 8.3%, 900 s → 0.6%, 1,800 s → 0.0%, 3,600 s → 0.0%. + /// Half of every delta collector's output was a fabricated zero (#2233, #2234). + /// + /// An hour clears the observed maximum with room to spare while still catching what the + /// guard is actually for: a server unreachable for hours, or a baseline restored from a store row + /// old enough that attributing its whole accrual to one interval would read as a spike. Note the + /// direction of the harm this replaces — a rejected gap returns 0, and a 0 is indistinguishable + /// from a genuinely idle interval, so the guard did not merely lose data, it invented quiet. + /// + public const int DefaultMaxGapSeconds = 3600; + /// /// How far back a restart re-seed reads when restoring baselines from a host's own store. /// - /// A correctness bound before it is a performance one. Every delta call site in this - /// assembly passes maxGapSeconds: 300 — all 36 of them — and the gap policy in - /// discards any baseline older than that and returns 0 - /// instead. A seed row from outside a ~5-minute window therefore cannot produce a delta no matter - /// what it cost to find, so reading it is work whose result is thrown away. + /// Fifteen minutes, and since became an hour this window + /// — not the gap policy — is what bounds restart recovery. It used to be the other way round: at a + /// 300 s policy every seed row older than five minutes was rejected on arrival, so most of this + /// window was work whose result was thrown away. Now every row it returns can produce a real + /// delta, which is the point. /// - /// Fifteen minutes rather than five: the seed runs at startup and the first collection lands - /// some seconds after it, so the window needs slack over the policy it serves, and a window that - /// merely errs generous costs nothing (a row the policy rejects seeds a baseline that is - /// immediately re-based, which is what an unseeded key does anyway). It still sits well inside one - /// store chunk, which is the property that matters: it lets TimescaleDB exclude the rest of a - /// multi-hundred-GB hypertable rather than scan every chunk on a 30-second command timeout — the - /// field failure in #1772. + /// Left at fifteen minutes deliberately. It sits well inside one store chunk, which is the + /// property that matters: it lets TimescaleDB exclude the rest of a multi-hundred-GB hypertable + /// rather than scan every chunk on a 30-second command timeout — the field failure in #1772. + /// Widening it to chase the hour-long policy would trade that back. /// public static readonly TimeSpan SeedLookback = TimeSpan.FromMinutes(15); @@ -54,6 +71,43 @@ public static DateTime SeedCutoff() /// private readonly ConcurrentDictionary>> _cache = new(); + /// + /// When this (server, collector) pair was last looked at, and the look before that: + /// serverId -> collectorName -> (current pass, previous pass). + /// + /// Needed by to answer "did this counter series begin + /// since we last looked?" for a key that has no history of its own. The PREVIOUS pass is the useful + /// one, and it is tracked separately from the per-key timestamps because a brand-new key has none. + /// + /// Advanced only when the collection time actually CHANGES, which is what makes it stable + /// across the many rows of one pass: a collector calls in per row, and if the first row rolled the + /// window forward every later row in the same pass would compare against its own pass and see a zero + /// gap — quietly disabling the credit for every row but the first. + /// + private readonly ConcurrentDictionary> _passes = new(); + + /// + /// Rolls the (server, collector) pass window forward when is a new + /// pass, and returns the previous pass — the boundary a series age is measured against. + /// + private DateTime? PreviousPass(int serverId, string collectorName, DateTime? collectionTime) + { + if (!collectionTime.HasValue) + { + return null; + } + + var byCollector = _passes.GetOrAdd(serverId, _ => new ConcurrentDictionary()); + var updated = byCollector.AddOrUpdate( + collectorName, + _ => (collectionTime.Value, null), + (_, existing) => existing.Current == collectionTime.Value + ? existing + : (collectionTime.Value, existing.Current)); + + return updated.Previous; + } + /// /// Removes all cached entries for a server (e.g., when the server tab is closed). /// Next collection will re-seed from database if needed. @@ -61,6 +115,10 @@ public static DateTime SeedCutoff() public void ClearServer(int serverId) { _cache.TryRemove(serverId, out _); + /* The pass window goes with the baselines it is interpreted against. Left behind, a re-added + server's first pass would measure a series age against a look from before it was removed and + credit a full counter to an interval that never happened. */ + _passes.TryRemove(serverId, out _); } /// @@ -70,6 +128,12 @@ public void ClearServer(int serverId) /// Gap detection: if collectionTime and maxGapSeconds are provided and the gap since the /// last cached value exceeds maxGapSeconds, returns 0 to avoid inflated deltas after restarts. /// Thread-safe via atomic AddOrUpdate. + /// All three of those zeros mean "no delta is knowable here", which a long cannot say + /// any other way — and none of them is the same claim as "this interval was idle". Use + /// when a caller has to tell them apart: the reported + /// interval is 0 in exactly these cases and non-zero whenever the delta is real, so a stored + /// (delta, interval) pair of (0, 0) reads as unknown while (0, n) reads as genuinely idle. That + /// pairing is what makes a zero interpretable downstream (#2234). /// public long CalculateDelta(int serverId, string collectorName, string key, long currentValue, DateTime? collectionTime = null, int maxGapSeconds = 0) @@ -85,7 +149,24 @@ public long CalculateDelta(int serverId, string collectorName, string key, long /// public long CalculateDeltaWithInterval(int serverId, string collectorName, string key, long currentValue, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + => Core(serverId, collectorName, key, currentValue, seriesAgeSeconds: null, out intervalSeconds, + collectionTime, maxGapSeconds); + + /// + public long CalculateDeltaWithSeriesAge(int serverId, string collectorName, string key, long currentValue, + int? seriesAgeSeconds, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + => Core(serverId, collectorName, key, currentValue, seriesAgeSeconds, out intervalSeconds, + collectionTime, maxGapSeconds); + + private long Core(int serverId, string collectorName, string key, long currentValue, + int? seriesAgeSeconds, out int intervalSeconds, DateTime? collectionTime, int maxGapSeconds) { + /* Read (and roll) the pass window BEFORE touching the key cache: the Add path below needs the + previous pass, and a key that is new has no timestamp of its own to supply it. Always called, + even when no series age was passed, so the window advances on every pass rather than only on + the passes that happen to use the hint. */ + var previousPass = PreviousPass(serverId, collectorName, collectionTime); + var serverCache = _cache.GetOrAdd(serverId, _ => new ConcurrentDictionary>()); var collectorCache = serverCache.GetOrAdd(collectorName, _ => new ConcurrentDictionary()); @@ -95,11 +176,36 @@ public long CalculateDeltaWithInterval(int serverId, string collectorName, strin collectorCache.AddOrUpdate( key, /* Add: first time seeing this key — store the baseline only and return 0. - All callers track cumulative counters (perfmon, wait stats, file IO, etc.). */ + All callers track cumulative counters (perfmon, wait stats, file IO, etc.). + + #2235 exception, and the ONLY case where a first sighting can report real work: when the + caller supplies a series age younger than the gap since our previous pass, the counter + demonstrably STARTED inside that gap, so its whole value accrued there and its baseline + was 0 rather than currentValue. Without this a recompiling plan reports 0 forever — it + presents a new plan_handle, hence a new key, on nearly every sighting. A real interval is + reported alongside it because this delta IS knowable; the (0, 0) pairing stays reserved + for the cases that genuinely are not. */ _ => { delta = 0; interval = 0; + + if (seriesAgeSeconds.HasValue && seriesAgeSeconds.Value >= 0 + && collectionTime.HasValue && previousPass.HasValue) + { + var gap = (collectionTime.Value - previousPass.Value).TotalSeconds; + + /* Bounded by the same gap policy as the reset branch: past it, attributing a whole + cumulative counter to one interval is the inflated spike that guard exists to + prevent — a plan compiled during an hour-long outage is not an hour of work in + the next minute. */ + if (gap > 0 && (maxGapSeconds <= 0 || gap <= maxGapSeconds) && seriesAgeSeconds.Value <= gap) + { + delta = currentValue; + interval = (int)gap; + } + } + return (currentValue, collectionTime); }, /* Update: compute delta atomically */ @@ -121,9 +227,23 @@ the wall-clock span this delta accrued over. */ ? (int)(collectionTime.Value - previous.Timestamp.Value).TotalSeconds : 0; - delta = currentValue < previous.Value - ? 0 /* counter reset (plan cache eviction/re-entry) — not real new work */ - : currentValue - previous.Value; + if (currentValue < previous.Value) + { + /* Counter reset (plan cache eviction/re-entry): the work between the two readings + is unknowable, not zero. Report no interval either, so the pair stays honest — + a 0 delta over a REAL interval is a claim that nothing happened for that long, + and this is the one case where that claim would be false. That invariant + (interval 0 <=> no delta knowable) is what lets a reader tell a fabricated zero + from an idle one, and every consumer already maps 0 to NULL via + NULLIF(sample_interval_seconds, 0). */ + delta = 0; + interval = 0; + } + else + { + delta = currentValue - previous.Value; + } + return (currentValue, collectionTime); }); diff --git a/PerformanceMonitor.Collectors/CollectorScheduleDefaults.cs b/PerformanceMonitor.Collectors/CollectorScheduleDefaults.cs index 8861a3ebe..885d4e142 100644 --- a/PerformanceMonitor.Collectors/CollectorScheduleDefaults.cs +++ b/PerformanceMonitor.Collectors/CollectorScheduleDefaults.cs @@ -78,6 +78,11 @@ the XE session on the monitored servers and disabling it DROPS the session there retention mirror the sibling XE collectors. */ ["long_query_completions"] = new(1, 30, DefaultEnabled: false), ["database_scoped_config"] = new(0, 30), + /* #2319: hourly, NOT the config family's on-load cadence — actual_state, readonly_reason and + current_storage_size_mb change BY THEMSELVES (the cap-hit transition to READ_ONLY is the point + of collecting this), and an on-load snapshot would miss the transition until the next + reconnect. One cheap row per database per hour; 30 days to match its config siblings. */ + ["query_store_health"] = new(60, 30), ["trace_flags"] = new(0, 30), ["running_jobs"] = new(5, 7), ["database_size_stats"] = new(60, 90), @@ -115,5 +120,59 @@ on one time axis without resampling. The fast-moving leading indicators an opera this collector adds is the slow CONSEQUENCE, and 90 days is the window that shows a PVS trend against the database-growth trend it explains. */ ["pvs_stats"] = new(60, 90), + + /* PostgreSQL. Same cadence and horizon as wait_stats, deliberately: it is the same kind of + signal (cumulative counters, delta-on-write) read at the same resolution, and matching the + two means a mixed-engine store shows one time axis without resampling. + + Enabled by default despite being Aurora-only, because the cost of it being wrong is + nothing: on a SQL Server target the engine gate drops it before dispatch with no log row, + and on non-Aurora PostgreSQL its own AppliesTo returns false. It only ever runs where + there is something to read. */ + ["pg_wait_stats"] = new(1, 30), + + /* Same cadence and horizon as query_stats, its SQL Server counterpart. */ + ["pg_statement_stats"] = new(1, 30), + + /* Freeze headroom moves slowly — autovacuum shifts it in steps, not continuously — so a + 5-minute read is ample, and 90 days is the horizon that shows an age trend against the + table-growth trend that usually explains it. Cheap: a handful of rows from a shared + catalog. */ + ["pg_wraparound_stats"] = new(5, 90), + + /* Per-minute, unlike its wraparound sibling: an xmin holder is the FAST-moving leading + indicator, and the thing an operator wants is the session or slot that appeared minutes + ago, before it has cost anything. At most five rows a cycle. 30 days matches the other + per-minute health series. */ + ["pg_xmin_horizon"] = new(1, 30), + + /* Per-minute: retained WAL on an abandoned slot grows at whatever rate the server generates + WAL, which on a busy writer fills a volume in hours, not days. 90 days of retention because + the question after an incident is "how long was that slot orphaned", and that answer has to + outlive the incident. A handful of rows a cycle. */ + ["pg_replication_slots"] = new(1, 90), + + /* Hourly, unlike every other PostgreSQL collector, because this one is a per-database + fan-out and on PostgreSQL that means one CONNECTION per database per cycle — a database + count that is fine hourly would be a connection storm per minute. Autovacuum also works on + the scale of minutes to hours, so a per-minute read would mostly re-record the same state. + 90 days matches database_size_stats: the useful reading is a bloat trend, not a spot check. */ + ["pg_autovacuum_stats"] = new(60, 90), + + /* Back to per-minute: pg_stat_io is cluster-wide (one connection, no fan-out) and returned + 25-37 rows per snapshot on the fleet, so the cost is the same order as pg_wait_stats. 30 days + matches the other rate collectors — the value here is correlating an I/O shift against a + deployment, which is a days-to-weeks question, not a quarterly one. */ + ["pg_io_stats"] = new(1, 30), + + /* Per-minute, and the cadence IS the limitation. Unlike SQL Server's blocked-process report — + where the engine itself records an event when blocking crosses a threshold, so evidence exists + whether or not anyone looked — PostgreSQL records nothing. This is a sample, so blocking shorter + than one minute is simply not seen. A minute is the floor worth paying for: + pg_blocking_pids() takes ShareLock on the lock manager partitions per call, and the whole + point of a blocking monitor is to not become the contention it reports. 30 days matches the + other per-minute series, and is the horizon that answers "is this the same chain every + Monday at open" — the question that turns a one-off into a pattern. */ + ["pg_blocking"] = new(1, 30), }; } diff --git a/PerformanceMonitor.Collectors/CollectorTargetEngine.cs b/PerformanceMonitor.Collectors/CollectorTargetEngine.cs new file mode 100644 index 000000000..e3426a002 --- /dev/null +++ b/PerformanceMonitor.Collectors/CollectorTargetEngine.cs @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +namespace PerformanceMonitor.Collectors; + +/// +/// Which database engine a definition's query dialect targets, and which engine a monitored server +/// actually is. The gate that keeps the two from being crossed is +/// — a definition +/// only runs when its equals the target's +/// . +/// Both sides default to , so every existing definition and every +/// existing target keep their present behaviour exactly: this enum adds a dimension, it does not +/// change a single dispatch decision until something opts into a different engine. +/// This is deliberately about SQL dialect and catalog surface, not about hosting. +/// Azure SQL DB, Managed Instance, and RDS for SQL Server are all — their +/// differences are already carried as flags on , because they run +/// the same T-SQL against the same DMVs. A different value here means the query text itself would +/// not parse on the other engine. +/// +public enum CollectorTargetEngine +{ + /// + /// Microsoft SQL Server in any of its hosted shapes (box, Azure SQL DB, Azure SQL Managed + /// Instance, AWS RDS for SQL Server). The default for definitions and targets alike. + /// + SqlServer = 0, + + /// + /// PostgreSQL, including Amazon Aurora PostgreSQL. Definitions marked with this read + /// pg_stat_* / pg_catalog surfaces and are never dispatched at a SQL Server target. + /// + PostgreSql = 1, +} diff --git a/PerformanceMonitor.Collectors/CollectorTargetFault.cs b/PerformanceMonitor.Collectors/CollectorTargetFault.cs new file mode 100644 index 000000000..447620adf --- /dev/null +++ b/PerformanceMonitor.Collectors/CollectorTargetFault.cs @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +namespace PerformanceMonitor.Collectors; + +/// +/// What went wrong on a monitored target, named in terms the collection loop cares about rather than +/// in one engine's error numbers. The runners classify a driver exception into one of these so the +/// same handling — SKIPPED vs YIELDED vs PERMISSIONS vs ERROR, reconnect or not — works for any +/// engine. +/// Today's SQL Server behaviour is the definition of correct here: this enum exists to +/// preserve it while making the same decisions expressible for PostgreSQL, whose failures arrive as +/// five-character SQLSTATEs rather than integer error numbers. +/// +public enum CollectorTargetFault +{ + /// + /// Not a recognized fault class. Handled as an ERROR, which is the correct default: an + /// unclassified failure must be loud, not quietly swallowed. + /// + Unclassified = 0, + + /// + /// The monitoring login lacks a grant this collector needs. Expected and benign for a + /// least-privilege login — logged as PERMISSIONS and skipped, never as an error. + /// SQL Server: 229, 297, 300, 8189. PostgreSQL: SQLSTATE 42501 insufficient_privilege. + /// + Permissions, + + /// + /// A deliberate short lock timeout fired instead of the collector queueing behind a blocking + /// chain. Evidence about the monitored server, not a monitoring failure, so it is excluded from + /// error rates and health bands. SQL Server: 1222, and only for a collector that declares + /// . PostgreSQL: SQLSTATE 55P03 + /// lock_not_available. + /// + LockTimeoutYield, + + /// + /// A server-side capture session the collector reads from is not there. SQL Server: an Extended + /// Events session (297/15151). PostgreSQL has no direct equivalent today, but the concept + /// generalizes to any absent server-side capture the collector expects. + /// + SessionMissing, + + /// + /// The object or function being read does not exist on this target — the normal answer when a + /// feature is version-gated or an extension was never created. Distinct from + /// because the remedy differs: install or upgrade, versus grant. + /// PostgreSQL: SQLSTATE 42P01 undefined_table / 42883 undefined_function. + /// SQL Server: 208 Invalid object name. + /// + ObjectMissing, + + /// + /// The feature exists but is switched off on this target, so it returns an error rather than an + /// empty set. Aurora does this for aurora_ccm_status() when the cluster cache manager is + /// disabled, and for Optimized Reads statistics when the feature is off — both raise rather than + /// return zero rows, which a naive collector records as a failure every cycle. + /// + FeatureDisabled, + + /// + /// The command exceeded its timeout. SQL Server surfaces this as error number -2. PostgreSQL: + /// SQLSTATE 57014 query_canceled, which is also what a statement_timeout produces. + /// + CommandTimeout, + + /// + /// The connection itself failed or died. Forces a reconnect and re-probe rather than just + /// failing the one collector. SQL Server: severity class 20 and above. PostgreSQL: SQLSTATE + /// class 08 connection_exception, and 57P01 admin_shutdown / 57P02 + /// crash_shutdown. + /// + ConnectionFatal, +} diff --git a/PerformanceMonitor.Collectors/CollectorTargetInfo.cs b/PerformanceMonitor.Collectors/CollectorTargetInfo.cs index e9283e37d..e3da92835 100644 --- a/PerformanceMonitor.Collectors/CollectorTargetInfo.cs +++ b/PerformanceMonitor.Collectors/CollectorTargetInfo.cs @@ -15,6 +15,17 @@ namespace PerformanceMonitor.Collectors; /// public sealed class CollectorTargetInfo { + /// + /// Which database engine this target actually is. Defaults to + /// , so every target the probes classify today — + /// and every bare new CollectorTargetInfo() in a test — keeps its present behaviour. + /// A definition is only dispatched when its + /// matches this; see . + /// The SQL Server hosting flags below ( and friends) are meaningful + /// only when this is . + /// + public CollectorTargetEngine Engine { get; init; } = CollectorTargetEngine.SqlServer; + /// True when the target is Azure SQL Database (engine edition 5). public bool IsAzureSqlDb { get; init; } @@ -50,4 +61,45 @@ public sealed class CollectorTargetInfo /// silently gates collection off. /// public bool HasMsdbAccess { get; init; } = true; + + /* ---- PostgreSQL facts. Meaningful only when Engine is PostgreSql; the SQL Server flags above + are correspondingly meaningless on a Postgres target. Kept flat alongside them for now + because there are few; if this list grows much further it wants its own sub-object rather + than more parallel properties. ---- */ + + /// + /// PostgreSQL major version (16, 17); 0 when unknown. Derived from + /// server_version_num / 10000 rather than parsing version() text, whose formatting + /// has changed across releases. + /// Definitions gate on this for the real 16→17 breaks: pg_stat_bgwriter loses five + /// columns to pg_stat_checkpointer and deletes two, pg_stat_statements renames + /// blk_*_time to shared_blk_*_time, and pg_stat_progress_vacuum renames two + /// columns. A fleet spanning both majors hits all of them. + /// + public int PostgresMajorVersion { get; init; } + + /// + /// The full server_version_num (e.g. 160011, 170007), for the minor-version gates a major + /// alone cannot express — aurora_stat_resource_usage() needs Aurora 16.9+/17.5+ and is + /// absent on 17.4, so a major-only check would call a function that is not there. + /// + public int PostgresVersionNum { get; init; } + + /// + /// True when the target is Amazon Aurora PostgreSQL, detected by the presence of + /// aurora_version in pg_proc. + /// This gates a large proprietary surface that stock PostgreSQL does not have at all, most + /// importantly aurora_stat_system_waits() — cumulative wait counters, which core PostgreSQL + /// simply does not provide in any version. + /// + public bool IsAurora { get; init; } + + /// + /// True when the target is in recovery, i.e. a read replica (pg_is_in_recovery()). + /// Not a routing hint: on Aurora every reader is a separate instance with its own + /// statistics — its own pg_stat_statements contents and its own wait profile — so a reader + /// is a distinct monitoring identity worth collecting from, not a shadow of the writer. Some + /// surfaces are writer-only and gate off this. + /// + public bool IsInRecovery { get; init; } } diff --git a/PerformanceMonitor.Collectors/DatabaseSizeStatsCollector.cs b/PerformanceMonitor.Collectors/DatabaseSizeStatsCollector.cs index 50b3f9797..2af09cf48 100644 --- a/PerformanceMonitor.Collectors/DatabaseSizeStatsCollector.cs +++ b/PerformanceMonitor.Collectors/DatabaseSizeStatsCollector.cs @@ -73,7 +73,12 @@ CREATE TABLE #file_space ( database_id int NOT NULL, file_id int NOT NULL, - used_size_mb decimal(19,2) NULL + used_size_mb decimal(19,2) NULL, + /* #2169: the file's CURRENT size, read in-database alongside SpaceUsed. sys.master_files.size is the + size recorded at configuration time and does NOT track autogrowth for tempdb, so a grown tempdb + reported used (current) against total (startup) and produced a used% above 100. Every database + benefits — master_files can lag any autogrowth — but tempdb is where it is guaranteed to. */ + current_size_mb decimal(19,2) NULL ); /* #1851: every failure below used to die in an empty CATCH, so a database that was mid-restore or @@ -106,11 +111,12 @@ ORDER BY BEGIN BEGIN TRY SET @sql = N'EXECUTE ' + QUOTENAME(@db_name) + N'.sys.sp_executesql N'' -INSERT #file_space (database_id, file_id, used_size_mb) +INSERT #file_space (database_id, file_id, used_size_mb, current_size_mb) SELECT DB_ID(), df.file_id, - CONVERT(decimal(19,2), FILEPROPERTY(df.name, N''''SpaceUsed'''') * 8.0 / 1024.0) + CONVERT(decimal(19,2), FILEPROPERTY(df.name, N''''SpaceUsed'''') * 8.0 / 1024.0), + CONVERT(decimal(19,2), df.size * 8.0 / 1024.0) FROM sys.database_files AS df;'';'; EXECUTE sys.sp_executesql @sql; @@ -138,7 +144,11 @@ INSERT @probe_failures (name, error_text) file_name = mf.name, physical_name = mf.physical_name, total_size_mb = - CONVERT(decimal(19,2), mf.size * 8.0 / 1024.0), + /* #2169: in-database current size when the probe got it, else master_files. Both operands of the + used% the viewer computes then come from the SAME snapshot, so used can no longer exceed total + on a database whose files grew since configuration (tempdb, always). A probe that failed leaves + this NULL and falls back — worse precision, never a wrong ratio direction. */ + CONVERT(decimal(19,2), COALESCE(fs.current_size_mb, mf.size * 8.0 / 1024.0)), used_size_mb = fs.used_size_mb, auto_growth_mb = diff --git a/PerformanceMonitor.Collectors/EnumeratedCollectorDriver.cs b/PerformanceMonitor.Collectors/EnumeratedCollectorDriver.cs index 1d5a9675a..e399393f9 100644 --- a/PerformanceMonitor.Collectors/EnumeratedCollectorDriver.cs +++ b/PerformanceMonitor.Collectors/EnumeratedCollectorDriver.cs @@ -203,9 +203,21 @@ public static class EnumeratedCollectorDriver /// public const string UnnamedItem = "(unnamed item)"; + /// + /// What an item abandoned at its wall-clock budget reports (#2150), {0} = the budget, already + /// rendered by . Shared so the enumerated loop and each host's per-database + /// (Azure SQL DB) loop say the same thing, and so the wording an operator greps for is one string. + /// + public const string WallClockBudgetErrorFormat = + "abandoned after exceeding its {0} per-database wall-clock budget; the range was not " + + "collected and will be re-read next cycle (the watermark did not advance)"; + /// parsed once (CA1863) — the const stays the greppable, pinnable text. private static readonly CompositeFormat s_probeFailureNote = CompositeFormat.Parse(ProbeFailureNoteFormat); + /// parsed once (CA1863). + private static readonly CompositeFormat s_wallClockBudget = CompositeFormat.Parse(WallClockBudgetErrorFormat); + /// parsed once (CA1863). private static readonly CompositeFormat s_unreadableFailureSet = CompositeFormat.Parse(UnreadableFailureSetErrorFormat); @@ -382,6 +394,13 @@ loosely and the malformed set still reports itself as a probe failure. */ /// surface the row-cap / byte-budget warning here — the context truncation signal persists until the /// next item's read resets it, so reading it post-flush is equivalent). /// Per-item skip log, invoked when one item fails (offline DB, timeout, permissions). + /// + /// Wall-clock ceiling for one item's watermark refresh plus its read (#2150), from + /// ICollectorDefinition.PerItemWallClockBudget. Null (every collector but query_store) leaves + /// the loop exactly as it was. Exceeding it abandons THAT item as a per-item failure and continues; + /// the WRITE is deliberately outside the budget, because abandoning a flush that is already underway + /// would trade a slow cycle for a partially-written one. + /// public static async Task RunAsync( IReadOnlyList items, Func? perItemWatermark, @@ -389,7 +408,8 @@ public static async Task RunAsync( Func, CancellationToken, Task> writeBatch, Action onItemComplete, Action onItemError, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + TimeSpan? perItemBudget = null) { var totalRows = 0; long sqlMs = 0; @@ -402,17 +422,24 @@ public static async Task RunAsync( List? batch = null; long itemSqlMs = 0; var sqlSlice = Stopwatch.StartNew(); + + /* #2150: the item's wall-clock budget. Null for every collector that declares none, in which + case itemToken IS cancellationToken and this loop is what it always was. */ + using var itemBudget = StartItemBudget(perItemBudget, cancellationToken); + var itemToken = itemBudget?.Token ?? cancellationToken; try { /* Per-database watermark refresh (query_store): its cutoff — including the 24h catch-up clamp — is computed HERE, inside the loop, so each database's commit advances only its - own watermark and an abort loses no other database's intervals. */ + own watermark and an abort loses no other database's intervals. Inside the budget on + purpose: it is a store read, and a store that has stopped answering is exactly the kind + of stall the budget exists to bound. */ if (perItemWatermark is not null) { - await perItemWatermark(item, cancellationToken); + await perItemWatermark(item, itemToken); } - batch = await readItem(item, cancellationToken); + batch = await readItem(item, itemToken); } catch (OutOfMemoryException) { @@ -422,6 +449,18 @@ not a routine one-database skip. There is no cross-item accumulator to clear — handler; the host classifies the run ERROR. */ throw; } + catch (Exception ex) when (ItemBudgetExpired(itemBudget, cancellationToken)) + { + /* #2150: THIS item ran out of wall clock. Reported as a per-item failure so the sweep + continues, which is the entire point — one database must not be able to starve the rest. + Ahead of the generic catch because a cancelled command does not reliably arrive as an + OperationCanceledException, so the generic filter cannot be trusted to claim it; and the + token check is what keeps a real shutdown out of this arm. `ex` is deliberately dropped + in favour of the budget message: whatever the provider raised on cancellation is an + artifact of HOW it was cancelled, not why. */ + _ = ex; + onItemError(item, ItemBudgetException(perItemBudget!.Value)); + } catch (Exception ex) when (ex is not OperationCanceledException) { /* One item failing is routine (an offline/mid-restore database, a permissions oddity, a @@ -460,4 +499,64 @@ truncation signal is still this item's — the next read resets it. */ return new EnumeratedRunResult(totalRows, sqlMs, storageMs); } + + /// + /// Starts one item's wall-clock budget (#2150), or returns null when the definition declares none — + /// which is every collector but query_store, so the unbounded path stays byte-identical. + /// + /// A LINKED source, so host shutdown still cancels the item promptly; the timer only adds a + /// second reason to stop. Callers must pass to the work + /// and dispose the source when the item ends. + /// + public static CancellationTokenSource? StartItemBudget(TimeSpan? budget, CancellationToken outer) + { + if (budget is not TimeSpan span || span <= TimeSpan.Zero) + { + return null; + } + + var source = CancellationTokenSource.CreateLinkedTokenSource(outer); + source.CancelAfter(span); + return source; + } + + /// + /// Did THIS item's budget fire, as opposed to the host shutting down? + /// + /// The distinction is the whole point: a budget expiry is a per-item fault to be reported and + /// skipped, while shutdown must propagate and stop the sweep. Shutdown deliberately WINS the ambiguous + /// case — if both are cancelled, this returns false and the exception propagates — because misreading a + /// shutdown as a per-item skip would have the loop keep collecting through it. + /// + /// Classifying on the TOKENS rather than the exception type is deliberate too. Cancelling a + /// SqlClient command mid-execute does not reliably surface as : + /// it commonly arrives as a provider exception ("Operation cancelled by user"), and which one depends on + /// whether the cancellation landed during the open or during the drain. The tokens know; the exception + /// type does not. + /// + public static bool ItemBudgetExpired(CancellationTokenSource? itemBudget, CancellationToken outer) => + itemBudget is not null + && itemBudget.IsCancellationRequested + && !outer.IsCancellationRequested; + + /// The exception handed to the per-item error hook for an abandoned item, so both loops report + /// it identically. A because that is what it is — and because the hosts' + /// hooks log ex.Message, which carries the whole explanation. + public static TimeoutException ItemBudgetException(TimeSpan budget) => + new(string.Format(CultureInfo.InvariantCulture, s_wallClockBudget, DescribeBudget(budget))); + + /// + /// Renders a budget the way the operator set it, choosing the unit rather than fixing one. + /// + /// Fixing it at minutes was the first cut, and a scratch harness caught it reporting a + /// sub-minute budget as "0.0-minute" — a message that names no number at all, on the one line an + /// operator has to work from. The shipped value is 10 minutes so it would never have shown in the + /// field; a test asserting the message merely CONTAINS "wall-clock budget" would not have shown it + /// either. Small values are the ones a person types while diagnosing, which is exactly when the + /// message matters most. + /// + public static string DescribeBudget(TimeSpan budget) => + budget < TimeSpan.FromMinutes(1) + ? string.Format(CultureInfo.InvariantCulture, "{0:0.###}-second", budget.TotalSeconds) + : string.Format(CultureInfo.InvariantCulture, "{0:0.#}-minute", budget.TotalMinutes); } diff --git a/PerformanceMonitor.Collectors/FileIoStatsCollector.cs b/PerformanceMonitor.Collectors/FileIoStatsCollector.cs index f6db5cde7..3b049dcfa 100644 --- a/PerformanceMonitor.Collectors/FileIoStatsCollector.cs +++ b/PerformanceMonitor.Collectors/FileIoStatsCollector.cs @@ -184,14 +184,14 @@ public override void WritePayload(Row row, ICollectorRowWriter writer, Collector { /* "{database}|{file}" delta key and the eight group names are the parity contract. */ var deltaKey = $"{row.DatabaseName}|{row.FileName}"; - var deltaReads = context.Deltas.CalculateDelta(context.ServerId, "file_io_reads", deltaKey, row.NumOfReads, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWrites = context.Deltas.CalculateDelta(context.ServerId, "file_io_writes", deltaKey, row.NumOfWrites, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaReadBytes = context.Deltas.CalculateDelta(context.ServerId, "file_io_read_bytes", deltaKey, row.ReadBytes, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWriteBytes = context.Deltas.CalculateDelta(context.ServerId, "file_io_write_bytes", deltaKey, row.WriteBytes, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaStallReadMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_read", deltaKey, row.IoStallReadMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaStallWriteMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_write", deltaKey, row.IoStallWriteMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaStallQueuedReadMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_queued_read", deltaKey, row.IoStallQueuedReadMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaStallQueuedWriteMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_queued_write", deltaKey, row.IoStallQueuedWriteMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaReads = context.Deltas.CalculateDelta(context.ServerId, "file_io_reads", deltaKey, row.NumOfReads, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWrites = context.Deltas.CalculateDelta(context.ServerId, "file_io_writes", deltaKey, row.NumOfWrites, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaReadBytes = context.Deltas.CalculateDelta(context.ServerId, "file_io_read_bytes", deltaKey, row.ReadBytes, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWriteBytes = context.Deltas.CalculateDelta(context.ServerId, "file_io_write_bytes", deltaKey, row.WriteBytes, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaStallReadMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_read", deltaKey, row.IoStallReadMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaStallWriteMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_write", deltaKey, row.IoStallWriteMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaStallQueuedReadMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_queued_read", deltaKey, row.IoStallQueuedReadMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaStallQueuedWriteMs = context.Deltas.CalculateDelta(context.ServerId, "file_io_stall_queued_write", deltaKey, row.IoStallQueuedWriteMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.DatabaseName) diff --git a/PerformanceMonitor.Collectors/ICollectorDefinition.cs b/PerformanceMonitor.Collectors/ICollectorDefinition.cs index f6ac74634..b33dd922d 100644 --- a/PerformanceMonitor.Collectors/ICollectorDefinition.cs +++ b/PerformanceMonitor.Collectors/ICollectorDefinition.cs @@ -6,6 +6,7 @@ * Licensed under the MIT License. See LICENSE file in the project root for full license information. */ +using System; using System.Collections.Generic; using System.Data.Common; using System.Threading; @@ -102,6 +103,28 @@ so the declaring collectors are enumerable off CollectorCatalog.All without the /// int? PerItemTextByteBudget { get; } + /// + /// WALL-CLOCK ceiling for one per-database unit of work — the watermark refresh, the command, and the + /// whole drain (#2150). Null (the common case) = unbounded, exactly as before. + /// + /// Why the command timeout is not this. CommandTimeout bounds the wait for a network + /// read, and SqlClient RESETS it on every read that arrives — so a result set that trickles rows + /// continuously never trips it, however long it takes in total. A 100-minute read under a 30-second + /// timeout is the documented behaviour, not a bug, which is why the field report in #2150 shows six + /// per-database passes of up to 99.8 minutes against a 30-second timeout. + /// + /// What exceeding it means. The item is abandoned and reported as a per-item FAILURE, and + /// the cycle continues to the next database — the same treatment an offline database gets. Nothing is + /// silently dropped: a collector with a watermark did not advance it, so the abandoned range is simply + /// re-read next cycle. For query_store that failure also feeds the #2111 consecutive-failure + /// count, so the window NARROWS on the next pass instead of retrying the same impossible width — a + /// bound that converges rather than one that just repeats. + /// + /// Host-enforced rather than definition-enforced, unlike the byte budget: only the host owns the + /// cancellation token and the loop, and the point is to bound the definition's own read. + /// + TimeSpan? PerItemWallClockBudget { get; } + /// /// Builds the T-SQL (and any bound parameters) for this cycle. Constant for most collectors; /// target-aware definitions branch on and diff --git a/PerformanceMonitor.Collectors/ICollectorDeltaCalculator.cs b/PerformanceMonitor.Collectors/ICollectorDeltaCalculator.cs index cedd6e5f5..014d49e97 100644 --- a/PerformanceMonitor.Collectors/ICollectorDeltaCalculator.cs +++ b/PerformanceMonitor.Collectors/ICollectorDeltaCalculator.cs @@ -26,4 +26,38 @@ long CalculateDelta(int serverId, string collectorName, string key, long current long CalculateDeltaWithInterval(int serverId, string collectorName, string key, long currentValue, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0); + + /// + /// As , but for a counter whose SERIES can restart under a + /// brand-new key — telling "this key is new to us" apart from "this counter is new to the world" + /// (#2235). + /// + /// The defect this exists for. query_stats keys its deltas on + /// plan_handle, which changes on every recompile, so a churning plan presents a fresh key on + /// nearly every sighting and the first sighting of a key reports 0. On a plan-churning instance that + /// discards most of the server's CPU: a query Datadog measured at ~43% of an 8-vCPU box read as 18 + /// executions and 2,824 ms over 168 hours. Worse, it is INVISIBLE — the reset branch below reports + /// interval = 0 precisely so a reader can tell a fabricated zero from an idle one, but that + /// branch needs the SAME key to reappear lower, and a recompile never does. Same class of harm as + /// the 300-second gap policy #2233 replaced: it did not merely lose data, it invented quiet. + /// + /// Why the caller cannot decide this itself. Only the implementation knows whether a key + /// is new, and only the caller knows how old the underlying series is. So the caller passes the age + /// and the implementation combines it with its own record of when it last looked. + /// + /// is how long ago the counter series began, measured on + /// the SOURCE's clock at collection time — an age, deliberately, not a timestamp. A + /// creation_time from a DMV is in the monitored server's local time while collection times are + /// UTC, so comparing the two directly is a timezone bug on every server that is not UTC; + /// DATEDIFF(SECOND, qs.creation_time, GETDATE()) is evaluated where both clocks are the same + /// and travels safely. Pass null when unknown, which behaves exactly as + /// . + /// + /// Default-implemented as a pass-through so existing implementers — including test doubles — + /// keep compiling and keep today's behaviour until they opt in. + /// + long CalculateDeltaWithSeriesAge(int serverId, string collectorName, string key, long currentValue, + int? seriesAgeSeconds, out int intervalSeconds, DateTime? collectionTime = null, int maxGapSeconds = 0) + => CalculateDeltaWithInterval(serverId, collectorName, key, currentValue, out intervalSeconds, + collectionTime, maxGapSeconds); } diff --git a/PerformanceMonitor.Collectors/ICollectorSchemaInfo.cs b/PerformanceMonitor.Collectors/ICollectorSchemaInfo.cs index 920f0ca89..f9d70101e 100644 --- a/PerformanceMonitor.Collectors/ICollectorSchemaInfo.cs +++ b/PerformanceMonitor.Collectors/ICollectorSchemaInfo.cs @@ -22,6 +22,16 @@ public interface ICollectorSchemaInfo /// Collector name as used in schedules and collection logs (e.g. "wait_stats"). string Name { get; } + /// + /// The engine whose dialect this definition's query text is written in. Defaults to + /// — a default interface implementation rather + /// than a required member, so the existing definitions and the test doubles that implement this + /// interface directly need no change. A definition is only dispatched at a target whose + /// matches; see + /// . + /// + CollectorTargetEngine TargetEngine => CollectorTargetEngine.SqlServer; + /// Destination table; hosts prepend their standard prefix columns when writing. string TargetTable { get; } diff --git a/PerformanceMonitor.Collectors/ITargetProvider.cs b/PerformanceMonitor.Collectors/ITargetProvider.cs new file mode 100644 index 000000000..da7ff34f6 --- /dev/null +++ b/PerformanceMonitor.Collectors/ITargetProvider.cs @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; + +namespace PerformanceMonitor.Collectors; + +/// +/// Everything engine-specific about *executing* a collector: opening a connection, building a +/// command with the collector's parameters, and naming a driver exception. One implementation per +/// . +/// This interface deliberately lives in the dependency-free collector library and speaks only +/// , so that library keeps its zero-PackageReference property. The +/// implementations live where the drivers already are (the Darling service references both +/// Microsoft.Data.SqlClient and Npgsql), which is why nothing here mentions either. +/// Collector definitions themselves need no changes to work across engines — they already +/// return query text plus parameters and read through . What was missing +/// was this: a way for the runner to obtain a connection and a command without naming a provider. +/// +public interface ITargetProvider +{ + /// Which engine this provider talks to. + CollectorTargetEngine Engine { get; } + + /// + /// Opens nothing — just constructs the connection. The caller owns it and is responsible for + /// OpenAsync and disposal, matching how the runner already works. + /// + DbConnection CreateConnection(string connectionString); + + /// + /// Builds a command for one collector query, mapping each to the + /// provider's own parameter type. A provider MUST throw on a parameter type it cannot map rather + /// than silently sending a default — a wrong parameter type is a wrong result set, which is worse + /// than a failure. + /// + DbCommand CreateCommand(CollectorQuery query, DbConnection connection, int commandTimeoutSeconds); + + /// + /// Names a driver exception in engine-neutral terms. Returns + /// for anything not recognized, so an unexpected + /// failure stays loud. + /// is passed in rather than inferred because whether a + /// lock timeout is a yield or an error is a property of the COLLECTOR, not of the engine: only a + /// definition that deliberately sets a short lock timeout may treat one as a yield. + /// + CollectorTargetFault Classify(Exception exception, bool yieldsOnLockTimeout); + + /// + /// Rewrites to point at a different database on the same + /// server, leaving every other setting alone. + /// This is what makes per-database fan-out possible on PostgreSQL at all. SQL Server has two + /// ways to reach another database — switch the connection's catalog, or stay put and prefix the + /// query (EXECUTE [db].sys.sp_executesql) — and the shared runner uses the first. PostgreSQL + /// has only the first: a connection is bound to one database for its lifetime and there is no + /// cross-database query at all. So the one shape both engines support is the one exposed here. + /// + string WithDatabase(string connectionString, string databaseName); + + /// + /// Where to ask for the fan-out database list, and what to ask. Returns the connection string the + /// enumeration should run on plus the query to run — one decision, because the two are coupled: SQL + /// Server reads sys.databases from master, while PostgreSQL reads pg_database, + /// which is a shared catalog readable from whichever database is already connected. + /// What to do when enumeration FAILS is deliberately not here. On Azure SQL DB an + /// inaccessible master has a meaningful fallback (collect from the one connected database) and a + /// re-probe throttle; on PostgreSQL a login that cannot read pg_database cannot monitor the + /// server at all, so there is nothing to fall back to. That policy stays with the runner. + /// + (string ConnectionString, CollectorQuery Query) BuildDatabaseListPlan( + string connectionString, IReadOnlyList? excludedDatabases); +} diff --git a/PerformanceMonitor.Collectors/LatchStatsCollector.cs b/PerformanceMonitor.Collectors/LatchStatsCollector.cs index 6c0e1209f..e99560bdf 100644 --- a/PerformanceMonitor.Collectors/LatchStatsCollector.cs +++ b/PerformanceMonitor.Collectors/LatchStatsCollector.cs @@ -20,7 +20,7 @@ namespace PerformanceMonitor.Collectors; /// exactly: single DMV, delta-based between snapshots. The /// Dashboard proc's server_start_time reset marker is intentionally dropped — the shared /// calculator detects a counter reset itself (a value drop -/// yields a 0 delta) and applies the 300 s gap policy, so no reset column is needed. Available on +/// yields a 0 delta) and applies the shared gap policy, so no reset column is needed. Available on /// SQL Server, Azure SQL Database, and Azure SQL Managed Instance (verified against MS Learn), so /// is unconditionally true, matching wait_stats. /// @@ -87,10 +87,10 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) { - /* Delta groups, key (latch_class), and the 300 s gap policy are the parity contract. */ - var deltaWaitingRequests = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_waiting_requests", row.LatchClass, row.WaitingRequestsCount, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_wait_time", row.LatchClass, row.WaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaMaxWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_max_wait", row.LatchClass, row.MaxWaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); + /* Delta groups, key (latch_class), and the shared gap policy are the parity contract. */ + var deltaWaitingRequests = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_waiting_requests", row.LatchClass, row.WaitingRequestsCount, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_wait_time", row.LatchClass, row.WaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaMaxWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "latch_stats_max_wait", row.LatchClass, row.MaxWaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.LatchClass) /* latch_class VARCHAR */ diff --git a/PerformanceMonitor.Collectors/LongQueryCompletionsCollector.cs b/PerformanceMonitor.Collectors/LongQueryCompletionsCollector.cs index 7d10b7f96..87fab92c6 100644 --- a/PerformanceMonitor.Collectors/LongQueryCompletionsCollector.cs +++ b/PerformanceMonitor.Collectors/LongQueryCompletionsCollector.cs @@ -137,8 +137,10 @@ public sealed class Row /* Server- vs database-scoped ring-buffer source is the only engine difference; the event/action shred is shared. The session only captures the three long-completion events, so the read shreds everything in the ring buffer newer than the watermark — the duration predicate lives in the - session DDL (BuildCreateSessionSql), not here. Customizable text columns (statement / batch_text - / object_name) are turned on in the DDL's SET clause, so they are present in the payload here. */ + session DDL (BuildCreateSessionSql), not here. The customizable text columns (statement / + batch_text) are turned on in the DDL's SET clause; object_name is rpc_completed's own default + data field (#2129 — SETting a collect_object_name there fails the CREATE), so all three are + present in the payload here. */ private const string ShredSelect = @" SELECT event_time = evt.value('(@timestamp)[1]', 'datetime2'), @@ -341,7 +343,8 @@ private static string ActionList(bool databaseScoped) /// MEMORY_PARTITION_MODE = NONE for a single readable ring buffer / AWS RDS compatibility, mirroring /// the deadlock session). The duration predicate is applied /// to the two COMPLETED events only; attention is captured unfiltered (see the class doc). The - /// customizable text/object columns are turned on via SET so they are actually collected. + /// customizable TEXT columns (statement / batch_text) are turned on via SET so they are actually + /// collected; object_name needs no SET — it is one of rpc_completed's default data fields (#2129). /// public static string BuildCreateSessionSql(bool databaseScoped, long thresholdMicroseconds) { @@ -351,14 +354,18 @@ public static string BuildCreateSessionSql(bool databaseScoped, long thresholdMi AWS RDS compatible). Azure database-scoped sessions do not accept MEMORY_PARTITION_MODE. */ var partitionMode = databaseScoped ? "" : "\n MEMORY_PARTITION_MODE = NONE,"; + /* #2129: rpc_completed SETs only collect_statement. object_name is one of that event's + DEFAULT data fields — the collect_object_name customizable attribute belongs to + sp_statement_completed, and SETting it here made the CREATE fail on every server. The + note lives in C# on purpose: the DDL string ships to every monitored server, and the + test pin asserts the bogus attribute appears NOWHERE in it, comment included. */ return $@" CREATE EVENT SESSION [{XeSessionName}] ON {scope} ADD EVENT sqlserver.rpc_completed ( SET - collect_statement = 1, - collect_object_name = 1 + collect_statement = 1 ACTION ({actions} ) diff --git a/PerformanceMonitor.Collectors/MemoryGrantsCollector.cs b/PerformanceMonitor.Collectors/MemoryGrantsCollector.cs index 4f9b2ce7f..c671b86ad 100644 --- a/PerformanceMonitor.Collectors/MemoryGrantsCollector.cs +++ b/PerformanceMonitor.Collectors/MemoryGrantsCollector.cs @@ -120,8 +120,8 @@ public override void WritePayload(Row row, ICollectorRowWriter writer, Collector { /* Composite delta key and group names are the parity contract — do not change casually. */ var deltaKey = $"{row.PoolId}_{row.ResourceSemaphoreId}"; - var deltaTimeouts = context.Deltas.CalculateDelta(context.ServerId, "memory_grants_timeouts", deltaKey, row.TimeoutErrorCount, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaForced = context.Deltas.CalculateDelta(context.ServerId, "memory_grants_forced", deltaKey, row.ForcedGrantCount, collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaTimeouts = context.Deltas.CalculateDelta(context.ServerId, "memory_grants_timeouts", deltaKey, row.TimeoutErrorCount, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaForced = context.Deltas.CalculateDelta(context.ServerId, "memory_grants_forced", deltaKey, row.ForcedGrantCount, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.ResourceSemaphoreId) /* resource_semaphore_id (appended as SHORT, matching the original) */ diff --git a/PerformanceMonitor.Collectors/PerfmonStatsCollector.cs b/PerformanceMonitor.Collectors/PerfmonStatsCollector.cs index afc03bee6..6ffa1ba0e 100644 --- a/PerformanceMonitor.Collectors/PerfmonStatsCollector.cs +++ b/PerformanceMonitor.Collectors/PerfmonStatsCollector.cs @@ -20,8 +20,8 @@ namespace PerformanceMonitor.Collectors; /// RemoteCollectorService.Perfmon.cs — the curated default counter list is parity brain and lives /// HERE; hosts may supply an override via /// (Lite: perfmon_counters.json). Counter names interpolate as escaped N'...' literals; one delta -/// group ("perfmon") keyed "{object}|{counter}|{instance}" with the 300 s gap; the constant 60 s -/// sample interval is written per row. +/// group ("perfmon") keyed "{object}|{counter}|{instance}" with the shared gap; the sample interval +/// written per row is the MEASURED gap since the previous sweep, not the configured cadence (#2234). /// public sealed class PerfmonStatsCollector : CollectorDefinitionBase { @@ -169,18 +169,28 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) { - /* Delta for per-second counters — gap detection at 5min (5x the 1-min collection interval) - prevents inflated deltas after app restarts. Group/key/gap are the parity contract. */ + /* Delta for per-second counters. Group/key/gap are the parity contract. + + The interval is MEASURED, not assumed. This wrote a literal 60 — the configured one-minute + cadence — on every row, while the fleet's actual gap between perfmon sweeps runs a median of + 299 s (p99 830 s, max 2,514 s over 99,717 samples), so anyone deriving a rate from it was up + to 5x high on a denominator the collector had invented. Nothing in-product divided by THIS + column (the perfmon MCP tools hand it to the caller and the Viewer carries it unplotted); the + NULLIF(sample_interval_seconds, 0) idiom guards query_stats' interval, which + QueryStatsCollector has always measured. Perfmon was the outlier (#2234). A 0 here means no + delta was knowable — first sighting, counter reset, or a gap past the policy — so callers must + treat 0 as unknown rather than dividing by it. */ var deltaKey = $"{row.ObjectName}|{row.CounterName}|{row.InstanceName}"; - var deltaCntrValue = context.Deltas.CalculateDelta(context.ServerId, "perfmon", deltaKey, row.CntrValue, - collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaCntrValue = context.Deltas.CalculateDeltaWithInterval(context.ServerId, "perfmon", deltaKey, + row.CntrValue, out var sampleIntervalSeconds, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer - .Value(row.ObjectName) /* object_name VARCHAR */ - .Value(row.CounterName) /* counter_name VARCHAR */ - .Value(row.InstanceName) /* instance_name VARCHAR */ - .Value(row.CntrValue) /* cntr_value BIGINT */ - .Value(deltaCntrValue) /* delta_cntr_value BIGINT */ - .Value(60); /* sample_interval_seconds — 1-minute collection interval */ + .Value(row.ObjectName) /* object_name VARCHAR */ + .Value(row.CounterName) /* counter_name VARCHAR */ + .Value(row.InstanceName) /* instance_name VARCHAR */ + .Value(row.CntrValue) /* cntr_value BIGINT */ + .Value(deltaCntrValue) /* delta_cntr_value BIGINT */ + .Value(sampleIntervalSeconds); /* sample_interval_seconds — measured, not the cadence */ } } diff --git a/PerformanceMonitor.Collectors/PgAutovacuumStatsCollector.cs b/PerformanceMonitor.Collectors/PgAutovacuumStatsCollector.cs new file mode 100644 index 000000000..03f8b1cec --- /dev/null +++ b/PerformanceMonitor.Collectors/PgAutovacuumStatsCollector.cs @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Per-table autovacuum state — whether autovacuum is keeping up, table by table. +/// Dead tuples on their own are not a finding. Every PostgreSQL table has some, and the number +/// that matters is not the count but the count relative to the threshold that triggers a vacuum: +/// autovacuum fires at autovacuum_vacuum_threshold + autovacuum_vacuum_scale_factor * reltuples, +/// so 500,000 dead tuples is routine on a 50-million-row table and a five-alarm fire on a 10,000-row +/// one. This collector computes that threshold per table and stores it alongside the counts, which is +/// what turns a number nobody can act on into a ratio anybody can. +/// The threshold has to be computed per table rather than read from the GUCs, because +/// ALTER TABLE ... SET (autovacuum_vacuum_scale_factor = ...) is common on exactly the big hot +/// tables where the global default is wrong. Reading the GUC alone would report a threshold the server +/// is not using — worse than reporting none, because it looks authoritative. +/// Runs once per database: pg_stat_user_tables shows only the connected database's tables, +/// with no cross-database equivalent. This is the first PostgreSQL collector on the per-database +/// fan-out path, and on PostgreSQL that path means one connection per database per cycle — which is why +/// the cadence is hourly rather than per-minute. +/// +public sealed class PgAutovacuumStatsCollector : PostgresCollectorDefinitionBase +{ + public static PgAutovacuumStatsCollector Instance { get; } = new(); + + private PgAutovacuumStatsCollector() + { + } + + public readonly record struct Row( + string SchemaName, + string TableName, + long LiveTuples, + long DeadTuples, + long ModsSinceAnalyze, + long InsertsSinceVacuum, + long VacuumThreshold, + long InsertVacuumThreshold, + long AnalyzeThreshold, + bool AutovacuumDisabled, + long TotalBytes, + DateTime? LastVacuum, + DateTime? LastAutovacuum, + DateTime? LastAnalyze, + DateTime? LastAutoanalyze, + long VacuumCount, + long AutovacuumCount, + long AnalyzeCount, + long AutoanalyzeCount); + + /* Version gating: + PG13+ : autovacuum_vacuum_insert_threshold / _scale_factor and n_ins_since_vacuum — the + insert-only path. Before it, an append-only table was never vacuumed by the dead-tuple + rule (it has no dead tuples) and so was never frozen either, which is one of the + classic routes to a wraparound emergency. Substituted with -1 below rather than + omitted, so the row shape does not change across a mixed-version fleet. + + The reloptions lookups are the reason this query is not two lines. Each per-table override is + read out of pg_class.reloptions via pg_options_to_table() and falls back to the GUC, mirroring + exactly what the autovacuum launcher itself does. Parsing the raw text[] instead would mean + re-implementing the option syntax; pg_options_to_table is the server's own parser and needs no + special grant. + + reltuples is -1, not 0, on a table that has never been analyzed (PG14+ distinguishes "empty" + from "unknown"). GREATEST(...,0) keeps that from producing a NEGATIVE threshold, which would + make a never-analyzed table look permanently overdue. + All four maintenance timestamps are `timestamp with time zone` and are converted with + AT TIME ZONE 'UTC' rather than ::timestamp. The cast form renders the instant in the SESSION's + TimeZone before dropping the offset, so it agrees with UTC only while every parameter group says + UTC — true across this fleet today, which is exactly what makes the bug invisible until it is not. + The store contract is naive UTC product-wide, so the conversion has to be explicit. */ + private static string BuildQueryText(int postgresMajorVersion) + { + var supportsInsertThreshold = postgresMajorVersion >= 13; + + var insertsSinceVacuum = supportsInsertThreshold ? "t.n_ins_since_vacuum::bigint" : "-1::bigint"; + var insertThreshold = supportsInsertThreshold + ? @"(coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_vacuum_insert_threshold')::bigint, + current_setting('autovacuum_vacuum_insert_threshold')::bigint) + + coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_vacuum_insert_scale_factor')::float8, + current_setting('autovacuum_vacuum_insert_scale_factor')::float8) + * GREATEST(c.reltuples, 0))::bigint" + : "-1::bigint"; + + /* Only tables with pending work. A table with no dead tuples, no modifications since its last + analyze, and no inserts since its last vacuum has had no writes since maintenance last ran: + there is nothing for autovacuum to do and nothing to report, and on a database with thousands + of mostly-static tables those rows would be the overwhelming majority of the volume. + + The insert clause is not optional decoration. An append-only table has NO dead tuples and NO + modifications, so the first two predicates both miss it — and an append-only table that never + gets vacuumed is never frozen either, which is one of the classic routes into a wraparound + emergency. Filtering on dead tuples alone would drop exactly the tables whose risk this + collector is meant to surface. + + autovacuum_enabled = false is kept regardless of activity: a table with autovacuum switched + off is a finding even while it is momentarily clean. */ + var insertActivityClause = supportsInsertThreshold ? "OR t.n_ins_since_vacuum > 0" : string.Empty; + return $@" +SELECT + t.schemaname AS schema_name, + t.relname AS table_name, + t.n_live_tup::bigint AS live_tuples, + t.n_dead_tup::bigint AS dead_tuples, + t.n_mod_since_analyze::bigint AS mods_since_analyze, + {insertsSinceVacuum} AS inserts_since_vacuum, + (coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_vacuum_threshold')::bigint, + current_setting('autovacuum_vacuum_threshold')::bigint) + + coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_vacuum_scale_factor')::float8, + current_setting('autovacuum_vacuum_scale_factor')::float8) + * GREATEST(c.reltuples, 0))::bigint AS vacuum_threshold, + {insertThreshold} AS insert_vacuum_threshold, + (coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_analyze_threshold')::bigint, + current_setting('autovacuum_analyze_threshold')::bigint) + + coalesce( + (SELECT option_value FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_analyze_scale_factor')::float8, + current_setting('autovacuum_analyze_scale_factor')::float8) + * GREATEST(c.reltuples, 0))::bigint AS analyze_threshold, + coalesce( + (SELECT lower(option_value) = 'false' FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_enabled'), false) AS autovacuum_disabled, + pg_total_relation_size(t.relid)::bigint AS total_bytes, + (t.last_vacuum AT TIME ZONE 'UTC') AS last_vacuum, + (t.last_autovacuum AT TIME ZONE 'UTC') AS last_autovacuum, + (t.last_analyze AT TIME ZONE 'UTC') AS last_analyze, + (t.last_autoanalyze AT TIME ZONE 'UTC') AS last_autoanalyze, + t.vacuum_count::bigint AS vacuum_count, + t.autovacuum_count::bigint AS autovacuum_count, + t.analyze_count::bigint AS analyze_count, + t.autoanalyze_count::bigint AS autoanalyze_count +FROM pg_stat_user_tables AS t +JOIN pg_class AS c + ON c.oid = t.relid +WHERE ( + t.n_dead_tup > 0 + OR t.n_mod_since_analyze > 0 + {insertActivityClause} + OR coalesce((SELECT lower(option_value) = 'false' FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'autovacuum_enabled'), false) + ) +ORDER BY t.n_dead_tup DESC"; + } + + public override string Name => "pg_autovacuum_stats"; + + public override string TargetTable => "pg_autovacuum_stats"; + + /// + /// Writers only. This is not a permissions or availability gate — the view is perfectly readable on a + /// standby — it is that on a standby every counter it reports is ZERO. + /// Measured on Aurora PostgreSQL 17.7, same cluster, same database, same 15 tables: the writer + /// reported 13,654,458 dead tuples and 150,790,506 live tuples, while the reader reported 0 for + /// n_dead_tup, n_mod_since_analyze, n_ins_since_vacuum AND n_live_tup. These are the writer's stats + /// collector's numbers and they are not replicated. + /// Left ungated, a reader target would produce zero rows, the activity filter would read that as + /// "no table has pending work", and the tool would report perfect autovacuum health for a cluster + /// 13 million dead tuples behind. A confidently wrong healthy answer is worse than no answer, which is + /// why this gates rather than collecting and hoping the consumer notices. + /// + public override bool AppliesTo(CollectorTargetInfo target) => !target.IsInRecovery; + + /// + /// pg_stat_user_tables is scoped to the connected database and PostgreSQL has no + /// cross-database read, so this is necessarily a fan-out. + /// + public override bool RunsPerDatabase(CollectorTargetInfo target) => true; + + public override CollectorQuery BuildQuery(CollectorContext context) + => new(BuildQueryText(context.Target.PostgresMajorVersion)); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + /* Not read from the result set: the per-database loop sets CurrentDatabaseName, and the + connection's database IS the row's database, so it is authoritative here in a way a value + parsed out of the payload could not be. */ + new CollectorColumn("database_name", CollectorColumnType.Varchar), + new CollectorColumn("schema_name", CollectorColumnType.Varchar), + new CollectorColumn("table_name", CollectorColumnType.Varchar), + new CollectorColumn("live_tuples", CollectorColumnType.BigInt), + new CollectorColumn("dead_tuples", CollectorColumnType.BigInt), + new CollectorColumn("mods_since_analyze", CollectorColumnType.BigInt), + /* -1 = the server predates the insert-only vacuum rule (PG13). */ + new CollectorColumn("inserts_since_vacuum", CollectorColumnType.BigInt), + /* The whole point of the collector: the count is meaningless without the line it has to cross. */ + new CollectorColumn("vacuum_threshold", CollectorColumnType.BigInt), + new CollectorColumn("insert_vacuum_threshold", CollectorColumnType.BigInt), + new CollectorColumn("analyze_threshold", CollectorColumnType.BigInt), + new CollectorColumn("autovacuum_disabled", CollectorColumnType.Boolean), + new CollectorColumn("total_bytes", CollectorColumnType.BigInt), + new CollectorColumn("last_vacuum", CollectorColumnType.Timestamp), + new CollectorColumn("last_autovacuum", CollectorColumnType.Timestamp), + new CollectorColumn("last_analyze", CollectorColumnType.Timestamp), + new CollectorColumn("last_autoanalyze", CollectorColumnType.Timestamp), + new CollectorColumn("vacuum_count", CollectorColumnType.BigInt), + new CollectorColumn("autovacuum_count", CollectorColumnType.BigInt), + new CollectorColumn("analyze_count", CollectorColumnType.BigInt), + new CollectorColumn("autoanalyze_count", CollectorColumnType.BigInt), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row( + SchemaName: reader.GetString(0), + TableName: reader.GetString(1), + LiveTuples: reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + DeadTuples: reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + ModsSinceAnalyze: reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + InsertsSinceVacuum: reader.IsDBNull(5) ? -1 : reader.GetInt64(5), + VacuumThreshold: reader.IsDBNull(6) ? -1 : reader.GetInt64(6), + InsertVacuumThreshold: reader.IsDBNull(7) ? -1 : reader.GetInt64(7), + AnalyzeThreshold: reader.IsDBNull(8) ? -1 : reader.GetInt64(8), + AutovacuumDisabled: !reader.IsDBNull(9) && reader.GetBoolean(9), + TotalBytes: reader.IsDBNull(10) ? -1 : reader.GetInt64(10), + LastVacuum: reader.IsDBNull(11) ? null : reader.GetDateTime(11), + LastAutovacuum: reader.IsDBNull(12) ? null : reader.GetDateTime(12), + LastAnalyze: reader.IsDBNull(13) ? null : reader.GetDateTime(13), + LastAutoanalyze: reader.IsDBNull(14) ? null : reader.GetDateTime(14), + VacuumCount: reader.IsDBNull(15) ? 0 : reader.GetInt64(15), + AutovacuumCount: reader.IsDBNull(16) ? 0 : reader.GetInt64(16), + AnalyzeCount: reader.IsDBNull(17) ? 0 : reader.GetInt64(17), + AutoanalyzeCount: reader.IsDBNull(18) ? 0 : reader.GetInt64(18))); + } + + return rows; + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* No deltas. Every column here is a level or a lifetime count read against a threshold — the + question is "how far past the line is this table now", not "how much moved since last time". + The vacuum/analyze counts are cumulative, but their useful reading is the timestamps beside + them, which say when maintenance last ran without any arithmetic. */ + writer + .Value(context.CurrentDatabaseName) + .Value(row.SchemaName) + .Value(row.TableName) + .Value(row.LiveTuples) + .Value(row.DeadTuples) + .Value(row.ModsSinceAnalyze) + .Value(row.InsertsSinceVacuum) + .Value(row.VacuumThreshold) + .Value(row.InsertVacuumThreshold) + .Value(row.AnalyzeThreshold) + .Value(row.AutovacuumDisabled) + .Value(row.TotalBytes) + .Value(row.LastVacuum) + .Value(row.LastAutovacuum) + .Value(row.LastAnalyze) + .Value(row.LastAutoanalyze) + .Value(row.VacuumCount) + .Value(row.AutovacuumCount) + .Value(row.AnalyzeCount) + .Value(row.AutoanalyzeCount); + } +} diff --git a/PerformanceMonitor.Collectors/PgBlockingCollector.cs b/PerformanceMonitor.Collectors/PgBlockingCollector.cs new file mode 100644 index 000000000..9571bfdd5 --- /dev/null +++ b/PerformanceMonitor.Collectors/PgBlockingCollector.cs @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Who is blocked, by whom, on what — the condition people actually call about. +/// +/// This is a SAMPLE, and that has to be said out loud. SQL Server has a blocked-process report: +/// the engine itself materialises a record when blocking exceeds a threshold, so a 6-second block leaves +/// evidence in a ring buffer whether or not anyone was looking. PostgreSQL has no equivalent — nothing is +/// recorded unless something asks. So this captures the state at the moment it runs, and +/// blocking shorter than the cadence is invisible. A reader who mistakes this for a +/// blocked-process-report equivalent will conclude "no blocking happened" from "no blocking was sampled". +/// +/// Cost control: pg_blocking_pids() is not free. It takes ShareLock on the lock manager +/// partitions per call, so calling it for every row on a 5,000-connection instance is exactly the monitoring +/// query that becomes the incident. It is evaluated only where wait_event_type = 'Lock' — the only +/// population that can have blockers — as a CASE in the select list rather than a WHERE filter, so one query +/// returns the whole picture and the expensive call is still bounded to the actually-blocked set. +/// +/// An edge list, not a rendered tree. pg_blocking_pids() returns an array; it is unnested +/// to one row per (blocked, blocker) pair. Rendering a chain here would bake in one view of it, and every +/// interesting question — root blocker, chain depth, fan-out — is cheap over edges and expensive to recover +/// from a string. The reader assembles the tree, the same division of labour as the other PostgreSQL reads +/// computing ratios this layer deliberately does not. +/// +/// Both sides of every edge carry their own state. A chain rooted in idle in transaction is +/// a different problem from one rooted in a long-running query, and the pid alone does not say which — the +/// remedy is "fix the application" versus "tune the query". Capturing only pids is the most common gap in +/// homegrown PostgreSQL blocking monitoring: you get a number, go looking, and by then it is gone. +/// +/// Runs on ANY PostgreSQL target INCLUDING standbys, deliberately unlike +/// . That collector gates off replicas because +/// pg_stat_user_tables reports zeros there; pg_stat_activity reports the standby's own backends, +/// and recovery conflicts are real blocking that only happens on a standby. +/// +/// Permissions degrade quietly. pg_monitor is required to see other backends' +/// query text; without it PostgreSQL substitutes <insufficient privilege> for backends the +/// login does not own. That is not an error and will not fail the collection — it produces a capture with +/// pids and no queries, which is nearly useless. The text is stored as returned so the condition is visible +/// in the data rather than hidden. +/// +public sealed class PgBlockingCollector : PostgresCollectorDefinitionBase +{ + public static PgBlockingCollector Instance { get; } = new(); + + private PgBlockingCollector() + { + } + + public readonly record struct Row( + long BlockedBackendId, + int BlockedPid, + long BlockingBackendId, + int BlockingPid, + string? DatabaseName, + string? BlockedUsername, + string? BlockedApplicationName, + string? BlockedClientAddr, + string? BlockedState, + string? BlockedWaitEventType, + string? BlockedWaitEvent, + string? BlockedQuery, + long BlockedXactDurationMs, + long BlockedQueryDurationMs, + string? BlockingUsername, + string? BlockingApplicationName, + string? BlockingClientAddr, + string? BlockingState, + string? BlockingWaitEventType, + string? BlockingWaitEvent, + string? BlockingQuery, + long BlockingXactDurationMs, + long BlockingQueryDurationMs, + int BlockedPidCount, + bool BlockingIsIdleInTransaction, + bool QueryTextMayBeTruncated); + + /* The synthetic backend identity is worth explaining because it looks like noise. A pid is reused: on a + busy instance the same number can be two different backends within one retention window, so a history + keyed on pid alone silently merges them. backend_start disambiguates, and packing the two into one + bigint (epoch seconds, then the zero-padded pid) gives a value that is stable for the life of a backend + and comparable across samples. Borrowed from pganalyze's collector, which solves the same problem the + same way — worth adopting rather than reinventing. + + track_activity_query_size caps pg_stat_activity.query (1 KB by default), so a long statement arrives + clipped with no marker. The length is compared against the setting to record WHETHER it may be clipped, + rather than joining out to pg_stat_statements for the full text: queryid is not on pg_stat_activity + before PG14, and even after, matching a live backend to a normalised entry is a different claim than + "this is what it ran". + + Durations are computed server-side in milliseconds rather than shipping timestamps, which sidesteps the + timestamptz-render trap entirely — there is no timestamp column here to get wrong. */ + private const string QueryText = @" +WITH activity AS +( + SELECT + (extract(epoch FROM coalesce(a.backend_start, pg_postmaster_start_time()))::bigint::text + || to_char(a.pid, 'FM0000000'))::bigint AS backend_id, + a.pid, + a.datname, + a.usename, + a.application_name, + a.client_addr, + a.state, + a.wait_event_type, + a.wait_event, + a.query, + (extract(epoch FROM (clock_timestamp() - a.xact_start)) * 1000)::bigint AS xact_duration_ms, + (extract(epoch FROM (clock_timestamp() - a.query_start)) * 1000)::bigint AS query_duration_ms, + /* The expensive call, bounded to backends already waiting on a lock. */ + CASE + WHEN coalesce(a.wait_event_type, '') = 'Lock' THEN pg_blocking_pids(a.pid) + END AS blocker_pids + FROM pg_stat_activity AS a + WHERE a.pid IS NOT NULL + AND a.pid <> pg_backend_pid() +), +edges AS +( + SELECT + blocked.backend_id AS blocked_backend_id, + blocked.pid AS blocked_pid, + unnest(blocked.blocker_pids) AS blocking_pid + FROM activity AS blocked + WHERE blocked.blocker_pids IS NOT NULL + AND cardinality(blocked.blocker_pids) > 0 +), +fan_out AS +( + SELECT blocking_pid, count(*)::int AS blocked_pid_count + FROM edges + GROUP BY blocking_pid +) +SELECT + e.blocked_backend_id, + e.blocked_pid, + /* A blocker that has since gone (or that pg_stat_activity does not show) still leaves a real edge, so + the join is LEFT and the blocker's own columns come back NULL rather than dropping the row. Losing the + edge would understate the chain, which is the opposite of useful. */ + coalesce(blocker.backend_id, 0) AS blocking_backend_id, + e.blocking_pid, + coalesce(blocked.datname, blocker.datname) AS database_name, + blocked.usename AS blocked_username, + blocked.application_name AS blocked_application_name, + blocked.client_addr::text AS blocked_client_addr, + blocked.state AS blocked_state, + blocked.wait_event_type AS blocked_wait_event_type, + blocked.wait_event AS blocked_wait_event, + blocked.query AS blocked_query, + coalesce(blocked.xact_duration_ms, -1) AS blocked_xact_duration_ms, + coalesce(blocked.query_duration_ms, -1) AS blocked_query_duration_ms, + blocker.usename AS blocking_username, + blocker.application_name AS blocking_application_name, + blocker.client_addr::text AS blocking_client_addr, + blocker.state AS blocking_state, + blocker.wait_event_type AS blocking_wait_event_type, + blocker.wait_event AS blocking_wait_event, + blocker.query AS blocking_query, + coalesce(blocker.xact_duration_ms, -1) AS blocking_xact_duration_ms, + coalesce(blocker.query_duration_ms, -1) AS blocking_query_duration_ms, + /* No coalesce: fan_out is grouped from the same edges CTE and joined INNER, so every blocking_pid + here necessarily has a row and the fallback was unreachable. */ + f.blocked_pid_count AS blocked_pid_count, + /* Stamped here rather than derived on read: the remedy for this root differs from every other root, and + a reader filtering rows must not be able to lose the distinction. */ + coalesce(blocker.state, '') = 'idle in transaction' AS blocking_is_idle_in_transaction, + /* Whether EITHER text may be clipped by track_activity_query_size. + + pg_size_bytes(), NOT ::int. current_setting() renders a memory GUC WITH ITS UNIT — this one comes + back as '8kB' on Aurora 17.7 and '4kB' on 16.11 (pg_settings.unit is 'B', and current_setting + scales to the largest whole unit), so current_setting(...)::int raises an + invalid-input-syntax-for-integer error and takes the WHOLE collection down with it, every cycle. + Verified against both majors rather than reasoned about. pg_size_bytes parses every unit form, so + this stays right if the value is ever set in MB or left at the 1kB default. + + octet_length(), NOT length() — the same unit mistake as above wearing a different disguise, and it + shipped in the first draft one line under a comment about getting units right. PostgreSQL truncates + query text at a BYTE boundary while length() counts CHARACTERS in the session encoding, so on + multi-byte text the comparison undercounts and the flag comes back false for a query that really was + clipped. Measured on live Aurora: repeat('あ',100) is length 100, octet_length 300 — a 3x undercount + against an 8192-byte limit. */ + ( + octet_length(coalesce(blocked.query, '')) + >= pg_size_bytes(current_setting('track_activity_query_size')) + OR octet_length(coalesce(blocker.query, '')) + >= pg_size_bytes(current_setting('track_activity_query_size')) + ) AS query_text_may_be_truncated +FROM edges AS e +JOIN activity AS blocked + ON blocked.pid = e.blocked_pid +LEFT JOIN activity AS blocker + ON blocker.pid = e.blocking_pid +JOIN fan_out AS f + ON f.blocking_pid = e.blocking_pid +ORDER BY f.blocked_pid_count DESC, e.blocking_pid, e.blocked_pid"; + + public override string Name => "pg_blocking"; + + public override string TargetTable => "pg_blocking_edges"; + + /// + /// Any PostgreSQL target, standbys included — recovery conflicts are real blocking and a standby is where + /// they happen. + /// + public override bool AppliesTo(CollectorTargetInfo target) => true; + + public override CollectorQuery BuildQuery(CollectorContext context) => new(QueryText); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("blocked_backend_id", CollectorColumnType.BigInt), + new CollectorColumn("blocked_pid", CollectorColumnType.Integer), + new CollectorColumn("blocking_backend_id", CollectorColumnType.BigInt), + new CollectorColumn("blocking_pid", CollectorColumnType.Integer), + new CollectorColumn("database_name", CollectorColumnType.Varchar), + new CollectorColumn("blocked_username", CollectorColumnType.Varchar), + new CollectorColumn("blocked_application_name", CollectorColumnType.Varchar), + new CollectorColumn("blocked_client_addr", CollectorColumnType.Varchar), + new CollectorColumn("blocked_state", CollectorColumnType.Varchar), + new CollectorColumn("blocked_wait_event_type", CollectorColumnType.Varchar), + new CollectorColumn("blocked_wait_event", CollectorColumnType.Varchar), + new CollectorColumn("blocked_query", CollectorColumnType.Varchar), + /* -1, not NULL and not 0: a backend with no open transaction has no duration to report, and 0 would + read as "started this instant". Same not-applicable sentinel the other PostgreSQL level columns use. + pg_stat_io is the documented exception, where NULL means something different. */ + new CollectorColumn("blocked_xact_duration_ms", CollectorColumnType.BigInt), + new CollectorColumn("blocked_query_duration_ms", CollectorColumnType.BigInt), + new CollectorColumn("blocking_username", CollectorColumnType.Varchar), + new CollectorColumn("blocking_application_name", CollectorColumnType.Varchar), + new CollectorColumn("blocking_client_addr", CollectorColumnType.Varchar), + new CollectorColumn("blocking_state", CollectorColumnType.Varchar), + new CollectorColumn("blocking_wait_event_type", CollectorColumnType.Varchar), + new CollectorColumn("blocking_wait_event", CollectorColumnType.Varchar), + new CollectorColumn("blocking_query", CollectorColumnType.Varchar), + new CollectorColumn("blocking_xact_duration_ms", CollectorColumnType.BigInt), + new CollectorColumn("blocking_query_duration_ms", CollectorColumnType.BigInt), + new CollectorColumn("blocked_pid_count", CollectorColumnType.Integer), + new CollectorColumn("blocking_is_idle_in_transaction", CollectorColumnType.Boolean), + new CollectorColumn("query_text_may_be_truncated", CollectorColumnType.Boolean), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var edges = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + edges.Add(new Row( + BlockedBackendId: reader.IsDBNull(0) ? 0 : reader.GetInt64(0), + BlockedPid: reader.IsDBNull(1) ? 0 : reader.GetInt32(1), + BlockingBackendId: reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + BlockingPid: reader.IsDBNull(3) ? 0 : reader.GetInt32(3), + DatabaseName: Text(reader, 4), + BlockedUsername: Text(reader, 5), + BlockedApplicationName: Text(reader, 6), + BlockedClientAddr: Text(reader, 7), + BlockedState: Text(reader, 8), + BlockedWaitEventType: Text(reader, 9), + BlockedWaitEvent: Text(reader, 10), + BlockedQuery: Text(reader, 11), + BlockedXactDurationMs: reader.IsDBNull(12) ? -1 : reader.GetInt64(12), + BlockedQueryDurationMs: reader.IsDBNull(13) ? -1 : reader.GetInt64(13), + BlockingUsername: Text(reader, 14), + BlockingApplicationName: Text(reader, 15), + BlockingClientAddr: Text(reader, 16), + BlockingState: Text(reader, 17), + BlockingWaitEventType: Text(reader, 18), + BlockingWaitEvent: Text(reader, 19), + BlockingQuery: Text(reader, 20), + BlockingXactDurationMs: reader.IsDBNull(21) ? -1 : reader.GetInt64(21), + BlockingQueryDurationMs: reader.IsDBNull(22) ? -1 : reader.GetInt64(22), + BlockedPidCount: reader.IsDBNull(23) ? 1 : reader.GetInt32(23), + BlockingIsIdleInTransaction: !reader.IsDBNull(24) && reader.GetBoolean(24), + QueryTextMayBeTruncated: !reader.IsDBNull(25) && reader.GetBoolean(25))); + } + + /* Zero rows is the overwhelmingly common case and the healthy one. It must never read as a failure — + and it does not, because the runner records a SUCCESS with 0 rows for a collector that legitimately + found nothing. */ + return edges; + } + + private static string? Text(DbDataReader reader, int ordinal) => + reader.IsDBNull(ordinal) ? null : reader.GetString(ordinal); + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + writer + .Value(row.BlockedBackendId) + .Value(row.BlockedPid) + .Value(row.BlockingBackendId) + .Value(row.BlockingPid) + .Value(row.DatabaseName) + .Value(row.BlockedUsername) + .Value(row.BlockedApplicationName) + .Value(row.BlockedClientAddr) + .Value(row.BlockedState) + .Value(row.BlockedWaitEventType) + .Value(row.BlockedWaitEvent) + .Value(row.BlockedQuery) + .Value(row.BlockedXactDurationMs) + .Value(row.BlockedQueryDurationMs) + .Value(row.BlockingUsername) + .Value(row.BlockingApplicationName) + .Value(row.BlockingClientAddr) + .Value(row.BlockingState) + .Value(row.BlockingWaitEventType) + .Value(row.BlockingWaitEvent) + .Value(row.BlockingQuery) + .Value(row.BlockingXactDurationMs) + .Value(row.BlockingQueryDurationMs) + .Value(row.BlockedPidCount) + .Value(row.BlockingIsIdleInTransaction) + .Value(row.QueryTextMayBeTruncated); + } +} diff --git a/PerformanceMonitor.Collectors/PgIoStatsCollector.cs b/PerformanceMonitor.Collectors/PgIoStatsCollector.cs new file mode 100644 index 000000000..67abf8738 --- /dev/null +++ b/PerformanceMonitor.Collectors/PgIoStatsCollector.cs @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// I/O broken down by who did it, to what, and why — pg_stat_io (PostgreSQL 16+). +/// Richer than sys.dm_io_virtual_file_stats, which attributes I/O to a FILE. This attributes +/// it to a (backend_type, object, context) triple, so "the database is doing 40k reads/sec" becomes +/// "autovacuum workers are doing 40k reads/sec against relations in the vacuum context" — a sentence you +/// can act on. The context dimension in particular has no SQL Server counterpart: it separates ordinary +/// buffer-pool traffic from sequential scans that deliberately bypass it (bulkread), from vacuum's +/// ring buffer, from WAL replay. +/// NULL is preserved and never coalesced to zero. PostgreSQL uses NULL for "this counter does +/// not apply to this combination" — the checkpointer performs no reads or hits, bulkread never +/// extends a relation, the normal context has no ring buffer to reuse. On Aurora the write-side +/// counters are NULL or permanently zero across the board, because backends there do not write data files; +/// the storage layer does. Storing 0 in any of those places would claim a measurement that was never +/// taken, and a consumer computing a write-latency average would divide by it. +/// Cluster-wide, so no per-database fan-out, and valid on a standby — a replica's own read traffic +/// and its walreplay context are exactly what you want when a reader is slow. Verified on Aurora +/// 16.11 and 17.7: 25–37 rows per snapshot, which is why a per-minute cadence is affordable here where it +/// would not be for a per-table collector. +/// +public sealed class PgIoStatsCollector : PostgresCollectorDefinitionBase +{ + public static PgIoStatsCollector Instance { get; } = new(); + + private PgIoStatsCollector() + { + } + + public readonly record struct Row( + string? BackendType, + string? ObjectType, + string? Context, + long? Reads, + double? ReadTimeMs, + long? Writes, + double? WriteTimeMs, + long? Writebacks, + double? WritebackTimeMs, + long? Extends, + double? ExtendTimeMs, + long? OpBytes, + long? Hits, + long? Evictions, + long? Reuses, + long? Fsyncs, + double? FsyncTimeMs, + DateTime? StatsReset); + + /* Two version concerns, both about keeping the stored shape constant: + + PG16+ : the view itself. Gated in AppliesTo rather than here. + PG18 : op_bytes was REMOVED and replaced by read_bytes / write_bytes / extend_bytes. Selecting + op_bytes on 18 would fail with "column does not exist" and take the whole collection + with it, so it is substituted. The replacement columns are deliberately NOT added + speculatively — they are a different measure (total bytes per operation class, not the + per-op block size) and deserve their own columns, decided against a real PG18 target + rather than guessed at now. + + Verified identical on Aurora 16.11 and 17.7: 18 columns, same names, same order. The enum VALUES do + differ between them — 17.7 showed a `walreplay` context and Aurora-specific backend types + ('aurora cache receiver process', 'aurora wal replay process', 'slotsync worker') that 16.11 did not + — which is why nothing here filters on them. A whitelist would silently drop rows. + + stats_reset is `timestamp with time zone`; AT TIME ZONE 'UTC' rather than ::timestamp, because the + cast renders in the SESSION's TimeZone and the store contract is naive UTC. */ + private static string BuildQueryText(int postgresMajorVersion) + { + var opBytes = postgresMajorVersion >= 18 ? "NULL::bigint" : "op_bytes"; + + return $@" +SELECT + backend_type AS backend_type, + object AS object_type, + context AS context, + reads AS reads, + read_time AS read_time_ms, + writes AS writes, + write_time AS write_time_ms, + writebacks AS writebacks, + writeback_time AS writeback_time_ms, + extends AS extends, + extend_time AS extend_time_ms, + {opBytes} AS op_bytes, + hits AS hits, + evictions AS evictions, + reuses AS reuses, + fsyncs AS fsyncs, + fsync_time AS fsync_time_ms, + (stats_reset AT TIME ZONE 'UTC') AS stats_reset +FROM pg_stat_io +ORDER BY backend_type, object, context"; + } + + public override string Name => "pg_io_stats"; + + public override string TargetTable => "pg_io_stats"; + + /// + /// pg_stat_io is PostgreSQL 16+. No Aurora gate and no recovery gate: it is a core view, and a + /// standby's own read traffic is a legitimate thing to monitor — unlike + /// , whose source reports zeros on a replica. + /// + public override bool AppliesTo(CollectorTargetInfo target) => target.PostgresMajorVersion >= 16; + + public override CollectorQuery BuildQuery(CollectorContext context) + => new(BuildQueryText(context.Target.PostgresMajorVersion)); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + /* The three dimensions. Together they are the row's identity, and the delta key any read has to + difference on. `object_type` rather than `object` because object is reserved-ish and reads badly + in a store query. */ + new CollectorColumn("backend_type", CollectorColumnType.Varchar), + new CollectorColumn("object_type", CollectorColumnType.Varchar), + new CollectorColumn("context", CollectorColumnType.Varchar), + /* Cumulative counters, stored raw with NULL intact. The windowed change is computed at read time + with the same positive-difference-per-interval rule the statement read uses, rather than by + keeping delta state for every triple: the row count is small but the NULL semantics matter more, + and a stored delta would have to invent a value for "not applicable". */ + new CollectorColumn("reads", CollectorColumnType.BigInt), + new CollectorColumn("read_time_ms", CollectorColumnType.Double), + /* NULL on Aurora, all majors: backends do not write data files there, the storage layer does. + Kept because a self-managed PostgreSQL target DOES populate them, and because a column that is + NULL for a documented reason is more useful than a missing one. */ + new CollectorColumn("writes", CollectorColumnType.BigInt), + new CollectorColumn("write_time_ms", CollectorColumnType.Double), + new CollectorColumn("writebacks", CollectorColumnType.BigInt), + new CollectorColumn("writeback_time_ms", CollectorColumnType.Double), + new CollectorColumn("extends", CollectorColumnType.BigInt), + new CollectorColumn("extend_time_ms", CollectorColumnType.Double), + /* The block size an operation moves (8192 in practice). PG18 removed it; NULL there. */ + new CollectorColumn("op_bytes", CollectorColumnType.BigInt), + /* hits is the buffer-pool hit count — the numerator of a real cache-hit ratio, alongside reads. */ + new CollectorColumn("hits", CollectorColumnType.BigInt), + new CollectorColumn("evictions", CollectorColumnType.BigInt), + /* reuses is ring-buffer reuse and applies only to the bulk/vacuum contexts. NOT eviction + pressure — conflating the two is the standard misreading of this view. */ + new CollectorColumn("reuses", CollectorColumnType.BigInt), + new CollectorColumn("fsyncs", CollectorColumnType.BigInt), + new CollectorColumn("fsync_time_ms", CollectorColumnType.Double), + /* The explicit reset signal, so a read does not have to infer one from a counter going backwards. */ + new CollectorColumn("stats_reset", CollectorColumnType.Timestamp), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row( + BackendType: reader.IsDBNull(0) ? null : reader.GetString(0), + ObjectType: reader.IsDBNull(1) ? null : reader.GetString(1), + Context: reader.IsDBNull(2) ? null : reader.GetString(2), + Reads: Long(reader, 3), + ReadTimeMs: Double(reader, 4), + Writes: Long(reader, 5), + WriteTimeMs: Double(reader, 6), + Writebacks: Long(reader, 7), + WritebackTimeMs: Double(reader, 8), + Extends: Long(reader, 9), + ExtendTimeMs: Double(reader, 10), + OpBytes: Long(reader, 11), + Hits: Long(reader, 12), + Evictions: Long(reader, 13), + Reuses: Long(reader, 14), + Fsyncs: Long(reader, 15), + FsyncTimeMs: Double(reader, 16), + StatsReset: reader.IsDBNull(17) ? null : reader.GetDateTime(17))); + } + + return rows; + + /* Nullable all the way through, deliberately. Every other Postgres collector here uses a -1 + sentinel for "not applicable", which suits a LEVEL that a consumer reads directly. These are + cumulative counters that get differenced, and -1 differenced against a real value produces a + garbage interval — so NULL, which propagates through the subtraction and drops out of the sum. */ + static long? Long(DbDataReader r, int ordinal) => r.IsDBNull(ordinal) ? null : r.GetInt64(ordinal); + static double? Double(DbDataReader r, int ordinal) => r.IsDBNull(ordinal) ? null : r.GetDouble(ordinal); + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + writer + .Value(row.BackendType) + .Value(row.ObjectType) + .Value(row.Context) + .Value(row.Reads) + .Value(row.ReadTimeMs) + .Value(row.Writes) + .Value(row.WriteTimeMs) + .Value(row.Writebacks) + .Value(row.WritebackTimeMs) + .Value(row.Extends) + .Value(row.ExtendTimeMs) + .Value(row.OpBytes) + .Value(row.Hits) + .Value(row.Evictions) + .Value(row.Reuses) + .Value(row.Fsyncs) + .Value(row.FsyncTimeMs) + .Value(row.StatsReset); + } +} diff --git a/PerformanceMonitor.Collectors/PgReplicationSlotsCollector.cs b/PerformanceMonitor.Collectors/PgReplicationSlotsCollector.cs new file mode 100644 index 000000000..f75e5372d --- /dev/null +++ b/PerformanceMonitor.Collectors/PgReplicationSlotsCollector.cs @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Replication slot state — an abandoned slot is one of the few PostgreSQL conditions that can take a +/// server down by itself, and it can do so two independent ways. +/// Disk exhaustion. A slot retains every WAL segment its consumer has not confirmed. With +/// max_slot_wal_keep_size at its default of -1 that retention is unbounded: an +/// inactive slot will hold WAL until the volume fills, and a full WAL volume stops the server. This is +/// the failure mode cannot see, which is why slots get their own +/// collector even though that one already reads them for the horizon. +/// Vacuum starvation. The same slot pins xmin/catalog_xmin, so nothing can be +/// reclaimed cluster-wide. Both counters are recorded here as well, so slot state is legible on its own +/// without a join. +/// The orphans are rarely deliberate: a removed CDC task, a finished blue/green deployment (which +/// creates one slot per database), a Debezium consumer that was decommissioned, or a failed major-version +/// upgrade. Nothing complains — the slot simply keeps its promise to retain WAL for a consumer that is +/// never coming back. +/// Core catalog only, so it runs on any PostgreSQL target. +/// +public sealed class PgReplicationSlotsCollector : PostgresCollectorDefinitionBase +{ + public static PgReplicationSlotsCollector Instance { get; } = new(); + + private PgReplicationSlotsCollector() + { + } + + public readonly record struct Row( + string SlotName, + string? SlotType, + string? Plugin, + string? DatabaseName, + bool IsActive, + long ActivePid, + bool IsTemporary, + bool TwoPhase, + string? WalStatus, + long SafeWalSizeBytes, + long RetainedWalBytes, + long XminAge, + long CatalogXminAge, + DateTime? InactiveSince, + string? InvalidationReason, + bool Conflicting); + + /* Version-gated because the most useful diagnostics are recent additions: + PG16+ : conflicting + PG17+ : inactive_since, invalidation_reason + + inactive_since in particular is what turns "this slot is inactive" into "this slot has been + inactive for three weeks" — the difference between a consumer between polls and an orphan. On 16 + the collector substitutes NULL rather than omitting the column, so the table shape stays constant + across the fleet and a chart does not change shape at an upgrade. + + Retained WAL is computed rather than read, because the column that would answer it directly + (safe_wal_size) is NULL whenever max_slot_wal_keep_size is -1, which is the DEFAULT. Relying on it + would mean reporting nothing on a stock server, precisely where the risk is unbounded. + + The LSN reference must switch on recovery state: pg_current_wal_lsn() ERRORS on a standby, so a + reader target would fail the whole collection. pg_last_wal_receive_lsn() is the standby's + equivalent. Aurora readers are legitimate targets here, so this is not hypothetical. */ + private static string BuildQueryText(int postgresMajorVersion) + { + var conflicting = postgresMajorVersion >= 16 ? "s.conflicting" : "false"; + /* AT TIME ZONE 'UTC', not a bare select and not ::timestamp. inactive_since is + `timestamp with time zone`, and two things go wrong if that is not converted HERE: + + * Npgsql maps a timestamptz read to DateTime with Kind=Utc, and refuses to write a Kind=Utc + DateTime into the store's `timestamp without time zone` column — so a bare select fails at + COPY time on any PG17 target with a slot that has ever been inactive. + * `::timestamp` would convert, but it renders the instant in the SESSION's TimeZone before + dropping the offset. The fleet's parameter groups all say UTC today, so it would agree + today and silently shift the moment one of them did not. + + AT TIME ZONE 'UTC' is the only form that is both correctly typed and timezone-independent. */ + var inactiveSince = postgresMajorVersion >= 17 + ? "(s.inactive_since AT TIME ZONE 'UTC')" + : "NULL::timestamp"; + var invalidationReason = postgresMajorVersion >= 17 ? "s.invalidation_reason" : "NULL::text"; + + return $@" +SELECT + s.slot_name AS slot_name, + s.slot_type AS slot_type, + s.plugin AS plugin, + s.database AS database_name, + s.active AS is_active, + coalesce(s.active_pid, 0)::bigint AS active_pid, + s.temporary AS is_temporary, + s.two_phase AS two_phase, + s.wal_status AS wal_status, + coalesce(s.safe_wal_size, -1)::bigint AS safe_wal_size_bytes, + coalesce( + (CASE + WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() + ELSE pg_current_wal_lsn() + END - s.restart_lsn)::bigint, -1) AS retained_wal_bytes, + coalesce(age(s.xmin)::bigint, -1) AS xmin_age, + coalesce(age(s.catalog_xmin)::bigint, -1) AS catalog_xmin_age, + {inactiveSince} AS inactive_since, + {invalidationReason} AS invalidation_reason, + coalesce({conflicting}, false) AS conflicting +FROM pg_replication_slots AS s +ORDER BY s.slot_name"; + } + + public override string Name => "pg_replication_slots"; + + /// + /// NOT pg_replication_slots — that name is taken by pg_catalog.pg_replication_slots, the + /// system view this collector READS, and a store table cannot share it. + /// pg_catalog is searched implicitly and FIRST, ahead of every entry in search_path, + /// so an unqualified reference to that name resolves to the system view no matter what the store + /// contains. It fails loudly in one place — CREATE INDEX on a view is 42809, which aborted the + /// whole migration and left the store unusable — and SILENTLY everywhere else: a reader's + /// FROM pg_replication_slots would have returned the MONITORING STORE's own (empty) slot list + /// instead of collected history, so the tool would always report no slots and the retention alert would + /// never fire. A muted outage predictor is worse than none, which is why the name changes rather than + /// every reference being schema-qualified and hoped over. + /// Name and TargetTable differing is established practice here (query_store → query_store_stats, + /// cpu_utilization → cpu_utilization_stats, and two more), so the collector keeps naming its source. + /// + public override string TargetTable => "pg_replication_slot_stats"; + + /// Core catalog only — any PostgreSQL target. + public override bool AppliesTo(CollectorTargetInfo target) => true; + + public override CollectorQuery BuildQuery(CollectorContext context) + => new(BuildQueryText(context.Target.PostgresMajorVersion)); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("slot_name", CollectorColumnType.Varchar), + new CollectorColumn("slot_type", CollectorColumnType.Varchar), + new CollectorColumn("plugin", CollectorColumnType.Varchar), + new CollectorColumn("database_name", CollectorColumnType.Varchar), + new CollectorColumn("is_active", CollectorColumnType.Boolean), + new CollectorColumn("active_pid", CollectorColumnType.BigInt), + new CollectorColumn("is_temporary", CollectorColumnType.Boolean), + new CollectorColumn("two_phase", CollectorColumnType.Boolean), + /* The single most diagnostic column: reserved = healthy, extended = WAL is being retained + BECAUSE of this slot (the disk-fill warning), unreserved = required WAL is already gone, + lost = the slot is unusable. */ + new CollectorColumn("wal_status", CollectorColumnType.Varchar), + /* -1 means "not applicable", which on a stock server is the norm: this column is NULL whenever + max_slot_wal_keep_size is -1. Stored as a sentinel rather than NULL so a consumer cannot + mistake "no limit configured" for "no data collected". */ + new CollectorColumn("safe_wal_size_bytes", CollectorColumnType.BigInt), + new CollectorColumn("retained_wal_bytes", CollectorColumnType.BigInt), + new CollectorColumn("xmin_age", CollectorColumnType.BigInt), + new CollectorColumn("catalog_xmin_age", CollectorColumnType.BigInt), + new CollectorColumn("inactive_since", CollectorColumnType.Timestamp), + new CollectorColumn("invalidation_reason", CollectorColumnType.Varchar), + new CollectorColumn("conflicting", CollectorColumnType.Boolean), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row( + SlotName: reader.GetString(0), + SlotType: reader.IsDBNull(1) ? null : reader.GetString(1), + Plugin: reader.IsDBNull(2) ? null : reader.GetString(2), + DatabaseName: reader.IsDBNull(3) ? null : reader.GetString(3), + IsActive: !reader.IsDBNull(4) && reader.GetBoolean(4), + ActivePid: reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + IsTemporary: !reader.IsDBNull(6) && reader.GetBoolean(6), + TwoPhase: !reader.IsDBNull(7) && reader.GetBoolean(7), + WalStatus: reader.IsDBNull(8) ? null : reader.GetString(8), + SafeWalSizeBytes: reader.IsDBNull(9) ? -1 : reader.GetInt64(9), + RetainedWalBytes: reader.IsDBNull(10) ? -1 : reader.GetInt64(10), + XminAge: reader.IsDBNull(11) ? -1 : reader.GetInt64(11), + CatalogXminAge: reader.IsDBNull(12) ? -1 : reader.GetInt64(12), + InactiveSince: reader.IsDBNull(13) ? null : reader.GetDateTime(13), + InvalidationReason: reader.IsDBNull(14) ? null : reader.GetString(14), + Conflicting: !reader.IsDBNull(15) && reader.GetBoolean(15))); + } + + return rows; + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* No deltas. Retained WAL is a level — it is the size of a hole, not work done — and the useful + reading is how big it is now plus whether it is still growing, which a trend of this column + shows directly. */ + writer + .Value(row.SlotName) + .Value(row.SlotType) + .Value(row.Plugin) + .Value(row.DatabaseName) + .Value(row.IsActive) + .Value(row.ActivePid) + .Value(row.IsTemporary) + .Value(row.TwoPhase) + .Value(row.WalStatus) + .Value(row.SafeWalSizeBytes) + .Value(row.RetainedWalBytes) + .Value(row.XminAge) + .Value(row.CatalogXminAge) + .Value(row.InactiveSince) + .Value(row.InvalidationReason) + .Value(row.Conflicting); + } +} diff --git a/PerformanceMonitor.Collectors/PgStatementStatsCollector.cs b/PerformanceMonitor.Collectors/PgStatementStatsCollector.cs new file mode 100644 index 000000000..022a8f1ca --- /dev/null +++ b/PerformanceMonitor.Collectors/PgStatementStatsCollector.cs @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Per-query-shape execution statistics for an Amazon Aurora PostgreSQL target — the Postgres +/// counterpart of . Reads aurora_stat_statements(), which is +/// pg_stat_statements plus columns only Aurora has. +/// Two things Aurora adds that are worth the dependency. It decomposes the opaque +/// shared_blks_read into where the block actually came from — +/// storage_blks_read (the distributed storage volume), orcache_blks_hit (the local NVMe +/// Optimized Reads tier), and local — which means a cache-hit ratio computed the community way is +/// arithmetically misleading on Aurora, because a "read" may have been a fast local hit. And it +/// reports peak memory per statement, which is the closest thing PostgreSQL has to SQL Server's +/// memory-grant data; core PostgreSQL has no grant concept at all. +/// Not gated on Aurora being present for the *statements* themselves — plain +/// pg_stat_statements would serve those — but this definition reads the Aurora-extended +/// function, so it is Aurora-only like its wait sibling. A stock-PostgreSQL variant reading the +/// vanilla view would be a separate definition. +/// +public sealed class PgStatementStatsCollector : PostgresCollectorDefinitionBase +{ + public static PgStatementStatsCollector Instance { get; } = new(); + + private PgStatementStatsCollector() + { + } + + public readonly record struct Row( + long QueryId, + long DatabaseId, + long UserId, + bool TopLevel, + long Calls, + double TotalExecTimeMs, + double MinExecTimeMs, + double MaxExecTimeMs, + double MeanExecTimeMs, + long RowsReturned, + long SharedBlocksHit, + long SharedBlocksRead, + long SharedBlocksDirtied, + long SharedBlocksWritten, + long TempBlocksRead, + long TempBlocksWritten, + double BlockReadTimeMs, + double BlockWriteTimeMs, + long StorageBlocksRead, + long OrcacheBlocksHit, + double StorageBlockReadTimeMs, + double OrcacheBlockReadTimeMs, + long WalRecords, + long WalFpi, + long WalBytes, + long TotalExecPeakMemBytes, + long MaxExecPeakMemBytes); + + /* Column names DIFFER between PostgreSQL 16 and 17 and our fleet spans both, so the query is + built per major rather than SELECT *-ed. Verified against live 16.11 and 17.7: + + 16.11 : blk_read_time, blk_write_time + 17.7 : shared_blk_read_time, shared_blk_write_time + + PG17 also adds jit_deform_time/_count, stats_since, and minmax_stats_since, which this + definition does not read. A SELECT * here would not error on either version — it would silently + shift every ordinal, which is how monitoring tools shipped broken PG17 collectors. + + Explicit casts pin the reader's types: wal_bytes is numeric in PostgreSQL (bigint cannot hold + its declared 10^20 range, though no real statement approaches it), and Npgsql's type checking + is strict enough that reading numeric with GetInt64 throws. */ + private static string BuildQueryText(int postgresMajorVersion) + { + var readTime = postgresMajorVersion >= 17 ? "shared_blk_read_time" : "blk_read_time"; + var writeTime = postgresMajorVersion >= 17 ? "shared_blk_write_time" : "blk_write_time"; + + return $@" +SELECT + queryid::bigint AS queryid, + dbid::bigint AS dbid, + userid::bigint AS userid, + toplevel AS toplevel, + calls::bigint AS calls, + total_exec_time AS total_exec_time, + min_exec_time AS min_exec_time, + max_exec_time AS max_exec_time, + mean_exec_time AS mean_exec_time, + rows::bigint AS rows_returned, + shared_blks_hit::bigint AS shared_blks_hit, + shared_blks_read::bigint AS shared_blks_read, + shared_blks_dirtied::bigint AS shared_blks_dirtied, + shared_blks_written::bigint AS shared_blks_written, + temp_blks_read::bigint AS temp_blks_read, + temp_blks_written::bigint AS temp_blks_written, + {readTime} AS blk_read_time, + {writeTime} AS blk_write_time, + storage_blks_read::bigint AS storage_blks_read, + orcache_blks_hit::bigint AS orcache_blks_hit, + storage_blk_read_time AS storage_blk_read_time, + orcache_blk_read_time AS orcache_blk_read_time, + wal_records::bigint AS wal_records, + wal_fpi::bigint AS wal_fpi, + wal_bytes::bigint AS wal_bytes, + total_exec_peakmem::bigint AS total_exec_peakmem, + max_exec_peakmem::bigint AS max_exec_peakmem +FROM aurora_stat_statements(false) +WHERE calls > 0"; + } + + public override string Name => "pg_statement_stats"; + + public override string TargetTable => "pg_statement_stats"; + + /// + /// Aurora only: this reads aurora_stat_statements(), the Aurora-extended function, not the + /// vanilla pg_stat_statements view. + /// + public override bool AppliesTo(CollectorTargetInfo target) => target.IsAurora; + + public override CollectorQuery BuildQuery(CollectorContext context) + => new(BuildQueryText(context.Target.PostgresMajorVersion)); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("queryid", CollectorColumnType.BigInt), + new CollectorColumn("database_id", CollectorColumnType.BigInt), + new CollectorColumn("user_id", CollectorColumnType.BigInt), + new CollectorColumn("toplevel", CollectorColumnType.Boolean), + new CollectorColumn("calls", CollectorColumnType.BigInt), + new CollectorColumn("total_exec_time_ms", CollectorColumnType.Double), + new CollectorColumn("min_exec_time_ms", CollectorColumnType.Double), + new CollectorColumn("max_exec_time_ms", CollectorColumnType.Double), + new CollectorColumn("mean_exec_time_ms", CollectorColumnType.Double), + new CollectorColumn("rows_returned", CollectorColumnType.BigInt), + new CollectorColumn("shared_blks_hit", CollectorColumnType.BigInt), + new CollectorColumn("shared_blks_read", CollectorColumnType.BigInt), + new CollectorColumn("shared_blks_dirtied", CollectorColumnType.BigInt), + new CollectorColumn("shared_blks_written", CollectorColumnType.BigInt), + new CollectorColumn("temp_blks_read", CollectorColumnType.BigInt), + new CollectorColumn("temp_blks_written", CollectorColumnType.BigInt), + new CollectorColumn("blk_read_time_ms", CollectorColumnType.Double), + new CollectorColumn("blk_write_time_ms", CollectorColumnType.Double), + /* Aurora-only I/O source split. Without these, blks_read is opaque: it may have been a + network round trip to the storage volume or a hit in the local NVMe tier. */ + new CollectorColumn("storage_blks_read", CollectorColumnType.BigInt), + new CollectorColumn("orcache_blks_hit", CollectorColumnType.BigInt), + new CollectorColumn("storage_blk_read_time_ms", CollectorColumnType.Double), + new CollectorColumn("orcache_blk_read_time_ms", CollectorColumnType.Double), + new CollectorColumn("wal_records", CollectorColumnType.BigInt), + new CollectorColumn("wal_fpi", CollectorColumnType.BigInt), + new CollectorColumn("wal_bytes", CollectorColumnType.BigInt), + /* The memory-grant analog. No SQL Server DMV gives per-query WAL bytes either, so both of + these are signals the SQL Server side cannot offer. */ + new CollectorColumn("total_exec_peakmem_bytes", CollectorColumnType.BigInt), + new CollectorColumn("max_exec_peakmem_bytes", CollectorColumnType.BigInt), + new CollectorColumn("delta_calls", CollectorColumnType.BigInt), + new CollectorColumn("delta_total_exec_time_ms", CollectorColumnType.BigInt), + new CollectorColumn("delta_rows", CollectorColumnType.BigInt), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row( + QueryId: reader.GetInt64(0), + DatabaseId: reader.GetInt64(1), + UserId: reader.GetInt64(2), + TopLevel: !reader.IsDBNull(3) && reader.GetBoolean(3), + Calls: reader.GetInt64(4), + TotalExecTimeMs: Dbl(reader, 5), + MinExecTimeMs: Dbl(reader, 6), + MaxExecTimeMs: Dbl(reader, 7), + MeanExecTimeMs: Dbl(reader, 8), + RowsReturned: reader.GetInt64(9), + SharedBlocksHit: reader.GetInt64(10), + SharedBlocksRead: reader.GetInt64(11), + SharedBlocksDirtied: reader.GetInt64(12), + SharedBlocksWritten: reader.GetInt64(13), + TempBlocksRead: reader.GetInt64(14), + TempBlocksWritten: reader.GetInt64(15), + BlockReadTimeMs: Dbl(reader, 16), + BlockWriteTimeMs: Dbl(reader, 17), + StorageBlocksRead: reader.GetInt64(18), + OrcacheBlocksHit: reader.GetInt64(19), + StorageBlockReadTimeMs: Dbl(reader, 20), + OrcacheBlockReadTimeMs: Dbl(reader, 21), + WalRecords: reader.GetInt64(22), + WalFpi: reader.GetInt64(23), + WalBytes: reader.GetInt64(24), + TotalExecPeakMemBytes: reader.GetInt64(25), + MaxExecPeakMemBytes: reader.GetInt64(26))); + } + + return rows; + + static double Dbl(DbDataReader r, int ordinal) => r.IsDBNull(ordinal) ? 0 : r.GetDouble(ordinal); + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* Delta key is (queryid, dbid, userid, toplevel) — the full pg_stat_statements identity, not + queryid alone. The same normalized statement run by a different user or against a different + database is a DIFFERENT entry with its own counters, so keying on queryid alone would + interleave several series into one and produce nonsense deltas. + + queryid itself is not stable across major versions (it is derived from a post-parse-analysis + tree, including internal object identifiers), so a mass reset after an upgrade is expected + behaviour rather than an anomaly — the delta calculator's counter-regression handling covers + it the same way it covers a restart. */ + var key = string.Create(CultureInfo.InvariantCulture, + $"{row.QueryId}|{row.DatabaseId}|{row.UserId}|{(row.TopLevel ? 1 : 0)}"); + + var deltaCalls = context.Deltas.CalculateDelta( + context.ServerId, "pg_statement_stats_calls", key, row.Calls, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + /* Time is stored as double milliseconds but the delta machinery is integral, so the delta is + taken on whole milliseconds. Sub-millisecond drift per interval is immaterial against the + totals this feeds, and keeping one delta calculator for both engines is worth more. */ + var deltaTotalTime = context.Deltas.CalculateDelta( + context.ServerId, "pg_statement_stats_time", key, (long)row.TotalExecTimeMs, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaRows = context.Deltas.CalculateDelta( + context.ServerId, "pg_statement_stats_rows", key, row.RowsReturned, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + writer + .Value(row.QueryId) + .Value(row.DatabaseId) + .Value(row.UserId) + .Value(row.TopLevel) + .Value(row.Calls) + .Value(row.TotalExecTimeMs) + .Value(row.MinExecTimeMs) + .Value(row.MaxExecTimeMs) + .Value(row.MeanExecTimeMs) + .Value(row.RowsReturned) + .Value(row.SharedBlocksHit) + .Value(row.SharedBlocksRead) + .Value(row.SharedBlocksDirtied) + .Value(row.SharedBlocksWritten) + .Value(row.TempBlocksRead) + .Value(row.TempBlocksWritten) + .Value(row.BlockReadTimeMs) + .Value(row.BlockWriteTimeMs) + .Value(row.StorageBlocksRead) + .Value(row.OrcacheBlocksHit) + .Value(row.StorageBlockReadTimeMs) + .Value(row.OrcacheBlockReadTimeMs) + .Value(row.WalRecords) + .Value(row.WalFpi) + .Value(row.WalBytes) + .Value(row.TotalExecPeakMemBytes) + .Value(row.MaxExecPeakMemBytes) + .Value(deltaCalls) + .Value(deltaTotalTime) + .Value(deltaRows); + } +} diff --git a/PerformanceMonitor.Collectors/PgWaitStatsCollector.cs b/PerformanceMonitor.Collectors/PgWaitStatsCollector.cs new file mode 100644 index 000000000..46c10ff4d --- /dev/null +++ b/PerformanceMonitor.Collectors/PgWaitStatsCollector.cs @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Cumulative wait statistics for an Amazon Aurora PostgreSQL target — the Postgres counterpart of +/// , with the same shape: cumulative counters in, deltas computed on +/// write, noise filtered out. +/// Core PostgreSQL has no cumulative wait accounting at all. There is no +/// sys.dm_os_wait_stats equivalent — only the instantaneous pg_stat_activity.wait_event, +/// and a proposal to add accumulation has been rejected twice on overhead grounds. The usual +/// workaround is a sampling extension (pg_wait_sampling, pgsentinel), and Aurora permits +/// exactly thirteen preloadable libraries, none of which are those. So on Aurora this function is not +/// merely the convenient source — it is the only one. +/// Aurora provides it as a built-in, no extension required, which is why this collector gates on +/// rather than on a version: on stock PostgreSQL there is +/// nothing to read and the collector must not run at all. +/// +public sealed class PgWaitStatsCollector : PostgresCollectorDefinitionBase +{ + public static PgWaitStatsCollector Instance { get; } = new(); + + private PgWaitStatsCollector() + { + } + + public readonly record struct Row( + int TypeId, + long EventId, + string? TypeName, + string? EventName, + long Waits, + long WaitTimeMicroseconds); + + /* Wait TYPES whose events are never a finding. Filtered by type rather than by event name so a + new background worker in a future Aurora release is excluded automatically instead of arriving + as a spike. + + Activity — the server process is idle. Every event here is a background sleep loop + (WalWriterMain, CheckpointerMain, AutoVacuumMain, AuroraServerlessMonitoringMain, ...) with + a fixed cycle, so wait_time grows at ~1 second per second of uptime forever. Measured on a + prod cluster: AuroraRuntimeMain alone at 8,563,569 seconds over 8,562,216 seconds of + uptime. Left in, it is 99%+ of the chart. + Client — waiting on the application's socket. ClientRead measured at 565,758,023 seconds on + one prod writer and 1,674,540,998 on another: that is idle connections, i.e. the app, not + the database. + Timeout — deliberate sleeps, including VacuumDelay (autovacuum's own cost-limit pause). + + Compared with the SQL Server side this is a type-level list, not the per-name + IgnoredWaitDefaults set, because Postgres wait types partition much more cleanly by intent. */ + /* Case-insensitive on purpose: Aurora renamed events between majors without changing their ids + (AutoVacuumMain on 16.11, AutovacuumMain on 17.7), so an ordinal set would filter on one major + and quietly stop filtering on the other. Deltas key on the numeric event_id for the same reason. */ + private static readonly HashSet s_ignoredWaitTypes = + new(StringComparer.OrdinalIgnoreCase) { "Activity", "Client", "Timeout" }; + + /* Signatures below are VERIFIED against live Aurora 16.11 and 17.7, not taken from the AWS + reference, which is wrong about one of them: + + aurora_stat_system_waits() -> (type_id, event_id, waits, wait_time) + aurora_stat_wait_type() -> (type_id, type_name) + aurora_stat_wait_event() -> (type_id, event_id, event_name) <- THREE columns + + The AWS docs describe aurora_stat_wait_event() as returning four columns including type_name. + It returns three, and type_id comes first. Getting that wrong does not error: the alias binds + event_id to type_id, the join matches nothing, and every event_name comes back NULL. That is + exactly how this was discovered. + + LEFT JOIN, never NATURAL JOIN (which the AWS example uses): the documented wait-type list omits + type 2, Aurora Limitless adds type 12, and an inner join silently drops any event whose type is + not in the lookup. A wait we cannot name is still a wait we must record. + + Explicit casts pin the reader's types. Without them the mapping depends on whether Aurora + returns int4 or int8 for a given column, and Npgsql's type checking is strict — GetInt64 on an + int4 column throws. + + No delta computed in SQL: wait_time is cumulative since instance start with NO reset function + anywhere in the Aurora API, so the only way to get an interval is snapshot-and-subtract, which + is what the shared delta machinery already does for SQL Server. A counter that went backwards + means the instance restarted. */ + private const string QueryText = @" +SELECT + w.type_id::int AS type_id, + w.event_id::bigint AS event_id, + t.type_name AS type_name, + e.event_name AS event_name, + w.waits::bigint AS waits, + w.wait_time::bigint AS wait_time_us +FROM aurora_stat_system_waits() AS w(type_id, event_id, waits, wait_time) +LEFT JOIN aurora_stat_wait_type() AS t(type_id, type_name) + ON t.type_id = w.type_id +LEFT JOIN aurora_stat_wait_event() AS e(type_id, event_id, event_name) + /* BOTH ids. event_id is unique across types today because Aurora packs the type into its high byte + (event_id >> 24 = type_id), so joining on event_id alone happens to be correct — but that is an + undocumented encoding, the function hands us type_id right there, and adding it costs nothing. + Without it, a future Aurora that reuses an event_id under a different type would silently produce a + row-multiplying join and double every wait figure for the affected events. */ + ON e.event_id = w.event_id + AND e.type_id = w.type_id +WHERE w.wait_time > 0"; + + public override string Name => "pg_wait_stats"; + + public override string TargetTable => "pg_wait_stats"; + + /// + /// Aurora only. The wait functions are Aurora built-ins; stock PostgreSQL has no cumulative wait + /// source at all, so there is nothing for this collector to read there. + /// + public override bool AppliesTo(CollectorTargetInfo target) => target.IsAurora; + + public override CollectorQuery BuildQuery(CollectorContext context) => new(QueryText); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("wait_type_id", CollectorColumnType.Integer), + new CollectorColumn("wait_event_id", CollectorColumnType.BigInt), + new CollectorColumn("wait_type", CollectorColumnType.Varchar), + new CollectorColumn("wait_event", CollectorColumnType.Varchar), + new CollectorColumn("waits", CollectorColumnType.BigInt), + new CollectorColumn("wait_time_us", CollectorColumnType.BigInt), + new CollectorColumn("delta_waits", CollectorColumnType.BigInt), + new CollectorColumn("delta_wait_time_us", CollectorColumnType.BigInt), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + var typeName = reader.IsDBNull(2) ? null : reader.GetString(2); + + /* Filter by type name, case-insensitively and on our own list rather than + context.IgnoredWaitTypes, which holds SQL Server wait-type names. An undecodable type + (NULL type_name) is deliberately KEPT: it means the lookup did not know the type, and + dropping it would hide exactly the new-wait-type case worth seeing. */ + if (typeName is not null && s_ignoredWaitTypes.Contains(typeName)) + { + continue; + } + + rows.Add(new Row( + TypeId: reader.GetInt32(0), + EventId: reader.GetInt64(1), + TypeName: typeName, + EventName: reader.IsDBNull(3) ? null : reader.GetString(3), + Waits: reader.GetInt64(4), + WaitTimeMicroseconds: reader.GetInt64(5))); + } + + return rows; + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* Delta key is the numeric event_id, NOT the event name. Wait-event NAME CASING DIFFERS + BETWEEN AURORA MAJORS — 16.11 emits AutoVacuumMain and BgWriterMain where 17.7 emits + AutovacuumMain and BgwriterMain — so a name-keyed delta would break its own history the + moment a cluster is upgraded, reading as one series ending and another beginning. The id + is stable, and event_id >> 24 == type_id holds (verified on every event on two clusters), + so the id also carries the type. + + The shared gap policy matches wait_stats: past it, emit no delta rather than a spike that + is really an interval measurement. */ + var key = row.EventId.ToString(System.Globalization.CultureInfo.InvariantCulture); + + var deltaWaits = context.Deltas.CalculateDelta( + context.ServerId, "pg_wait_stats_waits", key, row.Waits, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWaitTime = context.Deltas.CalculateDelta( + context.ServerId, "pg_wait_stats_time", key, row.WaitTimeMicroseconds, + collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + + writer + .Value(row.TypeId) /* wait_type_id INTEGER */ + .Value(row.EventId) /* wait_event_id BIGINT */ + .Value(row.TypeName) /* wait_type VARCHAR */ + .Value(row.EventName) /* wait_event VARCHAR */ + .Value(row.Waits) /* waits BIGINT */ + .Value(row.WaitTimeMicroseconds) /* wait_time_us BIGINT */ + .Value(deltaWaits) /* delta_waits BIGINT */ + .Value(deltaWaitTime); /* delta_wait_time_us BIGINT */ + } +} diff --git a/PerformanceMonitor.Collectors/PgWraparoundStatsCollector.cs b/PerformanceMonitor.Collectors/PgWraparoundStatsCollector.cs new file mode 100644 index 000000000..55f9b5f01 --- /dev/null +++ b/PerformanceMonitor.Collectors/PgWraparoundStatsCollector.cs @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +using System; + +namespace PerformanceMonitor.Collectors; + +/// +/// Transaction ID and MultiXact ID freeze headroom per database — the single most consequential thing +/// to monitor on a PostgreSQL server, and something SQL Server has no counterpart for at all. +/// PostgreSQL's transaction ids are 32-bit and wrap. Autovacuum normally keeps the oldest +/// unfrozen id far from the wrap point, but when it cannot the consequences escalate on a documented +/// ladder: an anti-wraparound autovacuum is forced at autovacuum_freeze_max_age (200M by +/// default), a failsafe mode that abandons cost limits and index cleanup engages at +/// vacuum_failsafe_age (1.6B), warnings begin around 40M remaining ids, and at 3M remaining the +/// server refuses to assign new transaction ids: writes and DDL stop, reads continue. That last +/// state is a full write outage that no failover fixes, because every replica shares the condition. +/// MultiXact ids are a second, independent counter that is separately fatal and almost +/// universally unmonitored. They are consumed when a row is locked by multiple transactions at once, so +/// a workload heavy on SELECT FOR UPDATE/FOR SHARE or on foreign-key checks burns them far +/// faster than plain transaction ids — a server can be comfortable on XID age and in trouble on +/// MultiXact age. Both live on pg_database, so both are collected here rather than pretending +/// one implies the other. +/// Not Aurora-gated: this reads only core catalog surfaces, so it works on any PostgreSQL target. +/// It is the first collector here that does. Aurora does add one escalation worth knowing about, though +/// it is observed rather than collected — Aurora restarts its read replicas as a cluster approaches +/// wraparound, so on Aurora read availability degrades before the writer goes read-only. +/// +public sealed class PgWraparoundStatsCollector : PostgresCollectorDefinitionBase +{ + public static PgWraparoundStatsCollector Instance { get; } = new(); + + private PgWraparoundStatsCollector() + { + } + + public readonly record struct Row( + string DatabaseName, + long FrozenXidAge, + long MinMultiXidAge, + long AutovacuumFreezeMaxAge, + long AutovacuumMultixactFreezeMaxAge, + bool AllowsConnections); + + /// + /// The hard ceiling on transaction id age. PostgreSQL stops assigning new ids about 3 million short + /// of 2^31, so this is the denominator for "percent of the way to a write outage" — as opposed to + /// percent of the way to an emergency vacuum, which is a different and much nearer threshold. + /// + public const long WraparoundCeiling = 2_147_483_648L; + + /// + /// The ids PostgreSQL holds back: it stops accepting new write transactions with roughly this many still + /// unconsumed, rather than running the counter to the wall. So the useful "how much runway is left" + /// figure is minus this minus the age — which is also what makes the + /// MCP tool's 99.86%-of-space writes-stop point agree with the stored column. + /// + public const long StopMargin = 3_000_000L; + + /* Reads pg_database, which is a SHARED catalog: one connection sees every database, so this needs + no per-database fan-out. Verified readable under pg_monitor on our fleet. + + age()/mxid_age() are the correct comparisons rather than arithmetic on the raw xid, because both + handle the modular wrap that makes a naive subtraction wrong near the boundary — which is exactly + the region where being wrong matters. + + datfrozenxid's TOAST caveat: a table's own relfrozenxid can be fine while its TOAST relation's is + not, and pg_database.datfrozenxid already accounts for both because it is the minimum across all + relations in the database. Per-relation attribution — which table is holding the floor — is a + per-database read and belongs in its own collector; this one answers "how much time is left", + which is the alerting question. */ + private const string QueryText = @" +SELECT + d.datname AS database_name, + age(d.datfrozenxid)::bigint AS frozen_xid_age, + mxid_age(d.datminmxid)::bigint AS min_multixid_age, + current_setting('autovacuum_freeze_max_age')::bigint AS autovacuum_freeze_max_age, + current_setting('autovacuum_multixact_freeze_max_age')::bigint AS autovacuum_multixact_freeze_max_age, + d.datallowconn AS allows_connections +FROM pg_database AS d +/* EVERY database, templates included. The cluster-wide stop limit derives from the oldest datfrozenxid in + pg_database, so excluding template0/template1 could understate cluster risk by exactly the amount that + matters — and template0 aging without ever being vacuumed is a real, documented way to get there, typically + after a major upgrade. + + The per-database autovacuum collector DOES exclude templates, and that is not inconsistent: it needs a + CONNECTION per database and template0 refuses them (datallowconn = false). This read needs no connection at + all — pg_database is a SHARED catalog, verified on live Aurora 17.7 where template0's datfrozenxid reads + fine despite datallowconn = false. allows_connections is already stored, so a consumer can still tell a + template row apart. */ +ORDER BY age(d.datfrozenxid) DESC"; + + public override string Name => "pg_wraparound_stats"; + + public override string TargetTable => "pg_wraparound_stats"; + + /// Core catalog only — every PostgreSQL target, Aurora or not. + public override bool AppliesTo(CollectorTargetInfo target) => true; + + public override CollectorQuery BuildQuery(CollectorContext context) => new(QueryText); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("database_name", CollectorColumnType.Varchar), + new CollectorColumn("frozen_xid_age", CollectorColumnType.BigInt), + new CollectorColumn("min_multixid_age", CollectorColumnType.BigInt), + new CollectorColumn("autovacuum_freeze_max_age", CollectorColumnType.BigInt), + new CollectorColumn("autovacuum_multixact_freeze_max_age", CollectorColumnType.BigInt), + /* Both percentages are stored rather than computed on read, because the DENOMINATORS are + per-server settings that can change: a stored percentage stays true to the configuration in + force when it was measured, where recomputing later against today's setting would silently + rewrite history after someone tunes autovacuum_freeze_max_age. */ + new CollectorColumn("pct_toward_emergency_vacuum", CollectorColumnType.Double), + new CollectorColumn("pct_toward_wraparound", CollectorColumnType.Double), + new CollectorColumn("pct_toward_multixact_emergency", CollectorColumnType.Double), + new CollectorColumn("pct_toward_multixact_wraparound", CollectorColumnType.Double), + new CollectorColumn("xids_remaining", CollectorColumnType.BigInt), + new CollectorColumn("multixids_remaining", CollectorColumnType.BigInt), + new CollectorColumn("allows_connections", CollectorColumnType.Boolean), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var rows = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row( + DatabaseName: reader.GetString(0), + FrozenXidAge: reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + MinMultiXidAge: reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + AutovacuumFreezeMaxAge: reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + AutovacuumMultixactFreezeMaxAge: reader.IsDBNull(4) ? 0 : reader.GetInt64(4), + AllowsConnections: !reader.IsDBNull(5) && reader.GetBoolean(5))); + } + + return rows; + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* No deltas. Age is not a counter that accumulates work — it is a distance from a wall, and it + falls when autovacuum freezes. A delta would describe the rate of approach, which is + interesting, but the alertable fact is the LEVEL, and a level is what a trend chart of this + column already shows. */ + writer + .Value(row.DatabaseName) + .Value(row.FrozenXidAge) + .Value(row.MinMultiXidAge) + .Value(row.AutovacuumFreezeMaxAge) + .Value(row.AutovacuumMultixactFreezeMaxAge) + .Value(Pct(row.FrozenXidAge, row.AutovacuumFreezeMaxAge)) + .Value(Pct(row.FrozenXidAge, WraparoundCeiling)) + .Value(Pct(row.MinMultiXidAge, row.AutovacuumMultixactFreezeMaxAge)) + .Value(Pct(row.MinMultiXidAge, WraparoundCeiling)) + /* To where writes STOP, not to the raw 2^31. PostgreSQL refuses new transactions with about + 3,000,000 ids still on the clock, so counting to the ceiling overstated the runway by that + margin — and disagreed with the MCP tool's own 99.86% figure, which already accounts for it. + Clamped at 0 so a database past the stop point reports "none left" rather than a negative. */ + .Value(Math.Max(0, WraparoundCeiling - StopMargin - row.FrozenXidAge)) + .Value(Math.Max(0, WraparoundCeiling - StopMargin - row.MinMultiXidAge)) + .Value(row.AllowsConnections); + } + + /// + /// Percentage of reached, or 0 when the ceiling is unknown. Returning 0 + /// for an unreadable setting is deliberate: an unknown denominator must not manufacture a + /// percentage, and it must not manufacture an alert either. + /// + private static double Pct(long age, long ceiling) + => ceiling > 0 ? (double)age / ceiling * 100.0 : 0.0; +} diff --git a/PerformanceMonitor.Collectors/PgXminHorizonCollector.cs b/PerformanceMonitor.Collectors/PgXminHorizonCollector.cs new file mode 100644 index 000000000..b03f82a13 --- /dev/null +++ b/PerformanceMonitor.Collectors/PgXminHorizonCollector.cs @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// What is holding back the xmin horizon — the reason autovacuum can run, report success, and still +/// reclaim nothing. +/// This is the highest-leverage composite signal in PostgreSQL monitoring, because four +/// unrelated causes present identically: dead tuples accumulate, autovacuum runs and logs success, +/// and the table never shrinks. A long-running or idle-in-transaction session, an abandoned replication +/// slot, a standby feeding back its xmin with hot_standby_feedback, and an orphaned prepared +/// transaction all produce exactly that picture — and the remedy is completely different for each. Kill +/// a session, drop a slot, disable feedback, or commit/rollback a two-phase transaction: doing the wrong +/// one is at best useless. +/// So attribution IS the deliverable. A collector that reported only "the horizon is old" would +/// leave the reader exactly where they started, which is why this emits the oldest holder per source +/// and marks which source is winning, rather than a single aggregate number. +/// The consequence compounds beyond bloat: a pinned horizon also blocks freezing, so an unattended +/// xmin holder is an upstream cause of the wraparound risk +/// measures. Core catalog surfaces only, so this runs on any PostgreSQL target. +/// +public sealed class PgXminHorizonCollector : PostgresCollectorDefinitionBase +{ + public static PgXminHorizonCollector Instance { get; } = new(); + + private PgXminHorizonCollector() + { + } + + public readonly record struct Row( + string Source, + long XminAge, + string? Holder, + string? Detail, + bool IsWinner); + + /* One UNION ALL branch per cause, reduced to the OLDEST holder per source by DISTINCT ON. That + bound matters: pg_stat_activity can carry hundreds of backends with an xmin, and storing all of + them every cycle would be a lot of rows to say one thing. Five rows per collection at most. + + Two branches come from pg_replication_slots deliberately. A slot's xmin holds back ordinary row + cleanup; its catalog_xmin holds back CATALOG cleanup specifically, which is what a logical + decoding slot pins, and the two can differ by a lot. Reporting only one would misattribute the + other. + + Every branch is independently empty-tolerant. No holder for a cause simply yields no row, and + zero rows overall is the healthy state — nothing is pinning the horizon. pg_stat_replication in + particular is expected to be empty on Aurora, where replicas read the same storage volume rather + than streaming WAL, so its absence must not read as an error. + + age() rather than arithmetic on the raw xid: modular wrap makes naive subtraction wrong exactly + near the boundary. */ + private const string QueryText = @" +WITH holders AS +( + SELECT + 'session'::text AS source, + /* The GREATER of the two ages. An idle-in-transaction writer under READ COMMITTED has RELEASED its + snapshot — backend_xmin is NULL — while still holding backend_xid, which pins the horizon just as + hard. Reading only backend_xmin made this collector blind to idle-in-transaction, which is its + single most-cited cause and the one the read surface leads with. */ + GREATEST( + coalesce(age(a.backend_xmin), 0), + coalesce(age(a.backend_xid), 0))::bigint AS xmin_age, + a.pid::text AS holder, + 'state=' || coalesce(a.state, '(none)') + || ' application=' || coalesce(a.application_name, '(none)') + /* AT TIME ZONE 'UTC', not ::text. These are timestamptz, so ::text renders in the SESSION + TimeZone — invisible on a UTC server, wrong everywhere else. The store's contract is naive UTC + and this detail string was the one place on the branch still breaking it. */ + || ' xact_start=' || coalesce((a.xact_start AT TIME ZONE 'UTC')::text, '(none)') + || ' query_start=' || coalesce((a.query_start AT TIME ZONE 'UTC')::text, '(none)') AS detail + FROM pg_stat_activity AS a + WHERE (a.backend_xmin IS NOT NULL OR a.backend_xid IS NOT NULL) + /* Never attribute the horizon to the collector's own snapshot. Darling's read sits in + pg_stat_activity with a backend_xmin like any other session, so without this it is a PERMANENT + 'session' holder: zero-rows-when-healthy becomes unreachable, and it silently pads the persistence + denominator so a real transient holder reads as chronic. */ + AND a.pid <> pg_backend_pid() + + UNION ALL + + SELECT + 'replication_slot'::text, + age(s.xmin)::bigint, + s.slot_name, + 'type=' || coalesce(s.slot_type, '(none)') + || ' active=' || coalesce(s.active::text, '(none)') + || ' database=' || coalesce(s.database, '(none)') + FROM pg_replication_slots AS s + WHERE s.xmin IS NOT NULL + + UNION ALL + + SELECT + 'replication_slot_catalog'::text, + age(s.catalog_xmin)::bigint, + s.slot_name, + 'type=' || coalesce(s.slot_type, '(none)') + || ' active=' || coalesce(s.active::text, '(none)') + || ' plugin=' || coalesce(s.plugin, '(none)') + FROM pg_replication_slots AS s + WHERE s.catalog_xmin IS NOT NULL + + UNION ALL + + SELECT + 'standby_feedback'::text, + age(r.backend_xmin)::bigint, + coalesce(r.application_name, r.client_addr::text, r.pid::text), + 'state=' || coalesce(r.state, '(none)') + || ' sync_state=' || coalesce(r.sync_state, '(none)') + FROM pg_stat_replication AS r + WHERE r.backend_xmin IS NOT NULL + + UNION ALL + + SELECT + 'prepared_transaction'::text, + age(p.transaction)::bigint, + p.gid, + 'prepared=' || coalesce((p.prepared AT TIME ZONE 'UTC')::text, '(none)') + || ' owner=' || coalesce(p.owner, '(none)') + || ' database=' || coalesce(p.database, '(none)') + FROM pg_prepared_xacts AS p +) +SELECT DISTINCT ON (source) + source, + xmin_age, + holder, + detail +FROM holders +WHERE xmin_age IS NOT NULL +ORDER BY source, xmin_age DESC"; + + public override string Name => "pg_xmin_horizon"; + + public override string TargetTable => "pg_xmin_horizon"; + + /// Core catalog surfaces only — any PostgreSQL target. + public override bool AppliesTo(CollectorTargetInfo target) => true; + + public override CollectorQuery BuildQuery(CollectorContext context) => new(QueryText); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("source", CollectorColumnType.Varchar), + new CollectorColumn("xmin_age", CollectorColumnType.BigInt), + new CollectorColumn("holder", CollectorColumnType.Varchar), + new CollectorColumn("detail", CollectorColumnType.Varchar), + /* Stamped at collection rather than derived on read, so a stored row always names the winner as + of the moment it was measured. Deriving it later would depend on which rows a query happened + to select, and a filtered read could crown a different winner than actually held the horizon. */ + new CollectorColumn("is_winner", CollectorColumnType.Boolean), + }; + + public override async ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + { + var holders = new List(); + + while (await reader.ReadAsync(cancellationToken)) + { + holders.Add(new Row( + Source: reader.GetString(0), + XminAge: reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + Holder: reader.IsDBNull(2) ? null : reader.GetString(2), + Detail: reader.IsDBNull(3) ? null : reader.GetString(3), + IsWinner: false)); + } + + if (holders.Count == 0) + { + /* Nothing pins the horizon. The healthy state, and the reason an empty result must never be + treated as a collection failure. */ + return holders; + } + + /* The winner is the single oldest holder across all sources — the one actually setting the + horizon. Ties resolve to the first source encountered, which is immaterial: if two causes are + at identical age, both need attention. */ + var oldest = holders.Max(h => h.XminAge); + var winnerStamped = false; + for (var i = 0; i < holders.Count; i++) + { + if (!winnerStamped && holders[i].XminAge == oldest) + { + holders[i] = holders[i] with { IsWinner = true }; + winnerStamped = true; + } + } + + return holders; + } + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + /* No deltas: like freeze age, an xmin age is a level. What matters is how old the horizon is + right now and what is holding it, not how fast it aged since the last sample. */ + writer + .Value(row.Source) + .Value(row.XminAge) + .Value(row.Holder) + .Value(row.Detail) + .Value(row.IsWinner); + } +} diff --git a/PerformanceMonitor.Collectors/PostgresCollectorDefinitionBase.cs b/PerformanceMonitor.Collectors/PostgresCollectorDefinitionBase.cs new file mode 100644 index 000000000..759238022 --- /dev/null +++ b/PerformanceMonitor.Collectors/PostgresCollectorDefinitionBase.cs @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +namespace PerformanceMonitor.Collectors; + +/// +/// Base for collector definitions whose query text is PostgreSQL — including Amazon Aurora +/// PostgreSQL. Exists for exactly one reason: is sealed to +/// here, so a Postgres definition cannot forget to +/// declare its dialect. +/// That footgun is otherwise real and silent. +/// defaults to — deliberately, so the 41 existing +/// definitions needed no edit — which means a new Postgres definition that derived from +/// and omitted the override would be advertised as T-SQL +/// and dispatched at SQL Server targets, where its query would fail every cycle. Deriving from this +/// class instead makes the dialect structural rather than a line someone has to remember. +/// Everything else is inherited unchanged: +/// stays available for gating WITHIN Postgres (server version floors, Aurora-only surfaces, whether +/// an extension is installed), because the engine check is composed on top by +/// . +/// +/// The definition's row type. +public abstract class PostgresCollectorDefinitionBase : CollectorDefinitionBase +{ + /// + /// Always . Sealed so a derived definition cannot + /// re-declare itself as another engine. + /// + public sealed override CollectorTargetEngine TargetEngine => CollectorTargetEngine.PostgreSql; +} diff --git a/PerformanceMonitor.Collectors/ProcedureStatsCollector.cs b/PerformanceMonitor.Collectors/ProcedureStatsCollector.cs index 86273f2d0..ac8f7a9d6 100644 --- a/PerformanceMonitor.Collectors/ProcedureStatsCollector.cs +++ b/PerformanceMonitor.Collectors/ProcedureStatsCollector.cs @@ -425,13 +425,13 @@ public override void WritePayload(Row row, ICollectorRowWriter writer, Collector /* Delta key: plan_handle to prevent cross-contamination when multiple plans exist for the same object; the db.schema.object fallback and the seven group names are the parity contract. */ var deltaKey = row.PlanHandle ?? $"{row.DatabaseName}.{row.SchemaName}.{row.ObjectName}"; - var deltaExec = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_exec", deltaKey, row.ExecutionCount, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWorker = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_worker", deltaKey, row.TotalWorkerTime, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaElapsed = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_elapsed", deltaKey, row.TotalElapsedTime, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaReads = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_reads", deltaKey, row.TotalLogicalReads, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWrites = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_writes", deltaKey, row.TotalLogicalWrites, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaPhysReads = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_phys_reads", deltaKey, row.TotalPhysicalReads, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaSpills = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_spills", deltaKey, row.TotalSpills, collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaExec = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_exec", deltaKey, row.ExecutionCount, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWorker = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_worker", deltaKey, row.TotalWorkerTime, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaElapsed = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_elapsed", deltaKey, row.TotalElapsedTime, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaReads = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_reads", deltaKey, row.TotalLogicalReads, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWrites = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_writes", deltaKey, row.TotalLogicalWrites, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaPhysReads = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_phys_reads", deltaKey, row.TotalPhysicalReads, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaSpills = context.Deltas.CalculateDelta(context.ServerId, "proc_stats_spills", deltaKey, row.TotalSpills, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.DatabaseName) diff --git a/PerformanceMonitor.Collectors/QueryStatsCollector.cs b/PerformanceMonitor.Collectors/QueryStatsCollector.cs index eb14b39f9..90c131496 100644 --- a/PerformanceMonitor.Collectors/QueryStatsCollector.cs +++ b/PerformanceMonitor.Collectors/QueryStatsCollector.cs @@ -83,6 +83,14 @@ public sealed class Row /// The statement's host object (schema.name) from sys.dm_exec_sql_text.objectid; /// NULL for ad-hoc/prepared text (#2012 stage 2 — splits INSERT...EXEC callers sharing a hash). public string? HostObjectName { get; set; } + + /// + /// #2235: seconds since this plan was compiled, as measured on the monitored server. Feeds the + /// delta calculator's series-age rule and is NOT stored — a recompile presents a new + /// plan_handle and therefore a new delta key, and without this the first sighting of that + /// key reports 0, which on a churning instance is most of the server's CPU. + /// + public int? CompileAgeSeconds { get; set; } } private const string SelectColumnsText = @" @@ -153,7 +161,14 @@ THEN ISNULL OBJECT_SCHEMA_NAME(st.objectid, st.dbid) + N'.' + OBJECT_NAME(st.objectid, st.dbid), N'Unknown' ) - END"; + END, + /* #2235: how long ago this plan was compiled, so the delta calculator can tell a key that is new + TO US from a counter that is new to the WORLD. Sent as an AGE rather than creation_time itself + because creation_time is in the monitored server's local time while collection times are UTC — + comparing them client-side is a timezone bug on every server that is not UTC. DATEDIFF is + evaluated where both clocks are the same one. Not stored: PayloadColumns is unchanged, this + exists only to inform the delta. */ + compile_age_seconds = DATEDIFF(SECOND, qs.creation_time, GETDATE())"; /* #1959: rank on the CHEAP DMV columns inside the derived table FIRST, and run the text apply, the NOT LIKE self-filter, and the (Darling-only) plan-XML render against the survivors ONLY. The optimizer @@ -388,9 +403,12 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec text — sys.dm_exec_sql_text.objectid resolved in the SELECT. This is what lets readers split INSERT...EXEC callers that share a query_hash. */ HostObjectName = reader.IsDBNull(42) ? null : reader.GetString(42), + /* #2235: compile age sits at 43 — inside SelectColumnsText, so it is present in BOTH + capture modes and its ordinal is fixed. That pushes the plan XML to 44. */ + CompileAgeSeconds = reader.IsDBNull(43) ? null : reader.GetInt32(43), /* query_plan_xml is the trailing column present only when CapturePlanXml spliced it - into the SELECT (ordinal 43); the short-circuit skips it entirely when off. */ - QueryPlanXml = context.CapturePlanXml && !reader.IsDBNull(43) ? reader.GetString(43) : null, + into the SELECT (ordinal 44); the short-circuit skips it entirely when off. */ + QueryPlanXml = context.CapturePlanXml && !reader.IsDBNull(44) ? reader.GetString(44) : null, }); } @@ -402,16 +420,30 @@ public override void WritePayload(Row row, ICollectorRowWriter writer, Collector /* Delta key = the dm_exec_query_stats row identity (sql_handle + offsets + plan_handle). Keying on plan_handle alone cross-contaminated multi-statement plans — parity contract. */ var deltaKey = $"{row.SqlHandle}:{row.StatementStartOffset}:{row.StatementEndOffset}:{row.PlanHandle}"; - var deltaExecCount = context.Deltas.CalculateDelta(context.ServerId, "query_stats_exec", deltaKey, row.ExecutionCount, collectionTime: context.CollectionTime, maxGapSeconds: 300); + + /* #2235: plan_handle is in the key above, and it changes on every recompile — so a churning plan + presents a NEW key on nearly every sighting, and a first sighting reports 0. That silently + discarded most of a plan-churning instance's CPU (a query Datadog measured at ~43% of the box + read as 18 executions and 2,824 ms over 168 hours), and it was invisible because the honest + "unknowable, not zero" path needs the SAME key to reappear lower, which a recompile never does. + Passing the compile age lets the calculator credit a plan whose counter demonstrably STARTED + since the previous pass, which is the recoverable half. The unrecoverable half is a plan + compiled AND evicted between two passes: it never appears in the DMV at all, so no keying + scheme can recover it — that ceiling is stated rather than left to be discovered. + + ALL EIGHT delta'd counters take the same rule. Crediting only some would make one row's metrics + disagree about how much work it did, which is worse than under-reporting all of them. */ + var age = row.CompileAgeSeconds; + var deltaExecCount = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_exec", deltaKey, row.ExecutionCount, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); /* Capture the collection interval alongside the CPU delta so the display can derive worker_time_per_second (peak CPU-ms per wall-clock second) over the window. */ - var deltaWorkerTime = context.Deltas.CalculateDeltaWithInterval(context.ServerId, "query_stats_worker", deltaKey, row.TotalWorkerTime, out var sampleIntervalSeconds, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaElapsedTime = context.Deltas.CalculateDelta(context.ServerId, "query_stats_elapsed", deltaKey, row.TotalElapsedTime, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaLogicalReads = context.Deltas.CalculateDelta(context.ServerId, "query_stats_reads", deltaKey, row.TotalLogicalReads, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaLogicalWrites = context.Deltas.CalculateDelta(context.ServerId, "query_stats_writes", deltaKey, row.TotalLogicalWrites, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaPhysicalReads = context.Deltas.CalculateDelta(context.ServerId, "query_stats_phys_reads", deltaKey, row.TotalPhysicalReads, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaRows = context.Deltas.CalculateDelta(context.ServerId, "query_stats_rows", deltaKey, row.TotalRows, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaSpills = context.Deltas.CalculateDelta(context.ServerId, "query_stats_spills", deltaKey, row.TotalSpills, collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaWorkerTime = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_worker", deltaKey, row.TotalWorkerTime, age, out var sampleIntervalSeconds, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaElapsedTime = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_elapsed", deltaKey, row.TotalElapsedTime, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaLogicalReads = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_reads", deltaKey, row.TotalLogicalReads, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaLogicalWrites = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_writes", deltaKey, row.TotalLogicalWrites, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaPhysicalReads = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_phys_reads", deltaKey, row.TotalPhysicalReads, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaRows = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_rows", deltaKey, row.TotalRows, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaSpills = context.Deltas.CalculateDeltaWithSeriesAge(context.ServerId, "query_stats_spills", deltaKey, row.TotalSpills, age, out _, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.DatabaseName) diff --git a/PerformanceMonitor.Collectors/QueryStoreBackfillState.cs b/PerformanceMonitor.Collectors/QueryStoreBackfillState.cs index ba25736b6..73b8051b1 100644 --- a/PerformanceMonitor.Collectors/QueryStoreBackfillState.cs +++ b/PerformanceMonitor.Collectors/QueryStoreBackfillState.cs @@ -37,6 +37,85 @@ public static class QueryStoreBackfillState /// State key prefix for a recorded clamp hole (value: ). public const string HoleKeyPrefix = "hole:"; + /// + /// The widest window a single backfill slice may hand the per-database query (#2102) — matched + /// to so NO path, live or backfill, ever windows wider + /// than the steady state the fleet proves. The backfill query aggregates and sorts its whole + /// window before the byte budget can bound anything (the same row-cap-is-not-a-cost-cap flaw + /// that wedged the live path), so an unchunked wide hole on a big database re-times-out forever + /// instead of draining. + /// + public static readonly TimeSpan MaxSliceSpan = TimeSpan.FromHours(1); + + /// + /// How recently the live path may have failed a server's query_store collection before the + /// backfill worker yields that server's slice (#2111). Two poll cycles: a failure inside the + /// current or previous cycle means the live path is struggling NOW, and a backfill slice + /// scanning the same QS internal tables on a MAXDOP-1 replica is exactly the contention that + /// keeps it struggling. The class doc's contract — "backfill can be slow forever without + /// delaying collection" — is what this enforces; holes wait, live recovers, backfill resumes. + /// + public static readonly TimeSpan YieldToLiveWindow = TimeSpan.FromMinutes(10); + + /// + /// True when the backfill worker should skip a server's slice this tick because its live + /// query_store collection failed within (#2111). Server-grain + /// on purpose: the contention is server-wide, and any database's live failure vouches for the + /// whole replica being contended. A pure function so the placement is pinnable in isolation, + /// like its siblings above. + /// + public static bool ShouldYieldToLive(DateTime? lastLiveFailureUtc, DateTime nowUtc) + => lastLiveFailureUtc is DateTime failure && nowUtc - failure < YieldToLiveWindow; + + /// + /// The narrowest window the adaptive shrink may reach (#2111 reserve, promoted on field + /// evidence): a member whose 1h window exceeds the command timeout halves per consecutive + /// failure toward this floor — 15 minutes fits inside a 60s read on every store the fleet has + /// shown us, and anything narrower than a flush interval would mostly return empty. + /// + public static readonly TimeSpan MinAdaptiveSpan = TimeSpan.FromMinutes(15); + + /// + /// The window a member gets after straight failures: + /// the full span halved per failure, floored at (the exponent is + /// capped so the shift math cannot wrap). Success resets the counter at the call sites, so a + /// recovered member is back at full span on its next cycle. Pure and pinned like its siblings — + /// the live clamp and the backfill slicing share it, so the two paths cannot drift on how fast + /// they back off. + /// + public static TimeSpan AdaptiveSpan(TimeSpan fullSpan, int consecutiveFailures) + { + if (consecutiveFailures <= 0) + { + return fullSpan; + } + + var halvings = Math.Min(consecutiveFailures, 6); + var shrunk = TimeSpan.FromTicks(fullSpan.Ticks >> halvings); + return shrunk < MinAdaptiveSpan ? MinAdaptiveSpan : shrunk; + } + + /// + /// Bounds one newest-first slice to the top of the remaining range: + /// returns the floor the slice should actually query, which is the requested floor once the + /// remainder is narrow enough. A pure function so the placement is pinnable in isolation, like + /// . The caller distinguishes "chunk exhausted" + /// (result > : an empty slice means only this CHUNK is quiet — + /// shrink the ceiling and keep walking) from "range exhausted" (result == + /// : an empty slice is terminal, exactly the pre-chunking semantics). + /// + public static DateTime BoundSliceFloor(DateTime floorUtc, DateTime ceilingUtc) + => BoundSliceFloor(floorUtc, ceilingUtc, MaxSliceSpan); + + /// The adaptive form (#2111 promoted): the caller passes + /// 's result so a server whose slices keep timing out digs in + /// progressively narrower chunks until one fits its command timeout. + public static DateTime BoundSliceFloor(DateTime floorUtc, DateTime ceilingUtc, TimeSpan span) + { + var chunkFloor = ceilingUtc - span; + return chunkFloor > floorUtc ? chunkFloor : floorUtc; + } + /// Encodes a hole range as from|to in round-trip format — deliberately not /// JSON, so the state row stays greppable and the codec dependency-free. public static string EncodeHole(DateTime fromUtc, DateTime toUtc) diff --git a/PerformanceMonitor.Collectors/QueryStoreCollector.cs b/PerformanceMonitor.Collectors/QueryStoreCollector.cs index dd5d1cefd..18b4497f4 100644 --- a/PerformanceMonitor.Collectors/QueryStoreCollector.cs +++ b/PerformanceMonitor.Collectors/QueryStoreCollector.cs @@ -326,6 +326,37 @@ WHERE actual_state IN (1, 2, 4) /// public const int MaxTextBytesPerDatabase = 64 * 1024 * 1024; + /// + /// The WALL-CLOCK ceiling for one database's pass (#2150), and the bound of last resort: the row cap + /// bounds ROWS, the byte budget bounds BYTES, and neither bounds TIME. + /// + /// The field report it exists for. Two Azure SQL DB elastic-pool databases, same day, + /// across the 3.3.0 → 3.4.0 upgrade: 198 passes at a median of 4.8 s before, then six passes of + /// 0.1, 37.6, 46.1, 82.1, 0.1 and 99.8 minutes after. Because a host's live collectors run one + /// after another, a single 100-minute pass starves every other collector on that server — which is the + /// actual mechanism behind #2148's "all collection stopped". + /// + /// Why nothing already caught it. The CommandTimeout was 30 s the whole time. It + /// bounds the wait for a network read and SqlClient resets it on each read that arrives, so a result + /// set that trickles rows never trips it — see . + /// + /// Why ten minutes. It has to sit far above every healthy observation and far below every + /// pathological one. Healthy: 4.8 s median and 31 s max across 198 field passes; 375–524 ms for the + /// staged query measured on a 212k-row Query Store; 6 s for the two fast post-upgrade passes. + /// Pathological: 37.6 minutes at the low end. Ten minutes is ~19× the worst healthy pass, ~10× the + /// Darling command-timeout default, and ~3.7× under the smallest pass this is meant to stop. + /// + /// It converges rather than repeating. A cut pass ships nothing, so the watermark does not + /// advance and the range is re-read — but the failure also feeds #2111's consecutive-failure count, so + /// the catch-up window halves per failure toward 15 minutes until a pass fits. A success resets it and + /// the database returns to full width. Without that this would be a bound that fires forever on the same + /// impossible width; with it, a database that cannot finish narrows until it can. + /// + public static readonly TimeSpan PerDatabaseWallClockBudget = TimeSpan.FromMinutes(10); + + /// + public override TimeSpan? PerItemWallClockBudget => PerDatabaseWallClockBudget; + /// /// The self-identification marker every collector query carries in its leading comment. Self rows /// are excluded CLIENT-SIDE in the shared read loop both paths use (#1565) — the old SQL-side @@ -476,7 +507,7 @@ public CollectorQuery BuildBackfillQuery(CollectorContext context, DateTime floo /// comparison — the #1960 invariant, mirror-imaged. Everything else (columns, slice aggregation, /// version gates) is byte-identical, so the reader contract cannot drift between live and backfill. /// - internal static string BuildPayloadBody(CollectorContext context, bool backfill = false) + internal static string BuildPayloadBody(CollectorContext context, bool backfill = false, string? databaseName = null) { /* Detect server version for version-gated columns. isNew = true for SQL Server 2017+ (product version > 13) or Azure SQL DB/MI. @@ -605,15 +636,15 @@ ordinal so the 55-column reader contract never moves. The inner fragments carry comma and sit at the END of the inner select list precisely because they can be empty; the outer ones keep their original trailing-comma form because they are never empty. */ string numPhysIoReadsAgg = isNew - ? $",\n {WeightedAverage("avg_num_physical_io_reads")},\n min_num_physical_io_reads = MIN(qsrs.min_num_physical_io_reads),\n max_num_physical_io_reads = MAX(qsrs.max_num_physical_io_reads)" + ? $",\n {WeightedAverage("avg_num_physical_io_reads")},\n min_num_physical_io_reads = MIN(qsrs.min_num_physical_io_reads),\n max_num_physical_io_reads = MAX(qsrs.max_num_physical_io_reads)" : ""; string logBytesAgg = isNew - ? $",\n {WeightedAverage("avg_log_bytes_used")},\n min_log_bytes_used = MIN(qsrs.min_log_bytes_used),\n max_log_bytes_used = MAX(qsrs.max_log_bytes_used)" + ? $",\n {WeightedAverage("avg_log_bytes_used")},\n min_log_bytes_used = MIN(qsrs.min_log_bytes_used),\n max_log_bytes_used = MAX(qsrs.max_log_bytes_used)" : ""; string tempdbAgg = isNew - ? $",\n {WeightedAverage("avg_tempdb_space_used")},\n min_tempdb_space_used = MIN(qsrs.min_tempdb_space_used),\n max_tempdb_space_used = MAX(qsrs.max_tempdb_space_used)" + ? $",\n {WeightedAverage("avg_tempdb_space_used")},\n min_tempdb_space_used = MIN(qsrs.min_tempdb_space_used),\n max_tempdb_space_used = MAX(qsrs.max_tempdb_space_used)" : ""; string numPhysIoReadsCols = isNew @@ -653,9 +684,64 @@ stays DESC even though the outer sort is now ASC (#1960): under oldest-first shi plans live, and Darling's stored-plan readers all guard `query_plan_text IS NOT NULL`. Not mirrored into the Dashboard proc: its "Download Plan" reads by exact collection_id, where per-row NULLs would break a real reader. */ - string planTextCol = context.CapturePlanXml - ? "query_plan_text = CASE WHEN ROW_NUMBER() OVER (PARTITION BY qsp.plan_id ORDER BY qsrs.last_execution_time DESC) = 1 THEN CONVERT(nvarchar(max), qsp.query_plan) ELSE CONVERT(nvarchar(max), NULL) END," - : "query_plan_text = CONVERT(nvarchar(1), NULL),"; + /* #2164: skip the XML for plans the store already holds. 97% of the plan XML shipped in a + three-hour fleet window was for plans held over an hour — the ROW_NUMBER gate ships each plan once + per PASS but re-ships it every pass forever, and since drain is 94-97% of a pass and is per-row LOB + cost, NOT fetching is worth far more than fetching less. The watermark is the highest plan_id whose + XML was actually STORED for this database; plan_id is monotonic within a database, so a higher id + is a plan we have never stored. Inlined as a parsed long (never operator input) because the body + nests inside sp_executesql on three paths and threading another parameter through all of them buys + nothing. Zero — absent, malformed, or expired — renders no predicate, so the conservative path is + byte-identical to the pre-#2164 query. + + NEVER on the backfill path. The watermark tracks the plans the LIVE window has stored, and backfill + digs the other way — into intervals older than anything collected, whose rows reference plans + compiled long ago and therefore numbered BELOW the live watermark. Applying it there would suppress + essentially every plan the backfill exists to fetch, silently: the slices would still ship runtime + stats, so a filled range would look complete while carrying no plan XML at all. + + KNOWN GAP, bounded by QueryStorePlanXmlState.RefreshAfter: plan_id is monotonic in COMPILE order, which is + not the same as "we have stored it". A plan compiled before monitoring began, dormant through every + collected window, then executed again, arrives with an id below the watermark and has its XML + suppressed until the refresh horizon expires. Bounding it is the reason that horizon exists. The + exact fix is a store-DERIVED watermark (the host asking its own plan dimension for the lowest + plan_id missing XML) rather than this collector-derived one; that needs host plumbing on both + products and is tracked separately. */ + /* #2210: the watermark now belongs to BuildPlanFetchQuery (the `watermark` parameter there, + resolved by the host via QueryStorePlanXmlState.Resolve). It no longer narrows anything in + this runtime-stats query, so there is nothing to compute here. */ + + /* #2210: this runtime-stats query no longer carries plan XML at all — the ROW_NUMBER-gated + CASE and its in-stream watermark predicate are DELETED, not reworked. BuildPlanFetchQuery is + the only thing that reads plan XML now: it fetches plans in plan_id order under a byte + budget and lands each plan ONCE per database LIFETIME instead of once per PASS. The shape + being replaced re-shipped every plan on every pass forever — measured at 5.0x redundancy + (871,196 plan-XML rows against 175,328 distinct database/plan pairs in a day, on a 33 GB + table). Both branches below now emit the same placeholder, so the payload is byte-identical + to Lite's regardless of the flag, and CapturePlanXml gates the separate BuildPlanFetchQuery + fetch rather than this query. Existing inline rows are NOT migrated by this change and stay + readable via the reader's existing NULL-guarded fallback; dropping the query_plan_text column + itself is a separate, later migration. */ + const string planTextCol = "query_plan_text = CONVERT(nvarchar(1), NULL),"; + + /* #2150: the LAST nvarchar(max) in this projection, and now the whole remaining cost of it. The cap + and ship order sit above these joins, so a Top-N Sort carries the text through the sort and reads + all of its input before emitting row one — choosing 50,000 rows materialized text for the entire + qualifying set. Measured with #2210's plan XML already gone and this column as the only + difference: time-to-first-row 4.67s vs 0.45s at 1,505 rows, 5.02s vs 0.57s at 4,037. Neither knob + bounds it (TOP (500) == TOP (50000); wall time flat from a 4 MB to a 256 MB client budget). + + Gated rather than removed, because Lite stores this text inline in DuckDB and reads it from + there — nulling it unconditionally would blind Lite, which is why this is a host flag and not a + deletion. The ORDINAL is identical either way, the same discipline the version-gated columns + above follow, so a host that has not built text storage is byte-compatible. + + The query_text JOIN deliberately STAYS when the column is nulled: it is one row per key and the + measurement above was taken with it in place, so removing it would be an unmeasured change riding + along on a measured one. */ + string queryTextCol = context.FetchQueryTextSeparately + ? "query_sql_text = CONVERT(nvarchar(1), NULL)," + : "query_sql_text = qst.query_sql_text,"; /* The replica-attribution column + its join (see hasReplicaAttribution above). Selected after every version-gated column, so pre-2022 targets read the nvarchar(1) NULL placeholder at the same @@ -681,7 +767,7 @@ fails the whole SELECT just as naming it in a select list would. When the gate i Leading comma: it splices into both the inner select list and the GROUP BY, and is empty on targets without the column. */ string replicaGroupKey = hasReplicaAttribution - ? ",\n qsrs.replica_group_id" + ? ",\n qsrs.replica_group_id" : ""; /* There is deliberately NO self-exclusion predicate in this query (#1565, actual-plan evidence @@ -786,18 +872,115 @@ boundary groups the same way (see ReadRowsAsync). */ its oldest shipped row, and the next slice's strict `< @ceiling_time` resumes with no hole or re-ship. Same TIES, same budget, same tie-group completion; only the window and the direction differ. */ + /* The interval pre-filter resolves candidate interval ids from the INTERVAL CATALOG + (sys.query_store_runtime_stats_interval, ~one row per interval of retained history — hundreds + of rows) rather than from runtime_stats itself (#2133; measured on the field store: 20 ms vs + 426 ms for the identical id set). end_time/start_time are datetimeoffset; the datetime2 + parameters promote with a zero offset, i.e. as the UTC instants they are — the same implicit + promotion the HAVING's last_execution_time comparison has always relied on. The catalog bound + is a SUPERSET (an interval can end after the cutoff while all its rows are older); the HAVING + below stays the exact row-level filter, so shipped semantics are unchanged. */ + /* #2312: the live path has two forms. Most cycles ship CLOSED intervals only — immutable, so + final on first collection — and skip the OPEN interval's cumulative snapshot, which is the + whole re-read bill on a big multi-tenant primary (40–110 s per run measured, every one of + those snapshots but the latest discarded by the read side's rn = 1). The host opts a cycle + back in via context.IncludeOpenInterval (default true = today's exact form) on the + QueryStoreOpenIntervalState cadence. SYSUTCDATETIME() promotes to datetimeoffset with a zero + offset against end_time — the same implicit UTC-instant promotion the @cutoff_time comparison + has always relied on — and being server-evaluated it adds no parameter, so the + single-parameter sp_executesql contract is unchanged. Correctness of the skip leans on the + cumulative-snapshot contract: a closed interval whose final content differs from our last + open-snapshot must carry executions newer than the watermark, so the standing HAVING readmits + it; one whose content did not change IS our last snapshot. */ var intervalPreFilter = backfill - ? @"f.last_execution_time > @floor_time - AND f.last_execution_time < @ceiling_time" - : "f.last_execution_time > @cutoff_time"; + ? @"i.end_time > @floor_time + AND i.start_time < @ceiling_time" + : context.IncludeOpenInterval + ? "i.end_time > @cutoff_time" + : @"i.end_time > @cutoff_time + AND i.end_time <= SYSUTCDATETIME()"; var intervalHaving = backfill ? @"MAX(qsrs.last_execution_time) > @floor_time - AND MAX(qsrs.last_execution_time) < @ceiling_time" + AND MAX(qsrs.last_execution_time) < @ceiling_time" : "MAX(qsrs.last_execution_time) > @cutoff_time"; var shipOrder = backfill ? "DESC" : "ASC"; + /* STAGED, not monolithic (#2133). Joining the slice aggregate straight into the + query_store_plan/query/text TVFs handed the optimizer nothing but fixed-guess cardinalities, + and the shape it picked re-materialized a TVF per probe — a fixed cost no window width could + reduce. Field bisection on an 82k-plan catalog (echo, SQL 2022): the aggregate alone ran in + 81 ms and each TVF scanned bare in ~300 ms, yet aggregate-JOIN-qsp could not finish in 30 s, + hinted or not; staged through the temp the same work totaled 524 ms (56 stage + 409 join). + That fixed cost is what wedged the big-catalog databases at EVERY catch-up width and made + #2125's shrink floor-pin instead of converge. The temp gives the final join REAL row counts — + and for that reason the old LOOP JOIN hint must NOT return: looping from the temp into the + TVFs is the same per-probe re-materialization by another name; the 524 ms join is unhinted, + chosen by the optimizer from true cardinalities. sp_QuickieStore stages for the same reason. + + Batch mechanics: SELECT INTO emits no result set, so the batch still returns exactly ONE + result set (the reader/byte-budget contract). Inside the on-prem [db].sys.sp_executesql + nesting the temp's scope dies with the invocation; on Azure's direct per-database path the + leading DROP TABLE IF EXISTS covers pooled-connection reuse. TOP ... WITH TIES, the ship + order, and the derived-watermark semantics live on the final SELECT, unchanged. + + BOTH statements carry OPTION(RECOMPILE) (review catch): split out on its own, the staging + statement would otherwise be cached via sp_executesql's parameterized text and sniffed + across live vs backfill windows of wildly different selectivity — the same fixed-guess + failure mode this rewrite removes, reintroduced one statement earlier. */ return $@"SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +DROP TABLE IF EXISTS #pm_qs_slice; + +SELECT /* PerformanceMonitorLite */ + qsrs.plan_id, + qsrs.runtime_stats_interval_id, + qsrs.execution_type_desc{replicaGroupKey}, + first_execution_time = MIN(qsrs.first_execution_time), + last_execution_time = MAX(qsrs.last_execution_time), + count_executions = SUM(qsrs.count_executions), + {WeightedAverage("avg_duration")}, + min_duration = MIN(qsrs.min_duration), + max_duration = MAX(qsrs.max_duration), + {WeightedAverage("avg_cpu_time")}, + min_cpu_time = MIN(qsrs.min_cpu_time), + max_cpu_time = MAX(qsrs.max_cpu_time), + {WeightedAverage("avg_logical_io_reads")}, + min_logical_io_reads = MIN(qsrs.min_logical_io_reads), + max_logical_io_reads = MAX(qsrs.max_logical_io_reads), + {WeightedAverage("avg_logical_io_writes")}, + min_logical_io_writes = MIN(qsrs.min_logical_io_writes), + max_logical_io_writes = MAX(qsrs.max_logical_io_writes), + {WeightedAverage("avg_physical_io_reads")}, + min_physical_io_reads = MIN(qsrs.min_physical_io_reads), + max_physical_io_reads = MAX(qsrs.max_physical_io_reads), + {WeightedAverage("avg_clr_time")}, + min_clr_time = MIN(qsrs.min_clr_time), + max_clr_time = MAX(qsrs.max_clr_time), + min_dop = MIN(qsrs.min_dop), + max_dop = MAX(qsrs.max_dop), + {WeightedAverage("avg_query_max_used_memory")}, + min_query_max_used_memory = MIN(qsrs.min_query_max_used_memory), + max_query_max_used_memory = MAX(qsrs.max_query_max_used_memory), + {WeightedAverage("avg_rowcount")}, + min_rowcount = MIN(qsrs.min_rowcount), + max_rowcount = MAX(qsrs.max_rowcount){numPhysIoReadsAgg}{logBytesAgg}{tempdbAgg} +INTO #pm_qs_slice +FROM sys.query_store_runtime_stats AS qsrs +WHERE qsrs.runtime_stats_interval_id IN +( + SELECT + i.runtime_stats_interval_id + FROM sys.query_store_runtime_stats_interval AS i + WHERE {intervalPreFilter} +) +GROUP BY + qsrs.plan_id, + qsrs.runtime_stats_interval_id, + qsrs.execution_type_desc{replicaGroupKey} +HAVING + {intervalHaving} +OPTION(RECOMPILE); + SELECT /* PerformanceMonitorLite */ TOP ({MaxRowsPerDatabase}) WITH TIES query_id = qsq.query_id, plan_id = qsp.plan_id, @@ -812,7 +995,7 @@ ELSE COALESCE( OBJECT_SCHEMA_NAME(qsq.object_id) + N'.' + OBJECT_NAME(qsq.object_id), N'Unknown') END, - query_sql_text = qst.query_sql_text, + {queryTextCol} query_hash = CONVERT(varchar(64), qsq.query_hash, 1), count_executions = qsrs.count_executions, avg_duration = qsrs.avg_duration, @@ -855,56 +1038,7 @@ ELSE COALESCE( {replicaRoleCol}, runtime_stats_interval_id = qsrs.runtime_stats_interval_id, interval_start_time_utc = CONVERT(datetime2, qsrsi.start_time AT TIME ZONE 'UTC') -FROM -( - SELECT - qsrs.plan_id, - qsrs.runtime_stats_interval_id, - qsrs.execution_type_desc{replicaGroupKey}, - first_execution_time = MIN(qsrs.first_execution_time), - last_execution_time = MAX(qsrs.last_execution_time), - count_executions = SUM(qsrs.count_executions), - {WeightedAverage("avg_duration")}, - min_duration = MIN(qsrs.min_duration), - max_duration = MAX(qsrs.max_duration), - {WeightedAverage("avg_cpu_time")}, - min_cpu_time = MIN(qsrs.min_cpu_time), - max_cpu_time = MAX(qsrs.max_cpu_time), - {WeightedAverage("avg_logical_io_reads")}, - min_logical_io_reads = MIN(qsrs.min_logical_io_reads), - max_logical_io_reads = MAX(qsrs.max_logical_io_reads), - {WeightedAverage("avg_logical_io_writes")}, - min_logical_io_writes = MIN(qsrs.min_logical_io_writes), - max_logical_io_writes = MAX(qsrs.max_logical_io_writes), - {WeightedAverage("avg_physical_io_reads")}, - min_physical_io_reads = MIN(qsrs.min_physical_io_reads), - max_physical_io_reads = MAX(qsrs.max_physical_io_reads), - {WeightedAverage("avg_clr_time")}, - min_clr_time = MIN(qsrs.min_clr_time), - max_clr_time = MAX(qsrs.max_clr_time), - min_dop = MIN(qsrs.min_dop), - max_dop = MAX(qsrs.max_dop), - {WeightedAverage("avg_query_max_used_memory")}, - min_query_max_used_memory = MIN(qsrs.min_query_max_used_memory), - max_query_max_used_memory = MAX(qsrs.max_query_max_used_memory), - {WeightedAverage("avg_rowcount")}, - min_rowcount = MIN(qsrs.min_rowcount), - max_rowcount = MAX(qsrs.max_rowcount){numPhysIoReadsAgg}{logBytesAgg}{tempdbAgg} - FROM sys.query_store_runtime_stats AS qsrs - WHERE qsrs.runtime_stats_interval_id IN - ( - SELECT - f.runtime_stats_interval_id - FROM sys.query_store_runtime_stats AS f - WHERE {intervalPreFilter} - ) - GROUP BY - qsrs.plan_id, - qsrs.runtime_stats_interval_id, - qsrs.execution_type_desc{replicaGroupKey} - HAVING - {intervalHaving} -) AS qsrs +FROM #pm_qs_slice AS qsrs JOIN sys.query_store_plan AS qsp ON qsp.plan_id = qsrs.plan_id JOIN sys.query_store_query AS qsq @@ -915,7 +1049,7 @@ LEFT JOIN sys.query_store_runtime_stats_interval AS qsrsi ON qsrsi.runtime_stats_interval_id = qsrs.runtime_stats_interval_id {replicaJoin} ORDER BY qsrs.last_execution_time {shipOrder} -OPTION(RECOMPILE, LOOP JOIN);"; +OPTION(RECOMPILE);"; } /// @@ -981,7 +1115,7 @@ why this is false there (re-clamping an already-clamped value changes nothing) a public override CollectorQuery BuildPerItemQuery(string item, CollectorContext context) { /* Double single quotes so the body survives nesting inside [db].sys.sp_executesql N'...' */ - var escapedBody = BuildPayloadBody(context).Replace("'", "''", StringComparison.Ordinal); + var escapedBody = BuildPayloadBody(context, databaseName: item).Replace("'", "''", StringComparison.Ordinal); var escapedDbName = item.Replace("]", "]]", StringComparison.Ordinal); var text = $@" @@ -993,6 +1127,236 @@ EXECUTE [{escapedDbName}].sys.sp_executesql return new CollectorQuery(text, BuildCutoffParameters(context)); } + /// + /// The plan-XML fetch for one database (#2210): plans above the watermark, in plan_id order, bounded + /// twice — coarsely by and exactly by a running byte total. + /// + /// SEPARATE from the runtime-stats query on purpose, and that separation is the fix rather than a + /// refactor. The runtime query ships ORDER BY qsrs.last_execution_time, so a budget cut truncates it + /// in TIME order and the plans whose XML landed are an arbitrary SUBSET of plan_ids — against which no + /// watermark value is safe, because receiving plan 500 while missing 300 skips 300 forever. That is why the + /// previous shape could not advance on a cut, and 97.8% of production passes are cut. Here rows arrive in + /// plan_id order, so a cut truncates a SUFFIX and the highest landed id is safe by construction. + /// + /// The two bounds are not redundant. The running total is exact but expensive to compute: it needs + /// DATALENGTH, and sys.query_store_plan.query_plan is decompressed BY the view on access, so an + /// unbounded candidate set pays a whole catalog's decompression to enforce a budget meant to prevent exactly + /// that. TOP (@candidate_plans) is evaluated on plan_id alone — no XML touched to sort or + /// filter — so the decompression is capped at K, sized per database by + /// from the previous pass's own bytes-per-plan. + /// + /// The budget test is running_bytes - plan_bytes < budget, i.e. admit a plan when the total + /// BEFORE it was still under. The obvious running_bytes <= budget is a per-database STALL: a single + /// plan larger than the whole budget has a running total that already exceeds it on its own row, so it is + /// excluded, every later row is excluded too (the total is monotonic), the pass ships nothing, the watermark + /// holds, and the next pass re-selects the same plan first — forever. One 13 MB plan against the 12 MB + /// default is enough, and it is the same never-advances failure this change exists to end, reached through + /// plan SIZE instead of cut ordering. Admitting the offender ships it alone, cuts after it, and moves the + /// watermark past it. + /// + /// The honest cost of that: worst-case bytes for one pass are budget + largest single plan, + /// not budget. The runtime-stats budget a few hundred lines up pays exactly the same price for the + /// same reason (measured: 19.6 MB shipped against a 12 MB budget when one very large plan carried a pass + /// past it), so "12 MB" is a floor on ship volume in both paths rather than a cap. + /// + /// Both bounds are inlined as parsed integers rather than parameters, matching the watermark predicate + /// above and for the same reason: the body nests inside sp_executesql, and the values are host-computed + /// longs that never touch operator input. + /// + /// A NULL query_plan — a plan too large to persist, or certain forced-plan-failure paths — + /// counts as ZERO bytes and STILL SHIPS, as a row with NULL text. Letting the NULL propagate through the + /// arithmetic instead would make the budget predicate NULL and filter the row out, and a window whose plans + /// are all NULL would then return nothing, hold the watermark, and re-select the same plans forever: the + /// same permanent stall as the oversized-plan case, reached through a different mechanism. Shipping the row + /// lets the watermark advance past a plan whose XML will never exist, which is correct — the store's readers + /// already guard query_plan_text IS NOT NULL because the runtime path has always been able to write + /// per-row NULLs there. + /// + /// NEVER on the backfill path, for the reason the watermark itself is not: backfill reads intervals + /// older than anything collected, whose plans are numbered BELOW the watermark, so a plan_id-ascending fetch + /// above the watermark would return nothing the backfill needs. Backfill plan XML stays on its own rows. + /// + /// The CONVERT happens ONCE, inside the candidate window, and the running total sums + /// DATALENGTH of that converted text rather than of the view column. The alternative — measure with + /// DATALENGTH(qsp.query_plan) in the window and join back to sys.query_store_plan for the text + /// — decompresses every shipped plan TWICE, and that is measured, not reasoned: on a 73,163-plan production + /// catalog, K=114 and a 12 MB budget, both shapes returned the same 114 rows and 1.7 MB, and the join-back + /// form took 274ms cold / 262ms warm against 133ms for this one. Plan-id-only with no XML touched was 114ms, + /// so this shape sits 19ms above the floor while the join-back form pays for the decompression twice. + /// + public CollectorQuery BuildPlanFetchQuery(string item, CollectorContext context, long watermark, int candidatePlans, long budgetBytes) + { + /* The invariant the doc comment spends a paragraph on, actually enforced rather than left to the caller: + this query exists only to fetch plan XML, so building it with plan capture off is a caller bug, not a + no-op to swallow. Cheap, and it makes CapturePlanXml the single gate for the whole feature — the + runtime query's plan-text CASE already reads the same flag. */ + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (!context.CapturePlanXml) + { + throw new InvalidOperationException( + "BuildPlanFetchQuery requires CapturePlanXml; a host that does not capture plan XML must not issue the plan fetch."); + } + + /* A non-positive budget would make the predicate `running_bytes - plan_bytes < 0`, which excludes even + the FIRST candidate (its running total before it is 0, and 0 < 0 is false) — the pass ships nothing, + the watermark holds, and the next pass re-selects the same plans. The oversized-plan stall for a third + time, from a third direction. CandidatePlanCount already floors a non-positive budget for its own + sizing; this method has to guard its own input rather than assume the caller passed that value through. */ + if (budgetBytes <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(budgetBytes), budgetBytes, "The plan-fetch byte budget must be positive; a zero or negative budget ships nothing and stalls the watermark."); + } + + /* Same failure, fourth route: TOP (0) returns no rows and TOP with a negative literal is a syntax error, + so a bad candidate count ships nothing and holds the watermark exactly like a bad budget. Every caller + today sources this from CandidatePlanCount, which floors at MinCandidatePlans — but "the only caller + happens to be safe" is the assumption this method has already been wrong about once. */ + if (candidatePlans <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(candidatePlans), candidatePlans, "The candidate plan count must be positive; TOP (0) ships nothing and stalls the watermark."); + } + + var escapedDbName = item.Replace("]", "]]", StringComparison.Ordinal); + var k = candidatePlans.ToString(System.Globalization.CultureInfo.InvariantCulture); + var budget = budgetBytes.ToString(System.Globalization.CultureInfo.InvariantCulture); + var floor = watermark.ToString(System.Globalization.CultureInfo.InvariantCulture); + + /* ROWS UNBOUNDED PRECEDING, not the RANGE default: RANGE would tie-group peers and, more to the point, + forces a spool. The frame is per-row precisely because the cut has to fall between two plans. */ + var body = $@"WITH candidates AS ( + SELECT TOP ({k}) + plan_id = qsp.plan_id, + query_plan_text = CONVERT(nvarchar(max), qsp.query_plan) + FROM sys.query_store_plan AS qsp + WHERE qsp.plan_id > {floor} + ORDER BY qsp.plan_id +), +budgeted AS ( + SELECT + plan_id = c.plan_id, + query_plan_text = c.query_plan_text, + plan_bytes = COALESCE(DATALENGTH(c.query_plan_text), 0), + running_bytes = SUM(COALESCE(DATALENGTH(c.query_plan_text), 0)) OVER (ORDER BY c.plan_id ROWS UNBOUNDED PRECEDING) + FROM candidates AS c +) +SELECT + plan_id = b.plan_id, + query_plan_text = b.query_plan_text +FROM budgeted AS b +WHERE b.running_bytes - b.plan_bytes < {budget} +ORDER BY b.plan_id +OPTION(RECOMPILE);"; + + var escapedBody = body.Replace("'", "''", StringComparison.Ordinal); + + var text = $@" +EXECUTE [{escapedDbName}].sys.sp_executesql + N'{escapedBody}';"; + + return new CollectorQuery(text, new List()); + } + + /// + /// Statement text for one database, resumed from a query_id watermark and cut by a byte budget + /// (#2150) — the sibling of , and the other half of taking + /// query_sql_text out of the runtime stream. + /// + /// Ordered by query_id, which is what makes a budget cut safe. The cut falls between + /// two statements, so everything up to it is stored and the highest stored id is a resume point with no + /// hole — the same suffix argument the plan fetch rests on. query_id is also already a stored + /// payload column on the runtime row, so this needs no new fact-table column and no migration to be + /// joinable. + /// + /// Simpler than the plan fetch on purpose. There is no candidate-window estimator here + /// because DATALENGTH(query_sql_text) is cheap: sys.query_store_plan.query_plan is + /// decompressed BY the view on access, which is what forces the plan side to bound how many plans a + /// windowed running total may touch, and query_sql_text has no such cost. A flat coarse bound + /// plus the exact running total is enough. There is no content hash either — plan XML can be rewritten + /// in place, whereas a statement's text is fixed for the life of its id. + /// + /// ROWS UNBOUNDED PRECEDING rather than the RANGE default, for the same reason as + /// the plan fetch: RANGE would tie-group peers and force a spool, and the frame has to be per-row + /// because the cut falls between two rows. + /// + public CollectorQuery BuildTextFetchQuery(string item, CollectorContext context, long watermark, int candidateTexts, long budgetBytes) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + /* Same enforcement as the plan fetch's CapturePlanXml gate: this query exists only because the + payload stopped carrying the text, so issuing it from a host that still ships the text inline is a + caller bug rather than a harmless extra round trip — it would fetch and store text nobody reads. */ + if (!context.FetchQueryTextSeparately) + { + throw new InvalidOperationException( + "BuildTextFetchQuery requires FetchQueryTextSeparately; a host that still ships query_sql_text inline must not issue the text fetch."); + } + + /* A non-positive budget makes the predicate `running_bytes - text_bytes < 0` exclude even the FIRST + candidate (its running total before it is 0, and 0 < 0 is false), so the pass ships nothing, the + watermark holds, and the next pass re-selects the same statements — a stall that looks like a + quiet database. */ + if (budgetBytes <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(budgetBytes), budgetBytes, "The text-fetch byte budget must be positive; a zero or negative budget ships nothing and stalls the watermark."); + } + + /* Same stall, other route: TOP (0) returns no rows and a negative literal is a syntax error. */ + if (candidateTexts <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(candidateTexts), candidateTexts, "The candidate text count must be positive; TOP (0) ships nothing and stalls the watermark."); + } + + var escapedDbName = item.Replace("]", "]]", StringComparison.Ordinal); + var k = candidateTexts.ToString(System.Globalization.CultureInfo.InvariantCulture); + var budget = budgetBytes.ToString(System.Globalization.CultureInfo.InvariantCulture); + var floor = watermark.ToString(System.Globalization.CultureInfo.InvariantCulture); + + var body = $@"WITH candidates AS ( + SELECT TOP ({k}) + query_id = qsq.query_id, + query_sql_text = qst.query_sql_text + FROM sys.query_store_query AS qsq + JOIN sys.query_store_query_text AS qst + ON qst.query_text_id = qsq.query_text_id + WHERE qsq.query_id > {floor} + ORDER BY qsq.query_id +), +budgeted AS ( + SELECT + query_id = c.query_id, + query_sql_text = c.query_sql_text, + text_bytes = COALESCE(DATALENGTH(c.query_sql_text), 0), + running_bytes = SUM(COALESCE(DATALENGTH(c.query_sql_text), 0)) OVER (ORDER BY c.query_id ROWS UNBOUNDED PRECEDING) + FROM candidates AS c +) +SELECT + query_id = b.query_id, + query_sql_text = b.query_sql_text +FROM budgeted AS b +WHERE b.running_bytes - b.text_bytes < {budget} +ORDER BY b.query_id +OPTION(RECOMPILE);"; + + var escapedBody = body.Replace("'", "''", StringComparison.Ordinal); + + var text = $@" +EXECUTE [{escapedDbName}].sys.sp_executesql + N'{escapedBody}';"; + + return new CollectorQuery(text, new List()); + } + /// /// The #2022 phase-2 backfill slice for one on-prem/RDS/MI database: /// in its backfill shape (newest-first DESC inside the two-sided window) nested in the same @@ -1068,7 +1432,12 @@ private static async Task ReadRowsAsync(string databaseName, DbDataReader reader the ROW COUNT, but a row carries two nvarchar(max) fields (query text + plan XML), so 50k rows can still be gigabytes. Accumulate the materialized text size and STOP reading at the budget, disposing the reader early, so one database can never balloon the process. */ - var budget = Instance.PerItemTextByteBudget ?? int.MaxValue; + /* #2164: an operator budget override wins over the compile-time default (the host supplies it + from the store knob; Lite passes null and keeps the const). Guarded to a positive value so a + corrupt/zero setting can never mean "ship nothing" — the clamp lives at the store read, and + this is the second line of defense. */ + var budget = (context.TextByteBudgetOverride is > 0 ? context.TextByteBudgetOverride : Instance.PerItemTextByteBudget) + ?? int.MaxValue; long textBytes = 0; /* #1960 boundary-group completion: once the budget trips, rows TIED at the trip row's @@ -1080,6 +1449,12 @@ last_execution_time still ship (they are adjacent under the query's ASC order), var budgetSpent = false; DateTime? cutBoundary = null; + /* #2164 watermark bookkeeping: counts ONLY plans whose XML actually landed in this batch, so a + budget-cut pass cannot claim coverage it does not have. Plans observed but not stored are + deliberately not tracked — see QueryStorePlanXmlState.RefreshAfter for why the observed maximum cannot be + used to detect a Query Store reset. */ + long maxStoredPlanId = 0; + while (await reader.ReadAsync(cancellationToken)) { var row = new Row @@ -1174,6 +1549,12 @@ and finish the boundary tie group — the host surfaces the WARNING. Rows are re everything past the cut stays ahead of the watermark and next cycle resumes exactly there: a bounded cycle costs latency, never data. */ textBytes += ((long)(row.QueryText?.Length ?? 0) + (row.QueryPlanText?.Length ?? 0)) * 2L; + + if (row.QueryPlanText is not null && row.PlanId > maxStoredPlanId) + { + maxStoredPlanId = row.PlanId; + } + if (!budgetSpent && textBytes >= budget) { budgetSpent = true; @@ -1184,8 +1565,42 @@ and finish the boundary tie group — the host surfaces the WARNING. Rows are re context.PerItemTextBytesShipped = textBytes; context.PerItemShippedBoundary = rows.Count > 0 ? rows[^1].LastExecutionTime : null; + + /* #2164: persist the plan-XML watermark for this database. + - Advance to the highest plan_id whose XML actually stored, never past it. + - Never move BACKWARD: a window whose newest-executing plan is older than the newest-COMPILED one + is an ordinary quiet window, not a reset, and lowering the watermark there would refetch the + whole catalog next cycle. (Treating it as a reset is the trap documented on + QueryStorePlanXmlState.RefreshAfter — it holds in most steady-state windows.) + - Never advance AT ALL on a budget-cut pass. Rows ship ordered by last_execution_time, NOT by + plan_id, so the cut drops an arbitrary set of plan_ids from the tail of the window — including + ids BELOW the highest one that did store. Advancing past them would suppress their XML on every + later pass (the ids no longer clear the watermark) even though it never shipped once. The cut is + already resumable on the time watermark, so declining to advance costs one repeated fetch and + nothing else. */ + if (context.CapturePlanXml && !string.IsNullOrEmpty(databaseName) && !budgetSpent && maxStoredPlanId > 0) + { + var standing = QueryStorePlanXmlState.Resolve(context.State, databaseName, context.CollectionTime); + + if (maxStoredPlanId > standing) + { + /* The stamp dates the last FULL fetch, and is carried FORWARD across advances rather than + renewed on each one. Re-stamping here would push the refresh horizon out every time a new + plan compiled, so on any database that keeps compiling — the busy ones, where a stale plan + is most likely to matter — the horizon would never fire and the watermark would effectively + be permanent. A standing watermark of 0 means this pass WAS the full fetch (absent or just + expired), so that is the one case that stamps now. */ + var stamp = standing > 0 + ? QueryStorePlanXmlState.ResolveStamp(context.State, databaseName) ?? context.CollectionTime + : context.CollectionTime; + + context.PendingState[QueryStorePlanXmlState.KeyFor(databaseName)] = + QueryStorePlanXmlState.Format(maxStoredPlanId, stamp); + } + } } + /// /// Reads a nullable int64, converting float/decimal Query Store values to long. /// Query Store runtime_stats columns are stored as float in the catalog but represent diff --git a/PerformanceMonitor.Collectors/QueryStoreHealthCollector.cs b/PerformanceMonitor.Collectors/QueryStoreHealthCollector.cs new file mode 100644 index 000000000..4066260ae --- /dev/null +++ b/PerformanceMonitor.Collectors/QueryStoreHealthCollector.cs @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Collectors; + +/// +/// Per-database Query Store health from sys.database_query_store_options (#2319) — the fields that +/// answer "is Query Store actually working, and how close to its cap is it", which +/// database_config's single is_query_store_on bit cannot: actual_state vs +/// desired_state (the classic silent failure is desired READ_WRITE with actual READ_ONLY after the +/// storage cap hit — readonly_reason says why), current vs max storage, the cleanup mode and +/// thresholds, and the runtime-stats interval length (the grain an investigation like #2312 needs to +/// interpret per-interval cost). +/// +/// The enumeration shape is 's verbatim: list databases +/// first (on-prem filters to non-AG or primary-replica databases the login can actually enter; Azure +/// lists all online), then EXECUTE [db].sys.sp_executesql per database on the same connection — +/// the proven per-database idiom. A database that fails is skipped with a warning by the host. +/// +/// The database list is deliberately NOT filtered to is_query_store_on = 1: +/// sys.database_query_store_options returns exactly one row even when Query Store is off +/// (actual_state_desc = 'OFF'), so every enumerated database yields one honest row and OFF is +/// recorded as OFF — an absent row means "not collected", never "off". The collector itself gates on +/// 2016+ via (the view does not exist before v13); WITHIN the view every +/// selected column exists from 2016 on, so there are no per-column version gates. +/// +/// Hourly rather than the config family's on-load cadence, because unlike the scoped-config knobs +/// (which only change when an operator changes them) actual_state, readonly_reason and +/// current_storage_size_mb change BY THEMSELVES — the cap-hit transition to READ_ONLY is the whole +/// point of collecting this, and an on-load snapshot would miss it until the next reconnect. +/// +public sealed class QueryStoreHealthCollector : CollectorDefinitionBase +{ + public static QueryStoreHealthCollector Instance { get; } = new(); + + private QueryStoreHealthCollector() + { + } + + public sealed class Row + { + public string DbName { get; set; } = ""; + public string? ActualState { get; set; } + public string? DesiredState { get; set; } + public int ReadonlyReason { get; set; } + public long CurrentStorageMb { get; set; } + public long MaxStorageMb { get; set; } + public string? SizeBasedCleanupMode { get; set; } + public long StaleQueryThresholdDays { get; set; } + public long MaxPlansPerQuery { get; set; } + public long IntervalLengthMinutes { get; set; } + } + + private const string OnPremDatabaseListQueryText = @" +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + +SELECT + d.name +FROM sys.databases AS d +LEFT JOIN sys.dm_hadr_database_replica_states AS drs + ON d.database_id = drs.database_id + AND drs.is_local = 1 +WHERE (d.database_id > 4 OR d.database_id = 2) +AND d.database_id < 32761 +AND d.name <> N'PerformanceMonitor' +AND d.state_desc = N'ONLINE' +AND HAS_DBACCESS(d.name) = 1 /*a least-privilege login without per-db access raised 916 per db per cycle (#1823); the sibling per-database collectors already self-skip this way. On-prem only: from master on Azure SQL DB this returns 0 for every user database.*/ +AND +( + drs.database_id IS NULL /*not in any AG*/ + OR drs.is_primary_replica = 1 /*primary replica*/ +) +/*EXCLUSION_FILTER*/ +ORDER BY d.name +OPTION(RECOMPILE);"; + + private const string AzureDatabaseListQueryText = @" +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; + +SELECT + d.name +FROM sys.databases AS d +WHERE (d.database_id > 4 OR d.database_id = 2) +AND d.database_id < 32761 +AND d.name <> N'PerformanceMonitor' +AND d.state_desc = N'ONLINE' +/*EXCLUSION_FILTER*/ +ORDER BY d.name +OPTION(RECOMPILE);"; + + /// + /// Query Store shipped in SQL Server 2016 (v13); sys.database_query_store_options does not exist + /// before it, so without this gate a pre-2016 target errors once per database per hour (review + /// catch). The same condition QueryStoreCollector gates on, so Lite and Darling skip identically; + /// 0 = version unknown = assume newest, and both Azure flavors always have the catalog. + /// + public override bool AppliesTo(CollectorTargetInfo target) => + target.SqlMajorVersion == 0 || target.SqlMajorVersion >= 13 || target.IsAzureSqlDb || target.IsAzureManagedInstance; + + public override string Name => "query_store_health"; + + public override string TargetTable => "query_store_health"; + + /// The config snapshots' prefix is config_id/capture_time in Lite's schema; Darling mirrors it. + public override string PrefixIdColumnName => "config_id"; + + public override string PrefixTimeColumnName => "capture_time"; + + /// Enumerating collector — the primary query is never used. + public override CollectorQuery BuildQuery(CollectorContext context) + => throw new NotSupportedException("query_store_health enumerates databases; BuildEnumerationQuery drives the cycle."); + + public override CollectorQuery? BuildEnumerationQuery(CollectorContext context) + { + var (exclusionClause, exclusionParameters) = DatabaseExclusionFilter.Build(context.ExcludedDatabases, "d.name"); + var text = (context.Target.IsAzureSqlDb ? AzureDatabaseListQueryText : OnPremDatabaseListQueryText) + .Replace("/*EXCLUSION_FILTER*/", exclusionClause, StringComparison.Ordinal); + + return new CollectorQuery(text, exclusionParameters); + } + + public override CollectorQuery BuildPerItemQuery(string item, CollectorContext context) + { + /* Use [dbname].sys.sp_executesql to run in database context (Azure SQL DB compatible). One row + always — the view answers for the database whether Query Store is on or off. */ + var text = $@" +EXECUTE [{item.Replace("]", "]]", StringComparison.Ordinal)}].sys.sp_executesql + N'SELECT + actual_state = qso.actual_state_desc, + desired_state = qso.desired_state_desc, + readonly_reason = qso.readonly_reason, + current_storage_size_mb = qso.current_storage_size_mb, + max_storage_size_mb = qso.max_storage_size_mb, + size_based_cleanup_mode = qso.size_based_cleanup_mode_desc, + stale_query_threshold_days = qso.stale_query_threshold_days, + max_plans_per_query = qso.max_plans_per_query, + interval_length_minutes = qso.interval_length_minutes + FROM sys.database_query_store_options AS qso + OPTION(RECOMPILE);'"; + + return new CollectorQuery(text); + } + + public override async ValueTask ReadItemAsync(string item, DbDataReader reader, List rows, CollectorContext context, CancellationToken cancellationToken) + { + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new Row + { + DbName = item, + ActualState = reader.IsDBNull(0) ? null : reader.GetString(0), + DesiredState = reader.IsDBNull(1) ? null : reader.GetString(1), + ReadonlyReason = reader.IsDBNull(2) ? 0 : Convert.ToInt32(reader.GetValue(2), CultureInfo.InvariantCulture), + CurrentStorageMb = reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3), CultureInfo.InvariantCulture), + MaxStorageMb = reader.IsDBNull(4) ? 0L : Convert.ToInt64(reader.GetValue(4), CultureInfo.InvariantCulture), + SizeBasedCleanupMode = reader.IsDBNull(5) ? null : reader.GetString(5), + StaleQueryThresholdDays = reader.IsDBNull(6) ? 0L : Convert.ToInt64(reader.GetValue(6), CultureInfo.InvariantCulture), + MaxPlansPerQuery = reader.IsDBNull(7) ? 0L : Convert.ToInt64(reader.GetValue(7), CultureInfo.InvariantCulture), + IntervalLengthMinutes = reader.IsDBNull(8) ? 0L : Convert.ToInt64(reader.GetValue(8), CultureInfo.InvariantCulture), + }); + } + } + + /// Never called — enumeration drives this collector. + public override ValueTask> ReadAsync(DbDataReader reader, CollectorContext context, CancellationToken cancellationToken) + => throw new NotSupportedException("query_store_health enumerates databases; ReadItemAsync drives row reads."); + + public override IReadOnlyList PayloadColumns { get; } = new[] + { + new CollectorColumn("database_name", CollectorColumnType.Varchar), + new CollectorColumn("actual_state", CollectorColumnType.Varchar), + new CollectorColumn("desired_state", CollectorColumnType.Varchar), + new CollectorColumn("readonly_reason", CollectorColumnType.Integer), + new CollectorColumn("current_storage_size_mb", CollectorColumnType.BigInt), + new CollectorColumn("max_storage_size_mb", CollectorColumnType.BigInt), + new CollectorColumn("size_based_cleanup_mode", CollectorColumnType.Varchar), + new CollectorColumn("stale_query_threshold_days", CollectorColumnType.BigInt), + new CollectorColumn("max_plans_per_query", CollectorColumnType.BigInt), + new CollectorColumn("interval_length_minutes", CollectorColumnType.BigInt), + }; + + public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) + { + writer + .Value(row.DbName) /* database_name VARCHAR */ + .Value(row.ActualState) /* actual_state VARCHAR */ + .Value(row.DesiredState) /* desired_state VARCHAR */ + .Value(row.ReadonlyReason) /* readonly_reason INTEGER */ + .Value(row.CurrentStorageMb) /* current_storage_size_mb BIGINT */ + .Value(row.MaxStorageMb) /* max_storage_size_mb BIGINT */ + .Value(row.SizeBasedCleanupMode) /* size_based_cleanup_mode VARCHAR */ + .Value(row.StaleQueryThresholdDays) /* stale_query_threshold_days BIGINT */ + .Value(row.MaxPlansPerQuery) /* max_plans_per_query BIGINT */ + .Value(row.IntervalLengthMinutes); /* interval_length_minutes BIGINT */ + } +} diff --git a/PerformanceMonitor.Collectors/QueryStoreOpenIntervalState.cs b/PerformanceMonitor.Collectors/QueryStoreOpenIntervalState.cs new file mode 100644 index 000000000..a60a885c6 --- /dev/null +++ b/PerformanceMonitor.Collectors/QueryStoreOpenIntervalState.cs @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace PerformanceMonitor.Collectors; + +/// +/// Per-database stamp of the last cycle that read the OPEN Query Store interval (#2312), the third +/// sibling of and — same +/// per-database key shape, same conservative-default rules, its own state owner. +/// +/// Why this exists. Query Store runtime stats are cumulative per-interval snapshots, and the +/// collector re-fetches the OPEN interval every cycle so the read side can collapse to the latest snapshot +/// (rn = 1). On a large multi-tenant primary that means re-aggregating the entire current interval's +/// slices across every tenant database, every cycle, each pass pricier as the interval fills — measured on +/// the prod fleet at 40–110 s per run around the clock, rising through the day with workload, with a +/// 554 s worst case (#2312). Closed intervals cost nothing extra: the time watermark's +/// MAX(last_execution_time) > @cutoff_time already excludes any interval fully collected. The +/// open interval is the whole bill, and most of its re-reads buy nothing a reader keeps — every snapshot +/// but the latest is discarded by rn = 1. +/// +/// The mechanism: most cycles ship only intervals that have CLOSED +/// (i.end_time <= SYSUTCDATETIME()) — they are immutable and therefore final on first +/// collection — and the open interval is included only when this stamp says its last inclusion is at +/// least ago. Correctness leans on the cumulative-snapshot contract twice +/// over: a newly CLOSED interval whose final content differs from our last open-snapshot must contain +/// executions newer than the watermark (counters only move with executions), so the standing time filter +/// picks it up; and one whose content did not change IS our last snapshot, so there is nothing to miss. +/// +/// Time-based, not cycle-counting, so the refresh survives cadence changes and delivered-vs- +/// configured drift (at fleet scale the delivered cadence runs well behind the configured one, and an +/// every-Nth-cycle rule would stretch with it). Include is the conservative default: an absent, +/// malformed or future-stamped row reads as "include the open interval now", so a first run, a restarted +/// host and a broken store all behave exactly like today's collector rather than silently going stale. +/// The same conservatism governs the write side: both hosts land the stamp only after that database's +/// read AND flush succeed, so a per-database fault the sweep tolerates re-includes next cycle instead of +/// spending the refresh window on a cycle that captured nothing. +/// +public static class QueryStoreOpenIntervalState +{ + /// + /// The collector name this state is stored under — its own owner, like the plan and text watermarks, + /// because a prefix pruned under the wrong owner silently deletes nothing and the three advance for + /// unrelated reasons. + /// + public const string StateCollectorName = "query_store_open_interval"; + + /// Prefix for the per-database state key. + public const string WatermarkKeyPrefix = "qsowm:"; + + /// + /// How stale the open interval's stored snapshot may grow before a cycle refreshes it. Fifteen minutes + /// against the 60-minute Query Store default interval means ~4 snapshots per interval instead of one + /// per cycle (~12 at the 5-minute cadence) — roughly two thirds of the open-interval spend removed — + /// while the CURRENT interval's view in any reader lags real time by at most this much. Readers of + /// closed history lose nothing at all. Deliberately NOT configurable yet: one number with a recorded + /// rationale beats a knob nobody can reason about, and the #2312 yardstick (multi-53 at ~50 s/run) + /// decides whether it ever needs to move. + /// + public static readonly TimeSpan RefreshEvery = TimeSpan.FromMinutes(15); + + /// The state key for one database. + public static string KeyFor(string databaseName) => WatermarkKeyPrefix + databaseName; + + /// + /// Whether this cycle should read the OPEN interval for one database. True — today's behavior — for an + /// absent, malformed or future stamp (clock skew must not pin the snapshot stale), or one at least + /// old. + /// + public static bool ShouldIncludeOpenInterval( + IReadOnlyDictionary? state, string databaseName, DateTime utcNow) + { + if (state is null || !state.TryGetValue(KeyFor(databaseName), out var raw) || string.IsNullOrWhiteSpace(raw)) + { + return true; + } + + if (!long.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var unixSeconds) + || unixSeconds <= 0) + { + return true; + } + + /* Same guard as QueryStoreTextState.TryParse: long.TryParse accepts values far outside + FromUnixTimeSeconds's year-0001..9999 range, and an out-of-range-but-numeric stamp is just + another flavor of corrupt row — the conservative include, not an exception. */ + try + { + var stamped = DateTimeOffset.FromUnixTimeSeconds(unixSeconds).UtcDateTime; + return stamped > utcNow || utcNow - stamped >= RefreshEvery; + } + catch (ArgumentOutOfRangeException) + { + return true; + } + } + + /// Formats the stamp for one inclusion of the open interval. + public static string Format(DateTime includedAtUtc) => + new DateTimeOffset(DateTime.SpecifyKind(includedAtUtc, DateTimeKind.Utc)).ToUnixTimeSeconds() + .ToString(CultureInfo.InvariantCulture); +} diff --git a/PerformanceMonitor.Collectors/QueryStorePerDatabaseState.cs b/PerformanceMonitor.Collectors/QueryStorePerDatabaseState.cs new file mode 100644 index 000000000..949dd6121 --- /dev/null +++ b/PerformanceMonitor.Collectors/QueryStorePerDatabaseState.cs @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; + +namespace PerformanceMonitor.Collectors; + +/// +/// Every collector_state key query_store owns that is keyed by DATABASE NAME (#2188) — the set both +/// hosts prune when a database is dropped or renamed. +/// +/// Nothing ever retired these. The #2164 plan-XML watermark writes one planwm: row per database +/// and the #2022/#2058 backfill worker writes done: and hole:, and while the worker deletes a +/// hole when it SERVICES or expires it, a dropped database will never service one — its hole can never be +/// dug and its tail can never drain. collector_state is a keyed registry rather than a hypertable +/// (pinned by CollectorStateContractTests), so no retention policy caught them either. +/// +/// Shared rather than one list per host, which is the whole reason this file exists. The two +/// stores prune with different dialects (Postgres anti-join, DuckDB NOT IN) and the SKUs write +/// different subsets — Lite never sets CollectorContext.CapturePlanXml, so it writes no +/// planwm: at all, while both write the backfill pair. A per-host list would make a fourth prefix a +/// two-place edit whose omission fails nothing: the rows would simply orphan on one SKU, invisibly, which is +/// the drift this product keeps paying for. Both hosts iterate THIS, so a prefix is pruned everywhere or +/// nowhere. Lite running the planwm: statement against rows it never writes costs one no-op delete +/// and buys the guarantee that enabling plan capture there cannot quietly create an unpruned orphan +/// class. +/// +/// Membership is a real decision, not a listing of every key: a key must be +/// <prefix><databaseName>, because both prunes reconstruct it that way to test it against +/// the live database list. A server-scoped key added here would match no database and be deleted on every +/// cycle — see , which records the keys deliberately left out so the +/// distinction is written down rather than rediscovered. +/// +public static class QueryStorePerDatabaseState +{ + /// + /// The (state owner, key prefix) pairs to prune, in the order the hosts run them. Owner and prefix + /// travel together because a prefix pruned under the wrong collector_name silently deletes + /// nothing, which is indistinguishable from having nothing to prune. + /// + public static readonly IReadOnlyList<(string Owner, string Prefix)> PrunableKeys = new[] + { + (QueryStorePlanXmlState.StateCollectorName, QueryStorePlanXmlState.WatermarkKeyPrefix), + (QueryStoreBackfillState.StateCollectorName, QueryStoreBackfillState.DoneKeyPrefix), + (QueryStoreBackfillState.StateCollectorName, QueryStoreBackfillState.HoleKeyPrefix), + /* #2150: the text watermark is keyed prefix + databaseName exactly like the plan watermark above, + so a dropped database's key must go with it. Paired with its OWN collector name rather than the + plan fetch's — the two watermarks are stored separately on purpose, and a prefix pruned under + the wrong owner silently deletes nothing. */ + (QueryStoreTextState.StateCollectorName, QueryStoreTextState.WatermarkKeyPrefix), + /* #2312: the open-interval refresh stamp, per database like the three above, under its own owner + for the same never-prune-under-the-wrong-name reason. */ + (QueryStoreOpenIntervalState.StateCollectorName, QueryStoreOpenIntervalState.WatermarkKeyPrefix), + }; + + /// + /// Key prefixes on the query_store state classes that are deliberately NOT pruned because they are not + /// keyed by database name. Empty today — every prefix either state class declares is per-database — and + /// it exists so that stays a recorded decision: the drift guard demands that every declared + /// *KeyPrefix appear in one list or the other, so a new server-scoped key is a deliberate entry + /// here rather than a test failure whose obvious "fix" is to add it to and + /// have it deleted every cycle. + /// + public static readonly IReadOnlyList NotKeyedByDatabase = System.Array.Empty(); +} diff --git a/PerformanceMonitor.Collectors/QueryStorePlanXmlState.cs b/PerformanceMonitor.Collectors/QueryStorePlanXmlState.cs new file mode 100644 index 000000000..ef38f6203 --- /dev/null +++ b/PerformanceMonitor.Collectors/QueryStorePlanXmlState.cs @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace PerformanceMonitor.Collectors; + +/// +/// What one plan-fetch pass earned: the watermark to persist, and whether the pass's rows actually arrived in +/// the plan_id order its ORDER BY promises (#2210). One value rather than two calls so a caller cannot +/// take the watermark without being handed the reason it may not have moved — the ordering guard is only +/// useful if the violation gets LOGGED, and a signal a caller can forget to ask for is one that eventually +/// nobody asks for. +/// +/// The plan_id to persist; the standing value when the pass earned no advance. +/// False when a descent was seen, meaning the advance was abandoned and the +/// caller should log a precondition violation rather than treat a static watermark as a quiet pass. +public readonly record struct PlanWatermarkAdvance(long Watermark, bool ArrivedInPlanIdOrder); + +/// +/// The persisted per-database plan-XML watermark (#2164) — the highest plan_id whose execution-plan +/// XML has actually been stored for a database, so collection stops re-shipping plans the store already +/// holds. 97% of the plan XML shipped in a three-hour fleet window was for plans held for over an hour, and +/// because streaming rows is 94-97% of a pass and costs per-row LOB bytes, not fetching beats fetching less. +/// +/// Owned by the HOST under its own , exactly like +/// and for the same reason: the query_store DEFINITION keeps declaring +/// no state keys, so CollectorStateContractTests stays honest and adding per-database state does not +/// silently become a two-host contract change. The keys are dynamic (one per database), which the host's +/// state read supports because it loads every row for a collector name rather than a declared key list — the +/// definition's StateKeys could not express these anyway. +/// +/// Lives in the shared collectors project rather than either host because it is watermark-shaped state +/// that must decode identically wherever it is read: a row written by Darling today has to keep meaning the +/// same thing after an upgrade, and Lite reads the same definition. +/// +public static class QueryStorePlanXmlState +{ + /// + /// The collector_state owner name for these rows — deliberately NOT the query_store definition's name, + /// which is the seam that lets the definition declare no state keys while the host still persists + /// per-database state for it. + /// + public const string StateCollectorName = "query_store_plan_xml"; + + /// + /// State key prefix; the remainder is the database name, because plan_id is only unique within one + /// database's Query Store and means nothing across databases. + /// + public const string WatermarkKeyPrefix = "planwm:"; + + /// + /// The target period for ONE FULL RE-VERIFICATION SWEEP of a database's plans — not an expiry, and + /// emphatically not a refetch trigger. QueryStorePlanMap.CursorSliceWidth derives the cursor's + /// per-pass id slice from it, so this constant sets the PACE of re-verification rather than a deadline + /// anything has to beat. + /// + /// It used to mean "after this long, drop the watermark to zero and refetch every plan's XML", and + /// that was measured to be unreachable on the catalogs it mattered most for: 2.2-15.1 GB of plan XML per + /// catalog on the production fleet, which at a 12 MB budget and 5-minute cadence is 15.9 to 107.5 HOURS of + /// walking — so a 1-day expiry meant the biggest catalogs restarted from their lowest plan_id forever and + /// never once reached their newest plans. The optimization could not converge on exactly the databases it + /// existed for. Raising the number does not fix that shape; the sweep has to stop being a byte-volume walk. + /// It now is one: hash-only, bounded by ROW count (77k ids at ~270 per pass), re-fetching XML solely where + /// something actually changed. + /// + /// THREE MECHANISMS, each owning one failure, none of them this constant on its own: + /// + /// A Query Store reset — the map's absent-content signal on the runtime stream + /// (TouchSql), recovering in one cycle. The ONLY thing permitted to zero a watermark.
+ /// • Dormant plans — the cursor finds a map row ABSENT at an id the watermark already passed, and + /// fetches it. No heuristic separates dormancy from a reset, because it does not have to: mass absence is + /// caught wholesale by the reset arm within a cycle.
+ /// • In-place XML rewrites — the cursor finds a stored plan_hash that DIFFERS from the live + /// one and re-fetches that plan alone. Across a day of fleet data this was 0 of 38,420 plan_ids, which is + /// why paying for it with a full walk was the wrong trade.
+ /// + /// ONE DAY remains the right pace for a hash-only sweep, for the reason the old value was chosen and + /// for a new one: the redundancy removed is per-pass, and a sweep bounded by rows rather than bytes finishes + /// comfortably inside a day on every catalog measured. + /// + /// Historical note on the three guarantees the old expiry claimed, kept because the reasoning still + /// explains why each mechanism above exists: + /// + /// 1. In-place XML rewrites. plan_id is monotonic and a plan's identity is stable (0 of 38,420 + /// plan_ids changed their plan hash in a day of fleet data), but nothing guarantees a feature like + /// memory-grant feedback never edits grant values inside the XML of a plan that keeps its id. The + /// expiry means that question does not have to be load-bearing. + /// + /// 2. A Query Store RESET. Clearing Query Store restarts plan_id at 1, so every new plan sorts + /// below a stale watermark and its XML would be suppressed. This is NOT what covers that any more — the + /// tempting detection test ("the highest plan_id seen this pass is below the standing watermark") is TRUE in + /// any ordinary quiet window, so it would drop the watermark constantly; but the map gives the payload a + /// signal it never had on its own. A plan_id at or below the watermark whose content the store has never + /// resolved is a RENUMBERED plan, which "no new plans this window" cannot produce, and + /// QueryStorePlanMap.TouchSql surfaces exactly those rows from the batch join it already performs. + /// That is the reset mechanism, it recovers in ONE CYCLE, and it is the only thing permitted to zero the + /// watermark. + /// + /// 3. The dormant-plan gap: plan_id is monotonic in COMPILE order, which is not the same as "we + /// have stored it", so a plan compiled before monitoring began and dormant through every collected + /// window arrives below the watermark. + /// + /// ONE DAY, not a week: the redundancy removed is per-pass (a 15-minute cadence re-ships a plan + /// ~96 times a day), so a daily full fetch already eliminates ~99% of it and a weekly one adds almost + /// nothing — while buying 7x the exposure on all three guarantees above, including a reset blackout + /// measured in days. + /// + /// That trade omits a term, named here because it is the one that will move this number: expiry + /// resets the watermark to zero, so "one expensive pass" is really a full budgeted catalog WALK. At a 12 MB + /// ship budget an 82k-plan catalog spends most of a day walking, which means the largest catalogs — the ones + /// this optimization matters most for — are close to continuously refetching, and shortening the horizon + /// makes that worse rather than safer. Once the stream signal above covers resets, the walk buys only the + /// in-place-rewrite case (speculative: 0 of 38,420 plan_ids changed hash in a day of fleet data) and dormant + /// plans (real, small), and a longer horizon is likely correct. Measure the walk cost on the worst catalog + /// before changing it. + ///
+ public static readonly TimeSpan RefreshAfter = TimeSpan.FromDays(1); + + /// The state key for one database. + public static string KeyFor(string databaseName) => WatermarkKeyPrefix + databaseName; + + /// + /// The average plan size assumed for a database with no previous pass to learn from. Deliberately near the + /// LARGE end of the measured fleet range (per-quartile averages of 162 / 80 / 39 / 15 KB across 2,166 + /// budget-cut passes on a 52-server fleet), because the estimate feeds a DIVISOR: over-estimating plan size + /// yields a SMALL candidate window, and small is the safe direction. A window that is too small merely + /// advances the watermark more slowly; one that is too large decompresses plans it will never ship, which + /// is the exact cost the window exists to bound. + /// + public const long FirstContactAvgPlanBytes = 160L * 1024L; + + /// + /// Floor on the candidate window, so progress is always possible. Even if the observed average is wildly + /// over-stated — one enormous plan in a quiet pass — a database must still be able to walk its catalog. + /// + public const int MinCandidatePlans = 32; + + /// + /// Ceiling on the candidate window. The smallest measured quartile average (15 KB) puts a 12 MB budget at + /// ~820 plans, so this leaves headroom for genuinely tiny plans while refusing to let a near-zero estimate + /// turn the window back into "the whole catalog" — which is the first-contact trap this window exists to + /// prevent. + /// + public const int MaxCandidatePlans = 2048; + + /// + /// How far past the budget the window reaches, in expected plans. The window is the COARSE bound and the + /// running byte total is the exact one, so the margin only has to cover the estimate being wrong in the + /// "plans are smaller than expected" direction — where extra plans genuinely fit the budget. + /// + /// Kept modest at 1.5x because margin is not free: a windowed running total is evaluated over every + /// row IN the window, so the server decompresses all K plans to compute it whether the budget is reached at + /// plan 5 or plan 500. Margin buys reachability and costs decompression, which is why the estimate errs + /// large and the margin stays small. + /// + public const double CandidatePlanMargin = 1.5; + + /// + /// The per-database average plan size to carry into the next pass, from a pass's own totals — free, because + /// both numbers are already in hand when a pass ends, and no probe can measure plan size without + /// decompressing the plans. Zero rows yields null: a quiet pass teaches nothing about plan size and must + /// leave the previous estimate standing rather than replace it with a divide-by-zero fallback. + /// + public static long? ObservedAvgPlanBytes(long planBytesShipped, int plansShipped) => + plansShipped <= 0 || planBytesShipped <= 0 ? null : planBytesShipped / plansShipped; + + /// + /// One database's carried plan-size estimate (#2312 Finding 1): the observed average the next pass + /// sizes its candidate window from, and whether the walk is mid-backlog (which biases the sample + /// small — the overload floors it). + /// AvgBytes of zero means "never learned"; callers pass null to CandidatePlanCount then. + /// + public readonly record struct PlanSizeEstimate(long AvgBytes, bool CatchUpInProgress); + + /// + /// Folds one pass's outcome into the carried estimate. The rules, each load-bearing: + /// a pass that shipped nothing teaches nothing about size (previous average stands) but DOES + /// prove the walk is caught up (nothing qualified past the watermark), so catch-up clears; + /// a pass cut by either bound — the candidate window consumed or the byte budget reached — + /// proves a backlog remains, so catch-up sets; an ordinary partial pass learns its average and + /// clears catch-up. Pure so the table is pinnable; the runner owns only the dictionary. + /// + /// Two counts on purpose (the review catch): is the RAW + /// row count — NULL-XML plans deliberately ship as rows so the watermark can pass unpersistable + /// plans, and the window/catch-up comparison wants exactly that count. But the average's divisor + /// is , the rows that actually carried XML: dividing real bytes + /// by a NULL-inflated count would understate the average, which INFLATES the next window — the + /// unsafe direction the whole estimator errs away from. + /// + public static PlanSizeEstimate Learn( + PlanSizeEstimate previous, long bytesShipped, int plansShipped, int plansMeasured, int candidateWindow, long budgetBytes) + { + if (plansShipped <= 0) + { + return new PlanSizeEstimate(previous.AvgBytes, CatchUpInProgress: false); + } + + var catchUp = plansShipped >= candidateWindow || bytesShipped >= budgetBytes; + var avg = ObservedAvgPlanBytes(bytesShipped, plansMeasured) ?? previous.AvgBytes; + return new PlanSizeEstimate(avg, catchUp); + } + + /// + /// How many plans one pass may CONSIDER: enough that the byte budget is the binding constraint, few enough + /// that the server never decompresses a catalog to discover which plans fit. + /// + /// This is the trap mitigation. SUM(DATALENGTH(query_plan)) OVER (ORDER BY plan_id) has to + /// materialize the XML to measure it — query_store_plan.query_plan is decompressed BY the TVF on + /// access — so an unbounded candidate set pays the whole catalog's decompression to enforce a budget meant + /// to prevent exactly that. Bounding the window first on the cheap columns costs nothing and caps it. + /// + /// Per-database rather than one fleet constant because measured plan size spans 11x (162 KB to 15 KB + /// by quartile). A constant sized for the small-plan end (~820) would decompress ~134 MB to ship 12 MB on + /// the large-plan end; one sized for the large end would never reach the budget on the small end. No single + /// value is both, which is what makes this adaptive rather than tunable. + /// + /// reports that a bound was applied, so the caller can LOG it. A window + /// silently pinned at its ceiling looks identical to one that fit, and that is how a cap becomes invisible. + /// + public static int CandidatePlanCount(long? observedAvgPlanBytes, long budgetBytes, out bool clamped) + => CandidatePlanCount(observedAvgPlanBytes, budgetBytes, catchUpInProgress: false, out clamped); + + /// + /// As above, with the catch-up guard: while — the watermark still below + /// the server's newest plan_id — the observed average is FLOORED at + /// rather than trusted. + /// + /// The estimator is biased during exactly that window, and measurably so: the average is computed over + /// the plans a pass actually shipped, which under plan_id-ascending shipping are the OLDEST ids in the + /// catalog. On one production catalog the plans the fetch shipped averaged 15 KB while the newest 300 plans + /// in the same catalog averaged 46 KB — a 3x under-estimate, which inflates K threefold and decompresses + /// that much more than the budget can ship. Flooring at the seed applies the same over-estimate-is-safe + /// logic the seed itself rests on, for the one window where the sample is known to be unrepresentative. + /// Once the first walk has converged the sample spans the catalog and the observed average is trusted. + /// + public static int CandidatePlanCount(long? observedAvgPlanBytes, long budgetBytes, bool catchUpInProgress, out bool clamped) + { + var avg = observedAvgPlanBytes is long observed && observed > 0 ? observed : FirstContactAvgPlanBytes; + + if (catchUpInProgress && avg < FirstContactAvgPlanBytes) + { + avg = FirstContactAvgPlanBytes; + } + + if (budgetBytes <= 0) + { + clamped = true; + return MinCandidatePlans; + } + + /* double for the margin, then ONE cap before the cast — at int.MaxValue rather than at + MaxCandidatePlans, deliberately. Capping at the bound here would pre-clamp the value and leave the + comparison below unable to tell a clamp from a natural landing, which is the false positive this + reports on. int.MaxValue only guards the cast itself, since the budget is operator input. */ + var wanted = (double)budgetBytes / avg * CandidatePlanMargin; + var unclamped = wanted >= int.MaxValue ? int.MaxValue : (int)Math.Ceiling(wanted); + var bounded = Math.Clamp(unclamped, MinCandidatePlans, MaxCandidatePlans); + + /* Reports that a bound CHANGED the answer, not that the answer happens to equal one. A window whose + measured size lands naturally on 32 or 2048 was sized by the measurement and needs no log line; saying + "clamped" there is a false positive against this contract, and a caller that logs on it teaches its + reader to ignore the message. */ + clamped = bounded != unclamped; + return bounded; + } + + /// + /// The watermark a pass earned, given the plan_ids whose XML actually landed. Under plan_id-ordered + /// shipping a budget cut truncates a SUFFIX, so the highest landed id is safe to keep even from a cut pass + /// — which is the whole point of the reordering (#2210): the previous design shipped in + /// last_execution_time order, where a cut left an arbitrary SUBSET and no value was safe, so the + /// watermark could not advance on 97.8% of passes and therefore never advanced at all. + /// + /// Defensive on the precondition rather than trusting it: a DESCENT anywhere in + /// abandons the advance entirely and reports itself through + /// . Honouring the leading ascending run instead + /// looks safer and is not — given {105, 101} it would advance to 105, and once ordering is broken + /// there is no longer any basis for inferring that every SELECTED plan below 105 landed, so a plan whose + /// XML never arrived gets suppressed until the refresh horizon. Ordering is what makes a cut a suffix; with + /// it gone the pass has earned nothing, and one lost pass of progress is the cheap side of that trade. + /// + /// The verdict and the signal come back TOGETHER, in one value, deliberately. Two separate functions + /// would let a caller take the watermark and never ask whether ordering held — a watermark that quietly + /// stops moving with nothing logged, which is precisely the failure this whole redesign exists to correct + /// and would be a poor thing to reintroduce one level up. + /// + /// Never moves backward: a pass that lands nothing, or only ids at or below the standing watermark, + /// returns the standing value. Lowering it would refetch the catalog, and "no new plans this window" is an + /// ordinary quiet pass, not a reset — the reset signal lives on the runtime stream, where a plan at or below + /// the watermark that the store has never resolved can actually be observed. + /// + public static PlanWatermarkAdvance AdvanceWatermark(long standing, IReadOnlyList landedPlanIdsInOrder) + { + if (landedPlanIdsInOrder is null || landedPlanIdsInOrder.Count == 0) + { + return new PlanWatermarkAdvance(standing, true); + } + + var advanced = standing; + var previous = long.MinValue; + + foreach (var planId in landedPlanIdsInOrder) + { + if (planId < previous) + { + return new PlanWatermarkAdvance(standing, false); + } + + previous = planId; + + if (planId > advanced) + { + advanced = planId; + } + } + + return new PlanWatermarkAdvance(advanced, true); + } + + /// + /// The watermark to apply for one database, or 0 — meaning "fetch every plan's XML" — for an absent, + /// malformed, EXPIRED or future-stamped one. Zero is the documented conservative path: absent is what a + /// first run, a restarted host and a broken store all look like, and all three must refetch rather than + /// skip. A future stamp means the clock moved backwards, which would otherwise pin the watermark for as + /// long as the skew lasts. + /// + public static long Resolve(IReadOnlyDictionary state, string databaseName, DateTime utcNow) + { + if (!TryParse(state, databaseName, out var planId, out var stamped)) + { + return 0; + } + + if (stamped > utcNow || utcNow - stamped >= RefreshAfter) + { + return 0; + } + + return planId; + } + + /// + /// The stored stamp — when this database last did a FULL plan-XML fetch — with no expiry applied, so a + /// write-back can carry it forward across an advance instead of renewing the refresh horizon. Null when + /// there is nothing parseable to carry, which the caller treats as "stamp now". + /// + public static DateTime? ResolveStamp(IReadOnlyDictionary state, string databaseName) => + TryParse(state, databaseName, out _, out var stamped) ? stamped : null; + + /// + /// Formats a watermark for storage: highest stored plan_id plus the stamp dating the last FULL fetch. + /// The stamp is a parameter rather than "now" precisely because it must survive advances — re-stamping + /// on every advance would push the horizon out forever on any database that keeps compiling plans, which + /// is the busy ones where a stale plan matters most, and the bounded refresh would never fire. + /// + public static string Format(long planId, DateTime fullFetchAtUtc) => + planId.ToString(CultureInfo.InvariantCulture) + ":" + + new DateTimeOffset(DateTime.SpecifyKind(fullFetchAtUtc, DateTimeKind.Utc)).ToUnixTimeSeconds() + .ToString(CultureInfo.InvariantCulture); + + private static bool TryParse( + IReadOnlyDictionary state, string databaseName, out long planId, out DateTime stamped) + { + planId = 0; + stamped = default; + + if (state is null || !state.TryGetValue(KeyFor(databaseName), out var raw) || string.IsNullOrWhiteSpace(raw)) + { + return false; + } + + var parts = raw.Split(':'); + if (parts.Length != 2 + || !long.TryParse(parts[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out planId) + || !long.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var stampedUnix) + || planId <= 0) + { + planId = 0; + return false; + } + + stamped = DateTimeOffset.FromUnixTimeSeconds(stampedUnix).UtcDateTime; + return true; + } +} diff --git a/PerformanceMonitor.Collectors/QueryStoreTextState.cs b/PerformanceMonitor.Collectors/QueryStoreTextState.cs new file mode 100644 index 000000000..10b4ec7fd --- /dev/null +++ b/PerformanceMonitor.Collectors/QueryStoreTextState.cs @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace PerformanceMonitor.Collectors; + +/// The result of advancing a text watermark: see . +public readonly record struct TextWatermarkAdvance(long Watermark, bool ArrivedInQueryIdOrder); + +/// +/// Per-database watermark for the query-text fetch (#2150), the sibling of +/// — same encoding, same conservative-zero rules, same +/// never-backward advance. +/// +/// Why this exists. The runtime-stats payload carried query_sql_text +/// (nvarchar(max)) inside a TOP ... WITH TIES ... ORDER BY last_execution_time projection. +/// A Top-N Sort carries every output column through the sort and reads ALL of its input before emitting +/// a row, so choosing the rows to ship materialized the text for the entire qualifying set. Measured on +/// a purpose-built Azure SQL DB store with the plan XML already removed by #2210 — the only difference +/// being that one column — time-to-first-row was 4.67s vs 0.45s at 1,505 rows / 12.8 MB of text +/// and 5.02s vs 0.57s at 4,037 rows / 34 MB, with full drain 8.06s vs 0.50s and +/// 16.95s vs 1.45s. Neither the row cap nor the client byte budget can bound that: TOP (500) +/// measured identical to TOP (50000), and wall time was flat across a 4 MB → 256 MB budget sweep, +/// because the server finishes before the client sees a byte. +/// +/// Why a watermarked fetch rather than a per-pass dedupe. That path was already tried on the +/// plan side and abandoned: #1556's ROW_NUMBER gate shipped each plan once per PASS, and #2164 +/// replaced it precisely because "the ROW_NUMBER gate ships each plan once per pass but re-ships it every +/// pass forever, and since drain is 94-97% of a pass and is per-row LOB cost, NOT fetching is worth far +/// more than fetching less." #2210 then took the column out of the stream entirely. +/// query_id is an identity, monotonic within a database, so the same shape applies: fetch a +/// statement's text ONCE, ever. +/// +/// Keyed on query_id, not query_text_id, and that is what keeps this cheap. +/// query_id is ALREADY a stored payload column on the runtime row, so readers get the join key for +/// free and the fact table needs no new column and no migration. Keying on query_text_id would have +/// required adding it to the payload — a schema change — to buy de-duplication across the handful of +/// query_ids that share one text (a query_id is per text PLUS context settings, so the two +/// are close to 1:1 in practice). Storing a rare duplicate is the cheaper side of that trade. +/// +/// What this deliberately does NOT mirror, and why. The plan side carries a whole candidate- +/// window estimator (FirstContactAvgPlanBytes, min/max clamps, an observed-average learning loop) +/// because SUM(DATALENGTH(query_plan)) OVER (ORDER BY plan_id) forces the server to DECOMPRESS +/// every plan in the window — sys.query_store_plan.query_plan is decompressed by the view on +/// access. sys.query_store_query_text.query_sql_text is not, so its DATALENGTH is cheap and +/// the window needs no estimate at all: a flat coarse bound plus the exact running-byte total is enough. +/// The plan side also re-verifies content hashes because plan XML can be rewritten in place; a +/// query_text_id maps to fixed text forever — a changed statement is a new id — so there is +/// nothing to re-verify and no content digest to track. +/// +public static class QueryStoreTextState +{ + /// + /// The collector name the watermark is stored under. Separate from the plan fetch's own state so the + /// two advance independently: they walk different catalogs at different rates, and sharing a key would + /// let a plan-side reset drop the text watermark (and vice versa) for no reason. + /// + public const string StateCollectorName = "query_store_text"; + + /// Prefix for the per-database state key. + public const string WatermarkKeyPrefix = "textwm:"; + + /// + /// How long a watermark stands before a full re-walk. Matched to the plan side's one day rather than + /// tuned separately, so an operator reasoning about one fetch reasons about both — and the term that + /// made the plan side's choice tight does not apply here: expiry means a budgeted catalog walk, and + /// text is roughly an order of magnitude smaller per row than plan XML (8.5 KB against 195 KB on the + /// measured store), so the walk this horizon triggers is correspondingly cheaper. + /// + /// The re-walk is not decoration. query_id is monotonic in FIRST-SEEN order, not in "we + /// have stored it", so two things arrive below a standing watermark: a statement first seen before + /// monitoring began and only executed later, and — the one that matters — a Query Store reset, which + /// renumbers ids from the start. Without a bounded horizon a reset would suppress every text forever. + /// + public static readonly TimeSpan RefreshAfter = TimeSpan.FromDays(1); + + /// + /// How many texts one pass may CONSIDER. A flat bound, not an estimate: the running byte total is the + /// exact constraint and DATALENGTH(query_sql_text) is cheap to evaluate, so this only has to be + /// large enough that the budget binds first and small enough that a pass never windows an entire + /// catalog. At the 12 MB default ship budget this covers texts averaging under ~2.5 KB, which is + /// comfortably below what a fragmenting literal-heavy statement produces. + /// + public const int CandidateTexts = 5_000; + + /// The state key for one database. + public static string KeyFor(string databaseName) => WatermarkKeyPrefix + databaseName; + + /// + /// The highest query_id landed, or the standing watermark when a pass lands nothing. + /// + /// Reports whether the ids arrived in query_id order, because that ordering is what + /// makes a budget cut a SUFFIX — everything up to the cut is stored, so the highest stored id is a + /// safe resume point. Out of order, that argument collapses and the caller must hold the watermark + /// rather than advance past statements whose text it never stored. + /// + /// Never moves backward. A pass landing nothing, or only ids at or below the standing watermark, + /// is an ordinary quiet pass — not a reset — and lowering the watermark would refetch the catalog. + /// + public static TextWatermarkAdvance AdvanceWatermark(long standing, IReadOnlyList landedQueryIdsInOrder) + { + if (landedQueryIdsInOrder is null || landedQueryIdsInOrder.Count == 0) + { + return new TextWatermarkAdvance(standing, true); + } + + var advanced = standing; + var previous = long.MinValue; + + foreach (var queryId in landedQueryIdsInOrder) + { + if (queryId < previous) + { + return new TextWatermarkAdvance(standing, false); + } + + previous = queryId; + + if (queryId > advanced) + { + advanced = queryId; + } + } + + return new TextWatermarkAdvance(advanced, true); + } + + /// + /// The watermark to apply for one database, or 0 — meaning "fetch every text" — for an absent, + /// malformed, EXPIRED or future-stamped one. Zero is the conservative path, and all three of a first + /// run, a restarted host and a broken store look identical from here: every one of them must refetch + /// rather than skip. A future stamp means the clock moved backwards, which would otherwise pin the + /// watermark for as long as the skew lasts. + /// + public static long Resolve(IReadOnlyDictionary state, string databaseName, DateTime utcNow) + { + if (!TryParse(state, databaseName, out var textId, out var stamped)) + { + return 0; + } + + if (stamped > utcNow || utcNow - stamped >= RefreshAfter) + { + return 0; + } + + return textId; + } + + /// + /// The stored stamp — when this database last did a FULL text fetch — with no expiry applied, so a + /// write-back can carry it forward across an advance instead of renewing the refresh horizon. Null + /// when there is nothing parseable to carry, which the caller treats as "stamp now". + /// + public static DateTime? ResolveStamp(IReadOnlyDictionary state, string databaseName) => + TryParse(state, databaseName, out _, out var stamped) ? stamped : null; + + /// + /// Formats a watermark for storage: highest stored query_id plus the stamp dating the last + /// FULL fetch. The stamp is a parameter rather than "now" precisely because it must survive advances — + /// re-stamping on every advance would push the horizon out forever on any database that keeps seeing + /// new statements, which is exactly where a reset would hurt most, and the bounded re-walk would never + /// fire. + /// + public static string Format(long textId, DateTime fullFetchAtUtc) => + textId.ToString(CultureInfo.InvariantCulture) + ":" + + new DateTimeOffset(DateTime.SpecifyKind(fullFetchAtUtc, DateTimeKind.Utc)).ToUnixTimeSeconds() + .ToString(CultureInfo.InvariantCulture); + + private static bool TryParse( + IReadOnlyDictionary state, string databaseName, out long textId, out DateTime stamped) + { + textId = 0; + stamped = default; + + if (state is null || !state.TryGetValue(KeyFor(databaseName), out var raw) || string.IsNullOrWhiteSpace(raw)) + { + return false; + } + + var split = raw.IndexOf(':'); + if (split <= 0 || split == raw.Length - 1) + { + return false; + } + + if (!long.TryParse(raw.AsSpan(0, split), NumberStyles.Integer, CultureInfo.InvariantCulture, out textId) + || textId < 0) + { + textId = 0; + return false; + } + + if (!long.TryParse(raw.AsSpan(split + 1), NumberStyles.Integer, CultureInfo.InvariantCulture, out var unix)) + { + textId = 0; + return false; + } + + try + { + stamped = DateTimeOffset.FromUnixTimeSeconds(unix).UtcDateTime; + } + catch (ArgumentOutOfRangeException) + { + textId = 0; + return false; + } + + return true; + } +} diff --git a/PerformanceMonitor.Collectors/SpinlockStatsCollector.cs b/PerformanceMonitor.Collectors/SpinlockStatsCollector.cs index c7b701353..d11c5a642 100644 --- a/PerformanceMonitor.Collectors/SpinlockStatsCollector.cs +++ b/PerformanceMonitor.Collectors/SpinlockStatsCollector.cs @@ -103,13 +103,13 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) { - /* Delta groups, key (spinlock_name), and the 300 s gap policy are the parity contract. + /* Delta groups, key (spinlock_name), and the shared gap policy are the parity contract. spins_per_collision is a computed ratio, not a cumulative counter — no delta (mirrors the Dashboard's collect.spinlock_stats table). */ - var deltaCollisions = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_collisions", row.SpinlockName, row.Collisions, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaSpins = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_spins", row.SpinlockName, row.Spins, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaSleepTime = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_sleep_time", row.SpinlockName, row.SleepTime, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaBackoffs = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_backoffs", row.SpinlockName, row.Backoffs, collectionTime: context.CollectionTime, maxGapSeconds: 300); + var deltaCollisions = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_collisions", row.SpinlockName, row.Collisions, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaSpins = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_spins", row.SpinlockName, row.Spins, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaSleepTime = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_sleep_time", row.SpinlockName, row.SleepTime, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaBackoffs = context.Deltas.CalculateDelta(context.ServerId, "spinlock_stats_backoffs", row.SpinlockName, row.Backoffs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.SpinlockName) /* spinlock_name VARCHAR */ diff --git a/PerformanceMonitor.Collectors/TempDbStatsCollector.cs b/PerformanceMonitor.Collectors/TempDbStatsCollector.cs index 016d1c601..c8c44720a 100644 --- a/PerformanceMonitor.Collectors/TempDbStatsCollector.cs +++ b/PerformanceMonitor.Collectors/TempDbStatsCollector.cs @@ -18,7 +18,8 @@ namespace PerformanceMonitor.Collectors; /// TempDB space usage from tempdb.sys.dm_db_file_space_usage plus the top tempdb-consuming /// session (two result sets → one row). Extracted verbatim from Lite's /// RemoteCollectorService.TempDb.cs. Always yields exactly one row — zeros when the result -/// sets are empty — matching the original collector's behavior. +/// sets are empty — matching the original collector's behavior. Not applicable to Azure SQL +/// Database; see . ///
public sealed class TempDbStatsCollector : CollectorDefinitionBase { @@ -44,7 +45,29 @@ public readonly record struct Row( public override string? WatermarkColumn => null; - public override bool AppliesTo(CollectorTargetInfo target) => true; + /// + /// Skips Azure SQL Database, which cannot serve this query at all. + /// + /// The first result set reads tempdb.sys.dm_db_file_space_usage — a THREE-part + /// reference out of the connected database — and on Azure SQL DB that requires + /// VIEW DATABASE PERFORMANCE STATE in tempdb. A non-administrative login cannot hold + /// it there (tempdb permissions are not persistable on Azure SQL DB, and in an elastic pool the + /// database is not even the permission boundary), so every cycle failed with error 262 — + /// "VIEW DATABASE PERFORMANCE STATE permission denied in database 'tempdb'" — reported from the + /// field at 11x consecutive, which is exactly the collection-health pollution the msdb gate on the + /// Agent collectors exists to prevent. + /// + /// Azure Managed Instance keeps collecting: it has a real tempdb and full DMV access, so the + /// gate is IsAzureSqlDb specifically, not "anything Azure" — the same distinction + /// and draw. + /// + /// The second result set (sys.dm_db_session_space_usage, two-part and in-database) + /// WOULD work on Azure SQL DB. Recovering that half needs an Azure-specific query variant rather + /// than a gate, so it is deliberately out of scope here: the immediate defect is a collector that + /// can only ever fail, and half a row would be a different contract than the one row this + /// collector promises. + /// + public override bool AppliesTo(CollectorTargetInfo target) => !target.IsAzureSqlDb; public override bool RunsPerDatabase(CollectorTargetInfo target) => false; diff --git a/PerformanceMonitor.Collectors/WaitStatsCollector.cs b/PerformanceMonitor.Collectors/WaitStatsCollector.cs index c67e8ddcc..8a1a1f909 100644 --- a/PerformanceMonitor.Collectors/WaitStatsCollector.cs +++ b/PerformanceMonitor.Collectors/WaitStatsCollector.cs @@ -89,10 +89,10 @@ public override async ValueTask> ReadAsync(DbDataReader reader, Collec public override void WritePayload(Row row, ICollectorRowWriter writer, CollectorContext context) { - /* Delta groups, keys, and the 300 s gap policy are the parity contract — do not reorder. */ - var deltaWaitingTasks = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_tasks", row.WaitType, row.WaitingTasks, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_time", row.WaitType, row.WaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); - var deltaSignalWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_signal", row.WaitType, row.SignalWaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: 300); + /* Delta groups, keys, and the shared gap policy are the parity contract — do not reorder. */ + var deltaWaitingTasks = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_tasks", row.WaitType, row.WaitingTasks, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_time", row.WaitType, row.WaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); + var deltaSignalWaitTimeMs = context.Deltas.CalculateDelta(context.ServerId, "wait_stats_signal", row.WaitType, row.SignalWaitTimeMs, collectionTime: context.CollectionTime, maxGapSeconds: CollectorDeltaCalculator.DefaultMaxGapSeconds); writer .Value(row.WaitType) /* wait_type VARCHAR */ diff --git a/PerformanceMonitor.Collectors/WatermarkPolicy.cs b/PerformanceMonitor.Collectors/WatermarkPolicy.cs index 768696fea..31881a315 100644 --- a/PerformanceMonitor.Collectors/WatermarkPolicy.cs +++ b/PerformanceMonitor.Collectors/WatermarkPolicy.cs @@ -17,10 +17,17 @@ namespace PerformanceMonitor.Collectors; /// one cycle tried to pull the entire backlog at once and drove the 0→13GB commit-limit blowout. /// /// -/// floors a stale watermark to now - 24h: a routine restart or a -/// brief outage never clamps (its watermark is minutes old), a multi-day outage survives with a -/// deliberate, logged, BOUNDED hole (the source still retains the older data; the viewer's windows -/// are 24h anyway). This is deliberately NOT applied to every timestamp watermark — for a ring-buffer +/// floors a stale watermark to now - 1h: a routine restart never +/// clamps (its watermark is minutes old), and anything longer survives as a deliberate, logged, +/// BOUNDED hole that the backfill worker (#2022/#2058) trickles in afterwards. The horizon was 24h +/// until the use1 migration wedge (#2102) proved a row cap is not a cost cap: the per-database query +/// aggregates and sorts the WHOLE window before TOP or the byte budget can bound anything, so its +/// cost grows with window width. A big database that missed one 60s cycle faced a wider window the +/// next cycle, which cost more and timed out again — a self-sustaining spiral the 24h clamp sat far +/// above and never interrupted. One hour is the envelope the fleet already proves every day (Query +/// Store's 900s flush cadence makes 15–60min effective windows the routine steady state), and the +/// clamp floor slides forward with now, so recovery is immediate no matter how stale the +/// watermark got. This is deliberately NOT applied to every timestamp watermark — for a ring-buffer /// or rolling-trace source the clamp is a no-op at best and, on a quiet default_trace whose /// 100MB ring can span days, a WRONG truncation of legitimate catch-up. It is therefore scoped to /// exactly ONE collector: query_store's per-database cutoff (the only unbounded-persisted source among @@ -40,13 +47,15 @@ namespace PerformanceMonitor.Collectors; public static class WatermarkPolicy { /// - /// The maximum catch-up horizon. 24h is chosen so routine outages never clamp while a multi-day - /// outage survives with a single logged, bounded hole. Exposed so a test pins the boundary. + /// The maximum catch-up horizon — the live path's one-query cost envelope, matched to + /// so no path ever windows wider than the + /// steady state the fleet proves (#2102). Everything older is the backfill worker's job. + /// Exposed so a test pins the boundary. /// - public static readonly TimeSpan MaxCatchup = TimeSpan.FromHours(24); + public static readonly TimeSpan MaxCatchup = TimeSpan.FromHours(1); /// - /// Floors a >24h-stale timestamp watermark to now - 24h; a null watermark (nothing + /// Floors a stale timestamp watermark to now - ; a null watermark (nothing /// collected yet — the definition's documented first-run window applies) stays null, and a /// watermark within the horizon is returned unchanged. Compare the result to the input to tell /// whether a clamp fired (the runner logs a WARNING when it does). diff --git a/PerformanceMonitor.Common/AbandonableStep.cs b/PerformanceMonitor.Common/AbandonableStep.cs new file mode 100644 index 000000000..f33c3306b --- /dev/null +++ b/PerformanceMonitor.Common/AbandonableStep.cs @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace PerformanceMonitor.Common; + +/// How one run of an ended. +public enum AbandonableStepOutcome +{ + /// The step finished within its deadline. + Completed, + + /// The step threw; the exception rides . + Faulted, + + /// The deadline elapsed first. The step's task is ABANDONED, not cancelled — it may still + /// be running; the in-flight guard keeps it from being relaunched until it truly ends. + Abandoned, + + /// A previously-abandoned run is still wedged, so this run never started. + SkippedStillRunning, + + /// The caller's token cancelled while waiting. + Cancelled, +} + +/// One run's outcome plus the fault when there was one. +public readonly record struct AbandonableStepResult(AbandonableStepOutcome Outcome, Exception? Exception = null); + +/// +/// A sequential background-loop step that may NOT hold the loop past a deadline (#2148). Born from the +/// field failure this class exists to make impossible: Lite's collection ladder ran its steps strictly +/// in sequence, one step wedged on an Azure elastic pool ~12 minutes after a 3.4.0 upgrade, and ALL +/// collection stopped — permanently, silently, with every step's exception armor intact, because the +/// armor bounded throws and nothing bounded a HANG. +/// +/// The discipline is the ladder's own scheduled-analysis idiom, extracted and made reusable: +/// against a deadline, and an in-flight guard cleared only when +/// the underlying task TRULY finishes — so an abandoned (possibly wedged) run is never overlapped by a +/// relaunch, and the moment it finally dies the step becomes runnable again on its own. Abandonment is +/// deliberately not cancellation: the wedged task already ignored cooperative signals by definition, +/// and the value here is that the LOOP keeps moving while the guard quarantines the stuck step. +/// +/// Outcomes are returned, never thrown (the caller is a loop whose next steps must run; it logs +/// each outcome at its own severity). The step delegate's synchronous throws are treated as +/// like any other fault, with the guard released. +/// +public sealed class AbandonableStep +{ + private int _inFlight; + + /// Whether a run is currently holding the guard — an abandoned run still counts until its + /// task truly ends. Exposed for the caller's logging/diagnostics, racy by nature. + public bool IsInFlight => Volatile.Read(ref _inFlight) == 1; + + /// + /// Runs unless a prior run is still wedged, waiting at most + /// before abandoning it and returning control to the loop. + /// (review catch on the #2148 PR) surfaces an exception thrown by a + /// run AFTER it was abandoned — without it that fault would be observed-but-discarded, and the one + /// exception that explains a wedge would never reach a log. Invoked only for faults the caller's + /// awaited path did NOT already receive; a fault landing in the microseconds between the deadline + /// decision and the abandonment flag can be missed (never doubled), which costs one log line, not + /// correctness — the caller already logged the abandonment itself. + /// Parameter order: is LAST, per CA1068. It was third + /// until #2193, which is the ordering the analyzer flags — and every call site already passed + /// by name, so the move cost nothing at the callers and the compiler + /// found all of them. + /// + public async Task RunAsync( + Func step, TimeSpan timeout, Action? onLateFault = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(step); + + if (Interlocked.CompareExchange(ref _inFlight, 1, 0) != 0) + { + return new AbandonableStepResult(AbandonableStepOutcome.SkippedStillRunning); + } + + Task work; + try + { + work = step(); + } + catch (Exception ex) + { + Interlocked.Exchange(ref _inFlight, 0); + return new AbandonableStepResult(AbandonableStepOutcome.Faulted, ex); + } + + var abandoned = new StrongBox(false); + + /* The guard clears when the task TRULY ends — completion, fault, or cancellation — never when + the deadline merely moves the loop on. Faults on the abandoned path are observed here (so an + abandoned-then-faulted task cannot surface as UnobservedTaskException) AND handed to + onLateFault, because a discarded exception from the wedged run is exactly the diagnostic the + field report needs. */ + _ = work.ContinueWith( + (t, state) => + { + var self = (AbandonableStep)state!; + var fault = t.Exception; /* observe unconditionally */ + if (fault is not null && Volatile.Read(ref abandoned.Value)) + { + try + { + onLateFault?.Invoke(fault.GetBaseException()); + } + catch + { + /* A throwing log callback must not take the continuation down. */ + } + } + + Interlocked.Exchange(ref self._inFlight, 0); + }, + this, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + + var finished = await Task.WhenAny(work, Task.Delay(timeout, cancellationToken)).ConfigureAwait(false); + + if (finished != work) + { + if (cancellationToken.IsCancellationRequested) + { + return new AbandonableStepResult(AbandonableStepOutcome.Cancelled); + } + + Volatile.Write(ref abandoned.Value, true); + return new AbandonableStepResult(AbandonableStepOutcome.Abandoned); + } + + try + { + await work.ConfigureAwait(false); + return new AbandonableStepResult(AbandonableStepOutcome.Completed); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return new AbandonableStepResult(AbandonableStepOutcome.Cancelled); + } + catch (Exception ex) + { + return new AbandonableStepResult(AbandonableStepOutcome.Faulted, ex); + } + } +} diff --git a/PerformanceMonitor.Common/CpuAttribution.cs b/PerformanceMonitor.Common/CpuAttribution.cs new file mode 100644 index 000000000..64b74fe90 --- /dev/null +++ b/PerformanceMonitor.Common/CpuAttribution.cs @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; + +namespace PerformanceMonitor.Common +{ + /// + /// The attributed-CPU denominator for the top-queries/procedures MCP reads (#2320, split from #2235) — + /// how much of the instance's actually-consumed CPU the returned ranking explains. Pre-#2290 the reads + /// explained ~10% of the box and nothing said so; a caller chased the visible 10% assuming it was + /// everything. And the number catches impossible claims at a glance: an external comparison died the + /// moment someone divided its worker_time sum by the box's available CPU-seconds and got 137%. Both + /// SKUs' tools hand the caller numerator, denominator, and ratio instead of leaving the division to be + /// re-derived — ONE computation here, so the two cannot disagree. + /// + /// The denominator is measured, not theoretical: the SQL process's average CPU% over the window + /// (the cpu_utilization series both stores already collect) × core count (server_properties) × window + /// seconds. When a piece is missing — no CPU samples, no properties snapshot, or the series covers too + /// little of the window — the ratio is OMITTED, never invented (#2320's explicit degrade rule). + /// + public static class CpuAttribution + { + /// The CPU series must span at least this fraction of the requested window for the + /// denominator to be honest — below it a server added (or monitoring resumed) mid-window would + /// deflate measured CPU-seconds and inflate the ratio. + public const double MinimumCoverageFraction = 0.9; + + /// Below this ratio the result carries the "not the whole story" note — the #2235 history + /// says real post-fix rankings explain roughly a third, so half is a generous line between "normal + /// plan-cache attribution loss" and "worth saying out loud". + public const double LowRatioThreshold = 0.5; + + /// Above this ratio the returned rows claim more CPU than the process measurably consumed — + /// the impossible-claim marker (137% is how the Datadog comparison died). Slack above 1.0 covers + /// sampling noise between the two series. + public const double OverAttributionThreshold = 1.1; + + /// + /// A null always comes with a saying why. + /// is usually null alongside it (the denominator could not be + /// measured) — EXCEPT the measured-zero case, where the zero is reported and only the ratio is + /// omitted, so the caller sees WHY dividing was refused. When the ratio is present the note is + /// null unless the ratio is low or impossible. + /// + public sealed record Result( + double RankedCpuSeconds, + double? SqlCpuSecondsInWindow, + double? AttributedCpuRatio, + string? Note); + + /// + /// is the summed windowed CPU of the rows the tool RETURNS + /// (post top-N, post filters) — the ratio answers "what does the caller-visible ranking explain", + /// not "what does the whole table hold". The sample aggregate (count / first / last / + /// ) comes from the store's cpu_utilization series windowed on + /// the SAME collection_time bounds as the ranking, so numerator and denominator share gaps. + /// + public static Result Compute( + double rankedCpuSeconds, + DateTime windowStartUtc, + DateTime windowEndUtc, + int sampleCount, + DateTime? firstSampleUtc, + DateTime? lastSampleUtc, + double? avgSqlCpuPercent, + int cpuCount) + { + var ranked = Math.Round(rankedCpuSeconds, 1); + var windowSeconds = (windowEndUtc - windowStartUtc).TotalSeconds; + + if (windowSeconds <= 0) + { + return new Result(ranked, null, null, + "the requested window is empty; ratio omitted"); + } + + if (sampleCount == 0 || avgSqlCpuPercent is null || firstSampleUtc is null || lastSampleUtc is null) + { + return new Result(ranked, null, null, + "no cpu_utilization samples in the window, so measured CPU-seconds cannot be computed; ratio omitted rather than invented"); + } + + if (cpuCount <= 0) + { + return new Result(ranked, null, null, + "core count unavailable (no server_properties snapshot), so measured CPU-seconds cannot be computed; ratio omitted rather than invented"); + } + + var coverageStart = firstSampleUtc.Value > windowStartUtc ? firstSampleUtc.Value : windowStartUtc; + var coverageEnd = lastSampleUtc.Value < windowEndUtc ? lastSampleUtc.Value : windowEndUtc; + var coverageFraction = Math.Max(0, (coverageEnd - coverageStart).TotalSeconds) / windowSeconds; + if (coverageFraction < MinimumCoverageFraction) + { + return new Result(ranked, null, null, + $"cpu_utilization covers only {Math.Round(coverageFraction * 100).ToString(CultureInfo.InvariantCulture)}% of the window; ratio omitted rather than computed against a partial denominator"); + } + + var sqlCpuSeconds = avgSqlCpuPercent.Value / 100.0 * cpuCount * windowSeconds; + if (sqlCpuSeconds <= 0) + { + return new Result(ranked, Math.Round(sqlCpuSeconds, 1), null, + "the SQL process's measured CPU in the window is zero; ratio omitted"); + } + + var ratio = rankedCpuSeconds / sqlCpuSeconds; + var pct = Math.Round(ratio * 100).ToString(CultureInfo.InvariantCulture); + + string? note = null; + if (ratio > OverAttributionThreshold) + { + note = $"the returned rows' CPU is {pct}% of the SQL process's measured CPU-seconds — more than the process consumed. Treat this as an impossible-claim marker: suspect double-counted deltas, clock skew between the two series, or a CPU-series gap before trusting the ranking's absolute numbers."; + } + else if (ratio < LowRatioThreshold) + { + note = $"the returned rows explain {pct}% of the SQL process's measured CPU-seconds in this window. The remainder is plans evicted between snapshots, statements outside the top-N or filters, zero-cost rows, and non-query CPU — a low ratio means the visible ranking is not the whole story."; + } + + return new Result(ranked, Math.Round(sqlCpuSeconds, 1), Math.Round(ratio, 3), note); + } + } +} diff --git a/PerformanceMonitor.Common/PerformanceMonitor.Common.csproj b/PerformanceMonitor.Common/PerformanceMonitor.Common.csproj index 5adf68aca..fb06642be 100644 --- a/PerformanceMonitor.Common/PerformanceMonitor.Common.csproj +++ b/PerformanceMonitor.Common/PerformanceMonitor.Common.csproj @@ -17,11 +17,11 @@ - + - + - + diff --git a/PerformanceMonitor.Common/ProvisioningVerdict.cs b/PerformanceMonitor.Common/ProvisioningVerdict.cs new file mode 100644 index 000000000..beb16aa4d --- /dev/null +++ b/PerformanceMonitor.Common/ProvisioningVerdict.cs @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; + +namespace PerformanceMonitor.Common; + +/// +/// The FinOps provisioning verdict for one server — over-provisioned, right-sized, or under-provisioned — +/// as a single shared predicate both apps call. +/// +/// Why this is shared rather than inlined. The rule this replaces was copy-pasted four times +/// (Darling's point-in-time and trend reads, Lite's two), and every copy carried the same defect: it tested +/// total_server_memory_mb / target_server_memory_mb > 0.95. Those are the perfmon Total and Target +/// Server Memory counters, and they converge at 1.0 the moment an instance is warmed — Target is what SQL +/// Server wants and Total is what it holds. Measured across a 42-server production fleet, that ratio ran +/// median 1.0000 (min 0.9997, max 1.0002), so the rule reported UNDER_PROVISIONED for every +/// server and OVER_PROVISIONED, whose arm needs the same ratio below 0.5, was unreachable at any +/// workload (#2246). One predicate, compiler-shared, is the same answer the collector gate surface reached +/// when it collapsed two drifting layers into one. +/// +/// Every threshold below was measured, not chosen. Fleet distributions over 24 hours: +/// +/// CPU p95 per server: min 0.0, p25 6.5, median 11.0, p95 48.6, max 51.0 — so +/// at 85 is comfortably clear of a healthy fleet and still reachable. +/// Worker ratio per server: median 0.246, max 0.635; zero servers over 0.8 and zero exhaustion +/// warnings, so at 0.8 does not false-fire here. +/// Workspace-grant utilization (granted / target): median 0.5%, p95 12.0%, max 18.8% — so +/// at 50 excludes nothing real today while still refusing to call +/// a server idle that is straining its semaphore. +/// Grant waiters, grant timeouts, forced grants and Memory Grants Pending: zero across +/// 2,938,711 grant rows and 1,000,560 counter rows, the entire retained history. This fleet has no +/// memory pressure, which is exactly why a correct memory term must be SILENT on it. +/// +/// +/// The honest gap. Because nothing in that history ever recorded grant pressure, there is no +/// positive control from the fleet: the measurements show the memory term stays quiet when it should, and +/// cannot show it fires when it should. That is established by construction in the tests instead, which +/// drive each pressure input directly. +/// +/// Pressure outranks idleness tests under-provisioning first. A server +/// can be quiet on CPU while queries queue for workspace memory, and calling that one over-provisioned would +/// recommend taking away the resource it is short of. +/// +public static class ProvisioningVerdict +{ + /// Sustained CPU at or above this p95 is under-provisioned. Fleet max p95 is 51.0. + public const decimal HighCpuP95Percent = 85m; + + /// Below this average CPU, with also satisfied, a server is a + /// downsizing candidate. + public const decimal IdleAvgCpuPercent = 15m; + + /// An idle server must also never have spiked past this. Guards against averaging away a + /// short daily peak that the smaller instance would not survive. + public const decimal IdleMaxCpuPercent = 40m; + + /// Worker-thread saturation. The Full Dashboard's view has always carried this term and the + /// app copies dropped it, so a genuinely worker-starved server was invisible to them (#2246). + public const double HighWorkerRatio = 0.8; + + /// A server using more than this share of its workspace-memory grant target is not idle, no + /// matter how quiet its CPU looks. Fleet max is 18.8%. + public const decimal IdleGrantUtilizationPercent = 50m; + + /// Over-provisioned: a downsizing candidate. + public const string OverProvisioned = "OVER_PROVISIONED"; + + /// Neither idle enough to shrink nor under pressure. + public const string RightSized = "RIGHT_SIZED"; + + /// Under pressure on CPU, workspace memory, or worker threads. + public const string UnderProvisioned = "UNDER_PROVISIONED"; + + /// + /// The verdict for one server from one window's measurements. + /// + /// Mean SQL Server CPU over the window. + /// Peak SQL Server CPU over the window. + /// 95th-percentile SQL Server CPU over the window. + /// Peak waiter_count from the resource semaphore. Any waiter at all + /// means a query could not get workspace memory when it asked. + /// Grant timeouts accrued over the window (delta, not cumulative). + /// Forced grants accrued over the window (delta, not cumulative). + /// Peak granted-over-target workspace memory, as a percentage. + /// The instance's worker-thread ceiling; 0 or negative means unknown, which + /// cannot imply saturation. + /// Workers in use at the latest sample. + public static string Evaluate( + decimal avgCpuPercent, + decimal maxCpuPercent, + decimal p95CpuPercent, + long maxGrantWaiters, + long grantTimeouts, + long forcedGrants, + decimal grantUtilizationPercent, + int maxWorkers, + int currentWorkers) + { + /* Any of these three is a query that asked for workspace memory and did not simply get it. They are + counts of events, not levels, so there is no threshold to tune — and on a fleet with no memory + pressure they are all zero, which is the point. */ + var memoryPressure = maxGrantWaiters > 0 || grantTimeouts > 0 || forcedGrants > 0; + + /* maxWorkers <= 0 means the sample never reported a ceiling. Unknown is not saturation: the same + rule the collector gates follow, where an unclassified target must never be gated off by + assumption. */ + var workerPressure = maxWorkers > 0 + && currentWorkers / (double)maxWorkers > HighWorkerRatio; + + if (p95CpuPercent > HighCpuP95Percent || memoryPressure || workerPressure) + { + return UnderProvisioned; + } + + if (avgCpuPercent < IdleAvgCpuPercent + && maxCpuPercent < IdleMaxCpuPercent + && grantUtilizationPercent < IdleGrantUtilizationPercent) + { + return OverProvisioned; + } + + return RightSized; + } + + /// + /// Why returned , in the operator's words. + /// + /// Shared for the same reason the verdict is: the UI used to re-derive the cause with + /// P95CpuPct > 85 ? "CPU..." : "memory ratio is {x} (threshold: 0.95)", so once the verdict + /// gained grant-pressure and worker-thread reasons, every one of those would have been explained as a + /// memory ratio that no longer decides anything — a fabricated cause citing a threshold the code does + /// not check. Deriving the text beside the decision is what stops that recurring. + /// + /// Checked in the same order checks them, so the reason names the condition + /// that actually fired first. + /// + public static string UnderProvisionedReason( + decimal p95CpuPercent, + long maxGrantWaiters, + long grantTimeouts, + long forcedGrants, + int maxWorkers, + int currentWorkers) + { + if (p95CpuPercent > HighCpuP95Percent) + { + return $"CPU p95 is {p95CpuPercent:N1}% (threshold: {HighCpuP95Percent:N0}%). " + + "This server may need more CPU capacity."; + } + + if (maxGrantWaiters > 0 || grantTimeouts > 0 || forcedGrants > 0) + { + return "Queries could not get the workspace memory they asked for: peak " + + $"{maxGrantWaiters} grant waiter(s), {grantTimeouts} grant timeout(s), " + + $"{forcedGrants} forced grant(s). This server may need more memory."; + } + + if (maxWorkers > 0 && currentWorkers / (double)maxWorkers > HighWorkerRatio) + { + return $"Worker threads are near the limit: {currentWorkers} of {maxWorkers} in use " + + $"(threshold: {HighWorkerRatio:P0}). This server may need more CPU capacity."; + } + + /* Reachable only if a caller asks for a reason on inputs that are not under-provisioned. Say so + rather than inventing a cause, which is the failure this method exists to end. */ + return "No under-provisioning condition is currently met."; + } +} diff --git a/PerformanceMonitor.Common/QueryStatExtremes.cs b/PerformanceMonitor.Common/QueryStatExtremes.cs new file mode 100644 index 000000000..6f9384793 --- /dev/null +++ b/PerformanceMonitor.Common/QueryStatExtremes.cs @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +namespace PerformanceMonitor.Common; + +/// +/// #2235: the min/max CPU and elapsed columns on the top-queries/top-procedures reads are LIFETIME +/// extremes for the plan's time in cache — sys.dm_exec_query_stats.max_worker_time is a +/// high-water mark the engine never lowers, so the collector snapshots it as-is (a max cannot be +/// delta'd) and a windowed MAX() over snapshots still returns the lifetime value. Same +/// semantics as max_dop, whose tool description has always warned about this; these columns +/// didn't, and the field report showed why that matters: 8 of 20 rows on one box had +/// max_cpu_ms EXCEEDING the whole window's total_cpu_ms, inviting a reader to quote +/// an extreme from an arbitrary earlier period as if it happened this week. +/// +/// Windowing the max for real is not derivable from what's collected once rows group across +/// plans (a per-series "did the high-water mark advance in-window" test drowns in false positives +/// when MIN/MAX mix different plans' marks), so the honest fix is the one shipped here: label the +/// semantics, and surface the self-evident tell — max exceeding the windowed total PROVES the +/// extreme predates the window. +/// +public static class QueryStatExtremes +{ + /// + /// The conditional annotation for one result row, or null when nothing needs saying. Fires only + /// on the provable case (an extreme larger than the whole window's total); a lifetime max that + /// happens to sit inside the window's total is indistinguishable from a windowed one and gets no + /// note. Unit-agnostic: compare like with like (both ms or both µs). + /// + public static string? LifetimeExtremeNote( + double totalCpu, double maxCpu, double totalElapsed, double maxElapsed) + { + var cpuExceeds = maxCpu > totalCpu; + var elapsedExceeds = maxElapsed > totalElapsed; + if (!cpuExceeds && !elapsedExceeds) + { + return null; + } + + var which = (cpuExceeds, elapsedExceeds) switch + { + (true, true) => "max_cpu_ms and max_elapsed_ms exceed", + (true, false) => "max_cpu_ms exceeds", + _ => "max_elapsed_ms exceeds", + }; + + return $"{which} this window's total — min/max (like max_dop) are lifetime extremes for the plan's time in cache, and this extreme predates the window; use total and avg for what happened in the window"; + } +} diff --git a/PerformanceMonitor.Common/QueryStoreReadonlyReason.cs b/PerformanceMonitor.Common/QueryStoreReadonlyReason.cs new file mode 100644 index 000000000..3c8ec5db2 --- /dev/null +++ b/PerformanceMonitor.Common/QueryStoreReadonlyReason.cs @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; +using System.Globalization; + +namespace PerformanceMonitor.Common +{ + /// + /// Decodes sys.database_query_store_options' readonly_reason bitmask into the documented + /// operator-facing labels. ONE table, shared by every surface that shows the value (both viewers' + /// Query Store grids and both MCP servers' get_query_store_health), because the labels here were + /// already miswritten from memory once during #2319 review — a single source is the fix, not care. + /// + /// readonly_reason is a COMBINABLE bitmask (this codebase already relies on that: + /// QueryStoreCollector tests bit 8 with an AND), so it is decoded bit by bit and joined — a switch + /// on exact values loses every multi-bit state. Labels are the documented ones for + /// sys.database_query_store_options; bits the documentation does not name are reported numerically + /// rather than guessed. + /// + public static class QueryStoreReadonlyReason + { + private static readonly (int Bit, string Label)[] Bits = + { + (1, "database is read-only"), + (2, "database is in single-user mode"), + (4, "database is in emergency mode"), + (8, "database is a secondary replica"), + (65536, "storage cap reached"), + (131072, "statement count reached internal memory limit"), + (262144, "persist backlog reached internal memory limit"), + (524288, "database reached disk size limit"), + }; + + /// Human-readable decode of the bitmask; empty string when 0 (not read-only). + public static string Decode(int readonlyReason) + { + if (readonlyReason == 0) + { + return ""; + } + + var parts = new List(); + var remaining = readonlyReason; + foreach (var (bit, label) in Bits) + { + if ((remaining & bit) != 0) + { + parts.Add(label); + remaining &= ~bit; + } + } + + if (remaining != 0) + { + parts.Add($"reason {remaining.ToString(CultureInfo.InvariantCulture)}"); + } + + return string.Join("; ", parts); + } + } +} diff --git a/PerformanceMonitor.Common/QueryStoreServerGate.cs b/PerformanceMonitor.Common/QueryStoreServerGate.cs new file mode 100644 index 000000000..530e4d187 --- /dev/null +++ b/PerformanceMonitor.Common/QueryStoreServerGate.cs @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; + +namespace PerformanceMonitor.Common; + +/// +/// Per-server mutual exclusion between the two independent loops that both run heavy Query Store text +/// extraction against the SAME monitored server: the regular per-tick query_store collection and the +/// #2058 first-contact backfill (#2165). +/// +/// The observed problem. The two loops had no coordination at all. On a 4-core multi-tenant box +/// mid-consolidation, a 64 MB backfill slice for a freshly restored database ran concurrently with the tick's +/// Query Store collection of a SIBLING database — a 12:50:58 backfill ship overlapping a 12:51:09 tick +/// completion — putting roughly 128 MB of Query Store text extraction in flight at once on the box least able +/// to afford it. The overlap is not bad luck: a big catalog arriving is exactly what triggers BOTH the backfill +/// and budget-bound tick passes, so the two loops are most likely to collide precisely when the server is +/// already drowning. +/// +/// Nothing ever waits, and that is deliberate. Both callers try-acquire with a zero timeout and +/// SKIP on failure. These are shared fleet loops: one server's in-flight slice can run to a 180-300 second +/// abandonment deadline, so a blocking acquire would let one slow server stall collection for every other +/// server — reintroducing the #2148 wedge this codebase already fixed once, through a lock instead of a hang. +/// A gate that can only ever skip cannot do that. +/// +/// Why skipping is safe for this collector specifically. Query Store collection is +/// watermark-driven (#1960): each pass resumes from the last shipped boundary rather than re-deriving a window, +/// so a skipped pass defers rows, it does not drop them. That is what makes "skip and retry" the right +/// behaviour here and why this gate must not be reused for a collector whose window is wall-clock derived — +/// for one of those, a skipped pass IS lost data. +/// +/// How the "tick wins, backfill defers" bias is actually realized. Not by preemption: the loser is +/// whichever loop arrives second, because stopping a statement already running against the monitored server +/// would mean killing it, and cancelling a Query Store read mid-flight buys nothing that waiting one cycle does +/// not. The bias comes from CADENCE instead — the tick retries on its own interval (about a minute) while the +/// backfill retries every five, so the tick recovers roughly five times faster from a collision, and a backfill +/// slice is byte-budgeted so it is short in the healthy case. Over any real window the tick therefore wins the +/// overwhelming majority of collisions without either loop ever blocking. +/// +/// One gate per server, held in each host's own keyed dictionary — Darling keys by int server id +/// and Lite by string — so this type is the shared primitive rather than the registry. Same reason +/// is shaped that way, and the two are siblings: that one bounds how long a step +/// may hold a loop, this one bounds what may run beside it. +/// +public sealed class QueryStoreServerGate +{ + /* A plain interlocked flag rather than a SemaphoreSlim, because this gate NEVER waits. A semaphore buys + blocking acquire, timeouts and async waits — all three of which this design deliberately refuses — while + costing a disposable kernel-backed object per monitored server that nothing ever disposes (the registries + are never pruned, by design, so one gate per server lives for the process). CompareExchange gives the one + operation actually needed, owns nothing, and cannot be released more times than it was taken. */ + private int _taken; + + /// + /// True while either loop holds this server's gate. Diagnostic only — never branch on it and then acquire, + /// which is a race; use , whose result IS the decision. + /// + public bool IsHeld => Volatile.Read(ref _taken) == 1; + + /// + /// Takes the server's gate if it is free, returning a lease to release it — or null when the other + /// loop holds it, which the caller must treat as "skip this server this cycle". + /// + /// Returns a disposable rather than exposing a bare release so the two cannot get out of step: every + /// acquisition site is a using, and an early return or a throw inside the guarded work + /// releases the gate on the way out. A leaked gate here would silently stop one server's Query Store + /// collection forever, which is a failure that looks like "that server has no Query Store data" rather than + /// like a bug. + /// + public IDisposable? TryAcquire() => + Interlocked.CompareExchange(ref _taken, 1, 0) == 0 ? new Lease(this) : null; + + /// + /// A lease that guards nothing, for a caller whose collector is not gated at all. + /// + /// Exists so null from keeps exactly ONE meaning — "the other loop + /// holds this server's gate, skip" — at a call site that decides whether to gate and whether it got the + /// gate in the same expression. Without it, "not gated" and "gate busy" would both be null and every such + /// site would need to re-test the predicate to tell a skip from a pass-through; getting that wrong silently + /// skips a collector nobody meant to gate. + /// + public static IDisposable NotGated { get; } = new NoOpLease(); + + private sealed class NoOpLease : IDisposable + { + public void Dispose() + { + /* Nothing held, nothing to release — and safe to dispose repeatedly, since it is a shared + singleton that every non-gated collector run disposes. */ + } + } + + private sealed class Lease : IDisposable + { + private QueryStoreServerGate? _gate; + + internal Lease(QueryStoreServerGate gate) => _gate = gate; + + /// + /// Releases once and only once. Idempotent because a using plus an explicit Dispose() — or + /// a double-dispose from any future refactor — would otherwise clear a flag a DIFFERENT loop had since + /// taken, letting both run against one server at the same time: the exact condition this class exists to + /// prevent. Interlocked because the tick and the backfill dispose their own leases on different threads. + /// + public void Dispose() => Interlocked.Exchange(ref _gate, null)?.Release(); + } + + private void Release() => Volatile.Write(ref _taken, 0); +} diff --git a/PerformanceMonitor.Common/ServerHealthBands.cs b/PerformanceMonitor.Common/ServerHealthBands.cs index 8f56e8cf0..d29e2aa4c 100644 --- a/PerformanceMonitor.Common/ServerHealthBands.cs +++ b/PerformanceMonitor.Common/ServerHealthBands.cs @@ -447,6 +447,7 @@ public static bool IsOnLoadCollector(string? collectorName) => "database_scoped_config", "index_object_stats", "plan_correction", + "query_store_health", }; /// @@ -619,4 +620,77 @@ collector with no inventory to compare against — stays exactly as it read befo : string.Format(CultureInfo.InvariantCulture, "{0} (all {1} runs)", lastNote, totalRuns); } } + + /// + /// Sweep-pressure verdict for get_collection_health (#2296): does the collection body's own execution + /// demand fit inside its fastest cadence? + /// + /// Why this number and not delivered-gap statistics: at fleet scale the delivered cadence + /// stretches for a benign reason — bounded sweep concurrency queues bodies, and the fleet-wide median + /// gap runs a multiple of the configured minute — so measuring gaps flags every server and drowns the + /// two that matter. What isolates a SATURATED server is the arithmetic behind its own watchdog line + /// ("collection body has not completed after Ns of EXECUTION — skipping relaunch"): the collectors' + /// summed average durations, amortized by cadence, exceed the sweep budget itself. Queueing cannot + /// inflate this number, because it is built from the collectors' own execution times. + /// + /// The consequence it names: a body that cannot fit its cadence finishes after the next + /// due time, every relaunch is skipped, and the server collects at a multiple of its configured + /// interval — while every collector, from its own point of view, is HEALTHY. That is precisely why + /// this signal exists: before it, half-rate collection was only visible by reading service-log + /// warnings (#2296 measured two servers at ~50 skip-warnings/hour with 40 of 40 collectors green). + /// + /// Pure and static like : the caller resolves each + /// collector's cadence (from the shared schedule defaults, matching the banding's parity choice) and + /// only the DECISION lives here, pinned by the same table in both suites. + /// + public static class SweepPressureClassifier + { + /* The verdict strings, same switch-friendly shape as the banding's. */ + public const string Ok = "OK"; + public const string AtRisk = "AT_RISK"; + public const string Saturated = "SATURATED"; + + /// + /// AT_RISK at 75% of budget: the amortized average leaves no headroom for variance — the slow + /// collectors on a busy hour are what push a 75% body over its cadence intermittently, which is + /// how saturation looks before it is constant. Chosen against the #2296 measurements: the two + /// saturated servers computed ~101%, the in-region fleet sits far below. + /// + public const double AtRiskBusyPercent = 75.0; + + /// SATURATED at 100%: the body mathematically cannot fit its cadence, so every cycle skips. + public const double SaturatedBusyPercent = 100.0; + + /// + /// Amortized execution demand and its verdict. Each scheduled collector contributes its average + /// duration divided by its cadence in minutes — milliseconds of work demanded per minute of wall + /// time for a body that runs collectors serially. A non-recurring collector + /// ( entry with frequency <= 0: on-load, unknown) contributes + /// nothing — it does not compete for the sweep. Percent is against the 60,000 ms one minute + /// holds; the fastest shipped cadence is one minute, which is what makes the minute the budget. + /// + public static SweepPressure Compute(IEnumerable<(string CollectorName, double AvgDurationMs, int FrequencyMinutes)> collectors) + { + double busyMsPerMinute = 0; + foreach (var (_, avgDurationMs, frequencyMinutes) in collectors) + { + if (frequencyMinutes <= 0 || avgDurationMs <= 0) + { + continue; + } + + busyMsPerMinute += avgDurationMs / frequencyMinutes; + } + + var busyPercent = busyMsPerMinute / 60_000.0 * 100.0; + var verdict = busyPercent >= SaturatedBusyPercent ? Saturated + : busyPercent >= AtRiskBusyPercent ? AtRisk + : Ok; + + return new SweepPressure(busyMsPerMinute, busyPercent, verdict); + } + } + + /// One server's sweep-pressure answer, as a single value so a caller cannot drop the verdict from its numbers. + public sealed record SweepPressure(double BusyMsPerMinute, double BusyPercent, string Verdict); } diff --git a/PerformanceMonitor.Common/Services/ServerIdHelper.cs b/PerformanceMonitor.Common/Services/ServerIdHelper.cs index 9ffba5e78..e5e0521b2 100644 --- a/PerformanceMonitor.Common/Services/ServerIdHelper.cs +++ b/PerformanceMonitor.Common/Services/ServerIdHelper.cs @@ -6,6 +6,8 @@ * Licensed under the MIT License. See LICENSE file in the project root for full license information. */ +using System; + namespace PerformanceMonitor.Common; /// @@ -47,13 +49,81 @@ public static int GetDeterministicHashCode(string value) /// read-write connections to the same host. Extracted verbatim from Lite's /// RemoteCollectorService.GetServerNameForStorage; every SKU MUST build storage names /// through this one implementation so the same server derives the same id everywhere. + /// + /// #2218 — engine and port, and why they are OPTIONAL rather than required. The name carried + /// neither, so a SQL Server and a PostgreSQL instance on one host collided into a single server_id and + /// interleaved their histories, as did two instances distinguished only by port. Both are now discriminators + /// — but only when they are actually present, and that is a correctness requirement, not tidiness. + /// + /// Lite derives server_id FRESH at runtime, everywhere, from this function, and has no stored-id + /// concept to fall back on: RemoteCollectorService.GetServerNameForStorage hashes it on every read. + /// So any change to what this returns for an EXISTING server re-keys that server in Lite and orphans all of + /// its collected history, silently. Making the new parameters optional — and appending nothing at their + /// defaults — is what keeps Lite's three-argument call byte-identical to what it produced before. The same + /// protection covers Darling's SQL Server targets, which pass no port. + /// + /// Darling's already-registered PostgreSQL and explicit-port servers do not re-key either, for a + /// different reason: their id comes from the store (StoredServerId), which is authoritative and is + /// only ever DERIVED for an entry that has no row yet. So the new discriminators change what a FRESH + /// registration derives, never what an existing one is called — which is the property that made #2158 + /// (identity assigned, not re-derived) a prerequisite for this rather than a sibling of it. + /// + /// Engine is folded to a short token rather than interpolated raw so an operator writing + /// "PostgreSQL", "postgres" or "Postgres" gets ONE identity instead of three. Only + /// non-SQL-Server engines append anything, since SQL Server is the historical default and appending for it + /// would re-key every server in both SKUs. /// - public static string BuildStorageName(string serverName, string? databaseName, bool readOnlyIntent) + public static string BuildStorageName( + string serverName, + string? databaseName, + bool readOnlyIntent, + string? engine = null, + int port = 0) { var name = string.IsNullOrWhiteSpace(databaseName) ? serverName : serverName + ":" + databaseName; + /* Engine BEFORE port and both before :RO, so the suffix order is fixed regardless of which + discriminators a caller supplies — two callers passing the same facts in a different order must + not produce two identities. */ + var engineToken = EngineToken(engine); + if (engineToken is not null) + { + name += ":" + engineToken; + } + + if (port > 0) + { + name += ":" + port.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + return readOnlyIntent ? name + ":RO" : name; } + + /// + /// The identity token for an engine, or null when it contributes nothing — which is the case for SQL + /// Server and for an unspecified engine (#2218). + /// + /// Null for SQL Server is load-bearing: it is the historical default, so emitting a token for it + /// would change every existing server's storage name in both SKUs and re-key the lot. Unrecognized values + /// also return null rather than being interpolated raw — a typo must not mint a new identity for a server + /// that already has one, and the engine gate elsewhere already rejects an unknown engine loudly. + /// + private static string? EngineToken(string? engine) + { + if (string.IsNullOrWhiteSpace(engine)) + { + return null; + } + + var trimmed = engine.Trim(); + if (trimmed.StartsWith("postgres", StringComparison.OrdinalIgnoreCase) + || trimmed.Equals("pg", StringComparison.OrdinalIgnoreCase)) + { + return "pg"; + } + + return null; + } } diff --git a/PerformanceMonitor.Notifications/AgAlertContexts.cs b/PerformanceMonitor.Notifications/AgAlertContexts.cs new file mode 100644 index 000000000..2213e7d7d --- /dev/null +++ b/PerformanceMonitor.Notifications/AgAlertContexts.cs @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Collections.Generic; + +namespace PerformanceMonitor.Notifications; + +/// +/// The discrete facts for a database-scoped AG alert (#2109): Database, Availability Group, and +/// Replica as fields a webhook consumer can read by name — the same names the prose detail already +/// speaks, now structured. Shared by both SKUs' AG evaluators for the same reason +/// AgAlertPolicy is: the fact NAMES are a wire contract downstream automation keys on, and +/// two hand-rolled copies would drift. Takes plain strings rather than AgDatabaseReading +/// because this project cannot see Common — the caller passes the reading's members. +/// +public static class AgAlertContexts +{ + /// One detail item headed by the database (matching the other per-database builders), + /// carrying the identity triple plus any alert-specific extras (e.g. Suspend Reason). + public static AlertContext ForDatabase( + string database, string agName, string replica, params (string Label, string Value)[] extras) + { + var fields = new List<(string, string)> + { + ("Database", database), + ("Availability Group", agName), + ("Replica", replica) + }; + fields.AddRange(extras); + + var context = new AlertContext(); + context.Details.Add(new AlertDetailItem { Heading = database, Fields = fields }); + return context; + } +} diff --git a/PerformanceMonitor.Notifications/AlertContext.cs b/PerformanceMonitor.Notifications/AlertContext.cs index 390ebd94e..bf42507d8 100644 --- a/PerformanceMonitor.Notifications/AlertContext.cs +++ b/PerformanceMonitor.Notifications/AlertContext.cs @@ -49,12 +49,32 @@ public class AlertContext /// never part of — only the identity members hashed by /// . /// +/// +/// How many events with this fingerprint are in the CURRENT read window (the rolling hour the +/// groupers counted). A gauge, not a total: it rises as events arrive and falls as they age out. +/// +/// +/// #2216: occurrences of this fingerprint accumulated across the whole incident — monotonic for as +/// long as the incident lasts, so a consumer that only sees throttled deliveries can still recover +/// how many events actually happened between two of them. null on any path with no +/// accumulator behind it (a host that does not persist occurrence state, or an alert whose incidents +/// are built outside the engine), which reads as "no total available" rather than a false zero. +/// Accumulated by IncidentOccurrenceAccumulator; see its remarks for the exactness bound. +/// +/// +/// #2216: when this fingerprint's current incident was first observed. The incident identity that +/// makes interpretable — a consumer seeing the total go +/// backwards can tell a genuine new incident (this moved) from a service restart or a dropped +/// store (this did not). +/// public sealed record AlertIncident( string DedupKey, IReadOnlyList InvolvedObjects, int OccurrenceCount = 1, string? WaitRange = null, - IReadOnlyList? DetailFields = null); + IReadOnlyList? DetailFields = null, + long? TotalOccurrences = null, + DateTime? IncidentStartedUtc = null); /// /// A forensic label/value pair carried on an for #1141 Per-event delivery @@ -116,8 +136,19 @@ public record FieldDto(string Label, string Value); /// JSON mirror of (#1140). The trailing optional Incidents /// member on keeps the round-trip backward-compatible: legacy /// contextJson written before this field existed deserializes Incidents to null. +/// +/// #2216's two members are trailing and nullable for the same reason: a history row written before +/// they existed rehydrates them as null, which is exactly "this alert carried no total" rather than +/// a fabricated zero. remains unpersisted. +/// /// -public record AlertIncidentDto(string DedupKey, List InvolvedObjects, int OccurrenceCount = 1, string? WaitRange = null); +public record AlertIncidentDto( + string DedupKey, + List InvolvedObjects, + int OccurrenceCount = 1, + string? WaitRange = null, + long? TotalOccurrences = null, + DateTime? IncidentStartedUtc = null); /// /// JSON mirror of / @@ -198,6 +229,11 @@ public record RcsiInactionFiguresDto( /// written before #1882 has no such property and deserializes to null, which is the same thing the /// extractor produces for a server that does not attribute replicas — so an old row and a /// non-AG row are indistinguishable, as they should be. +/// (#2138 gap 3) follows the same appended-and-defaulted +/// discipline — and it MUST be mirrored here, not just on the record: both apps render their +/// copy-paste command from the DESERIALIZED action, so a flag dropped by this DTO never reaches the +/// pasted surface at all, and the future auto-force bot reading persisted actions would see false +/// for every flagged target (review catch on #2140). /// public record ForcePlanTargetDto( string Database, @@ -208,7 +244,8 @@ public record ForcePlanTargetDto( double LatestCpuPerExecUs, double BestCpuPerExecUs, double RegressionFactor, - string? ReplicaRole = null); + string? ReplicaRole = null, + bool ParameterSensitivityCoFired = false); /// /// JSON mirror of . is persisted @@ -294,14 +331,33 @@ public static string Serialize(AlertContext context) d.Body, d.IsCodeBlock, ToDto(d.Remediation))), - context.Incidents?.ConvertAll(i => new AlertIncidentDto( - i.DedupKey, - new List(i.InvolvedObjects), - i.OccurrenceCount, - i.WaitRange))); + ToDto(context.Incidents)); return JsonSerializer.Serialize(dto); } + /// + /// #2302: just the incidents array, for the generic webhook's {{incidents_json}} token — + /// the SAME projection embeds in the + /// persisted ContextJson (one shape for every consumer, this method and the full write cannot + /// drift because both go through the same mapping). "[]" for a null context or an alert + /// with no fingerprintable incident, so a template's "incidents": {{incidents_json}} + /// stays well-formed JSON either way. + /// + public static string SerializeIncidents(AlertContext? context) + { + var incidents = ToDto(context?.Incidents); + return incidents is null ? "[]" : JsonSerializer.Serialize(incidents); + } + + private static List? ToDto(List? incidents) => + incidents?.ConvertAll(i => new AlertIncidentDto( + i.DedupKey, + new List(i.InvolvedObjects), + i.OccurrenceCount, + i.WaitRange, + i.TotalOccurrences, + i.IncidentStartedUtc)); + /// /// Serializes a single to JSON for persistence on a /// finding row (recommendations rebuild D2). Reuses the SAME private @@ -378,7 +434,9 @@ public static bool TryDeserialize(string? json, out AlertContext context) i.DedupKey ?? string.Empty, i.InvolvedObjects ?? new List(), i.OccurrenceCount, - i.WaitRange)); + i.WaitRange, + TotalOccurrences: i.TotalOccurrences, + IncidentStartedUtc: i.IncidentStartedUtc)); } } return true; @@ -406,7 +464,8 @@ public static bool TryDeserialize(string? json, out AlertContext context) t.LatestCpuPerExecUs, t.BestCpuPerExecUs, t.RegressionFactor, - t.ReplicaRole)); + t.ReplicaRole, + t.ParameterSensitivityCoFired)); } List? dbConfigTargets = null; @@ -510,7 +569,8 @@ public static bool TryDeserialize(string? json, out AlertContext context) t.LatestCpuPerExecUs, t.BestCpuPerExecUs, t.RegressionFactor, - t.ReplicaRole)); + t.ReplicaRole, + t.ParameterSensitivityCoFired)); } } diff --git a/PerformanceMonitor.Notifications/AlertIncidentRenderer.cs b/PerformanceMonitor.Notifications/AlertIncidentRenderer.cs index f18ace9d6..105aba925 100644 --- a/PerformanceMonitor.Notifications/AlertIncidentRenderer.cs +++ b/PerformanceMonitor.Notifications/AlertIncidentRenderer.cs @@ -6,7 +6,9 @@ * Licensed under the MIT License. See LICENSE file in the project root for full license information. */ +using System; using System.Collections.Generic; +using System.Globalization; namespace PerformanceMonitor.Notifications; @@ -63,6 +65,16 @@ public static AlertDetailItem BuildItem(AlertIncident incident, string heading, incident.InvolvedObjects.Count > 0 ? string.Join(", ", incident.InvolvedObjects) : "(unresolved)")); if (incident.OccurrenceCount > 1) item.Fields.Add(("Occurrences", incident.OccurrenceCount.ToString())); + /* #2216: the monotonic total, as its OWN fact rather than a correction to "Occurrences". + Automation keys on the fact name (see class remarks), so redefining the existing one from + a window gauge to a total would silently change what every current consumer reads — the + two facts answer different questions and both are emitted. Emitted whenever an + accumulator produced one, INCLUDING when it equals the window count: a consumer polling + for a total must not have the field vanish on the incident's first delivery. */ + if (incident.TotalOccurrences is long total) + item.Fields.Add(("Total Occurrences", total.ToString(CultureInfo.InvariantCulture))); + if (incident.IncidentStartedUtc is DateTime started) + item.Fields.Add(("Incident Since", started.ToString("yyyy-MM-dd HH:mm:ss'Z'", CultureInfo.InvariantCulture))); if (!string.IsNullOrEmpty(incident.WaitRange)) item.Fields.Add(("Wait Range", incident.WaitRange)); return item; diff --git a/PerformanceMonitor.Notifications/DeadlockObjectExtractor.cs b/PerformanceMonitor.Notifications/DeadlockObjectExtractor.cs index 84d349470..360b9dd88 100644 --- a/PerformanceMonitor.Notifications/DeadlockObjectExtractor.cs +++ b/PerformanceMonitor.Notifications/DeadlockObjectExtractor.cs @@ -26,6 +26,38 @@ public static class DeadlockObjectExtractor private static readonly string[] s_lockTypes = { "objectlock", "pagelock", "keylock", "ridlock", "rowgrouplock" }; + /// + /// Returns the distinct database names across the graph's processes (currentdbname), or an + /// empty list when the XML is blank, unparseable, or carries none. Never throws. The same attribute + /// the excluded-database filter reads — this is the discrete "Database" fact's source (#2109), kept + /// separate from the object-name fingerprint parse because a graph can name objects in databases no + /// process was running in (cross-database deadlocks), and the fact answers "where did this happen", + /// not "what was locked". + /// + public static IReadOnlyList DatabasesFromGraphXml(string? graphXml) + { + if (string.IsNullOrWhiteSpace(graphXml)) + return Array.Empty(); + + try + { + var doc = XElement.Parse(graphXml); + var names = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var process in doc.Descendants("process")) + { + var db = process.Attribute("currentdbname")?.Value; + if (!string.IsNullOrWhiteSpace(db)) + names.Add(db.Trim()); + } + + return names.Count == 0 ? Array.Empty() : names.ToList(); + } + catch + { + return Array.Empty(); + } + } + /// /// Returns the distinct object names across all lock resources in the graph, or an empty list when /// the XML is blank, unparseable, or carries no named objects. Never throws. diff --git a/PerformanceMonitor.Notifications/IAlertSettings.cs b/PerformanceMonitor.Notifications/IAlertSettings.cs index 8f77acc99..9db809f1d 100644 --- a/PerformanceMonitor.Notifications/IAlertSettings.cs +++ b/PerformanceMonitor.Notifications/IAlertSettings.cs @@ -61,7 +61,9 @@ same automation need with no process-execution surface in a signed binary. */ /// /// The JSON request body, with {{metric}}, {{server}}, {{value}}, /// {{threshold}}, {{severity}}, {{context}} and {{timestamp}} - /// placeholders substituted per alert. Empty falls back to + /// placeholders substituted per alert, plus the #2302 automation tokens: + /// {{context_json}} / {{incidents_json}} (raw JSON values, substituted unquoted) + /// and {{dedup_key}} (the PagerDuty-shape correlation key). Empty falls back to /// . /// string GenericWebhookBodyTemplate { get; } diff --git a/PerformanceMonitor.Notifications/LowDiskAlertGate.cs b/PerformanceMonitor.Notifications/LowDiskAlertGate.cs index 13eceede5..0451ad5db 100644 --- a/PerformanceMonitor.Notifications/LowDiskAlertGate.cs +++ b/PerformanceMonitor.Notifications/LowDiskAlertGate.cs @@ -53,7 +53,13 @@ public static class LowDiskAlertGate /// by Lite and Dashboard so the two apps grade low-disk identically. /// public static bool IsCriticallyLow(double freePercent, double freeGb) => - freePercent <= CriticalFreePercent || freeGb <= CriticalFreeGb; + IsCriticallyLow(freePercent, freeGb, CriticalFreePercent, CriticalFreeGb); + + /// #2107: the configurable form — both apps pass their settings' critical floors; the + /// parameterless overload keeps the shipped constants for callers with no settings in reach + /// (and for the tests pinning the defaults). + public static bool IsCriticallyLow(double freePercent, double freeGb, double criticalFreePercent, double criticalFreeGb) => + freePercent <= criticalFreePercent || freeGb <= criticalFreeGb; /// /// Returns true when a low-disk alert should fire this cycle. diff --git a/PerformanceMonitor.Notifications/PerformanceMonitor.Notifications.csproj b/PerformanceMonitor.Notifications/PerformanceMonitor.Notifications.csproj index c72e7ea07..4abc114fd 100644 --- a/PerformanceMonitor.Notifications/PerformanceMonitor.Notifications.csproj +++ b/PerformanceMonitor.Notifications/PerformanceMonitor.Notifications.csproj @@ -13,7 +13,7 @@ - + diff --git a/PerformanceMonitor.Notifications/WebhookAlertService.cs b/PerformanceMonitor.Notifications/WebhookAlertService.cs index f7778f2d4..f0e684d43 100644 --- a/PerformanceMonitor.Notifications/WebhookAlertService.cs +++ b/PerformanceMonitor.Notifications/WebhookAlertService.cs @@ -149,7 +149,7 @@ the alert log on first touch per key (#1145), unless the store is null (no seedi if (_settings.GenericWebhookEnabled && !string.IsNullOrWhiteSpace(_settings.GenericWebhookUrl)) { - sent |= await TrySendGenericAlertAsync(metricName, serverName, currentValue, thresholdValue, context); + sent |= await TrySendGenericAlertAsync(metricName, serverName, currentValue, thresholdValue, serverId, context); } if (_settings.PagerDutyEnabled && !string.IsNullOrWhiteSpace(_settings.PagerDutyRoutingKey)) @@ -229,7 +229,11 @@ the alert log on first touch per key (#1145), unless the store is null (no seedi if (!TryParseHeaders(headersJson, out var headers, out var headerError)) return headerError; - var payload = BuildGenericPayload("Test Notification", "", "Webhook configuration verified", "", branding, isTest: true, bodyTemplate: bodyTemplate); + /* Same stand-in context as Save-time validation, for the same reason: the Test button must + exercise the raw tokens with quote-bearing structure or a mis-quoted one test-sends clean. */ + var payload = BuildGenericPayload( + "Test Notification", "", "Webhook configuration verified", "", branding, + isTest: true, bodyTemplate: bodyTemplate, context: ValidationStandInContext()); if (!IsWellFormedJson(payload, out var bodyError)) return bodyError; @@ -332,6 +336,12 @@ internal static string BuildTeamsPayload( facts.Add(new { name = "Time (Local)", value = localNow.ToString("yyyy-MM-dd HH:mm:ss") }); } + /* #2108: each Fields-carrying detail item becomes its OWN section further down, so a + multi-incident alert reads as labeled, self-contained units instead of one flat fact + list where a victim's fields and its fingerprint's fields drift apart. Advice prose and + remediation-T-SQL items stay folded into the lead section's facts — they are commentary + on the whole alert, not incidents. */ + var itemSections = new List(); if (context?.Details != null) { foreach (var detail in context.Details) @@ -358,10 +368,18 @@ which skip Fields when Body is present. */ continue; } + var itemFacts = new List(); foreach (var (label, value) in detail.Fields) { - facts.Add(new { name = label, value }); + itemFacts.Add(new { name = label, value }); } + + itemSections.Add(new + { + activityTitle = detail.Heading, + facts = itemFacts, + markdown = true + }); } } @@ -379,6 +397,7 @@ which skip Fields when Body is present. */ markdown = true } }; + sections.AddRange(itemSections); if (!isTest && branding.SnoozeHint is not null) { @@ -561,6 +580,7 @@ private async Task TrySendGenericAlertAsync( string serverName, string currentValue, string thresholdValue, + string serverId, AlertContext? context) { try @@ -576,7 +596,7 @@ private async Task TrySendGenericAlertAsync( var payload = BuildGenericPayload( metricName, serverName, currentValue, thresholdValue, _branding, - context: context, bodyTemplate: _settings.GenericWebhookBodyTemplate); + context: context, bodyTemplate: _settings.GenericWebhookBodyTemplate, serverId: serverId); if (!IsWellFormedJson(payload, out var bodyError)) { @@ -631,6 +651,20 @@ private void RecordGenericFailure(string error) /// The escaping goes through (see ) — NOT /// JsonEncodedText.Encode, which throws on the lone surrogates SQL Server names can carry — and the /// two surrounding quotes are stripped to leave exactly the escaped INNER text a token inside a literal needs. + /// + /// #2302: the three automation tokens are the exception, and two of them deliberately BYPASS the + /// escaping. {{context_json}} / {{incidents_json}} are raw JSON VALUES substituted + /// unquoted ("context": {{context_json}}) — escaping them would turn structure back into the + /// flattened string the token exists to replace. Their shape is the SAME + /// projection persisted as the alert-history ContextJson, so a + /// consumer parses one shape whether it reads the webhook or the history row — except that code-block + /// bodies and remediation payloads are redacted first (): the copy-paste + /// T-SQL follows the same never-on-a-webhook rule as every other channel here. {{dedup_key}} is + /// an ordinary escaped string carrying the same key the PagerDuty channel derives — including its + /// stable metric+server fallback when an alert has no incident — so tickets correlate across channels. + /// A template that quotes a raw token anyway produces malformed JSON and is caught by the caller's + /// well-formedness check, surfacing as a config error rather than a silent bad post. + /// /// internal static string BuildGenericPayload( string metricName, @@ -640,7 +674,8 @@ internal static string BuildGenericPayload( AlertBranding branding, bool isTest = false, AlertContext? context = null, - string? bodyTemplate = null) + string? bodyTemplate = null, + string serverId = "") { var (_, badgeText, _) = AlertSeverity.ForMetric(metricName, context?.SeverityOverride); var template = string.IsNullOrWhiteSpace(bodyTemplate) ? DefaultGenericBodyTemplate : bodyTemplate!; @@ -649,6 +684,14 @@ internal static string BuildGenericPayload( ? $"Webhook configuration is working correctly. Sent by {branding.EditionName}." : RenderContextForTemplate(context, branding); + /* Keyed on the numeric serverId the fan-out passes — the same identity the LIVE PagerDuty path + feeds DerivePagerDutyDedupKey — so the two channels' keys are equal for the same alert. The + serverName arm is THIS channel's own fallback for callers with no id (the settings-window test + send); it is not a guarantee PagerDuty's path shares, so a caller wanting cross-channel + correlation must pass the id. */ + var dedupKey = DerivePagerDutyDedupKey( + string.IsNullOrEmpty(serverId) ? serverName : serverId, metricName, context); + var values = new Dictionary(StringComparer.Ordinal) { ["metric"] = EscapeForJson(isTest ? $"TEST — {metricName}" : metricName), @@ -658,6 +701,13 @@ internal static string BuildGenericPayload( ["severity"] = EscapeForJson(isTest ? "TEST" : badgeText), ["context"] = EscapeForJson(contextText), ["timestamp"] = EscapeForJson(DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)), + /* Raw JSON values — never EscapeForJson (see the doc comment). "{}" / "[]" rather than empty + so a template's `"context": {{context_json}}` stays well-formed on a context-less alert. + Serialized from a REDACTED copy: the copy-paste remediation T-SQL never leaves the process + on any webhook channel, and a raw token is not an exception to that rule. */ + ["context_json"] = context is null ? "{}" : AlertContextSerializer.Serialize(RedactForWebhook(context)), + ["incidents_json"] = AlertContextSerializer.SerializeIncidents(context), + ["dedup_key"] = EscapeForJson(dedupKey), }; /* Single pass: a MatchEvaluator's output is NOT re-scanned, so a value that itself contains the @@ -667,8 +717,10 @@ literal text of another token (e.g. a server name "{{timestamp}}") is left verba return s_genericPlaceholders.Replace(template, m => values[m.Groups[1].Value]); } + /* context_json before context: alternation is ordered, and while the closing \}\} would force a + backtrack to the right answer anyway, longest-first means correctness never leans on it. */ private static readonly System.Text.RegularExpressions.Regex s_genericPlaceholders = - new(@"\{\{(metric|server|value|threshold|severity|context|timestamp)\}\}", + new(@"\{\{(metric|server|value|threshold|severity|context_json|incidents_json|dedup_key|context|timestamp)\}\}", System.Text.RegularExpressions.RegexOptions.Compiled); /// @@ -708,6 +760,49 @@ private static string RenderContextForTemplate(AlertContext? context, AlertBrand return parts.Count == 0 ? $"Sent by {branding.EditionName}" : string.Join(" | ", parts); } + /// + /// The webhook posture applied to structure (#2302 review catch): every channel in this file replaces + /// copy-paste remediation T-SQL with before anything leaves the process — + /// Teams, Slack, PagerDuty's custom_details, and this channel's own {{context}} flattening — and + /// a raw-JSON token is not an exception. Code-block items keep their heading and the flag (so a consumer + /// can see a remediation EXISTS) but carry the hint as their body and no Remediation payload; the + /// typed payload is likewise stripped from every item defensively. Returns a COPY — the same context + /// instance flows on to the other channels, and mutating it here would redact their email too. + /// + private static AlertContext RedactForWebhook(AlertContext context) + { + var redacted = new AlertContext { Incidents = context.Incidents }; + foreach (var detail in context.Details) + { + if (detail.IsCodeBlock) + { + redacted.Details.Add(new AlertDetailItem + { + Heading = detail.Heading, + Body = TsqlWebhookHint, + IsCodeBlock = true + }); + continue; + } + + if (detail.Remediation is null) + { + redacted.Details.Add(detail); + continue; + } + + redacted.Details.Add(new AlertDetailItem + { + Heading = detail.Heading, + Fields = detail.Fields, + Body = detail.Body, + IsCodeBlock = false + }); + } + + return redacted; + } + /// /// Escapes a value for interpolation INSIDE a JSON string literal — the escaped inner text, without the /// surrounding quotes. HTML-sensitive and non-ASCII characters escape to \uXXXX; over-escaping is @@ -744,14 +839,42 @@ private static string EscapeForJson(string? value) } /* Render with placeholder-shaped stand-ins: substitution is what can break the JSON, so validating - the raw template would miss a token sitting outside a string literal. */ + the raw template would miss a token sitting outside a string literal. The stand-in CONTEXT is + what makes the raw tokens honest here (#2310 review catch): with a null context they render to + the quote-free `{}` / `[]`, so a mis-quoted raw token — `"context": "{{context_json}}"` — + validates clean and only breaks on the first real alert that carries structure. */ var rendered = BuildGenericPayload( "Test Notification", "Test Server", "0", "0", - new AlertBranding("Performance Monitor", null), isTest: true, bodyTemplate: bodyTemplate); + new AlertBranding("Performance Monitor", null), isTest: true, bodyTemplate: bodyTemplate, + context: ValidationStandInContext()); return IsWellFormedJson(rendered, out var bodyError) ? null : bodyError; } + /// + /// The stand-in context Save-time validation and the settings Test send render the raw tokens with. + /// Its serialization is guaranteed to contain double quotes (every JSON property name carries them), + /// so quoting a raw token in a template breaks at Save/Test — where + /// the operator can see and fix it — instead of validating clean against the trivial {} and + /// failing silently into the log on the first deadlock alert with real structure. One incident, so + /// {{incidents_json}} is exercised the same way. + /// + internal static AlertContext ValidationStandInContext() => new() + { + Details = + { + new AlertDetailItem + { + Heading = "Validation", + Fields = { ("Check", "stand-in \"quoted\" value") } + } + }, + Incidents = new List + { + new("0000000000000000", new List { "validation.dbo.stand_in" }) + } + }; + /// /// True when the text is the built-in default body template (newline-insensitive), or empty. The settings /// windows pre-fill the body box with when nothing is stored, so diff --git a/PerformanceMonitor.PlanAnalysis/PerformanceMonitor.PlanAnalysis.csproj b/PerformanceMonitor.PlanAnalysis/PerformanceMonitor.PlanAnalysis.csproj index 773e270e6..578c43c24 100644 --- a/PerformanceMonitor.PlanAnalysis/PerformanceMonitor.PlanAnalysis.csproj +++ b/PerformanceMonitor.PlanAnalysis/PerformanceMonitor.PlanAnalysis.csproj @@ -21,7 +21,7 @@ - + diff --git a/PerformanceMonitor.Ui/ChartStyle.cs b/PerformanceMonitor.Ui/ChartStyle.cs index 53223ece3..ac99d4685 100644 --- a/PerformanceMonitor.Ui/ChartStyle.cs +++ b/PerformanceMonitor.Ui/ChartStyle.cs @@ -239,12 +239,25 @@ public static void StyleScatter(ScottPlot.Plottables.Scatter scatter) // the line unfilled. /* NaN Ys are #1944's injected gap markers, and Enumerable.Min/Max PROPAGATE NaN - one gap would read as minY=maxY=NaN and silently kill the gradient fill for the entire series. - The fill must survive a gap (ScottPlot splits it at the break); only real values rank. */ + Only real values rank. */ var realYs = pts.Where(p => !double.IsNaN(p.Y)).Select(p => p.Y).ToList(); double minY = realYs.Count > 0 ? realYs.Min() : 0.0; double maxY = realYs.Count > 0 ? realYs.Max() : 0.0; + /* #2324: a series carrying gap markers gets NO fill at all — line-only. The earlier reading + here ("the fill must survive a gap; ScottPlot splits it at the break") is true of the LINE + and false of the FILL: reproduced headlessly against ScottPlot 5.1.59, one NaN in a + FillY + ColorPositions series renders the ribbon as OPAQUE BLACK polygons with straight + chord edges crossing the gap — Scatter.Render's fill path closes its SKPath contours + through the break, and its fill paint under ColorPositions is hardcoded `Colors.Black` + with the gradient shader expected to paint over it, which a NaN-bearing series defeats. + That black buried every other series on 3.4.0's gapped charts (the field report's one + healthy tab was the one whose data happened to have no gaps; 3.3.0 predates gap markers). + Lines and markers handle NaN contours correctly, so line-only is the honest degradation: + a chart showing an outage keeps its break, and continuous data keeps the full ribbon. */ + bool hasGapMarkers = realYs.Count != pointCount; bool canFill = pointCount >= 2 && maxY > minY + && !hasGapMarkers && !double.IsNaN(minY) && !double.IsNaN(maxY) && !double.IsInfinity(minY) && !double.IsInfinity(maxY); scatter.ColorPositions.Clear(); diff --git a/PerformanceMonitor.Ui/DataGridFilterManager.cs b/PerformanceMonitor.Ui/DataGridFilterManager.cs index 8ae7d905c..f690cb03e 100644 --- a/PerformanceMonitor.Ui/DataGridFilterManager.cs +++ b/PerformanceMonitor.Ui/DataGridFilterManager.cs @@ -25,6 +25,7 @@ public interface IDataGridFilterManager Dictionary Filters { get; } void SetFilter(ColumnFilterState filterState); void UpdateFilterButtonStyles(); + void ClearFilters(); } /// @@ -76,6 +77,31 @@ public void SetFilter(ColumnFilterState filterState) UpdateFilterButtonStyles(); } + /// + /// #2306: drops every filter and restores the unfiltered data (sort preserved, funnel icons dimmed). + /// The caller that matters is a SERVER SWITCH on a cross-server surface: a DatabaseName filter set + /// against server A silently zeroes server B's grid while count indicators — computed from the + /// unfiltered list — stay full, and Refresh cannot clear it because + /// deliberately re-applies active filters. That re-apply is correct for refresh-on-the-same-server + /// and is untouched; only an explicit context change goes through here. + /// + public void ClearFilters() + { + if (_filters.Count == 0) + { + return; + } + + _filters.Clear(); + + if (_unfilteredData is not null) + { + SetItemsSourcePreservingSort(_unfilteredData); + } + + UpdateFilterButtonStyles(); + } + private bool HasActiveFilters() { return _filters.Count > 0 && _filters.Values.Any(f => f.IsActive); diff --git a/PerformanceMonitor.Ui/PerformanceMonitor.Ui.csproj b/PerformanceMonitor.Ui/PerformanceMonitor.Ui.csproj index eef7ce10a..33e9461b3 100644 --- a/PerformanceMonitor.Ui/PerformanceMonitor.Ui.csproj +++ b/PerformanceMonitor.Ui/PerformanceMonitor.Ui.csproj @@ -16,7 +16,7 @@ - + diff --git a/README.md b/README.md index 3129b29e3..93e7083b9 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Data starts flowing within 1–5 minutes. That's it. No installation on your ser ### Lite Collectors -38 collectors run on independent, configurable schedules (the long-running-query completion trace is opt-in and ships disabled): +42 collectors run on independent, configurable schedules (the long-running-query completion trace is opt-in and ships disabled): | Collector | Default | Source | |---|---|---| @@ -123,6 +123,7 @@ Data starts flowing within 1–5 minutes. That's it. No installation on your ser | query_stats | 1 min | `sys.dm_exec_query_stats` (deltas) | | procedure_stats | 1 min | `sys.dm_exec_procedure_stats` (deltas) | | cpu_utilization | 1 min | `sys.dm_os_ring_buffers` scheduler monitor | +| database_states | 1 min | `sys.databases` state per database — feeds the database offline/unhealthy alert (not Azure SQL DB) | | file_io_stats | 1 min | `sys.dm_io_virtual_file_stats` (deltas) | | memory_stats | 1 min | `sys.dm_os_sys_memory` + memory counters | | memory_grant_stats | 1 min | `sys.dm_exec_query_memory_grants` | @@ -146,6 +147,7 @@ Data starts flowing within 1–5 minutes. That's it. No installation on your ser | running_jobs | 5 min | `msdb` job history with duration vs avg/p95 | | database_size_stats | 1 hour | `sys.master_files` + `FILEPROPERTY` + `dm_os_volume_stats` | | pvs_stats | 1 hour | `sys.dm_tran_persistent_version_store_stats` + `sys.databases` (ADR persistent version store size and cleanup state per database; SQL Server 2019+ only, always collected on Azure SQL DB) | +| query_store_health | 1 hour | `sys.database_query_store_options` per database (actual vs desired state, readonly_reason, storage used vs cap, cleanup thresholds, runtime-stats interval length; SQL Server 2016+, one row per database with OFF recorded explicitly) | | server_properties | on connect | `SERVERPROPERTY()` hardware and licensing metadata | | index_object_stats | Daily | `sys.dm_db_partition_stats` + `sys.dm_db_index_usage_stats` + `sys.dm_db_index_operational_stats` | | server_config | On connect | `sys.configurations` | @@ -205,7 +207,7 @@ Configuration is a single JSON file with no schedule knobs. See the **[Darling o | Alerts (tray + email + webhooks) | Yes | Email + webhooks (headless) | Yes | | Themes | Dark and light | Dark and light | Dark and light | | Portability | Single executable | Portable service + viewer zip | Server-bound | -| MCP server (LLM integration) | Built-in (74 tools) | On request | Built into Dashboard (66 tools) | +| MCP server (LLM integration) | Built-in (77 tools) | On request | Built into Dashboard (66 tools) | --- @@ -341,7 +343,7 @@ claude mcp add --transport http --scope user sql-monitor http://localhost:5151/ ### Available Tools -**Lite** exposes 74 tools; **Darling** exposes the analysis + data-read surface on request; the deprecated **Dashboard** exposes 66 (see [deprecated/Dashboard/README.md](deprecated/Dashboard/README.md)). Core tools are shared. +**Lite** exposes 77 tools; **Darling** exposes the analysis + data-read surface on request; the deprecated **Dashboard** exposes 66 (see [deprecated/Dashboard/README.md](deprecated/Dashboard/README.md)). Core tools are shared. | Category | Tools | |---|---| @@ -358,7 +360,7 @@ claude mcp add --transport http --scope user sql-monitor http://localhost:5151/ | TempDB | `get_tempdb_trend` | | Perfmon | `get_perfmon_stats`, `get_perfmon_trend` | | Jobs | `get_running_jobs` | -| Configuration | `get_server_config`, `get_database_config`, `get_database_scoped_config`, `get_trace_flags` | +| Configuration | `get_server_config`, `get_database_config`, `get_database_scoped_config`, `get_query_store_health`, `get_trace_flags` | | Server Info | `get_server_properties`, `get_database_sizes` | | Object/Index Stats | `get_table_index_sizes`, `get_index_usage`, `get_object_locking` | | Sessions | `get_session_stats` | diff --git a/deprecated/Dashboard.Tests/Dashboard.Tests.csproj b/deprecated/Dashboard.Tests/Dashboard.Tests.csproj index e8c32b284..b36d3f555 100644 --- a/deprecated/Dashboard.Tests/Dashboard.Tests.csproj +++ b/deprecated/Dashboard.Tests/Dashboard.Tests.csproj @@ -9,12 +9,12 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/deprecated/Dashboard.Tests/packages.lock.json b/deprecated/Dashboard.Tests/packages.lock.json index d22c2ece9..a7111190c 100644 --- a/deprecated/Dashboard.Tests/packages.lock.json +++ b/deprecated/Dashboard.Tests/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0-windows7.0": { "Microsoft.NET.Test.Sdk": { @@ -49,16 +49,6 @@ "Microsoft.Identity.Client.Extensions.Msal": "4.78.0" } }, - "CredentialManagement": { - "type": "Transitive", - "resolved": "1.0.2", - "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" - }, - "Hardcodet.NotifyIcon.Wpf": { - "type": "Transitive", - "resolved": "2.0.1", - "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" - }, "HarfBuzzSharp": { "type": "Transitive", "resolved": "8.3.1.1", @@ -103,21 +93,6 @@ "resolved": "18.8.1", "contentHash": "Eclse/ZZjr4lmWzZFNN9h/OluhKL+SK/QbUyKUewgX139aGeyMEO/DkMPwuFs2MixvanTnz6891rF8UHDg+W4Q==" }, - "Microsoft.Data.SqlClient": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", - "dependencies": { - "Microsoft.Bcl.Cryptography": "9.0.13", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", - "Microsoft.Extensions.Caching.Memory": "9.0.13", - "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", - "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", - "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" - } - }, "Microsoft.Data.SqlClient.Extensions.Abstractions": { "type": "Transitive", "resolved": "7.0.2", @@ -126,20 +101,6 @@ "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)" } }, - "Microsoft.Data.SqlClient.Extensions.Azure": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", - "dependencies": { - "Azure.Core": "1.51.1", - "Azure.Identity": "1.18.0", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Extensions.Caching.Memory": "8.0.1", - "Microsoft.Identity.Client": "4.84.2", - "Microsoft.Identity.Client.Broker": "4.84.2" - } - }, "Microsoft.Data.SqlClient.Internal.Logging": { "type": "Transitive", "resolved": "7.0.2", @@ -175,15 +136,6 @@ "Microsoft.Extensions.Primitives": "9.0.13" } }, - "Microsoft.Extensions.Configuration": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", - "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Primitives": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -231,17 +183,6 @@ "Microsoft.Extensions.Primitives": "10.0.10" } }, - "Microsoft.Extensions.Configuration.Json": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Configuration.UserSecrets": { "type": "Transitive", "resolved": "10.0.10", @@ -308,35 +249,6 @@ "resolved": "10.0.10", "contentHash": "jSOCVxEwCd4Aq925kJVz1kSO1EpX2OHYKL04qVREXkDU7Ce3pVDdHPYm+fEy8y/th2kJf/DAstRHpJAqoNWP8w==" }, - "Microsoft.Extensions.Hosting": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", - "dependencies": { - "Microsoft.Extensions.Configuration": "10.0.10", - "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", - "Microsoft.Extensions.Configuration.Binder": "10.0.10", - "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", - "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", - "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", - "Microsoft.Extensions.Configuration.Json": "10.0.10", - "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", - "Microsoft.Extensions.Diagnostics": "10.0.10", - "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", - "Microsoft.Extensions.FileProviders.Physical": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Logging.Configuration": "10.0.10", - "Microsoft.Extensions.Logging.Console": "10.0.10", - "Microsoft.Extensions.Logging.Debug": "10.0.10", - "Microsoft.Extensions.Logging.EventLog": "10.0.10", - "Microsoft.Extensions.Logging.EventSource": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "Transitive", "resolved": "10.0.10", @@ -349,24 +261,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.10" } }, - "Microsoft.Extensions.Logging": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", "resolved": "10.0.10", @@ -584,28 +478,10 @@ "resolved": "5.0.0", "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" }, - "ModelContextProtocol": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "X/KDNZDP9Zgs7YXXxxpKiDMWWPXYrs764lVgc6vEM4pCg87bYz4VaceeiKW91XDTGgUvIbmRYXgWqyNpV32xRg==", - "dependencies": { - "Microsoft.Extensions.Caching.Abstractions": "10.0.10", - "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "ModelContextProtocol.Core": "[2.0.0]" - } - }, - "ModelContextProtocol.AspNetCore": { - "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "dXrB7sBpQjUQU0UcdyFPJbOTFw7yaceD+OgAZVAeBveRzbiBlg89jEygAtcgOwd/L+O+YpM98zfwaXz067NFDQ==", - "dependencies": { - "ModelContextProtocol": "[2.0.0]" - } - }, "ModelContextProtocol.Core": { "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "piFR0HtA/2Oc1tgk96EE5Tye6qA2sg3WGRAXBhUqo/BWikdEYEs2UuqtmwLrQZJUge1nUOPgGHYsb15VIBK8iw==", + "resolved": "2.1.0", + "contentHash": "cU/urrhRxE4/iSyBIJI7QOaFqSP1FOEnwEHsct9n6t6/XluCAFD9iqnrPkBAsEYr+f/G4tVQ21U+6wN/6fQvOg==", "dependencies": { "Microsoft.Extensions.AI.Abstractions": "10.8.3", "Microsoft.Extensions.Logging.Abstractions": "10.0.10" @@ -718,17 +594,6 @@ "SkiaSharp.NativeAssets.Linux.NoDependencies": "3.119.0" } }, - "ScottPlot.WPF": { - "type": "Transitive", - "resolved": "5.1.59", - "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", - "dependencies": { - "OpenTK": "4.9.4", - "OpenTK.GLWpfControl": "4.3.3", - "ScottPlot": "5.1.59", - "SkiaSharp.Views.WPF": "3.119.0" - } - }, "SkiaSharp": { "type": "Transitive", "resolved": "3.119.0", @@ -811,11 +676,6 @@ "resolved": "10.0.1", "contentHash": "BZC4mhdL569AXV56ep9YO6ShjhxFXGP7SwVX0Bc/e0dJPWnS6aBEXZJXqh64RVx8HquqWHkJUINBydLRQ1yq0g==" }, - "Velopack": { - "type": "Transitive", - "resolved": "1.2.0", - "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" - }, "xunit.analyzers": { "type": "Transitive", "resolved": "1.27.0", @@ -904,7 +764,7 @@ "dependencies": { "CredentialManagement": "[1.0.2, )", "Microsoft.Extensions.Logging.Abstractions": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )" + "ModelContextProtocol": "[2.1.0, )" } }, "performancemonitor.notifications": { @@ -940,8 +800,8 @@ "Microsoft.Extensions.Configuration": "[10.0.10, )", "Microsoft.Extensions.Configuration.Json": "[10.0.10, )", "Microsoft.Extensions.Hosting": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )", - "ModelContextProtocol.AspNetCore": "[2.0.0, )", + "ModelContextProtocol": "[2.1.0, )", + "ModelContextProtocol.AspNetCore": "[2.1.0, )", "PerformanceMonitor.Alerting": "[1.0.0, )", "PerformanceMonitor.Analysis": "[1.0.0, )", "PerformanceMonitor.Common": "[1.0.0, )", @@ -951,6 +811,159 @@ "ScottPlot.WPF": "[5.1.59, )", "Velopack": "[1.2.0, )" } + }, + "CredentialManagement": { + "type": "CentralTransitive", + "requested": "[1.0.2, )", + "resolved": "1.0.2", + "contentHash": "VkP04/jFXaxT3TkcRhzETYtOrznQxRmQ2J1XJdbXz47Bir7hIzPR7mFZk4GJQ4An4gozW+vonpf+iqTHomAkQw==" + }, + "Hardcodet.NotifyIcon.Wpf": { + "type": "CentralTransitive", + "requested": "[2.0.1, )", + "resolved": "2.0.1", + "contentHash": "dtxmeZXzV2GzSm91aZ3hqzgoeVoARSkDPVCYfhVUNyyKBWYxMgNC0EcLiSYxD4Uc4alq/2qb3SmV8DgAENLRLQ==" + }, + "Microsoft.Data.SqlClient": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "zwv76lANFQQI6Gmp6ntkzMWIWVqm8Wf4Mz00AeGCk1n8HCi5afi6bNynSe18uI0xeL0n6J+Myjk9AiIsL5oSqw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "9.0.13", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.SNI.runtime": "[6.0.2, 7.0.0)", + "Microsoft.Extensions.Caching.Memory": "9.0.13", + "Microsoft.IdentityModel.JsonWebTokens": "8.16.0", + "Microsoft.IdentityModel.Protocols.OpenIdConnect": "8.16.0", + "Microsoft.SqlServer.Server": "[1.0.0, 2.0.0)" + } + }, + "Microsoft.Data.SqlClient.Extensions.Azure": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", + "dependencies": { + "Azure.Core": "1.51.1", + "Azure.Identity": "1.18.0", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Extensions.Caching.Memory": "8.0.1", + "Microsoft.Identity.Client": "4.84.2", + "Microsoft.Identity.Client.Broker": "4.84.2" + } + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "plJWK2zpWuuyxI8F8s2scx6Je7N1Ajjs6HvYUGKwRnDMWIVIz9FHwAkiT7ASgrvAOd10T0FPVlh9BzAJJME+jg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "uvJ6sHwjgrkMEJOgiC76G0mcZGXerwyyWkwX34EOjCbxKG6TCtfAoqDKAMsCvEBf9HxjlGQEgqsSMOGCmGBf+A==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10" + } + }, + "Microsoft.Extensions.Hosting": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "tL9FkfV64GPUDSPvwrgyw42LVzsnVAnyrqJEuZVJbODgrQ3eL63zmzEcVWoCHzfgqUhWggzbgAyUCnz/zfI3Pg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.10", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.10", + "Microsoft.Extensions.Configuration.Binder": "10.0.10", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.10", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.10", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.10", + "Microsoft.Extensions.Configuration.Json": "10.0.10", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.10", + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Diagnostics": "10.0.10", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.10", + "Microsoft.Extensions.FileProviders.Physical": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging.Configuration": "10.0.10", + "Microsoft.Extensions.Logging.Console": "10.0.10", + "Microsoft.Extensions.Logging.Debug": "10.0.10", + "Microsoft.Extensions.Logging.EventLog": "10.0.10", + "Microsoft.Extensions.Logging.EventSource": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" + } + }, + "ModelContextProtocol": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "Oa4rU7EL9C2qyFjQj1dx+ysGMzfWDRpM8RRaUMmLGs5vPvfJ9xyz4ZtyF4ychY+Nx1b/auGCqIQLqSz/IpPkKA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", + "ModelContextProtocol.Core": "[2.1.0]" + } + }, + "ModelContextProtocol.AspNetCore": { + "type": "CentralTransitive", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "yhJ8bBXIgrX0mAgRYRgzcbH3bLdv3MDSkG52utRW9EAAtQrPw/g7Q/T6EurxKV+L+Zefv8VVUYcNbXEOd9GgfA==", + "dependencies": { + "ModelContextProtocol": "[2.1.0]" + } + }, + "ScottPlot.WPF": { + "type": "CentralTransitive", + "requested": "[5.1.59, )", + "resolved": "5.1.59", + "contentHash": "d6Mv5PFtp+SUH2r8vBCb/mKsR6kobsOX/8/oZYzZl+k3a9jv+BmxVZAkr1Vshq6457812HcNGppoNJ1pSJk5zQ==", + "dependencies": { + "OpenTK": "4.9.4", + "OpenTK.GLWpfControl": "4.3.3", + "ScottPlot": "5.1.59", + "SkiaSharp.Views.WPF": "3.119.0" + } + }, + "Velopack": { + "type": "CentralTransitive", + "requested": "[1.2.0, )", + "resolved": "1.2.0", + "contentHash": "Rz67gJL619fSBS6omaSINUxyDuwhIxkm5mmubf7uLd5Qgi6LLKaKCha+QFP6n+Bw/UjA0vutnH4JQfYzn6ANtw==" } } } diff --git a/deprecated/Dashboard/Dashboard.csproj b/deprecated/Dashboard/Dashboard.csproj index e2c46d84e..21afe42d1 100644 --- a/deprecated/Dashboard/Dashboard.csproj +++ b/deprecated/Dashboard/Dashboard.csproj @@ -7,7 +7,7 @@ PerformanceMonitorDashboard.Program PerformanceMonitorDashboard SQL Server Performance Monitor Dashboard - 3.3.0 + 3.5.0 3.3.0.0 3.3.0.0 3.3.0 @@ -40,17 +40,17 @@ - - - - - - - - - - - + + + + + + + + + + + diff --git a/deprecated/Dashboard/packages.lock.json b/deprecated/Dashboard/packages.lock.json index eedd2a4f5..3c7d85bd7 100644 --- a/deprecated/Dashboard/packages.lock.json +++ b/deprecated/Dashboard/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0-windows7.0": { "CredentialManagement": { @@ -99,22 +99,22 @@ }, "ModelContextProtocol": { "type": "Direct", - "requested": "[2.0.0, )", - "resolved": "2.0.0", - "contentHash": "X/KDNZDP9Zgs7YXXxxpKiDMWWPXYrs764lVgc6vEM4pCg87bYz4VaceeiKW91XDTGgUvIbmRYXgWqyNpV32xRg==", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "Oa4rU7EL9C2qyFjQj1dx+ysGMzfWDRpM8RRaUMmLGs5vPvfJ9xyz4ZtyF4ychY+Nx1b/auGCqIQLqSz/IpPkKA==", "dependencies": { "Microsoft.Extensions.Caching.Abstractions": "10.0.10", "Microsoft.Extensions.Hosting.Abstractions": "10.0.10", - "ModelContextProtocol.Core": "[2.0.0]" + "ModelContextProtocol.Core": "[2.1.0]" } }, "ModelContextProtocol.AspNetCore": { "type": "Direct", - "requested": "[2.0.0, )", - "resolved": "2.0.0", - "contentHash": "dXrB7sBpQjUQU0UcdyFPJbOTFw7yaceD+OgAZVAeBveRzbiBlg89jEygAtcgOwd/L+O+YpM98zfwaXz067NFDQ==", + "requested": "[2.1.0, )", + "resolved": "2.1.0", + "contentHash": "yhJ8bBXIgrX0mAgRYRgzcbH3bLdv3MDSkG52utRW9EAAtQrPw/g7Q/T6EurxKV+L+Zefv8VVUYcNbXEOd9GgfA==", "dependencies": { - "ModelContextProtocol": "[2.0.0]" + "ModelContextProtocol": "[2.1.0]" } }, "ScottPlot.WPF": { @@ -359,24 +359,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.10" } }, - "Microsoft.Extensions.Logging": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection": "10.0.10", - "Microsoft.Extensions.Logging.Abstractions": "10.0.10", - "Microsoft.Extensions.Options": "10.0.10" - } - }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.10", - "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" - } - }, "Microsoft.Extensions.Logging.Configuration": { "type": "Transitive", "resolved": "10.0.10", @@ -548,8 +530,8 @@ }, "ModelContextProtocol.Core": { "type": "Transitive", - "resolved": "2.0.0", - "contentHash": "piFR0HtA/2Oc1tgk96EE5Tye6qA2sg3WGRAXBhUqo/BWikdEYEs2UuqtmwLrQZJUge1nUOPgGHYsb15VIBK8iw==", + "resolved": "2.1.0", + "contentHash": "cU/urrhRxE4/iSyBIJI7QOaFqSP1FOEnwEHsct9n6t6/XluCAFD9iqnrPkBAsEYr+f/G4tVQ21U+6wN/6fQvOg==", "dependencies": { "Microsoft.Extensions.AI.Abstractions": "10.8.3", "Microsoft.Extensions.Logging.Abstractions": "10.0.10" @@ -765,7 +747,7 @@ "dependencies": { "CredentialManagement": "[1.0.2, )", "Microsoft.Extensions.Logging.Abstractions": "[10.0.10, )", - "ModelContextProtocol": "[2.0.0, )" + "ModelContextProtocol": "[2.1.0, )" } }, "performancemonitor.notifications": { @@ -789,6 +771,26 @@ "PerformanceMonitor.PlanAnalysis": "[1.0.0, )", "ScottPlot.WPF": "[5.1.59, )" } + }, + "Microsoft.Extensions.Logging": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "Tf6z5HsL0VDYRTfvsoNrTGHGheCwkTsZBA2FFh5ATJUbkAwug+FFNISJK2gjpUNemlAOoWllAK52HOWCjto3EQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" + } } } } diff --git a/deprecated/Installer.Core/Installer.Core.csproj b/deprecated/Installer.Core/Installer.Core.csproj index 3b160b6dd..78460e049 100644 --- a/deprecated/Installer.Core/Installer.Core.csproj +++ b/deprecated/Installer.Core/Installer.Core.csproj @@ -19,8 +19,8 @@ - - + + diff --git a/deprecated/Installer.Core/packages.lock.json b/deprecated/Installer.Core/packages.lock.json index e1a07d878..f0dfeab10 100644 --- a/deprecated/Installer.Core/packages.lock.json +++ b/deprecated/Installer.Core/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0": { "Microsoft.Data.SqlClient": { @@ -147,14 +147,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.3" } }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.3", - "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" - } - }, "Microsoft.Extensions.Options": { "type": "Transitive", "resolved": "10.0.3", @@ -296,8 +288,18 @@ "resolved": "9.0.13", "contentHash": "dxJhkuoaelvWy588wPXjShNks+ZMiSgXnN75/u+DPbER5PqKrLPDftE0BvGM7nDK/scQAVlD+gRXlCAAjWi58Q==" }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.3", + "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" + } + }, "System.Security.Cryptography.ProtectedData": { - "type": "Transitive", + "type": "CentralTransitive", + "requested": "[10.0.10, )", "resolved": "9.0.13", "contentHash": "t8S9IDpjJKsLpLkeBdW8cWtcPyYqrGu93Dej1RO6WwuL/lkFSqWlan3rMJfortqz1mRIh+sys2AFsSA6jWJ3Jg==" } diff --git a/deprecated/Installer.Tests/Installer.Tests.csproj b/deprecated/Installer.Tests/Installer.Tests.csproj index 41d0ea2fb..a8b5c9747 100644 --- a/deprecated/Installer.Tests/Installer.Tests.csproj +++ b/deprecated/Installer.Tests/Installer.Tests.csproj @@ -10,13 +10,13 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/deprecated/Installer.Tests/packages.lock.json b/deprecated/Installer.Tests/packages.lock.json index c63f0d42c..5f09b29ec 100644 --- a/deprecated/Installer.Tests/packages.lock.json +++ b/deprecated/Installer.Tests/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0": { "Microsoft.Data.SqlClient": { @@ -95,20 +95,6 @@ "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)" } }, - "Microsoft.Data.SqlClient.Extensions.Azure": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", - "dependencies": { - "Azure.Core": "1.51.1", - "Azure.Identity": "1.18.0", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Extensions.Caching.Memory": "8.0.1", - "Microsoft.Identity.Client": "4.84.2", - "Microsoft.Identity.Client.Broker": "4.84.2" - } - }, "Microsoft.Data.SqlClient.Internal.Logging": { "type": "Transitive", "resolved": "7.0.2", @@ -181,14 +167,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.3" } }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.3", - "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" - } - }, "Microsoft.Extensions.Options": { "type": "Transitive", "resolved": "10.0.3", @@ -378,11 +356,6 @@ "resolved": "9.0.13", "contentHash": "dxJhkuoaelvWy588wPXjShNks+ZMiSgXnN75/u+DPbER5PqKrLPDftE0BvGM7nDK/scQAVlD+gRXlCAAjWi58Q==" }, - "System.Security.Cryptography.ProtectedData": { - "type": "Transitive", - "resolved": "9.0.13", - "contentHash": "t8S9IDpjJKsLpLkeBdW8cWtcPyYqrGu93Dej1RO6WwuL/lkFSqWlan3rMJfortqz1mRIh+sys2AFsSA6jWJ3Jg==" - }, "xunit.analyzers": { "type": "Transitive", "resolved": "1.27.0", @@ -456,6 +429,36 @@ "Microsoft.Data.SqlClient": "[7.0.2, )", "Microsoft.Data.SqlClient.Extensions.Azure": "[7.0.2, )" } + }, + "Microsoft.Data.SqlClient.Extensions.Azure": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", + "dependencies": { + "Azure.Core": "1.51.1", + "Azure.Identity": "1.18.0", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Extensions.Caching.Memory": "8.0.1", + "Microsoft.Identity.Client": "4.84.2", + "Microsoft.Identity.Client.Broker": "4.84.2" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.3", + "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" + } + }, + "System.Security.Cryptography.ProtectedData": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "9.0.13", + "contentHash": "t8S9IDpjJKsLpLkeBdW8cWtcPyYqrGu93Dej1RO6WwuL/lkFSqWlan3rMJfortqz1mRIh+sys2AFsSA6jWJ3Jg==" } } } diff --git a/deprecated/Installer/PerformanceMonitorInstaller.csproj b/deprecated/Installer/PerformanceMonitorInstaller.csproj index 6998a7f2a..29d7f2e48 100644 --- a/deprecated/Installer/PerformanceMonitorInstaller.csproj +++ b/deprecated/Installer/PerformanceMonitorInstaller.csproj @@ -31,7 +31,7 @@ - + diff --git a/deprecated/Installer/packages.lock.json b/deprecated/Installer/packages.lock.json index 39ad2a429..6fcafc034 100644 --- a/deprecated/Installer/packages.lock.json +++ b/deprecated/Installer/packages.lock.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "dependencies": { "net10.0": { "Microsoft.Data.SqlClient": { @@ -66,20 +66,6 @@ "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)" } }, - "Microsoft.Data.SqlClient.Extensions.Azure": { - "type": "Transitive", - "resolved": "7.0.2", - "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", - "dependencies": { - "Azure.Core": "1.51.1", - "Azure.Identity": "1.18.0", - "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", - "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", - "Microsoft.Extensions.Caching.Memory": "8.0.1", - "Microsoft.Identity.Client": "4.84.2", - "Microsoft.Identity.Client.Broker": "4.84.2" - } - }, "Microsoft.Data.SqlClient.Internal.Logging": { "type": "Transitive", "resolved": "7.0.2", @@ -152,14 +138,6 @@ "Microsoft.Extensions.Logging.Abstractions": "10.0.3" } }, - "Microsoft.Extensions.Logging.Abstractions": { - "type": "Transitive", - "resolved": "10.0.3", - "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" - } - }, "Microsoft.Extensions.Options": { "type": "Transitive", "resolved": "10.0.3", @@ -301,17 +279,42 @@ "resolved": "9.0.13", "contentHash": "dxJhkuoaelvWy588wPXjShNks+ZMiSgXnN75/u+DPbER5PqKrLPDftE0BvGM7nDK/scQAVlD+gRXlCAAjWi58Q==" }, - "System.Security.Cryptography.ProtectedData": { - "type": "Transitive", - "resolved": "9.0.13", - "contentHash": "t8S9IDpjJKsLpLkeBdW8cWtcPyYqrGu93Dej1RO6WwuL/lkFSqWlan3rMJfortqz1mRIh+sys2AFsSA6jWJ3Jg==" - }, "installer.core": { "type": "Project", "dependencies": { "Microsoft.Data.SqlClient": "[7.0.2, )", "Microsoft.Data.SqlClient.Extensions.Azure": "[7.0.2, )" } + }, + "Microsoft.Data.SqlClient.Extensions.Azure": { + "type": "CentralTransitive", + "requested": "[7.0.2, )", + "resolved": "7.0.2", + "contentHash": "mJhONie3MuVXvSfBbtqGQAOGgeQDbOTfO5d0ZYo3KE4Vb7mua4O23lPJrsTYzvWn5od2CPAh0dHMZEWtJXcpxQ==", + "dependencies": { + "Azure.Core": "1.51.1", + "Azure.Identity": "1.18.0", + "Microsoft.Data.SqlClient.Extensions.Abstractions": "[7.0.2, 8.0.0)", + "Microsoft.Data.SqlClient.Internal.Logging": "[7.0.2, 8.0.0)", + "Microsoft.Extensions.Caching.Memory": "8.0.1", + "Microsoft.Identity.Client": "4.84.2", + "Microsoft.Identity.Client.Broker": "4.84.2" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "10.0.3", + "contentHash": "lxl0WLk7ROgBFAsjcOYjQ8/DVK+VMszxGBzUhgtQmAsTNldLL5pk9NG/cWTsXHq0lUhUEAtZkEE7jOGOA8bGKQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3" + } + }, + "System.Security.Cryptography.ProtectedData": { + "type": "CentralTransitive", + "requested": "[10.0.10, )", + "resolved": "9.0.13", + "contentHash": "t8S9IDpjJKsLpLkeBdW8cWtcPyYqrGu93Dej1RO6WwuL/lkFSqWlan3rMJfortqz1mRIh+sys2AFsSA6jWJ3Jg==" } }, "net10.0/win-x64": { diff --git a/docs/how-collection-works.md b/docs/how-collection-works.md index 4b4331b92..e231d0742 100644 --- a/docs/how-collection-works.md +++ b/docs/how-collection-works.md @@ -1,207 +1,179 @@ # How Collection Works -A tour of the collection pipeline for people who know SQL but don't know this codebase. Read this, then read three SQL files, and you'll understand 80% of what Performance Monitor is doing on your server. +A tour of the collection pipeline for people who know SQL but don't know this codebase. Read this, then read two or three collector definitions, and you'll understand 80% of what Performance Monitor is doing on your servers. -This doc covers both editions. Full Edition first (SQL Agent → `PerformanceMonitor` database → Dashboard reads), Lite Edition second (WPF app → DuckDB file → same app reads). The shapes are similar; the surface area is different. +There is **one collection brain and two storage engines**. Every collector — the exact T-SQL sent to a monitored server, the result-row mapping, the delta rules, the default cadence, the retention horizon — is defined once in the shared `PerformanceMonitor.Collectors` library. **Lite** writes those rows to DuckDB; **Darling** writes the same rows to PostgreSQL. A collector change lands once and both editions get it. ---- +> The **Full / Dashboard edition is deprecated** (SQL Agent jobs running T-SQL stored procedures into a `PerformanceMonitor` database). It still builds and existing installs keep working, but it does not share the collector library and is not described here. Its docs live with its code: [deprecated/Dashboard/README.md](../deprecated/Dashboard/README.md), [deprecated/Installer/README.md](../deprecated/Installer/README.md), and the `install/*.sql` scripts at the repo root belong to it. -## Full Edition +--- -### The minute loop +## The shared collector library -Everything happens inside one SQL Agent job: +**Project**: [`PerformanceMonitor.Collectors`](../PerformanceMonitor.Collectors/) -| Job | What it runs | -| --- | --- | -| `PerformanceMonitor - Collection` | `EXEC collect.scheduled_master_collector @debug = 0;` on a 1-minute schedule (`Every 1 Minute`) | -| `PerformanceMonitor - Data Retention` | `EXEC config.data_retention @debug = 1;` once a day | -| `PerformanceMonitor - Hung Job Monitor` | Kills the Collection job if it's been stuck past its max duration | +This library is deliberately dependency-free — it has **zero PackageReferences**. No `Microsoft.Data.SqlClient`, no DuckDB, no Npgsql. Definitions emit SQL *text* and read results through `System.Data.Common.DbDataReader`; the host SKU supplies the connection and the row writer. That is what makes one definition serve both storage engines. -When the Collection job fires, it calls the **scheduled master collector** — the dispatcher. The dispatcher is the heartbeat of the whole system. Every minute it wakes up, figures out which collectors are due, and runs them one at a time. +### What a collector definition looks like -### The dispatcher +Each collector is a sealed singleton deriving from `CollectorDefinitionBase`: -**File**: [`install/42_scheduled_master_collector.sql`](../install/42_scheduled_master_collector.sql) +```csharp +public sealed class WaitStatsCollector : CollectorDefinitionBase +{ + public static readonly WaitStatsCollector Instance = new(); -At the core of the dispatcher is a cursor over `config.collection_schedule` that picks up anything due: + public override string Name => "wait_stats"; + public override string TargetTable => "wait_stats"; + public override IReadOnlyList PayloadColumns => ...; -```sql -SELECT - cs.schedule_id, - cs.collector_name, - cs.frequency_minutes, - cs.max_duration_minutes -FROM config.collection_schedule AS cs -WHERE cs.enabled = 1 -AND ( - @force_run_all = 1 - OR cs.next_run_time <= SYSDATETIME() - OR cs.next_run_time IS NULL - ) -ORDER BY - cs.next_run_time; + public override CollectorQuery BuildQuery(CollectorContext context) => ...; // the T-SQL + public override ValueTask> ReadAsync(DbDataReader reader, ...); + public override void WritePayload(WaitStatsRow row, ICollectorRowWriter writer, ...); +} ``` -For each row, the dispatcher has a big `IF/ELSE IF` block that maps `collector_name` to a specific stored procedure: +The four members that matter: **`BuildQuery`** returns the SQL text plus parameters, **`ReadAsync`** maps the reader into typed rows, **`WritePayload`** hands each row's columns to whichever writer the SKU supplied, and **`PayloadColumns`** declares the table shape that the storage layer generates DDL from. Definitions are stateless and thread-safe; per-cycle state rides on `CollectorContext`. -```sql -ELSE IF @collector_name = N'default_trace_collector' -BEGIN - EXECUTE collect.default_trace_collector @debug = @debug; -END; -ELSE IF @collector_name = N'blocking_deadlock_analyzer' -BEGIN - EXECUTE collect.blocking_deadlock_analyzer @debug = @debug; -END; --- ...etc -``` +Optional behaviours a definition can opt into, all with sensible defaults on the base class: -Each collector runs inside its own `BEGIN TRY / BEGIN CATCH` block — a failure in one doesn't stop the rest of the cycle. After each run (success or failure), the dispatcher bumps `last_run_time` and `next_run_time = last_run_time + frequency_minutes` so the next tick knows when that collector is eligible again. +| Member | Purpose | +| --- | --- | +| `AppliesTo(target)` | Skip the collector on targets that can't serve it (Azure SQL DB, missing msdb, version floors) | +| `RunsPerDatabase(target)` | Run once per database instead of once per server (Azure SQL DB has no cross-database DMV reach) | +| `WatermarkColumn` / `NumericWatermarkColumn` / `PerDatabaseWatermarkColumn` | Incremental collection — only pull rows newer than what's already stored | +| `BuildEnumerationQuery` / `BuildPerItemQuery` | Two-phase collection: list the items (usually databases), then query each one | +| `BuildSupplementalQuery` | A best-effort second result set that enriches the primary rows; failure never fails the collector | +| `EmitsProbeFailures` | The definition returns a trailing `(item, error)` result set so per-item failures get summarized instead of lost | +| `YieldsOnLockTimeout` | Treat a lock timeout as "come back later," not an error (only `query_snapshots`) | +| `StateKeys` | Persist a cursor the SQL can't derive — e.g. `default_trace_events` remembering its last trace file | +| `CommandTimeoutSecondsOverride` | Raise the 60-second default (only `index_object_stats`, at 300s) | +| `PerItemTextByteBudget` | Stop a text-heavy drain mid-read and defer the backlog rather than ballooning memory | + +### The three registration tables + +There is **no DI container for collectors**. A new definition is wired up by adding it to three static tables, and a test pins them against each other so you can't add one and forget the others: + +| Concern | File | Shape | +| --- | --- | --- | +| Schema catalog — drives DDL generation | [`CollectorCatalog.cs`](../PerformanceMonitor.Collectors/CollectorCatalog.cs) | `IReadOnlyList All` — 48 `XCollector.Instance` entries (41 SQL Server + 7 PostgreSQL) | +| Cadence, retention, default-enabled | [`CollectorScheduleDefaults.cs`](../PerformanceMonitor.Collectors/CollectorScheduleDefaults.cs) | `record Entry(int FrequencyMinutes, int RetentionDays, bool DefaultEnabled = true)` — 48 entries | +| Runtime dispatch (Darling) | [`DarlingWorker.cs`](../Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs) | `s_dispatch` — 48 typed lambdas | + +The catalog is deliberately **engine-mixed**: the schema generator walks it to create tables and one +store can hold both engines' data, so splitting it per engine would fragment DDL generation. What keeps +dispatch honest is a separate gate — each definition declares a `TargetEngine`, and both SKUs drop +wrong-engine collectors before dispatch, so a PostgreSQL definition is never sent to a SQL Server target +or the reverse. + +`FrequencyMinutes = 0` means **collect once on connect** — used for config snapshots (`server_config`, `database_config`, `database_scoped_config`, `trace_flags`, `server_properties`) that only change across restarts. `DefaultEnabled: false` ships a collector off; only `long_query_completions` does, because enabling it creates an Extended Events session on the target. -Before any of this, the dispatcher also does two self-heal steps: +--- -- **Ensures config tables exist** (`config.ensure_config_tables`) — lets you recover from an accidentally-dropped table without reinstalling. -- **Detects server restarts** — if `sqlserver_start_time` has changed since last run, it captures a fresh snapshot of server properties. Config values only change across restarts, so this is the efficient moment to grab them. +## Darling: the 24/7 service -### What a collector looks like +**Project**: [`Darling/PerformanceMonitor.Darling.Service`](../Darling/PerformanceMonitor.Darling.Service/) · operator guide: [`Darling/README.md`](../Darling/README.md) -Pick any `install/NN_collect_*.sql` file — they all follow the same shape. A minimal example: +### The sweep loop -**File**: [`install/29_collect_default_trace.sql`](../install/29_collect_default_trace.sql) +`DarlingWorker` is a `BackgroundService`. It ticks every **15 seconds** — that is the *scheduling* tick, not a collection interval. On each tick it walks every enabled server and runs whatever is due. -```sql -ALTER PROCEDURE - collect.default_trace_collector -( - @hours_back integer = 2, - @include_memory_events bit = 1, - @include_autogrow_events bit = 1, - @include_object_events bit = 1, - -- ...more flags - @debug bit = 0 -) -AS -BEGIN - BEGIN TRY - -- 1. Validate parameters - IF @hours_back <= 0 OR @hours_back > 168 - BEGIN - RAISERROR(N'@hours_back must be between 1 and 168 hours', 16, 1); - RETURN; - END; - - -- 2. Detect first run (empty target table, no prior success in config.collection_log) - IF NOT EXISTS (SELECT 1/0 FROM collect.default_trace_events) - AND NOT EXISTS (SELECT 1/0 FROM config.collection_log WHERE collector_name = N'default_trace_collector' AND collection_status = N'SUCCESS') - BEGIN - SET @cutoff_time = CONVERT(datetime2(7), '19000101'); -- grab everything on first run - END; - - -- 3. Query the DMV / system view - INSERT INTO collect.default_trace_events (...) - SELECT ... - FROM sys.fn_trace_gettable(@trace_path, @max_files) AS ft - WHERE ft.StartTime >= @cutoff_time - AND - AND NOT EXISTS (); - - -- 4. Log success to config.collection_log - INSERT INTO config.collection_log (...) VALUES (..., 'SUCCESS', @rows_collected, ...); - END TRY - BEGIN CATCH - -- 5. Log failure with error message - INSERT INTO config.collection_log (...) VALUES (..., 'ERROR', 0, @error_message); - THROW; - END CATCH; -END; -``` +Per-server sweeps run concurrently behind a semaphore sized by `MaxConcurrentSweeps` (default 4, clamped 1–16, resizable at runtime). Within one server, a semaphore serializes the scheduled sweep against an on-demand `snapshot_now`, so a user-triggered snapshot never interleaves with the regular cadence. -Every collector does exactly these five things: **validate, detect first-run, pull from DMV, insert with dedupe, log**. Once you've read one, you've read all thirty. The differences are the source DMV, the filter conditions, and the shape of the destination table. +`RunDueCollectorsAsync` iterates the collector names, resolves each one's effective schedule, and runs anything whose next-due time has passed, then sets `NextDue = now + FrequencyMinutes`. On reconnect, first-due times are seeded from the persisted `MAX(collection_time)` per collector plus a per-server jitter, so restarting the service resumes the real cadence instead of re-phasing every collector to the same instant. -### The schedule table +### Running one collector -**File**: [`install/03_create_config_tables.sql`](../install/03_create_config_tables.sql) (table definition) +[`DarlingCollectorRunner.RunAsync`](../Darling/PerformanceMonitor.Darling.Service/DarlingCollectorRunner.cs) has three execution paths, chosen by what the definition declares: -`config.collection_schedule` is the single source of truth for *what runs and when*. It has one row per collector: +1. **Plain** — one query, optional supplemental query, map rows, write. +2. **Per-database** — open a connection per database and run the same query in each (Azure SQL DB). +3. **Enumerate-then-iterate** — run the enumeration query to get an item list, optionally probe it, then run the per-item query for each. -| Column | Meaning | -| --- | --- | -| `collector_name` | The name the dispatcher's `IF/ELSE` block matches on | -| `enabled` | Bit flag — off means the dispatcher skips this row entirely | -| `frequency_minutes` | How often to run. `0` means "on connect / daily / special" (see below) | -| `last_run_time` | When the collector last started — updated by the dispatcher | -| `next_run_time` | When the collector is next eligible — `last_run_time + frequency_minutes` | -| `max_duration_minutes` | Kill switch for the hung-job monitor | -| `retention_days` | How long to keep data in the target `collect.*` table | +Rows are written to PostgreSQL through `PgCollectorRowWriter` using **binary COPY**. Large text — query text and plan XML — is diverted to hash-keyed dimension tables (`query_text_dim`, `query_plan_dim`) instead of being stored inline, because inline payload was 94% of one 250 GB field store. -You can edit this table directly, but **don't**. The supported knobs are: +### Error isolation -- **`config.apply_collection_preset`** — bulk-sets `frequency_minutes` for all collectors at once (presets: `Aggressive`, `Balanced`, `Low-Impact`). -- **Individual `UPDATE` statements on `enabled`** — turn specific collectors on or off. +Every run is wrapped so that one failure never stops the sweep. It writes exactly one row to `collect.collection_log` and returns zero rows. That row *is* the heartbeat — there is no separate heartbeat table. -**File**: [`install/41_schedule_management.sql`](../install/41_schedule_management.sql) has the preset procedure and some helper procs for listing / resetting the schedule. +| Status | Meaning | +| --- | --- | +| `SUCCESS` | Completed, including a legitimate zero rows | +| `PERMISSIONS` | A grant is missing — the collector is skipped, not broken | +| `SESSION_MISSING` | An expected Extended Events session isn't there | +| `YIELDED` | Lock timeout on a collector that opted into yielding; excluded from error rates and health bands | +| `ERROR` | Anything else. Fatal or timeout additionally forces a reconnect and re-probe on the next tick | -### Where does the data go? +Health is *derived* from that log by the shared `CollectorHealthClassifier` (`NEVER_RUN`, `NO_PERMISSIONS`, `FAILING`, `STALE`, `WARNING`, `HEALTHY`). Its thresholds are **relative to each collector's own cadence**, with the old flat values as floors — `FAILING` at `max(24h, 2 × interval)`, `STALE` at `max(4h, 1.5 × interval)` — so a 60-minute collector isn't judged like a 1-minute one. The on-connect collectors are exempt from staleness. One classifier is shared by Lite, the viewer, and the service so the three can't drift. -Each collector writes to a table in the `collect` schema — `collect.query_stats`, `collect.default_trace_events`, `collect.wait_stats`, etc. Same shape each time: a `collection_time datetime2` column, plus whatever the DMV gave us, plus whatever we computed. +Observability writes are deliberately failure-isolated: they log at debug and never throw, because an observability write must never break the collection loop. -Some tables use `COMPRESS()` on large text/XML columns (query text, plan XML) — stored as `varbinary(max)` and wrapped in `DECOMPRESS()` on read. That's why query text looks like gibberish if you `SELECT * FROM collect.query_stats` directly — read through `v_query_stats` instead, which handles the decompression. +### The store -### The Dashboard read path +Schema is **generated from the catalog**, not hand-written — there is no migration framework and no `.sql` files. [`PgSchemaGenerator`](../Darling/PerformanceMonitor.Darling.Storage/PgSchemaGenerator.cs) walks `CollectorCatalog.All` to emit DDL, and [`PgMigrations`](../Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs) is an append-only ladder of versioned rungs applied once each under an advisory lock. -The Dashboard is a WPF app. It connects to the `PerformanceMonitor` database and issues SELECT queries. No collection happens in the app — the Dashboard is purely a reader. Every time you pick a time range, change a tab, or hit refresh, the app runs a SQL query against `collect.*` tables or `v_*` views, pulls rows into a `List`, and binds that list to a WPF DataGrid or a ScottPlot chart. +Two schemas: **`collect`** is service-written and user-read; **`config`** is the operator's write surface (server list, alert thresholds, schedule overrides, commands). Every fact table starts with the same four columns — -The query layer lives in `Dashboard/Services/DatabaseService.*.cs` — split by concern (`DatabaseService.QueryPerformance.cs`, `DatabaseService.SystemEvents.cs`, etc.). Each file is just SQL in C# strings. If the Dashboard is showing you something, there's a method somewhere in that folder returning it. +```sql +collection_id bigint NOT NULL, -- or deadlock_id, config_id, … per collector +collection_time timestamp NOT NULL, -- or capture_time on config snapshots; the partition column +server_id integer NOT NULL, +server_name text NOT NULL +``` -### Retention +— followed by nullable payload columns. Fact tables have **no primary key** (a hypertable's unique constraint must include the partition column, and COPY ingest doesn't want one) and are indexed `(server_id,