From 4a9a5cab641f7a2a2e8699eb9129d1c5a91bd9d3 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 15:44:29 -0400 Subject: [PATCH 01/10] Build all macOS ARM64 Private Preview packages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../sqlsim-macos-arm64-private-preview.yml | 106 ++++++++++++++++-- BUILDING.md | 33 ++++++ sqlsimstudio/MACOS-PRIVATE-PREVIEW.md | 58 ++++++++++ sqlsimstudio/build.ps1 | 3 + 4 files changed, 191 insertions(+), 9 deletions(-) create mode 100644 sqlsimstudio/MACOS-PRIVATE-PREVIEW.md diff --git a/.github/workflows/sqlsim-macos-arm64-private-preview.yml b/.github/workflows/sqlsim-macos-arm64-private-preview.yml index 85e80c1..91b80d8 100644 --- a/.github/workflows/sqlsim-macos-arm64-private-preview.yml +++ b/.github/workflows/sqlsim-macos-arm64-private-preview.yml @@ -1,4 +1,4 @@ -name: sqlsim macOS ARM64 Private Preview +name: sqlsimtools macOS ARM64 Private Preview on: workflow_dispatch: @@ -8,7 +8,13 @@ on: paths: - '.github/workflows/sqlsim-macos-arm64-private-preview.yml' - 'sqlsim/**' + - 'sqlpysim/**' + - 'sqlsimstudio/**' - 'release/**' + - '.github/skills/sqlsim/**' + - '.github/skills/sqlpysim/**' + - '.github/skills/sqlsimstudio/**' + - '.github/agents/sqlsimtools-workload-builder.agent.md' permissions: contents: read @@ -26,7 +32,19 @@ jobs: sw_vers - name: Install build prerequisites - run: brew install cmake unixodbc + run: brew install cmake unixodbc openssl pandoc + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: '10.0.x' + + - name: Install Python driver + working-directory: sqlpysim + run: python -m pip install -r requirements.txt - name: Build and test sqlsim working-directory: sqlsim @@ -34,25 +52,95 @@ jobs: bash ./build-macos.sh --clean bash ./smoke-test-macos.sh --skip-build - - name: Stage archive + - name: Stage sqlsim working-directory: sqlsim run: | bash ./publish-macos.sh --skip-build --release-dir '${{ runner.temp }}/stage/sqlsim' - - name: Validate staged package + - name: Stage sqlpysim + working-directory: sqlpysim + shell: pwsh + run: | + ./publish.ps1 -ReleaseDir '${{ runner.temp }}/stage/sqlpysim' + if ($LASTEXITCODE -ne 0) { throw 'sqlpysim publishing failed.' } + + - name: Build Studio with both engines + working-directory: sqlsimstudio + shell: pwsh + run: | + ./build.ps1 -Runtime osx-arm64 + if ($LASTEXITCODE -ne 0) { throw 'Studio publishing failed.' } + + - name: Create and extract candidate archives + env: + CANDIDATE_SHA: ${{ github.sha }} + CANDIDATE_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + mkdir -p "$RUNNER_TEMP/artifacts" "$RUNNER_TEMP/verify" + printf 'Commit: %s\nBuild: %s\nPlatform: macOS 15+ ARM64\nValidation: offline only; live SQL testing required\n' \ + "$CANDIDATE_SHA" "$CANDIDATE_RUN" > "$RUNNER_TEMP/artifacts/BUILD-INFO.txt" + for tool in sqlsim sqlpysim sqlsimstudio; do + stage="$RUNNER_TEMP/stage/$tool" + if [[ "$tool" == "sqlsimstudio" ]]; then + stage="$GITHUB_WORKSPACE/sqlsimstudio/build/osx-arm64" + fi + cp "$RUNNER_TEMP/artifacts/BUILD-INFO.txt" "$stage/BUILD-INFO.txt" + archive="$RUNNER_TEMP/artifacts/$tool-macos-arm64-private-preview.tar.gz" + tar -C "$stage" -czf "$archive" . + mkdir -p "$RUNNER_TEMP/verify/$tool" + tar -C "$RUNNER_TEMP/verify/$tool" -xzf "$archive" + done + cd "$RUNNER_TEMP/artifacts" + shasum -a 256 ./*.tar.gz > SHA256SUMS + shasum -a 256 -c SHA256SUMS + + - name: Validate extracted CLI packages shell: pwsh run: | ./release/validate-package-layout.ps1 ` - -InstallRoot '${{ runner.temp }}/stage/sqlsim' ` + -InstallRoot '${{ runner.temp }}/verify/sqlsim' ` -Tool sqlsim ` -SqlsimExecutable 'sqlsim/arm64/sqlsim' + if ($LASTEXITCODE -ne 0) { throw 'sqlsim archive validation failed.' } + ./release/validate-package-layout.ps1 ` + -InstallRoot '${{ runner.temp }}/verify/sqlpysim' ` + -Tool sqlpysim + if ($LASTEXITCODE -ne 0) { throw 'sqlpysim archive validation failed.' } - - name: Create archive + - name: Validate extracted Studio package + shell: pwsh run: | - tar -C '${{ runner.temp }}/stage/sqlsim' -czf '${{ runner.temp }}/sqlsim-macos-arm64-private-preview.tar.gz' . + $studio = '${{ runner.temp }}/verify/sqlsimstudio' + foreach ($path in @( + 'sqlsimstudio', 'sqlsim/sqlsim', 'sqlpysim/sqlpysim.py', + 'sqlpysim/requirements.txt', 'LICENSE', 'THIRD-PARTY-NOTICES.md', + 'sqlsimstudio.md', 'sqlsimstudio.html', 'CONFIGURATION.md', + 'MACOS-PRIVATE-PREVIEW.md', 'examples', + '.github/skills/sqlsimstudio/SKILL.md', 'BUILD-INFO.txt' + )) { + if (-not (Test-Path -LiteralPath (Join-Path $studio $path))) { + throw "Studio archive is missing $path" + } + } + & lipo -verify_arch arm64 (Join-Path $studio 'sqlsimstudio') + if ($LASTEXITCODE -ne 0) { throw 'Studio is not an ARM64 executable.' } + & lipo -verify_arch arm64 (Join-Path $studio 'sqlsim/sqlsim') + if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim is not an ARM64 executable.' } + & (Join-Path $studio 'sqlsim/sqlsim') --version + if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim could not start.' } + & python (Join-Path $studio 'sqlpysim/sqlpysim.py') --version + if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlpysim could not start.' } + Push-Location sqlsimstudio + try { + ./smoke-test.ps1 -SkipBuild -StartupOnly ` + -StudioExecutable (Join-Path $studio 'sqlsimstudio') + if ($LASTEXITCODE -ne 0) { throw 'Studio startup smoke test failed.' } + } finally { + Pop-Location + } - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: sqlsim-macos-arm64-private-preview - path: ${{ runner.temp }}/sqlsim-macos-arm64-private-preview.tar.gz + name: sqlsimtools-macos-arm64-private-preview + path: ${{ runner.temp }}/artifacts/ if-no-files-found: error diff --git a/BUILDING.md b/BUILDING.md index 95b4513..954f1df 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -86,6 +86,39 @@ Studio builds are self-contained and include the matching native sqlsim binary. See [sqlsimstudio/README.md](sqlsimstudio/README.md) for the supported platform matrix and package contents. +### macOS ARM64 Private Preview candidates + +The [macOS preview workflow](.github/workflows/sqlsim-macos-arm64-private-preview.yml) +builds all three tools on a macOS 15 Apple Silicon runner using their existing +build and publish scripts. To build an invited-tester candidate without +creating a tag or public release: + +```powershell +gh workflow run sqlsim-macos-arm64-private-preview.yml --repo microsoft/sqlsimtools --ref main +``` + +The `sqlsimtools-macos-arm64-private-preview` Actions artifact contains: + +| Archive | Intended use | +| --- | --- | +| `sqlsim-macos-arm64-private-preview.tar.gz` | Complete native CLI overlay, including its macOS preview guide | +| `sqlpysim-macos-arm64-private-preview.tar.gz` | Complete Python source overlay, including its macOS preview guide | +| `sqlsimstudio-macos-arm64-private-preview.tar.gz` | Self-contained Studio with both engines and its macOS preview guide | + +The artifact also contains `SHA256SUMS` and `BUILD-INFO.txt`; each archive +includes the same build receipt. Distribute packages from the same successful +run to the invited cohort. Downloading Actions artifacts requires signing in +to GitHub, and the artifacts expire according to repository retention settings. +Private Preview describes support status, not access control: artifacts in +this public repository are not confidential. + +The workflow extracts each archive before validation, runs CLI help/version +and layout checks, verifies Studio and its native engine are ARM64, and runs +Studio's offline startup smoke test. This does not validate live SQL +connectivity, workload execution, Finder launch, signing, or notarization. +Testers should follow the packaged preview guides and report sanitized results +in their existing request issues. macOS candidates are not public release assets. + ## Prepare for source testing After building sqlsim and preparing sqlpysim, run the shared test preflight diff --git a/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md b/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md new file mode 100644 index 0000000..8540d55 --- /dev/null +++ b/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md @@ -0,0 +1,58 @@ +# sqlsim Studio macOS ARM64 Private Preview + +This candidate is for invited Apple Silicon testers, not a supported public +release. Automated checks cover packaging, engine startup, and Studio's local +HTTP startup. Live database connections, browser integration, and workload +execution on testers' Macs still need validation. + +## Requirements and launch + +- Apple Silicon Mac running macOS 15 or later. +- Microsoft ODBC Driver 18 for the bundled sqlsim engine. Follow the dependency + instructions in the separate sqlsim package's `sqlsim/MACOS-PRIVATE-PREVIEW.md`. +- A current Safari, Microsoft Edge, Google Chrome, or Firefox browser. +- Optional: Python 3.11 or later and Homebrew OpenSSL for sqlpysim. + +Extract this archive into its own folder. Keep the bundled `sqlsim` and +`sqlpysim` subfolders next to the Studio executable: + +```bash +chmod +x ./sqlsimstudio ./sqlsim/sqlsim +./sqlsimstudio +``` + +Studio is self-contained; no .NET installation is required. The candidate is +not Developer ID signed or notarized. If macOS blocks it, report the exact +message and stop; do not disable Gatekeeper or remove quarantine recursively. + +To enable the optional Python engine, create its environment in the extracted +Studio folder before launching: + +```bash +brew install openssl +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install -r ./sqlpysim/requirements.txt +./sqlsimstudio +``` + +## First test + +Use a non-production database you have permission to load test. + +1. Record `sw_vers`, `uname -m`, the Studio version, and `BUILD-INFO.txt`. +2. Confirm that Studio opens in your browser and detects the sqlsim engine. +3. Choose a server, database, and authentication mode. For Azure SQL, start + with an access token; Entra interactive authentication on macOS is not + validated. Never post a token or other credential in an issue. +4. Test the connection and run `SELECT 1` with one thread and one iteration. +5. Confirm results and console output appear. Then try a small repeated + workload and cancellation before increasing concurrency. +6. If requested, repeat with sqlpysim and compare the experience, not just + raw timing numbers. +7. Shut down through Studio's shutdown control. + +Report feedback in your existing Private Preview request issue. Include the +build commit, chip, macOS version, browser, engine, target SQL product, +authentication mode, and sanitized reproduction steps. Include no passwords, +tokens, customer data, or credential-bearing connection strings. diff --git a/sqlsimstudio/build.ps1 b/sqlsimstudio/build.ps1 index 76680ab..ac59de7 100644 --- a/sqlsimstudio/build.ps1 +++ b/sqlsimstudio/build.ps1 @@ -74,6 +74,9 @@ if ($pandoc) { Write-Warning 'pandoc was not found; sqlsimstudio.html was not generated.' } Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'CONFIGURATION.md') -Destination $output -Force +if ($Runtime -eq 'osx-arm64') { + Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'MACOS-PRIVATE-PREVIEW.md') -Destination $output -Force +} Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'examples') -Destination $output -Recurse -Force $studioSkillDirectory = Join-Path $output '.github\skills\sqlsimstudio' New-Item -ItemType Directory -Path $studioSkillDirectory -Force | Out-Null From 8d6d8dd8be93f6f935b91ebd3e097b1cbd5ca937 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 15:46:13 -0400 Subject: [PATCH 02/10] Use a portable temporary directory when publishing Python docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sqlpysim/publish.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlpysim/publish.ps1 b/sqlpysim/publish.ps1 index 79b4638..fe69743 100644 --- a/sqlpysim/publish.ps1 +++ b/sqlpysim/publish.ps1 @@ -222,7 +222,7 @@ nav#TOC a { "@ # Create temp CSS file - $tempCssPath = Join-Path $env:TEMP "sqlpysim-style.css" + $tempCssPath = Join-Path ([IO.Path]::GetTempPath()) ("sqlpysim-style-{0}.css" -f [guid]::NewGuid().ToString('N')) $customCss | Out-File -FilePath $tempCssPath -Encoding utf8 # Python/SQL themed logo SVG (embedded for standalone HTML) From 8ffedc58a60cc71b28650563b52a3c6651c2e068 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 15:47:56 -0400 Subject: [PATCH 03/10] Correct lipo architecture verification argument order Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/sqlsim-macos-arm64-private-preview.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sqlsim-macos-arm64-private-preview.yml b/.github/workflows/sqlsim-macos-arm64-private-preview.yml index 91b80d8..6148db3 100644 --- a/.github/workflows/sqlsim-macos-arm64-private-preview.yml +++ b/.github/workflows/sqlsim-macos-arm64-private-preview.yml @@ -122,9 +122,9 @@ jobs: throw "Studio archive is missing $path" } } - & lipo -verify_arch arm64 (Join-Path $studio 'sqlsimstudio') + & lipo (Join-Path $studio 'sqlsimstudio') -verify_arch arm64 if ($LASTEXITCODE -ne 0) { throw 'Studio is not an ARM64 executable.' } - & lipo -verify_arch arm64 (Join-Path $studio 'sqlsim/sqlsim') + & lipo (Join-Path $studio 'sqlsim/sqlsim') -verify_arch arm64 if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim is not an ARM64 executable.' } & (Join-Path $studio 'sqlsim/sqlsim') --version if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim could not start.' } From e49a9c88af193f25ff6ee8fbf0d07c08a89c8a82 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 15:50:18 -0400 Subject: [PATCH 04/10] Report Studio startup exit diagnostics in smoke tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sqlsimstudio/smoke-test.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sqlsimstudio/smoke-test.ps1 b/sqlsimstudio/smoke-test.ps1 index ca52b22..4e18f3e 100644 --- a/sqlsimstudio/smoke-test.ps1 +++ b/sqlsimstudio/smoke-test.ps1 @@ -54,6 +54,9 @@ try { } $line = $readTask.Result + if ($null -eq $line -and $process.HasExited) { + throw "Studio exited with code $($process.ExitCode) before startup: $($process.StandardError.ReadToEnd())" + } if ($line -like 'SQLSIM_STUDIO_URL=*') { $baseUrl = $line.Substring('SQLSIM_STUDIO_URL='.Length) } elseif ($line -eq 'SQLSIM_STUDIO_BOOTSTRAP_READY=1') { From 12669b57cc98ecc1b5becb168f7c41d6841377c6 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 15:54:21 -0400 Subject: [PATCH 05/10] Use system ICU for the macOS ARM64 Studio preview Keep app-local ICU unchanged for other targets and document macOS globalization differences. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sqlsimstudio/MACOS-PRIVATE-PREVIEW.md | 5 +++++ sqlsimstudio/THIRD-PARTY-NOTICES.md | 3 ++- sqlsimstudio/server/SqlsimStudio.csproj | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md b/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md index 8540d55..af84a82 100644 --- a/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md +++ b/sqlsimstudio/MACOS-PRIVATE-PREVIEW.md @@ -5,6 +5,11 @@ release. Automated checks cover packaging, engine startup, and Studio's local HTTP startup. Live database connections, browser integration, and workload execution on testers' Macs still need validation. +The macOS ARM64 build uses macOS's system ICU globalization libraries rather +than the app-local ICU pinned in Windows and Linux builds. Culture-sensitive +sorting and comparisons may differ across platforms; include unexpected +differences in your feedback. + ## Requirements and launch - Apple Silicon Mac running macOS 15 or later. diff --git a/sqlsimstudio/THIRD-PARTY-NOTICES.md b/sqlsimstudio/THIRD-PARTY-NOTICES.md index 231640b..74a5b51 100644 --- a/sqlsimstudio/THIRD-PARTY-NOTICES.md +++ b/sqlsimstudio/THIRD-PARTY-NOTICES.md @@ -27,5 +27,6 @@ Microsoft.Data.SqlClient and its runtime dependencies are included in self-conta - Project: https://github.com/microsoft/icu - Package: https://www.nuget.org/packages/Microsoft.ICU.ICU4C.Runtime/72.1.0.3 -- Purpose: App-local globalization support for self-contained Linux and macOS packages +- Purpose: App-local globalization support for Windows and Linux packages. + The macOS ARM64 preview uses the operating system's ICU libraries instead. - License: Unicode-DFS-2016 and third-party ICU notices distributed with the package diff --git a/sqlsimstudio/server/SqlsimStudio.csproj b/sqlsimstudio/server/SqlsimStudio.csproj index 933aec2..c1e42dc 100644 --- a/sqlsimstudio/server/SqlsimStudio.csproj +++ b/sqlsimstudio/server/SqlsimStudio.csproj @@ -18,7 +18,7 @@ - + @@ -27,7 +27,7 @@ - + From 1d9d9edaeccf1d0d6ebca5426d4a3e6b64fb41b6 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 16:35:38 -0400 Subject: [PATCH 06/10] Ship a maintained macOS tester kit and platform support plan Reuse canonical suites, gate live and load testing, record JSON results, and validate extracted kits on Apple Silicon CI. Document live-backend options without selecting or provisioning one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../sqlsim-macos-arm64-private-preview.yml | 64 ++++-- BACKLOG.md | 7 + BUILDING.md | 9 + MACOS-SUPPORT-PLAN.md | 133 ++++++++++++ release/README.md | 3 + release/build-macos-tester-kit.ps1 | 64 ++++++ release/macos-tester-README.md | 137 +++++++++++++ release/run-macos-tests.ps1 | 190 ++++++++++++++++++ release/test-macos-tester-kit.ps1 | 111 ++++++++++ sqlsim/smoke-test-linux.sh | 28 ++- sqlsim/smoke-test-macos.sh | 26 ++- sqlsimstudio/full-test-suite.ps1 | 35 +++- sqlsimstudio/smoke-test.ps1 | 36 +++- 13 files changed, 807 insertions(+), 36 deletions(-) create mode 100644 MACOS-SUPPORT-PLAN.md create mode 100644 release/build-macos-tester-kit.ps1 create mode 100644 release/macos-tester-README.md create mode 100644 release/run-macos-tests.ps1 create mode 100644 release/test-macos-tester-kit.ps1 diff --git a/.github/workflows/sqlsim-macos-arm64-private-preview.yml b/.github/workflows/sqlsim-macos-arm64-private-preview.yml index 6148db3..bd82617 100644 --- a/.github/workflows/sqlsim-macos-arm64-private-preview.yml +++ b/.github/workflows/sqlsim-macos-arm64-private-preview.yml @@ -2,6 +2,19 @@ name: sqlsimtools macOS ARM64 Private Preview on: workflow_dispatch: + pull_request: + branches: + - main + paths: + - '.github/workflows/sqlsim-macos-arm64-private-preview.yml' + - 'sqlsim/**' + - 'sqlpysim/**' + - 'sqlsimstudio/**' + - 'release/**' + - '.github/skills/sqlsim/**' + - '.github/skills/sqlpysim/**' + - '.github/skills/sqlsimstudio/**' + - '.github/agents/sqlsimtools-workload-builder.agent.md' push: branches: - main @@ -32,7 +45,7 @@ jobs: sw_vers - name: Install build prerequisites - run: brew install cmake unixodbc openssl pandoc + run: brew install cmake unixodbc openssl pandoc coreutils jq - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -90,9 +103,6 @@ jobs: mkdir -p "$RUNNER_TEMP/verify/$tool" tar -C "$RUNNER_TEMP/verify/$tool" -xzf "$archive" done - cd "$RUNNER_TEMP/artifacts" - shasum -a 256 ./*.tar.gz > SHA256SUMS - shasum -a 256 -c SHA256SUMS - name: Validate extracted CLI packages shell: pwsh @@ -130,14 +140,44 @@ jobs: if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlsim could not start.' } & python (Join-Path $studio 'sqlpysim/sqlpysim.py') --version if ($LASTEXITCODE -ne 0) { throw 'Bundled sqlpysim could not start.' } - Push-Location sqlsimstudio - try { - ./smoke-test.ps1 -SkipBuild -StartupOnly ` - -StudioExecutable (Join-Path $studio 'sqlsimstudio') - if ($LASTEXITCODE -ne 0) { throw 'Studio startup smoke test failed.' } - } finally { - Pop-Location - } + + - name: Assemble self-contained tester kit + shell: pwsh + run: | + ./release/build-macos-tester-kit.ps1 ` + -SqlsimRoot '${{ runner.temp }}/verify/sqlsim' ` + -SqlpysimRoot '${{ runner.temp }}/verify/sqlpysim' ` + -StudioRoot '${{ runner.temp }}/verify/sqlsimstudio' ` + -OutputDirectory '${{ runner.temp }}/tester-kit' + + - name: Archive and extract tester kit + run: | + tar -C "$RUNNER_TEMP/tester-kit" -czf "$RUNNER_TEMP/artifacts/sqlsimtools-macos-arm64-tester-kit.tar.gz" . + mkdir -p "$RUNNER_TEMP/verify/tester-kit" + tar -C "$RUNNER_TEMP/verify/tester-kit" -xzf "$RUNNER_TEMP/artifacts/sqlsimtools-macos-arm64-tester-kit.tar.gz" + + - name: Validate extracted tester kit and runner safety gates + shell: pwsh + run: | + ./scripts/check-suite-parity.ps1 -Quiet + if ($LASTEXITCODE -ne 0) { throw 'Shared suite inventories diverged.' } + ./release/test-macos-tester-kit.ps1 ` + -KitRoot '${{ runner.temp }}/verify/tester-kit' ` + -ValidationDirectory '${{ runner.temp }}/tester-kit-validation' + if ($LASTEXITCODE -ne 0) { throw 'Tester kit validation failed.' } + + - name: Checksum all candidate archives + run: | + cd "$RUNNER_TEMP/artifacts" + shasum -a 256 ./*.tar.gz > SHA256SUMS + shasum -a 256 -c SHA256SUMS + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: macos-tester-kit-offline-results + path: ${{ runner.temp }}/tester-kit-validation/offline-results/ + if-no-files-found: ignore - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/BACKLOG.md b/BACKLOG.md index 8e5235b..e0cc9f7 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -23,6 +23,13 @@ Azure SQL Managed Instance, and SQL database in Microsoft Fabric. ### Testing and validation +- **Maintain macOS as a tested platform, not just a volunteer preview.** + Follow [MACOS-SUPPORT-PLAN.md](MACOS-SUPPORT-PLAN.md): continuously validate + extracted packages and the shared tester kit on Apple Silicon; evaluate + maintainer-owned live-testing backends before selecting or provisioning one; + and complete clean-install, browser, signing/notarization, and release gates + before promoting each tool. Tester feedback supplements rather than replaces + a repeatable maintainer-owned validation baseline. - **Close the sqlpysim workload and live-telemetry gaps.** Agreed as the next piece of work, in this order, because the three share plumbing and together restore what Studio currently disables for the Python engine: diff --git a/BUILDING.md b/BUILDING.md index 954f1df..1911979 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -104,6 +104,7 @@ The `sqlsimtools-macos-arm64-private-preview` Actions artifact contains: | `sqlsim-macos-arm64-private-preview.tar.gz` | Complete native CLI overlay, including its macOS preview guide | | `sqlpysim-macos-arm64-private-preview.tar.gz` | Complete Python source overlay, including its macOS preview guide | | `sqlsimstudio-macos-arm64-private-preview.tar.gz` | Self-contained Studio with both engines and its macOS preview guide | +| `sqlsimtools-macos-arm64-tester-kit.tar.gz` | Self-contained installation of all tools plus shared offline, smoke, and full suites, fixtures, and a JSON-reporting runner | The artifact also contains `SHA256SUMS` and `BUILD-INFO.txt`; each archive includes the same build receipt. Distribute packages from the same successful @@ -119,6 +120,14 @@ connectivity, workload execution, Finder launch, signing, or notarization. Testers should follow the packaged preview guides and report sanitized results in their existing request issues. macOS candidates are not public release assets. +The tester kit extracts into a new folder and does not require a checkout. +Its [README](release/macos-tester-README.md) documents dependencies, offline +checks, explicit live/load-test consent, and reports. CI runs the offline tier +from the extracted kit, not from source-tree paths. Relevant pull requests also +run the macOS workflow. The long-term live-testing backend is deliberately +undecided; see [MACOS-SUPPORT-PLAN.md](MACOS-SUPPORT-PLAN.md) for the maintained +platform plan and promotion gates. + ## Prepare for source testing After building sqlsim and preparing sqlpysim, run the shared test preflight diff --git a/MACOS-SUPPORT-PLAN.md b/MACOS-SUPPORT-PLAN.md new file mode 100644 index 0000000..ce07f3d --- /dev/null +++ b/MACOS-SUPPORT-PLAN.md @@ -0,0 +1,133 @@ +# Maintained macOS ARM64 support plan + +## Objective and current status + +macOS must become a maintained build, packaging, and validation target, not a +platform supported only by a one-time volunteer campaign. The initial scope is +Apple Silicon with macOS 15 or later; Intel Macs are not in scope. + +All three tools remain **Private Preview**. Building a candidate or collecting +positive feedback does not promote support status. The maintainers own the +gates below; volunteers provide additional real-machine, browser, installation, +and SQL-environment coverage. + +The current implementation provides: + +- A native Apple Silicon build of sqlsim and a source package of sqlpysim. +- A self-contained ARM64 Studio with both engines. +- Extracted-package layout/architecture/entry-point checks. +- A self-contained tester kit assembled from the same candidate packages and + the canonical suites, with explicit live/load-test consent and JSON reports. +- A macOS workflow for relevant PRs, pushes to main, and manual dispatch. +- Automated offline execution of the extracted tester kit before upload. + +These checks do not establish live database correctness, signing/notarization, +Finder launch, or browser-specific behavior. + +## Stage 1: continuously maintain offline validation + +The maintainer acceptance gate for changes affecting macOS is a successful +[macOS workflow](.github/workflows/sqlsim-macos-arm64-private-preview.yml). +Do not bypass it because another platform passes. + +On every relevant PR and main update: + +1. Build native ARM64 sqlsim and Studio with the existing per-tool scripts. +2. Publish the three packages and extract them into clean directories. +3. Validate architecture, manifests, licenses, agent guidance, help/version, + and startup. +4. Assemble the tester kit from those exact extracted packages and canonical + suites; archive it, extract it again, and run its offline tier. +5. Check suite syntax, CLI inventory parity, and the runner's refusal of + incomplete credentials or unacknowledged live/load testing. +6. Preserve reports, build receipts, and checksums. Never upload credentials. + +Use one maintained Unix CLI suite for Linux and macOS; fix portability in the +source suite rather than editing generated kits. Keep platform-specific +assertions explicit and preserve the existing Windows/Bash test inventories. +Keep Studio's PowerShell suites shared across platforms. + +Repository branch-protection settings are not changed by this work. A +maintainer should decide whether to require the macOS check once its PR-triggered +workflow has a stable baseline and its check name is available. + +## Stage 2: establish maintainer-owned live automation + +**The live-testing backend has not been selected or provisioned.** Evaluate +these options before implementing live CI: + +| Option | Advantages | Costs and constraints | +| --- | --- | --- | +| GitHub-hosted Apple Silicon + dedicated Azure SQL test database | No Mac hardware ownership; isolated disposable data; direct Azure SQL/authentication coverage | Cloud cost, network/firewall setup, least-privilege identity, token renewal; does not cover every SQL Server/MI/Fabric feature | +| GitHub-hosted Apple Silicon + existing reachable SQL Server test instance | Reuses SQL Server fixtures; covers engine-specific features | Secure network access from ephemeral runners, isolated database, identity and availability ownership | +| Maintainer-controlled/self-hosted Apple Silicon runner | Real local installation/browser testing; access to a private test network | Hardware/cloud-Mac cost, patching, runner isolation, credential protection; never execute untrusted fork PR code on a privileged runner | + +Record the selected backend, owner, budget, network design, credential +mechanism, and data-reset policy before enabling it. Do not widen database +firewalls or expose a local SQL instance merely to make CI pass. + +The eventual live workflow must: + +- Be opt-in initially and run only trusted, reviewed commits. +- Use a protected GitHub environment and scoped credentials, preferably a + short-lived identity where the selected backend supports it. +- Run the same packaged tester kit, never a separate CI-only test inventory. +- Run smoke first, then full suites against disposable test data with explicit + concurrency limits and separate approval for scale tests. +- Fail on test failures and distinguish product defects, harness failures, + permission/feature skips, and infrastructure failures. +- Produce sanitized, commit-linked reports without credential files or raw + customer data; clean up test objects and credentials even on cancellation. + +Required initial live coverage is SQL authentication and Azure SQL access-token +authentication for both engines and Studio, on the appropriate selected +targets. If the first backend cannot exercise both, leave the missing coverage +open and plan a second target. Kerberos, managed identity, service principal, +MI, and Fabric must be tracked individually rather than inferred from another +authentication mode or product. + +Until this infrastructure exists, maintainers must explicitly record the live +validation gap. Volunteer results can inform fixes but are not a substitute +for repeatable release validation. + +## Stage 3: installation, browser, and distribution readiness + +Before broader availability: + +- Decide and document Developer ID signing and notarization for native + executables and Studio; arrange certificate ownership/renewal and protected + signing credentials. +- Validate download/extraction/quarantine behavior on a clean Mac without + disabling Gatekeeper. +- Exercise Studio launch, reconnect, shutdown, both engines, monitoring, + cancellation, results, and A/B comparison in real macOS browsers. WebKit + emulation on Windows is not Safari-on-macOS evidence. +- Decide whether to add a `.app` launcher or retain the documented terminal + launch experience; package that choice consistently. +- Document ODBC, Python/OpenSSL, minimum macOS, and architecture requirements. +- Keep the system-ICU exception for macOS explicit and test culture-sensitive + comparisons rather than promising identical collation across platforms. +- Define reproducible candidate labels, artifact retention, update/rollback, + and the eventual versioned release-asset path. + +## Promotion and ongoing maintenance + +Promotion is an explicit maintainer decision, separately for each tool. Require: + +1. Green native build, packaging, and extracted-kit offline gates. +2. Recorded maintainer-controlled smoke/full live results on the release + candidate, with SQL/authentication coverage and skips listed. +3. Clean-machine installation and actual macOS browser evidence where relevant. +4. Signing/notarization and distribution decisions completed and documented. +5. No unresolved installation, credential-handling, data-integrity, + cancellation, or stability blockers. +6. Updated platform matrices, manuals, release notes, skills, and release + packaging/validation for the promoted tools. + +After promotion, keep macOS on relevant PR/main validation and the release +pipeline, revalidate driver/.NET/Python changes, and run the selected live +gate before every release. Evaluate newer macOS/Python versions explicitly; +do not silently raise the current minimum or imply an untested version works. +Choose a periodic live/clean-install cadence when the backend and its budget +are approved. Volunteers remain valuable for diverse environments, but the +maintainer owns release confidence and the regression baseline. diff --git a/release/README.md b/release/README.md index 3c916ea..c7e41fc 100644 --- a/release/README.md +++ b/release/README.md @@ -31,6 +31,9 @@ as evidence that a release artifact is correct. | --- | --- | | `publish.ps1` | Runs the sqlsim and sqlpysim publishers for local staging | | `build-sqlsimtools-bundle.ps1` | Assembles the Windows sqlsimtools bundle containing Studio and both engines | +| `build-macos-tester-kit.ps1` | Assembles a self-contained macOS test installation from validated candidate packages and canonical suites | +| `run-macos-tests.ps1` | Tester-kit runner; defaults to offline checks, gates live/load testing, and writes JSON plus logs | +| `macos-tester-README.md` | Instructions shipped as the tester kit's README | | `validate-package-layout.ps1` | Validates an extracted tool package and its documented entry points | | `check-release-version.ps1` | Confirms version parity across every shipped tool | | `stage-release-assets.ps1` | Stages the five versioned public release assets; the README links to the latest release page | diff --git a/release/build-macos-tester-kit.ps1 b/release/build-macos-tester-kit.ps1 new file mode 100644 index 0000000..c13b5da --- /dev/null +++ b/release/build-macos-tester-kit.ps1 @@ -0,0 +1,64 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$SqlsimRoot, + [Parameter(Mandatory)][string]$SqlpysimRoot, + [Parameter(Mandatory)][string]$StudioRoot, + [Parameter(Mandatory)][string]$OutputDirectory +) + +$ErrorActionPreference = 'Stop' +$repository = Split-Path -Parent $PSScriptRoot +if (Test-Path -LiteralPath $OutputDirectory) { + throw "Tester kit output must be a new directory: $OutputDirectory" +} +$output = [IO.Path]::GetFullPath($OutputDirectory) +New-Item -ItemType Directory -Path $output | Out-Null + +function Copy-KitTree { + param([string]$Source, [string]$Destination) + + $sourceRoot = (Resolve-Path -LiteralPath $Source).Path + foreach ($file in Get-ChildItem -LiteralPath $sourceRoot -Recurse -Force -File) { + $relative = [IO.Path]::GetRelativePath($sourceRoot, $file.FullName) + $target = Join-Path $Destination $relative + if (Test-Path -LiteralPath $target) { + if ((Get-FileHash -LiteralPath $file.FullName).Hash -ne (Get-FileHash -LiteralPath $target).Hash) { + throw "Conflicting tester kit file: $relative" + } + continue + } + New-Item -ItemType Directory -Path (Split-Path -Parent $target) -Force | Out-Null + Copy-Item -LiteralPath $file.FullName -Destination $target + } +} + +Copy-KitTree $SqlsimRoot $output +Copy-KitTree $SqlpysimRoot $output +Copy-KitTree $StudioRoot (Join-Path $output 'sqlsimstudio') + +$sources = @( + 'sqlsim/smoke-test-macos.sh', + 'sqlsim/smoke-test-linux.sh', + 'sqlsim/full-test-suite-linux.sh', + 'sqlsim/testing/macOS/verify-macos-arm64.sh', + 'sqlsim/testing/test-print-statements.sql', + 'sqlsim/testing/replay-test-data.xml', + 'sqlsim/testing/clean-replay.xml', + 'sqlpysim/smoke-test-linux.sh', + 'sqlpysim/full-test-suite-linux.sh', + 'sqlsimstudio/smoke-test.ps1', + 'sqlsimstudio/full-test-suite.ps1' +) +foreach ($relative in $sources) { + $target = Join-Path $output $relative + New-Item -ItemType Directory -Path (Split-Path -Parent $target) -Force | Out-Null + Copy-Item -LiteralPath (Join-Path $repository $relative) -Destination $target +} +Copy-KitTree (Join-Path $repository 'sqlpysim/tests') (Join-Path $output 'sqlpysim/tests') +Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'run-macos-tests.ps1') -Destination (Join-Path $output 'run-tests.ps1') +Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'macos-tester-README.md') -Destination (Join-Path $output 'README.md') +Copy-Item -LiteralPath (Join-Path $repository 'LICENSE') -Destination (Join-Path $output 'LICENSE') + +Write-Host "Self-contained macOS tester kit staged: $output" diff --git a/release/macos-tester-README.md b/release/macos-tester-README.md new file mode 100644 index 0000000..e5551e8 --- /dev/null +++ b/release/macos-tester-README.md @@ -0,0 +1,137 @@ +# macOS ARM64 Private Preview tester kit + +This is a self-contained test installation of sqlsim, sqlpysim, and sqlsim +Studio, plus their maintained test suites, examples, and fixtures. You do not +need a source checkout or to extract the three individual packages first. +Extract the kit into a new, writable folder; do not overlay an existing project. + +The build commit and Actions run are in `BUILD-INFO.txt`. Use kits from one +successful run, not a mixture of different candidates. + +## Requirements + +- Apple Silicon, macOS 15 or later. +- Homebrew and PowerShell 7 (`pwsh`) for the test runner and Studio suites. +- Python 3.11 or later, OpenSSL, and the pinned Python driver for Python/Studio + testing. +- Microsoft ODBC Driver 18 for live sqlsim/Studio tests. +- `coreutils` (provides `gtimeout`) for the native live smoke suite, and `jq` + for the native full suite. + +Install the test-runner dependencies: + +```bash +brew install powershell python@3.11 openssl coreutils jq +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install -r ./sqlpysim/requirements.txt +``` + +Install ODBC Driver 18 using the instructions in +`sqlsim/MACOS-PRIVATE-PREVIEW.md`. Studio's platform-specific launch and signing +limitations are in `sqlsimstudio/MACOS-PRIVATE-PREVIEW.md`. If macOS blocks an +executable, report the exact message; do not disable Gatekeeper or remove +quarantine recursively. + +## 1. Offline checks first + +From the extracted kit folder, with the Python environment activated: + +```bash +pwsh -NoProfile -File ./run-tests.ps1 -Tier Offline -Python python +``` + +This runs the existing native macOS smoke checks, Python help/version and unit +tests, and Studio startup plus its full offline validation suite. No configured +database is contacted. Some native rejection tests probe a reserved, +unreachable address to validate error reporting. + +To test just one tool, add `-Tool sqlsim`, `-Tool sqlpysim`, or +`-Tool sqlsimstudio`. Python/Studio selections require the Python environment; +the native-only selection does not. + +## 2. Live smoke tests + +Only use a dedicated, non-production database you are authorized to test. +These are the repository's real suites, not a single-query demonstration: +even smoke tests can create test procedures/temporary objects, run dozens of +concurrent connections, and require catalog/statistics permissions. The native +smoke suite includes a 100-thread test. Non-Azure tests inherit the suites' +existing trust-server-certificate behavior. Review the suites before running +against your target. + +Create a local credential file with only the password or access token on its +first line. Keep it outside the kit and its results folder, restrict access +(`chmod 600 /path/to/credential-file`), and never attach it to an issue. Do not +put the secret itself on the command line. + +For SQL authentication: + +```bash +pwsh -NoProfile -File ./run-tests.ps1 \ + -Tier Smoke -Tool sqlsim \ + -Server 'your-test-server,1433' -Database YourDedicatedTestDatabase \ + -Username YourTestUser -PasswordFile /path/to/password-file \ + -AllowLiveTests +``` + +For Azure SQL, use a fresh access token rather than Entra interactive sign-in: + +```bash +pwsh -NoProfile -File ./run-tests.ps1 \ + -Tier Smoke -Tool sqlpysim -Python python \ + -Server your-test-server.database.windows.net -Database YourDedicatedTestDatabase \ + -TokenFile /path/to/access-token-file -AllowLiveTests +``` + +For Studio, substitute `-Tool sqlsimstudio -Python python`. Its smoke suite +tests both bundled engines, file and mixed workloads, results, and replay. +This does not replace manual Safari/Chrome/Edge/Firefox testing or Finder launch +validation. Follow the included example guides for interactive testing. + +## 3. Optional full suites + +Full suites are opt-in. They generate substantial load, exercise error paths, +and can create/drop named test objects. They must not be run against a database +containing valuable data. Feature/permission differences across SQL Server, +Azure SQL, and Fabric can produce failures or explicit skips; report these +instead of treating them as proof of a product bug. + +Use the same target/authentication options as smoke, but select: + +```text +-Tier Full -AllowLiveTests -AllowLoadTests +``` + +The sqlsim runner passes `--skip-scale` by default, excluding its 1024-thread +tests. To opt into those as well, add `-IncludeScale`; the rest of the full suite +still includes significant concurrency and performance checks without it. +The wrapper stops at the first failed suite, preserves its exit code in the +report, and exits nonzero. Individual suites retain their existing failure and +skip behavior; read their logs as well as the overall runner status. + +The CLI Bash filenames retain `-linux.sh` because these are the same maintained +Unix suites, not divergent macOS copies. They run from the kit's installed +paths without compilation. Live execution on macOS remains unverified until +recorded against an actual SQL target. + +## Reports and feedback + +Each invocation creates a fresh `results/-/` directory with +`results.json` plus separate stdout/stderr logs for each check. A failed run +still writes a JSON report. You can choose a new output directory with +`-ReportDirectory /path/to/new-results-directory`. + +Reply in your existing Private Preview request issue with: + +- Mac chip, macOS version, tool selection, and build commit. +- Tier and SQL target product/version (no private hostname needed). +- Authentication mode, not the credentials. +- What passed, failed, or was skipped. +- Sanitized report/log excerpts and reproduction steps. + +The kit uploads nothing. Review all logs before attaching them: they can contain +server/database names, SQL text, result rows, and error details. Never share +passwords, tokens, credential files, credential-bearing connection strings, or +customer data. Report vulnerabilities privately under the repository's +[security policy](https://github.com/microsoft/sqlsimtools/security/policy). diff --git a/release/run-macos-tests.ps1 b/release/run-macos-tests.ps1 new file mode 100644 index 0000000..dcfa65f --- /dev/null +++ b/release/run-macos-tests.ps1 @@ -0,0 +1,190 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param( + [ValidateSet('sqlsim', 'sqlpysim', 'sqlsimstudio')] + [string[]]$Tool = @('sqlsim', 'sqlpysim', 'sqlsimstudio'), + [ValidateSet('Offline', 'Smoke', 'Full')] + [string]$Tier = 'Offline', + [string]$Server, + [string]$Database, + [string]$Username, + [string]$PasswordFile, + [string]$TokenFile, + [string]$Python = 'python3', + [string]$ReportDirectory, + [switch]$AllowLiveTests, + [switch]$AllowLoadTests, + [switch]$IncludeScale +) + +$ErrorActionPreference = 'Stop' +if (-not $IsMacOS -or [Runtime.InteropServices.RuntimeInformation]::OSArchitecture -ne 'Arm64') { + throw 'This tester kit requires macOS on Apple Silicon.' +} +if ($Tool.Count -eq 0) { throw 'Select at least one tool.' } +if ($Tier -eq 'Offline') { + if ($Server -or $Database -or $Username -or $PasswordFile -or $TokenFile -or $AllowLiveTests -or $AllowLoadTests -or $IncludeScale) { + throw 'Offline tests do not accept live target, credential, or load-testing options.' + } +} else { + if (-not $AllowLiveTests -or -not $Server -or -not $Database) { + throw 'Live tests require AllowLiveTests, Server, and Database. Use a dedicated test database.' + } + if ([bool]$TokenFile -eq [bool]$PasswordFile -or ($PasswordFile -and -not $Username) -or ($TokenFile -and $Username)) { + throw 'Choose exactly one: TokenFile, or Username with PasswordFile.' + } + $secretPath = (Resolve-Path -LiteralPath $(if ($TokenFile) { $TokenFile } else { $PasswordFile })).Path + if (-not (Get-Content -LiteralPath $secretPath -TotalCount 1)) { + throw 'The credential file must contain a non-empty first line.' + } + if ($Tier -eq 'Full' -and -not $AllowLoadTests) { + throw 'Full suites require AllowLoadTests as well as AllowLiveTests; they generate substantial load and create test objects.' + } +} +if ($IncludeScale -and ($Tier -ne 'Full' -or $Tool -notcontains 'sqlsim')) { + throw 'IncludeScale applies only to the sqlsim Full suite.' +} + +$root = $PSScriptRoot +$bash = (Get-Command bash -CommandType Application -ErrorAction Stop).Source +$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop).Source +$pythonPath = $null +if ($Tool -contains 'sqlpysim' -or $Tool -contains 'sqlsimstudio') { + $pythonPath = (Get-Command $Python -CommandType Application -ErrorAction Stop).Source +} +if (-not $ReportDirectory) { + $ReportDirectory = Join-Path $root ('results/{0}-{1}' -f (Get-Date -Format 'yyyyMMdd-HHmmss'), [guid]::NewGuid().ToString('N')) +} +$reportRoot = [IO.Path]::GetFullPath($ReportDirectory) +if (Test-Path -LiteralPath $reportRoot) { throw "ReportDirectory must be a new directory: $reportRoot" } +New-Item -ItemType Directory -Path $reportRoot -Force | Out-Null +$results = [Collections.Generic.List[object]]::new() +$started = [DateTime]::UtcNow +$failure = $null + +function Invoke-KitCheck { + param([string]$Name, [string]$Executable, [string[]]$Arguments, [string]$WorkingDirectory) + + Write-Host "Running $Name..." + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $Executable + $start.WorkingDirectory = $WorkingDirectory + $start.UseShellExecute = $false + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { $start.ArgumentList.Add($argument) } + foreach ($variable in @('SQLSIM_SERVER', 'SQLSIM_DATABASE', 'SQLSIM_USERNAME', 'SQLSIM_PASSWORD', 'SQLSIM_ACCESS_TOKEN', + 'SQLSIM_STUDIO_SQLSIM_PATH', 'SQLSIM_STUDIO_PYTHON_PATH')) { + $start.Environment.Remove($variable) | Out-Null + } + if ($pythonPath) { $start.Environment['SQLSIM_STUDIO_PYTHON_PATH'] = $pythonPath } + $stdoutPath = Join-Path $reportRoot "$Name.stdout.log" + $stderrPath = Join-Path $reportRoot "$Name.stderr.log" + $stdout = [IO.File]::Create($stdoutPath) + $stderr = [IO.File]::Create($stderrPath) + $process = $null + try { + $process = [Diagnostics.Process]::Start($start) + $outTask = $process.StandardOutput.BaseStream.CopyToAsync($stdout) + $errTask = $process.StandardError.BaseStream.CopyToAsync($stderr) + $process.WaitForExit() + $outTask.GetAwaiter().GetResult() + $errTask.GetAwaiter().GetResult() + $results.Add([pscustomobject]@{ + name = $Name + status = if ($process.ExitCode -eq 0) { 'passed' } else { 'failed' } + exitCode = $process.ExitCode + stdout = [IO.Path]::GetFileName($stdoutPath) + stderr = [IO.Path]::GetFileName($stderrPath) + }) + if ($process.ExitCode -ne 0) { + throw "$Name failed with exit code $($process.ExitCode). See $stdoutPath and $stderrPath" + } + Write-Host "PASS: $Name" + } finally { + if ($process) { + if (-not $process.HasExited) { $process.Kill($true); $process.WaitForExit() } + $process.Dispose() + } + $stdout.Dispose() + $stderr.Dispose() + } +} + +try { + if ($pythonPath) { + Invoke-KitCheck 'python-prerequisites' $pythonPath @('-c', 'import sys; assert sys.version_info >= (3, 11), "Python 3.11+ required"; import mssql_python') $root + } + $auth = @() + if ($Tier -ne 'Offline') { + $auth = @('--server', $Server, '--database', $Database) + if ($TokenFile) { $auth += @('--token-file', $secretPath) } + else { $auth += @('--username', $Username, '--password-file', $secretPath) } + } + if ($Tool -contains 'sqlsim') { + $directory = Join-Path $root 'sqlsim' + $binary = Join-Path $directory 'arm64/sqlsim' + if ($Tier -eq 'Offline') { + Invoke-KitCheck 'sqlsim-offline' $bash @('smoke-test-macos.sh', '--skip-build', '--sqlsim', $binary) $directory + } elseif ($Tier -eq 'Smoke') { + Invoke-KitCheck 'sqlsim-smoke' $bash (@('smoke-test-linux.sh', '--skip-build', '--sqlsim', $binary) + $auth) $directory + } else { + $arguments = @('full-test-suite-linux.sh', '--sqlsim', $binary) + $auth + if (-not $IncludeScale) { $arguments += '--skip-scale' } + Invoke-KitCheck 'sqlsim-full' $bash $arguments $directory + } + } + if ($Tool -contains 'sqlpysim') { + $directory = Join-Path $root 'sqlpysim' + if ($Tier -eq 'Offline') { + Invoke-KitCheck 'sqlpysim-help' $pythonPath @('sqlpysim.py', '--help') $directory + Invoke-KitCheck 'sqlpysim-version' $pythonPath @('sqlpysim.py', '--version') $directory + Invoke-KitCheck 'sqlpysim-unit-tests' $pythonPath @('-m', 'unittest', 'discover', '-s', 'tests', '-p', 'test_*.py', '-v') $directory + } else { + $suite = if ($Tier -eq 'Smoke') { 'smoke-test-linux.sh' } else { 'full-test-suite-linux.sh' } + Invoke-KitCheck "sqlpysim-$($Tier.ToLowerInvariant())" $bash (@($suite, '--python', $pythonPath) + $auth) $directory + } + } + if ($Tool -contains 'sqlsimstudio') { + $directory = Join-Path $root 'sqlsimstudio' + $executable = Join-Path $directory 'sqlsimstudio' + $arguments = @('-NoProfile', '-File', (Join-Path $directory 'smoke-test.ps1'), + '-SkipBuild', '-StudioExecutable', $executable) + if ($Tier -eq 'Offline') { + Invoke-KitCheck 'studio-startup' $pwsh ($arguments + '-StartupOnly') $directory + Invoke-KitCheck 'studio-offline-full' $pwsh @('-NoProfile', '-File', (Join-Path $directory 'full-test-suite.ps1'), + '-SkipBuild', '-StudioExecutable', $executable, '-OfflineOnly') $directory + } else { + $studioAuth = @('-Server', $Server, '-Database', $Database, '-SecretFile', $secretPath) + if ($TokenFile) { $studioAuth += @('-Authentication', 'token') } + else { $studioAuth += @('-Authentication', 'sql', '-Username', $Username) } + Invoke-KitCheck 'studio-smoke' $pwsh ($arguments + $studioAuth) $directory + if ($Tier -eq 'Full') { + Invoke-KitCheck 'studio-full' $pwsh (@('-NoProfile', '-File', (Join-Path $directory 'full-test-suite.ps1'), + '-SkipBuild', '-StudioExecutable', $executable) + $studioAuth) $directory + } + } + } +} catch { + $failure = $_ + Write-Warning $_.Exception.Message +} finally { + $report = [ordered]@{ + schemaVersion = 1 + tier = $Tier + tools = $Tool + startedUtc = $started.ToString('o') + completedUtc = [DateTime]::UtcNow.ToString('o') + os = [Runtime.InteropServices.RuntimeInformation]::OSDescription + architecture = [Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString() + build = @(Get-Content -LiteralPath (Join-Path $root 'BUILD-INFO.txt')) + status = if ($failure) { 'failed' } else { 'passed' } + error = if ($failure) { $failure.Exception.Message } else { $null } + checks = @($results.ToArray()) + } + $report | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $reportRoot 'results.json') -Encoding utf8 + Write-Host "Report: $reportRoot" + Write-Host 'Review logs for sensitive data before sharing. No results are uploaded automatically.' +} +if ($failure) { throw $failure } diff --git a/release/test-macos-tester-kit.ps1 b/release/test-macos-tester-kit.ps1 new file mode 100644 index 0000000..04dc764 --- /dev/null +++ b/release/test-macos-tester-kit.ps1 @@ -0,0 +1,111 @@ +#!/usr/bin/env pwsh + +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$KitRoot, + [Parameter(Mandatory)][string]$ValidationDirectory +) + +$ErrorActionPreference = 'Stop' +$root = (Resolve-Path -LiteralPath $KitRoot).Path +if (Test-Path -LiteralPath $ValidationDirectory) { throw 'ValidationDirectory must be a new directory.' } +New-Item -ItemType Directory -Path $ValidationDirectory -Force | Out-Null +$runner = Join-Path $root 'run-tests.ps1' +$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop).Source +$bash = (Get-Command bash -CommandType Application -ErrorAction Stop).Source +$dummySecret = Join-Path $ValidationDirectory 'synthetic-token.txt' +$emptySecret = Join-Path $ValidationDirectory 'empty-token.txt' +Set-Content -LiteralPath $dummySecret -Value 'synthetic-noncredential' +Set-Content -LiteralPath $emptySecret -Value '' -NoNewline + +function Assert-Rejected { + param([string]$Name, [string[]]$Arguments, [string]$Expected) + + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $pwsh + $start.UseShellExecute = $false + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in (@('-NoProfile', '-File', $runner) + $Arguments)) { + $start.ArgumentList.Add($argument) + } + $process = [Diagnostics.Process]::Start($start) + try { + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(10000)) { + $process.Kill($true) + $process.WaitForExit() + throw "$Name did not reject input promptly." + } + $text = $stdout.GetAwaiter().GetResult() + $stderr.GetAwaiter().GetResult() + if ($process.ExitCode -eq 0 -or $text -notmatch $Expected -or $text -match 'Running .+\.\.\.') { + throw "$Name did not reject input before running suites: $text" + } + Write-Host "PASS: $Name" + } finally { + if (-not $process.HasExited) { $process.Kill($true); $process.WaitForExit() } + $process.Dispose() + } +} + +Assert-Rejected 'live consent required' @('-Tier', 'Smoke', '-Server', 'test.invalid', '-Database', 'test') 'AllowLiveTests' +Assert-Rejected 'explicit live target required' @('-Tier', 'Smoke', '-AllowLiveTests') 'Server, and Database' +Assert-Rejected 'offline cannot become live' @('-Tier', 'Offline', '-Server', 'test.invalid') 'Offline tests do not accept' +Assert-Rejected 'ambiguous credentials rejected' @('-Tier', 'Smoke', '-AllowLiveTests', '-Server', 'test.invalid', + '-Database', 'test', '-Username', 'test', '-PasswordFile', $dummySecret, '-TokenFile', $dummySecret) 'Choose exactly one' +Assert-Rejected 'SQL username required' @('-Tier', 'Smoke', '-AllowLiveTests', '-Server', 'test.invalid', + '-Database', 'test', '-PasswordFile', $dummySecret) 'Choose exactly one' +Assert-Rejected 'empty credential rejected' @('-Tier', 'Smoke', '-AllowLiveTests', '-Server', 'test.invalid', + '-Database', 'test', '-TokenFile', $emptySecret) 'non-empty first line' +Assert-Rejected 'full load consent required' @('-Tier', 'Full', '-AllowLiveTests', '-Server', 'test.invalid', + '-Database', 'test', '-TokenFile', $dummySecret) 'Full suites require AllowLoadTests' +Assert-Rejected 'scale cannot bypass full tier' @('-Tier', 'Smoke', '-AllowLiveTests', '-Server', 'test.invalid', + '-Database', 'test', '-TokenFile', $dummySecret, '-IncludeScale') 'IncludeScale applies only' + +$failureReports = Join-Path $ValidationDirectory 'expected-failure-results' +$falseExecutable = (Get-Command false -CommandType Application -ErrorAction Stop).Source +& $pwsh -NoProfile -File $runner -Tier Offline -Tool sqlpysim -Python $falseExecutable -ReportDirectory $failureReports +if ($LASTEXITCODE -eq 0) { throw 'A failed prerequisite was reported as success.' } +$failureReport = Get-Content -LiteralPath (Join-Path $failureReports 'results.json') -Raw | ConvertFrom-Json +if ($failureReport.status -ne 'failed' -or -not $failureReport.error -or $failureReport.checks.Count -ne 1 -or + $failureReport.checks[0].name -ne 'python-prerequisites' -or $failureReport.checks[0].exitCode -eq 0) { + throw 'A failed check did not produce the required failure report.' +} +Write-Host 'PASS: failed prerequisites produce a nonzero exit and a JSON failure report' + +foreach ($relative in @('sqlsim/smoke-test-macos.sh', 'sqlsim/smoke-test-linux.sh', 'sqlsim/full-test-suite-linux.sh', + 'sqlpysim/smoke-test-linux.sh', 'sqlpysim/full-test-suite-linux.sh')) { + $path = Join-Path $root $relative + & $bash -n $path + if ($LASTEXITCODE -ne 0) { throw "Shell syntax validation failed: $relative" } + $help = & $bash $path --help | Out-String + if ($LASTEXITCODE -ne 0 -or $help -notmatch 'Usage:') { throw "Suite help failed: $relative" } + if ($relative -notmatch 'smoke-test-macos') { + $inventory = @(& $bash $path --list-tests) + if ($LASTEXITCODE -ne 0 -or $inventory.Count -lt 10) { throw "Missing test inventory: $relative" } + $fileParsing = @(& $bash $path --token-file $dummySecret --list-tests) + if ($LASTEXITCODE -ne 0 -or $fileParsing.Count -ne $inventory.Count) { + throw "Credential-file parsing changed the test inventory: $relative" + } + } +} + +$reports = Join-Path $ValidationDirectory 'offline-results' +& $pwsh -NoProfile -File $runner -Tier Offline -ReportDirectory $reports +if ($LASTEXITCODE -ne 0) { throw 'Extracted tester kit offline run failed.' } +$report = Get-Content -LiteralPath (Join-Path $reports 'results.json') -Raw | ConvertFrom-Json +$expectedChecks = @('python-prerequisites', 'sqlsim-offline', 'sqlpysim-help', 'sqlpysim-version', + 'sqlpysim-unit-tests', 'studio-startup', 'studio-offline-full') +if ($report.status -ne 'passed' -or $report.tier -ne 'Offline' -or + $report.tools.Count -ne 3 -or $report.checks.Count -ne $expectedChecks.Count -or + @($report.checks | Where-Object { $_.status -ne 'passed' -or $_.exitCode -ne 0 }).Count -ne 0 -or + @(Compare-Object ($expectedChecks | Sort-Object) ($report.checks.name | Sort-Object)).Count -ne 0) { + throw 'Offline report does not contain exactly the seven required passing checks.' +} +foreach ($check in $report.checks) { + foreach ($log in @($check.stdout, $check.stderr)) { + if (-not (Test-Path -LiteralPath (Join-Path $reports $log))) { throw "Missing report log: $log" } + } +} +Write-Host 'Extracted macOS tester kit validation passed.' diff --git a/sqlsim/smoke-test-linux.sh b/sqlsim/smoke-test-linux.sh index 0d99734..6f30e23 100644 --- a/sqlsim/smoke-test-linux.sh +++ b/sqlsim/smoke-test-linux.sh @@ -10,6 +10,7 @@ # bash smoke-test-linux.sh --skip-build --server HOST[,PORT] --database DB \ # (--username USER --password-file FILE | --token-file FILE) # bash smoke-test-linux.sh --list-tests +# macOS packages: add --skip-build --sqlsim ./arm64/sqlsim # # Live tests also accept the environment interface used by CI: # SQLSIM_SERVER, SQLSIM_DATABASE, SQLSIM_ACCESS_TOKEN, @@ -81,6 +82,12 @@ cli_token="" while (($# > 0)); do case "$1" in --skip-build) skip_build=true; shift ;; + --sqlsim) + if [[ $# -lt 2 || -z "$2" ]]; then + echo "ERROR: --sqlsim requires an executable path." >&2 + exit 2 + fi + binary="$2"; shift 2 ;; --server) cli_server="$2"; shift 2 ;; --database) cli_database="$2"; shift 2 ;; --username) cli_username="$2"; shift 2 ;; @@ -110,7 +117,7 @@ if [[ "$skip_build" == false ]]; then fi if [[ ! -x "$binary" ]]; then - echo "ERROR: Linux sqlsim binary not found at $binary" >&2 + echo "ERROR: sqlsim binary not found at $binary" >&2 exit 1 fi @@ -208,7 +215,15 @@ fi signal_output="" signal_status=0 set +e -signal_output="$(timeout --preserve-status --signal=INT 1 "$binary" -S 192.0.2.1,1433 -E -l 30 2>&1)" +timeout_command=timeout +if [[ "$(uname -s)" == "Darwin" ]]; then + timeout_command=gtimeout +fi +if ! command -v "$timeout_command" >/dev/null 2>&1; then + echo "ERROR: $timeout_command is required for the signal test (macOS: brew install coreutils)." >&2 + exit 1 +fi +signal_output="$("$timeout_command" --preserve-status --signal=INT 1 "$binary" -S 192.0.2.1,1433 -E -l 30 2>&1)" signal_status=$? set -e if [[ $signal_status -eq 130 && "$signal_output" == *"Received termination signal"* ]]; then @@ -223,7 +238,14 @@ fi live_tests=0 live_skipped=0 if [[ -n "${SQLSIM_SERVER:-}" ]]; then - bash "$script_dir/check-prerequisites-linux.sh" --require-driver + if [[ "$(uname -s)" == "Darwin" ]]; then + if ! command -v odbcinst >/dev/null 2>&1 || ! odbcinst -q -d | grep -q 'ODBC Driver 18 for SQL Server'; then + echo "ERROR: Microsoft ODBC Driver 18 is required for live macOS tests; see MACOS-PRIVATE-PREVIEW.md." >&2 + exit 1 + fi + else + bash "$script_dir/check-prerequisites-linux.sh" --require-driver + fi server="$SQLSIM_SERVER" database="${SQLSIM_DATABASE:-tempdb}" diff --git a/sqlsim/smoke-test-macos.sh b/sqlsim/smoke-test-macos.sh index ff63774..2b5629b 100644 --- a/sqlsim/smoke-test-macos.sh +++ b/sqlsim/smoke-test-macos.sh @@ -6,15 +6,23 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" binary="$script_dir/build/macos-arm64/sqlsim" skip_build=false -if [[ "${1:-}" == "--skip-build" ]]; then - skip_build=true - shift -fi - -if (($# > 0)); then - echo "Usage: bash smoke-test-macos.sh [--skip-build]" >&2 - exit 2 -fi +while (($# > 0)); do + case "$1" in + --skip-build) skip_build=true; shift ;; + --sqlsim) + if [[ $# -lt 2 || -z "$2" ]]; then + echo "ERROR: --sqlsim requires an executable path." >&2 + exit 2 + fi + binary="$2"; shift 2 ;; + -h|--help) + echo "Usage: bash smoke-test-macos.sh [--skip-build] [--sqlsim PATH]" + exit 0 ;; + *) + echo "Usage: bash smoke-test-macos.sh [--skip-build] [--sqlsim PATH]" >&2 + exit 2 ;; + esac +done if [[ "$skip_build" == false ]]; then bash "$script_dir/build-macos.sh" diff --git a/sqlsimstudio/full-test-suite.ps1 b/sqlsimstudio/full-test-suite.ps1 index 5cd3bfa..b5e7a0f 100644 --- a/sqlsimstudio/full-test-suite.ps1 +++ b/sqlsimstudio/full-test-suite.ps1 @@ -61,10 +61,23 @@ param( [string]$StudioExecutable, [switch]$SkipBuild, [switch]$OfflineOnly, - [switch]$StopOnError + [switch]$StopOnError, + [ValidateSet('integrated', 'sql', 'token')] + [string]$Authentication = 'integrated', + [string]$Username, + [string]$SecretFile ) $ErrorActionPreference = 'Stop' +$testSecret = $null +if (-not $OfflineOnly -and $Authentication -ne 'integrated') { + if (-not $SecretFile -or ($Authentication -eq 'sql' -and -not $Username)) { + throw 'SQL authentication requires Username and SecretFile; token authentication requires SecretFile.' + } + $testSecret = Get-Content -LiteralPath $SecretFile -TotalCount 1 + if (-not $testSecret) { throw 'SecretFile must contain a non-empty first line.' } +} +$liveAuthentication = $false $studioRoot = $PSScriptRoot $projectPath = Join-Path $studioRoot 'server\SqlsimStudio.csproj' $dllPath = Join-Path $studioRoot 'server\bin\Release\net10.0\sqlsimstudio.dll' @@ -78,9 +91,19 @@ $settingsDirectory = Join-Path ([IO.Path]::GetTempPath()) ("sqlsimstudio-full-se # /api/engines is that sqlsim was not found. $sqlsimCandidates = if ($StudioExecutable) { $studioDirectory = Split-Path -Parent (Resolve-Path $StudioExecutable).Path - @((Join-Path $studioDirectory 'sqlsim\sqlsim.exe'), (Join-Path $studioDirectory '..\sqlsim\x64\sqlsim.exe')) + if ($IsWindows) { + @((Join-Path $studioDirectory 'sqlsim\sqlsim.exe'), (Join-Path $studioDirectory '..\sqlsim\x64\sqlsim.exe')) + } else { + @(Join-Path $studioDirectory 'sqlsim/sqlsim') + } } else { - @(Join-Path $studioRoot '..\sqlsim\build\x64\Release\sqlsim.exe') + if ($IsWindows) { + @(Join-Path $studioRoot '..\sqlsim\build\x64\Release\sqlsim.exe') + } elseif ($IsMacOS) { + @(Join-Path $studioRoot '../sqlsim/build/macos-arm64/sqlsim') + } else { + @(Join-Path $studioRoot '../sqlsim/build/linux-x64/sqlsim') + } } $sqlsimExpected = [bool]$env:SQLSIM_STUDIO_SQLSIM_PATH -or [bool]($sqlsimCandidates | Where-Object { Test-Path -LiteralPath $_ }) @@ -524,6 +547,11 @@ try { trustServerCertificate = $true; stopOnError = $true telemetry = 'final'; statisticsSource = 'none' } + if ($liveAuthentication) { + $payload.authentication = $Authentication + $payload.username = if ($Authentication -eq 'sql') { $Username } else { $null } + $payload.secret = $testSecret + } foreach ($key in $Overrides.Keys) { $payload[$key] = $Overrides[$key] } return $payload } @@ -1057,6 +1085,7 @@ try { # ----------------------------------------------------------------------- if (-not $OfflineOnly) { + $liveAuthentication = $true Write-Section '9. Connection failure reporting' Test-Case 'connection test to an unreachable port reports a specific failure' { diff --git a/sqlsimstudio/smoke-test.ps1 b/sqlsimstudio/smoke-test.ps1 index 4e18f3e..9e93352 100644 --- a/sqlsimstudio/smoke-test.ps1 +++ b/sqlsimstudio/smoke-test.ps1 @@ -4,10 +4,24 @@ param( [switch]$SkipBuild, [switch]$StartupOnly, [string]$StudioExecutable, - [string]$QueryStoreDatabase + [string]$QueryStoreDatabase, + [string]$Server = 'localhost', + [string]$Database = 'tempdb', + [ValidateSet('integrated', 'sql', 'token')] + [string]$Authentication = 'integrated', + [string]$Username, + [string]$SecretFile ) $ErrorActionPreference = 'Stop' +$testSecret = $null +if (-not $StartupOnly -and $Authentication -ne 'integrated') { + if (-not $SecretFile -or ($Authentication -eq 'sql' -and -not $Username)) { + throw 'SQL authentication requires Username and SecretFile; token authentication requires SecretFile.' + } + $testSecret = Get-Content -LiteralPath $SecretFile -TotalCount 1 + if (-not $testSecret) { throw 'SecretFile must contain a non-empty first line.' } +} $studioRoot = $PSScriptRoot $projectPath = Join-Path $studioRoot 'server\SqlsimStudio.csproj' $dllPath = Join-Path $studioRoot 'server\bin\Release\net10.0\sqlsimstudio.dll' @@ -353,11 +367,11 @@ try { $baseRunPayload = @{ engine = 'sqlsim' - server = 'localhost' - database = 'tempdb' - authentication = 'integrated' - username = $null - secret = $null + server = $Server + database = $Database + authentication = $Authentication + username = if ($Authentication -eq 'sql') { $Username } else { $null } + secret = $testSecret threads = 1 iterations = 1 quiet = $true @@ -410,7 +424,7 @@ try { -not $connectionTest.serverVersion -or -not $connectionTest.edition -or -not $connectionTest.driver -or - $connectionTest.database -ne 'tempdb' -or + $connectionTest.database -ne $Database -or $null -eq $connectionTest.elapsedMs) { throw "Studio connection test did not report live server details: $($connectionTest | ConvertTo-Json -Depth 6 -Compress)" } @@ -520,9 +534,13 @@ try { $replayPayload = $baseRunPayload.Clone() $replayPayload.mode = 'replay' - $replayPayload.database = 'master' + $replayPayload.database = $Database $replayPayload.inputFileName = 'studio-smoke.xml' - $replayPayload.inputContent = Get-Content (Join-Path $studioRoot '..\sqlsim\testing\replay-test-data.xml') -Raw + [xml]$replayFixture = Get-Content (Join-Path $studioRoot '..\sqlsim\testing\replay-test-data.xml') -Raw + foreach ($recordedDatabase in $replayFixture.SelectNodes('//action[@name="database_name"]/value')) { + $recordedDatabase.InnerText = $Database + } + $replayPayload.inputContent = $replayFixture.OuterXml $replayPayload.replayMode = 'fast' $replayRun = Invoke-StudioRun -Payload $replayPayload -Name 'Replay' From aa58f6228a1849185e28ce9be61737a769d386b2 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 16:38:01 -0400 Subject: [PATCH 07/10] Resolve one executable when PATH contains multiple installations Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release/run-macos-tests.ps1 | 6 +++--- release/test-macos-tester-kit.ps1 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/release/run-macos-tests.ps1 b/release/run-macos-tests.ps1 index dcfa65f..83e23dd 100644 --- a/release/run-macos-tests.ps1 +++ b/release/run-macos-tests.ps1 @@ -47,11 +47,11 @@ if ($IncludeScale -and ($Tier -ne 'Full' -or $Tool -notcontains 'sqlsim')) { } $root = $PSScriptRoot -$bash = (Get-Command bash -CommandType Application -ErrorAction Stop).Source -$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop).Source +$bash = (Get-Command bash -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source +$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source $pythonPath = $null if ($Tool -contains 'sqlpysim' -or $Tool -contains 'sqlsimstudio') { - $pythonPath = (Get-Command $Python -CommandType Application -ErrorAction Stop).Source + $pythonPath = (Get-Command $Python -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source } if (-not $ReportDirectory) { $ReportDirectory = Join-Path $root ('results/{0}-{1}' -f (Get-Date -Format 'yyyyMMdd-HHmmss'), [guid]::NewGuid().ToString('N')) diff --git a/release/test-macos-tester-kit.ps1 b/release/test-macos-tester-kit.ps1 index 04dc764..a337353 100644 --- a/release/test-macos-tester-kit.ps1 +++ b/release/test-macos-tester-kit.ps1 @@ -11,8 +11,8 @@ $root = (Resolve-Path -LiteralPath $KitRoot).Path if (Test-Path -LiteralPath $ValidationDirectory) { throw 'ValidationDirectory must be a new directory.' } New-Item -ItemType Directory -Path $ValidationDirectory -Force | Out-Null $runner = Join-Path $root 'run-tests.ps1' -$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop).Source -$bash = (Get-Command bash -CommandType Application -ErrorAction Stop).Source +$pwsh = (Get-Command pwsh -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source +$bash = (Get-Command bash -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source $dummySecret = Join-Path $ValidationDirectory 'synthetic-token.txt' $emptySecret = Join-Path $ValidationDirectory 'empty-token.txt' Set-Content -LiteralPath $dummySecret -Value 'synthetic-noncredential' @@ -64,7 +64,7 @@ Assert-Rejected 'scale cannot bypass full tier' @('-Tier', 'Smoke', '-AllowLiveT '-Database', 'test', '-TokenFile', $dummySecret, '-IncludeScale') 'IncludeScale applies only' $failureReports = Join-Path $ValidationDirectory 'expected-failure-results' -$falseExecutable = (Get-Command false -CommandType Application -ErrorAction Stop).Source +$falseExecutable = (Get-Command false -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source & $pwsh -NoProfile -File $runner -Tier Offline -Tool sqlpysim -Python $falseExecutable -ReportDirectory $failureReports if ($LASTEXITCODE -eq 0) { throw 'A failed prerequisite was reported as success.' } $failureReport = Get-Content -LiteralPath (Join-Path $failureReports 'results.json') -Raw | ConvertFrom-Json From 5face70fd9c45938f33c84e6f43adaa5046b887d Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 16:43:15 -0400 Subject: [PATCH 08/10] Make Studio offline lifecycle tests portable and deterministic Use native Unix suspend/resume signals, isolate offline acceptance probes from configured targets, and stop accepted validation runs before continuing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sqlsimstudio/full-test-suite.ps1 | 60 +++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/sqlsimstudio/full-test-suite.ps1 b/sqlsimstudio/full-test-suite.ps1 index b5e7a0f..d189fbb 100644 --- a/sqlsimstudio/full-test-suite.ps1 +++ b/sqlsimstudio/full-test-suite.ps1 @@ -408,7 +408,7 @@ Test-Case 'Studio survives the machine sleeping' { # The closest a test can get to closing a laptop lid: suspend every thread # so no time passes for the process while the clock keeps running. This is # the failure that started the redesign, and it must stay fixed. - if (-not ('Native.ProcessControl' -as [type])) { + if ($IsWindows -and -not ('Native.ProcessControl' -as [type])) { Add-Type -Namespace Native -Name ProcessControl -MemberDefinition @' [System.Runtime.InteropServices.DllImport("ntdll.dll")] public static extern int NtSuspendProcess(System.IntPtr processHandle); @@ -426,9 +426,21 @@ public static extern int NtResumeProcess(System.IntPtr processHandle); $session = [Microsoft.PowerShell.Commands.WebRequestSession]::new() Invoke-WebRequest -Uri $bootstrapUrl -WebSession $session -MaximumRedirection 5 | Out-Null - $null = [Native.ProcessControl]::NtSuspendProcess($proc.Handle) + if ($IsWindows) { + $status = [Native.ProcessControl]::NtSuspendProcess($proc.Handle) + if ($status -ne 0) { return "suspending Studio failed: $status" } + } else { + & /bin/kill -STOP $proc.Id + if ($LASTEXITCODE -ne 0) { return 'suspending Studio with SIGSTOP failed' } + } Start-Sleep -Seconds 15 - $null = [Native.ProcessControl]::NtResumeProcess($proc.Handle) + if ($IsWindows) { + $status = [Native.ProcessControl]::NtResumeProcess($proc.Handle) + if ($status -ne 0) { return "resuming Studio failed: $status" } + } else { + & /bin/kill -CONT $proc.Id + if ($LASTEXITCODE -ne 0) { return 'resuming Studio with SIGCONT failed' } + } Start-Sleep -Seconds 3 if ($proc.HasExited) { return 'Studio exited while the machine was asleep' } @@ -552,6 +564,10 @@ try { $payload.username = if ($Authentication -eq 'sql') { $Username } else { $null } $payload.secret = $testSecret } + if ($OfflineOnly) { + $payload.server = '192.0.2.1,1' + $payload.loginTimeoutSeconds = 1 + } foreach ($key in $Overrides.Keys) { $payload[$key] = $Overrides[$key] } return $payload } @@ -563,6 +579,22 @@ try { if ($response.Status -ne [Net.HttpStatusCode]::BadRequest) { return "expected 400, got $($response.Status)" } + + function Stop-ValidationRun { + $stop = Invoke-StudioApi -Method 'DELETE' -Path '/api/runs/current' + if ($stop.Status -notin [Net.HttpStatusCode]::Accepted, [Net.HttpStatusCode]::NotFound) { + return "stopping the validation run returned $($stop.Status)" + } + $deadline = [DateTime]::UtcNow.AddSeconds(10) + do { + Start-Sleep -Milliseconds 100 + $state = (Invoke-StudioApi -Method GET -Path '/api/runs/current').Json.state + } while ($state -notin 'completed', 'failed', 'stopped' -and [DateTime]::UtcNow -lt $deadline) + if ($state -notin 'completed', 'failed', 'stopped') { + return "the validation run did not stop: $state" + } + return $null + } if (-not $response.Json.error) { return 'rejection carried no error message' } if ($ExpectedError -and $response.Json.error -notmatch $ExpectedError) { return "expected error matching '$ExpectedError', got '$($response.Json.error)'" @@ -800,16 +832,10 @@ try { $groups = @(@{ fileName = 'g.sql'; content = 'SELECT 1'; threads = 1; iterations = 1; thinkTimeMs = 0 }) $response = Invoke-StudioApi -Payload (New-RunPayload @{ engine = 'sqlpysim'; mode = 'workload'; query = $null; workloadGroups = $groups }) - if ($response.Status -eq [Net.HttpStatusCode]::BadRequest) { + if ($response.Status -ne [Net.HttpStatusCode]::Accepted) { return "mixed workload was rejected for sqlpysim: $($response.Json.error)" } - # Leave nothing running: a live run would refuse the next request. - $deadline = [DateTime]::UtcNow.AddSeconds(30) - do { - Start-Sleep -Milliseconds 200 - $state = (Invoke-StudioApi -Method GET -Path '/api/runs/current').Json.state - } while ($state -notin 'completed', 'failed', 'stopped' -and [DateTime]::UtcNow -lt $deadline) - return $null + return Stop-ValidationRun } Test-Case 'sqlpysim with replay is rejected' { @@ -895,18 +921,10 @@ try { $response = Invoke-StudioApi -Payload (New-RunPayload @{ engine = 'sqlpysim'; authentication = 'service-principal'; username = 'client-id'; secret = 'not-a-real-secret' }) - if ($response.Status -eq [Net.HttpStatusCode]::BadRequest) { + if ($response.Status -ne [Net.HttpStatusCode]::Accepted) { return "service principal was rejected for sqlpysim: $($response.Json.error)" } - $deadline = [DateTime]::UtcNow.AddSeconds(30) - do { - Start-Sleep -Milliseconds 200 - $run = (Invoke-StudioApi -Method GET -Path '/api/runs/current').Json - } while ($run.state -notin 'completed', 'failed', 'stopped' -and [DateTime]::UtcNow -lt $deadline) - if ($run.state -notin 'completed', 'failed') { - return "expected the authentication attempt to finish, got '$($run.state)'" - } - return $null + return Stop-ValidationRun } Test-Case 'an application name does not reach sqlpysim' -RequiresServer { From 1736b3e96e4faa4f7f3fa5b8261dc98f200b74de Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 16:46:49 -0400 Subject: [PATCH 09/10] Expose validation cleanup helper to Studio test cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sqlsimstudio/full-test-suite.ps1 | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/sqlsimstudio/full-test-suite.ps1 b/sqlsimstudio/full-test-suite.ps1 index d189fbb..3d33bfe 100644 --- a/sqlsimstudio/full-test-suite.ps1 +++ b/sqlsimstudio/full-test-suite.ps1 @@ -580,21 +580,6 @@ try { return "expected 400, got $($response.Status)" } - function Stop-ValidationRun { - $stop = Invoke-StudioApi -Method 'DELETE' -Path '/api/runs/current' - if ($stop.Status -notin [Net.HttpStatusCode]::Accepted, [Net.HttpStatusCode]::NotFound) { - return "stopping the validation run returned $($stop.Status)" - } - $deadline = [DateTime]::UtcNow.AddSeconds(10) - do { - Start-Sleep -Milliseconds 100 - $state = (Invoke-StudioApi -Method GET -Path '/api/runs/current').Json.state - } while ($state -notin 'completed', 'failed', 'stopped' -and [DateTime]::UtcNow -lt $deadline) - if ($state -notin 'completed', 'failed', 'stopped') { - return "the validation run did not stop: $state" - } - return $null - } if (-not $response.Json.error) { return 'rejection carried no error message' } if ($ExpectedError -and $response.Json.error -notmatch $ExpectedError) { return "expected error matching '$ExpectedError', got '$($response.Json.error)'" @@ -602,6 +587,22 @@ try { return $null } + function Stop-ValidationRun { + $stop = Invoke-StudioApi -Method 'DELETE' -Path '/api/runs/current' + if ($stop.Status -notin [Net.HttpStatusCode]::Accepted, [Net.HttpStatusCode]::NotFound) { + return "stopping the validation run returned $($stop.Status)" + } + $deadline = [DateTime]::UtcNow.AddSeconds(10) + do { + Start-Sleep -Milliseconds 100 + $state = (Invoke-StudioApi -Method GET -Path '/api/runs/current').Json.state + } while ($state -notin 'completed', 'failed', 'stopped' -and [DateTime]::UtcNow -lt $deadline) + if ($state -notin 'completed', 'failed', 'stopped') { + return "the validation run did not stop: $state" + } + return $null + } + # ----------------------------------------------------------------------- # 2. Authentication boundary # ----------------------------------------------------------------------- From 071c03cccd967b01533f3acdfe71bacbf30a4dc2 Mon Sep 17 00:00:00 2001 From: Bob Ward Date: Tue, 20 Oct 2026 16:55:39 -0400 Subject: [PATCH 10/10] Bound offline kit checks and use direct POSIX lifecycle signals Keep partial diagnostics readable if a check stalls, report offline timeouts, and avoid spawning a native command while Studio is suspended. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../sqlsim-macos-arm64-private-preview.yml | 1 + release/macos-tester-README.md | 3 +++ release/run-macos-tests.ps1 | 15 ++++++++++++--- sqlsimstudio/full-test-suite.ps1 | 17 ++++++++++++----- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sqlsim-macos-arm64-private-preview.yml b/.github/workflows/sqlsim-macos-arm64-private-preview.yml index bd82617..012fdac 100644 --- a/.github/workflows/sqlsim-macos-arm64-private-preview.yml +++ b/.github/workflows/sqlsim-macos-arm64-private-preview.yml @@ -157,6 +157,7 @@ jobs: tar -C "$RUNNER_TEMP/verify/tester-kit" -xzf "$RUNNER_TEMP/artifacts/sqlsimtools-macos-arm64-tester-kit.tar.gz" - name: Validate extracted tester kit and runner safety gates + timeout-minutes: 15 shell: pwsh run: | ./scripts/check-suite-parity.ps1 -Quiet diff --git a/release/macos-tester-README.md b/release/macos-tester-README.md index e5551e8..10fba6a 100644 --- a/release/macos-tester-README.md +++ b/release/macos-tester-README.md @@ -46,6 +46,9 @@ tests, and Studio startup plus its full offline validation suite. No configured database is contacted. Some native rejection tests probe a reserved, unreachable address to validate error reporting. +Each offline check has a five-minute limit. A timeout stops that check's process +tree, fails the run, and records `timedOut: true` alongside its logs. + To test just one tool, add `-Tool sqlsim`, `-Tool sqlpysim`, or `-Tool sqlsimstudio`. Python/Studio selections require the Python environment; the native-only selection does not. diff --git a/release/run-macos-tests.ps1 b/release/run-macos-tests.ps1 index 83e23dd..a0b57e8 100644 --- a/release/run-macos-tests.ps1 +++ b/release/run-macos-tests.ps1 @@ -81,23 +81,32 @@ function Invoke-KitCheck { if ($pythonPath) { $start.Environment['SQLSIM_STUDIO_PYTHON_PATH'] = $pythonPath } $stdoutPath = Join-Path $reportRoot "$Name.stdout.log" $stderrPath = Join-Path $reportRoot "$Name.stderr.log" - $stdout = [IO.File]::Create($stdoutPath) - $stderr = [IO.File]::Create($stderrPath) + $stdout = [IO.FileStream]::new($stdoutPath, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::Read, 1) + $stderr = [IO.FileStream]::new($stderrPath, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::Read, 1) $process = $null try { $process = [Diagnostics.Process]::Start($start) $outTask = $process.StandardOutput.BaseStream.CopyToAsync($stdout) $errTask = $process.StandardError.BaseStream.CopyToAsync($stderr) + $timedOut = $false + if ($Tier -eq 'Offline') { + $timedOut = -not $process.WaitForExit(300000) + if ($timedOut -and -not $process.HasExited) { $process.Kill($true) } + } $process.WaitForExit() $outTask.GetAwaiter().GetResult() $errTask.GetAwaiter().GetResult() $results.Add([pscustomobject]@{ name = $Name - status = if ($process.ExitCode -eq 0) { 'passed' } else { 'failed' } + status = if ($process.ExitCode -eq 0 -and -not $timedOut) { 'passed' } else { 'failed' } exitCode = $process.ExitCode + timedOut = $timedOut stdout = [IO.Path]::GetFileName($stdoutPath) stderr = [IO.Path]::GetFileName($stderrPath) }) + if ($timedOut) { + throw "$Name exceeded the five-minute offline check limit. See $stdoutPath and $stderrPath" + } if ($process.ExitCode -ne 0) { throw "$Name failed with exit code $($process.ExitCode). See $stdoutPath and $stderrPath" } diff --git a/sqlsimstudio/full-test-suite.ps1 b/sqlsimstudio/full-test-suite.ps1 index 3d33bfe..5affc45 100644 --- a/sqlsimstudio/full-test-suite.ps1 +++ b/sqlsimstudio/full-test-suite.ps1 @@ -408,12 +408,14 @@ Test-Case 'Studio survives the machine sleeping' { # The closest a test can get to closing a laptop lid: suspend every thread # so no time passes for the process while the clock keeps running. This is # the failure that started the redesign, and it must stay fixed. - if ($IsWindows -and -not ('Native.ProcessControl' -as [type])) { + if (-not ('Native.ProcessControl' -as [type])) { Add-Type -Namespace Native -Name ProcessControl -MemberDefinition @' [System.Runtime.InteropServices.DllImport("ntdll.dll")] public static extern int NtSuspendProcess(System.IntPtr processHandle); [System.Runtime.InteropServices.DllImport("ntdll.dll")] public static extern int NtResumeProcess(System.IntPtr processHandle); +[System.Runtime.InteropServices.DllImport("libc", EntryPoint = "kill", SetLastError = true)] +public static extern int SendSignal(int processId, int signal); '@ } @@ -430,16 +432,21 @@ public static extern int NtResumeProcess(System.IntPtr processHandle); $status = [Native.ProcessControl]::NtSuspendProcess($proc.Handle) if ($status -ne 0) { return "suspending Studio failed: $status" } } else { - & /bin/kill -STOP $proc.Id - if ($LASTEXITCODE -ne 0) { return 'suspending Studio with SIGSTOP failed' } + # Darwin and Linux assign different numbers to STOP and CONT. + $stopSignal = if ($IsMacOS) { 17 } else { 19 } + $continueSignal = if ($IsMacOS) { 19 } else { 18 } + if ([Native.ProcessControl]::SendSignal($proc.Id, $stopSignal) -ne 0) { + return "SIGSTOP failed: $([Runtime.InteropServices.Marshal]::GetLastPInvokeError())" + } } Start-Sleep -Seconds 15 if ($IsWindows) { $status = [Native.ProcessControl]::NtResumeProcess($proc.Handle) if ($status -ne 0) { return "resuming Studio failed: $status" } } else { - & /bin/kill -CONT $proc.Id - if ($LASTEXITCODE -ne 0) { return 'resuming Studio with SIGCONT failed' } + if ([Native.ProcessControl]::SendSignal($proc.Id, $continueSignal) -ne 0) { + return "SIGCONT failed: $([Runtime.InteropServices.Marshal]::GetLastPInvokeError())" + } } Start-Sleep -Seconds 3