From b8456215dafd9c5ef1ca0de4af7c37c5c04f88e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:27:59 +0530 Subject: [PATCH 01/31] Add OneBranch release pipelines for mssql-python-odbc Adds the build and release pipelines for the standalone mssql-python-odbc package: per-platform build stages (Windows x64/arm64, macOS universal2, Linux manylinux_2_28 + musllinux_1_2 for x86_64/aarch64), a consolidate job that gathers all 7 py3-none wheels, the build orchestration pipeline, and the ESRP-based official release pipeline. The build stages run no compilation (data-only package); Linux builds run in tdslibrs ACR PyPA containers and the musllinux tag is auto-detected via libc_ver. Infra follow-ups (ESRP onboarding of the mssql-python-odbc name, registering the ADO build definition to set odbcBuildDefinitionId, and sharing the ESRP variable group) are marked with TODO(infra) comments. --- .../build-release-odbc-pipeline.yml | 210 ++++++++++++++++ .../jobs/consolidate-odbc-artifacts-job.yml | 68 ++++++ .../official-release-odbc-pipeline.yml | 227 ++++++++++++++++++ .../stages/build-odbc-linux-stage.yml | 163 +++++++++++++ .../stages/build-odbc-macos-stage.yml | 107 +++++++++ .../stages/build-odbc-windows-stage.yml | 125 ++++++++++ 6 files changed, 900 insertions(+) create mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml create mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/stages/build-odbc-linux-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-macos-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-windows-stage.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml new file mode 100644 index 000000000..260fd1c3c --- /dev/null +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -0,0 +1,210 @@ +# ========================================================================================= +# OneBranch Build Pipeline for mssql-python-odbc +# ========================================================================================= +# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. +# +# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is +# NO compiled Python extension, so every wheel is tagged `py3-none-` and a +# SINGLE build per platform covers all supported Python versions (3.10+). That is why +# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces +# only 7 wheels total: +# - Windows: x64, arm64 +# - macOS: universal2 +# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) +# +# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml +# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). +# +# INFRA PREREQUISITES (see companion release pipeline for the definition ID): +# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). +# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. +# - Ensure the reused pools/images are available: Python-1ES-pool +# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the +# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. +# ========================================================================================= + +# Build number format: YYDDD.r +name: $(Year:YY)$(DayOfYear)$(Rev:.r) + +# Only trigger when ODBC-package inputs change (path filters are literal prefixes, +# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. +trigger: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +pr: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + + - name: runSdlTasks + displayName: 'Run SDL Security Tasks' + type: boolean + default: true + + # Windows: one wheel per architecture (NO per-Python matrix). + - name: odbcWindowsConfigs + type: object + default: + - arch: 'x64' + - arch: 'arm64' + + # macOS: single universal2 wheel. + - name: odbcMacosConfigs + type: object + default: + - name: 'universal2' + + # Linux: one wheel per libc/arch combo. + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + +variables: + # Scheduled builds run as Official; manual/PR builds use the parameter. + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the + # per-stage EsrpMalwareScanning tasks. + - template: /OneBranchPipelines/variables/signing-variables.yml@self + - group: 'ESRP Federated Creds (AME)' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + + # This pipeline compiles NO first-party native code — it only packages the + # pre-built ODBC driver binaries, which are signed and governed by the Microsoft + # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; + # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). + binskim: + enabled: false + justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' + apiscan: + enabled: false + justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' + armory: + enabled: false + codeql: + enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' + + credscan: + enabled: ${{ parameters.runSdlTasks }} + policheck: + enabled: ${{ parameters.runSdlTasks }} + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: ${{ parameters.runSdlTasks }} + sbom: + enabled: ${{ parameters.runSdlTasks }} + packageName: 'mssql-python-odbc' + tsa: + enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + # WINDOWS BUILD STAGES (2: x64, arm64) + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # MACOS BUILD STAGE (1: universal2) + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # CONSOLIDATE STAGE (waits for all 7 platform stages) + - stage: Consolidate + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..8ac829631 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,68 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download ALL artifacts from the current build (one per platform stage). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" + else + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + fi + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_Consolidate_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml new file mode 100644 index 000000000..3a9898c25 --- /dev/null +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -0,0 +1,227 @@ +# ========================================================================================= +# OneBranch Official Release Pipeline for mssql-python-odbc +# ========================================================================================= +# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to +# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group +# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python +# release (same inputs and FolderLocation); the only differences are the build pipeline +# source and that this package has no debug symbols to publish. +# +# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the +# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# ========================================================================================= + +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC + +# Manual trigger only — releases should be deliberate. +trigger: none +pr: none + +parameters: + - name: releaseToPyPI + displayName: 'Release to PyPI (Production)' + type: boolean + default: false # Safety: default false to prevent accidental releases. + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Shared ESRP identity — same group used by the mssql-python release pipeline. + - group: 'ESRP Federated Creds (AME)' + # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # once it is registered (the mssql-python build definition is 2199). Left as 0 so the + # pipeline compiles; the download tasks will fail at runtime until this is set. + - name: odbcBuildDefinitionId + value: '0' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + + # Reference to the ODBC build pipeline (select a run in the UI when releasing). + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + + # Artifacts were already scanned during the build; this pipeline only republishes. + binskim: + enabled: false + justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: ReleasePackages + displayName: 'Release mssql-python-odbc to PyPI' + + jobs: + # Job 1: Download + validate the consolidated wheels (custom pool). + - job: PrepAndValidate + displayName: 'Download and Validate Artifacts' + + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Consolidated ODBC Artifacts' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + - task: PowerShell@2 + displayName: 'List and Verify Downloaded Wheels' + inputs: + targetType: 'inline' + script: | + $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" + if (-not (Test-Path $wheelsPath)) { + Write-Error "Wheel directory not found at: $wheelsPath" + exit 1 + } + $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" + Write-Host "Total wheel files found: $($wheels.Count)" + + $allValid = $true + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($wheel in $wheels) { + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) + $entryCount = $zip.Entries.Count + $zip.Dispose() + Write-Host " OK $($wheel.Name) ($entryCount entries)" + } catch { + Write-Error " BAD $($wheel.Name): $_" + $allValid = $false + } + } + if (-not $allValid) { + Write-Error "One or more wheel files are corrupted" + exit 1 + } + if ($wheels.Count -ne 7) { + Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + } + Write-Host "All wheels verified successfully!" + + # Dry-run summary (when release is disabled). + - ${{ if eq(parameters.releaseToPyPI, false) }}: + - task: PowerShell@2 + displayName: 'Dry Run - Release Skipped' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "DRY RUN MODE - No Release Performed" + Write-Host "Package: mssql-python-odbc" + Write-Host "====================================" + Write-Host "To perform an actual release:" + Write-Host "1. Set 'releaseToPyPI' parameter to true" + Write-Host "2. Re-run pipeline" + + # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). + # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. + - ${{ if eq(parameters.releaseToPyPI, true) }}: + - job: PyPIRelease + displayName: 'ESRP Release to PyPI' + dependsOn: PrepAndValidate + + templateContext: + type: releaseJob + isProduction: true + + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Artifacts from ODBC Build Pipeline' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + # ESRP release task — identical to the mssql-python release pipeline + # (same shared ESRP identity and PyPI publish inputs). + - task: EsrpRelease@9 + displayName: 'ESRP Release to PyPI' + inputs: + connectedservicename: '$(ESRPConnectedServiceName)' + usemanagedidentity: true + keyvaultname: '$(AuthAKVName)' + signcertname: '$(AuthSignCertName)' + clientid: '$(EsrpClientId)' + Intent: 'PackageDistribution' + ContentType: 'PyPI' + ContentSource: 'Folder' + FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' + WaitForReleaseCompletion: true + Owners: '$(owner)' + Approvers: '$(approver)' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMAN' + DomainTenantId: '$(DomainTenantId)' + + - task: PowerShell@2 + displayName: 'Release Summary' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "ESRP Release Completed" + Write-Host "Package: mssql-python-odbc" + Write-Host "Target: PyPI" + Write-Host "====================================" + Write-Host "Next steps:" + Write-Host "1. Verify release in the ESRP portal" + Write-Host "2. Wait for approval workflow completion" + Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..798b3c341 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,163 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel + cd /workspace + $PY setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..9b47388e3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,107 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..7a812de45 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,125 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). + targetArch: ${{ parameters.architecture }} + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 From c19a261ed16c625e4f35f9111941600fa9bcd3d4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:43:11 +0530 Subject: [PATCH 02/31] Address PR review: fail-fast on wheel count, armory justification, Linux twine check + container cleanup, reword infra TODOs --- OneBranchPipelines/build-release-odbc-pipeline.yml | 1 + .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++--- OneBranchPipelines/official-release-odbc-pipeline.yml | 9 +++++---- OneBranchPipelines/stages/build-odbc-linux-stage.yml | 8 +++++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml index 260fd1c3c..b64dc936b 100644 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -144,6 +144,7 @@ extends: justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' armory: enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' codeql: enabled: false justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 8ac829631..0225d34a2 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -54,10 +54,10 @@ jobs: WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then - echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" - else - echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." displayName: 'Consolidate ODBC wheels' - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 3a9898c25..90eaedddb 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -8,7 +8,7 @@ # source and that this package has no debug symbols to publish. # # This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# `mssql-python-odbc` project name lands and the two infra values below are set. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC @@ -28,7 +28,7 @@ variables: - template: /OneBranchPipelines/variables/onebranch-variables.yml@self # Shared ESRP identity — same group used by the mssql-python release pipeline. - group: 'ESRP Federated Creds (AME)' - # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' # once it is registered (the mssql-python build definition is 2199). Left as 0 so the # pipeline compiles; the download tasks will fail at runtime until this is set. - name: odbcBuildDefinitionId @@ -44,7 +44,7 @@ resources: # Reference to the ODBC build pipeline (select a run in the UI when releasing). pipelines: - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. trigger: none extends: @@ -141,7 +141,8 @@ extends: exit 1 } if ($wheels.Count -ne 7) { - Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" + exit 1 } Write-Host "All wheels verified successfully!" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 798b3c341..d0a9883cd 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -123,9 +123,10 @@ stages: # Pick any available CPython from the PyPA image. PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) echo "Using: $($PY --version)" - $PY -m pip install -q -U pip setuptools wheel + $PY -m pip install -q -U pip setuptools wheel twine cd /workspace $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl echo "Produced wheels:"; ls -lh /workspace/dist/*.whl ' displayName: 'Build ODBC wheel in container' @@ -135,6 +136,11 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Linux Artifact' inputs: From 62910b4b92233601caa258ead365d7d5417bee92 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:58:20 +0530 Subject: [PATCH 03/31] Address review: preflight guard for odbcBuildDefinitionId placeholder; drop unused targetArch var on Windows stage --- OneBranchPipelines/official-release-odbc-pipeline.yml | 11 +++++++++++ .../stages/build-odbc-windows-stage.yml | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 90eaedddb..ef06b0895 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -99,6 +99,17 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: + - task: PowerShell@2 + displayName: 'Preflight: verify odbcBuildDefinitionId is configured' + inputs: + targetType: 'inline' + script: | + if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { + Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." + exit 1 + } + Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" + - task: DownloadPipelineArtifact@2 displayName: 'Download Consolidated ODBC Artifacts' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 7a812de45..57de838ed 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -48,8 +48,6 @@ stages: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' # OneBranch-required variable (unused on Windows stages). LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' - # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). - targetArch: ${{ parameters.architecture }} steps: # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. From 38b9ff9fbae948078f9e62e3355adee23a98cf17 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:33:46 +0530 Subject: [PATCH 04/31] Fold mssql-python-odbc build+release into existing pipelines Per review feedback, build and release the standalone mssql-python-odbc package from the existing Build-Release-Package-Pipeline (def 2199) and official release pipeline instead of separate ADO definitions. Build pipeline: add odbcWindows/Macos/LinuxConfigs params (2+1+4=7 data-only wheels), reference the 3 odbc stage templates alongside the mssql-python stages, and add a separate ConsolidateOdbc stage publishing drop_ConsolidateOdbc_ConsolidateArtifacts (distinct from mssql-python's drop_Consolidate_ConsolidateArtifacts). Release pipeline: add releasePackage parameter (mssql-python | mssql-python-odbc); select the consolidated artifact accordingly and skip symbol publishing + mssql-py-core version validation for odbc. Delete the redundant standalone orchestrators (build-release-odbc-pipeline.yml, official-release-odbc-pipeline.yml). This removes the need to register a new ADO build definition and the odbcBuildDefinitionId placeholder. Both packages now build from def 2199; the odbc stages consume setup_odbc.py from PR #663, so #663 must merge before these stages can build. --- .../build-release-odbc-pipeline.yml | 211 ---------------- .../build-release-package-pipeline.yml | 94 ++++++- .../jobs/consolidate-odbc-artifacts-job.yml | 5 +- .../official-release-odbc-pipeline.yml | 239 ------------------ .../official-release-pipeline.yml | 56 ++-- 5 files changed, 136 insertions(+), 469 deletions(-) delete mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml delete mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml deleted file mode 100644 index b64dc936b..000000000 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ /dev/null @@ -1,211 +0,0 @@ -# ========================================================================================= -# OneBranch Build Pipeline for mssql-python-odbc -# ========================================================================================= -# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. -# -# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is -# NO compiled Python extension, so every wheel is tagged `py3-none-` and a -# SINGLE build per platform covers all supported Python versions (3.10+). That is why -# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces -# only 7 wheels total: -# - Windows: x64, arm64 -# - macOS: universal2 -# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) -# -# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml -# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). -# -# INFRA PREREQUISITES (see companion release pipeline for the definition ID): -# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). -# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. -# - Ensure the reused pools/images are available: Python-1ES-pool -# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the -# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. -# ========================================================================================= - -# Build number format: YYDDD.r -name: $(Year:YY)$(DayOfYear)$(Rev:.r) - -# Only trigger when ODBC-package inputs change (path filters are literal prefixes, -# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. -trigger: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -pr: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -parameters: - - name: oneBranchType - displayName: 'OneBranch Template Type' - type: string - values: - - 'Official' - - 'NonOfficial' - default: 'NonOfficial' - - - name: runSdlTasks - displayName: 'Run SDL Security Tasks' - type: boolean - default: true - - # Windows: one wheel per architecture (NO per-Python matrix). - - name: odbcWindowsConfigs - type: object - default: - - arch: 'x64' - - arch: 'arm64' - - # macOS: single universal2 wheel. - - name: odbcMacosConfigs - type: object - default: - - name: 'universal2' - - # Linux: one wheel per libc/arch combo. - - name: odbcLinuxConfigs - type: object - default: - - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } - - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } - -variables: - # Scheduled builds run as Official; manual/PR builds use the parameter. - - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: 'Official' - ${{ else }}: - value: '${{ parameters.oneBranchType }}' - - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the - # per-stage EsrpMalwareScanning tasks. - - template: /OneBranchPipelines/variables/signing-variables.yml@self - - group: 'ESRP Federated Creds (AME)' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - -extends: - template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress - suppressionSet: default - - # This pipeline compiles NO first-party native code — it only packages the - # pre-built ODBC driver binaries, which are signed and governed by the Microsoft - # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; - # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). - binskim: - enabled: false - justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' - apiscan: - enabled: false - justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' - armory: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' - codeql: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' - - credscan: - enabled: ${{ parameters.runSdlTasks }} - policheck: - enabled: ${{ parameters.runSdlTasks }} - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: ${{ parameters.runSdlTasks }} - sbom: - enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python-odbc' - tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - # WINDOWS BUILD STAGES (2: x64, arm64) - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # MACOS BUILD STAGE (1: universal2) - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # CONSOLIDATE STAGE (waits for all 7 platform stages) - - stage: Consolidate - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..984e52bcc 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -138,6 +138,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -338,8 +365,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -466,3 +495,64 @@ extends: # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 0225d34a2..42db132ff 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -64,5 +64,8 @@ jobs: displayName: 'Publish Consolidated ODBC Artifacts' inputs: targetPath: '$(ob_outputDirectory)' - artifact: 'drop_Consolidate_ConsolidateArtifacts' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml deleted file mode 100644 index ef06b0895..000000000 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -# ========================================================================================= -# OneBranch Official Release Pipeline for mssql-python-odbc -# ========================================================================================= -# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to -# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group -# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python -# release (same inputs and FolderLocation); the only differences are the build pipeline -# source and that this package has no debug symbols to publish. -# -# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two infra values below are set. -# ========================================================================================= - -name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC - -# Manual trigger only — releases should be deliberate. -trigger: none -pr: none - -parameters: - - name: releaseToPyPI - displayName: 'Release to PyPI (Production)' - type: boolean - default: false # Safety: default false to prevent accidental releases. - -variables: - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Shared ESRP identity — same group used by the mssql-python release pipeline. - - group: 'ESRP Federated Creds (AME)' - # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' - # once it is registered (the mssql-python build definition is 2199). Left as 0 so the - # pipeline compiles; the download tasks will fail at runtime until this is set. - - name: odbcBuildDefinitionId - value: '0' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - - # Reference to the ODBC build pipeline (select a run in the UI when releasing). - pipelines: - - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. - trigger: none - -extends: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - - # Artifacts were already scanned during the build; this pipeline only republishes. - binskim: - enabled: false - justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' - credscan: - enabled: true - policheck: - enabled: true - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: true - tsa: - enabled: true - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - - stage: ReleasePackages - displayName: 'Release mssql-python-odbc to PyPI' - - jobs: - # Job 1: Download + validate the consolidated wheels (custom pool). - - job: PrepAndValidate - displayName: 'Download and Validate Artifacts' - - pool: - type: windows - isCustom: true - name: Python-1ES-pool - demands: - - imageOverride -equals PYTHON-1ES-MMS2022 - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - - steps: - - task: PowerShell@2 - displayName: 'Preflight: verify odbcBuildDefinitionId is configured' - inputs: - targetType: 'inline' - script: | - if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { - Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." - exit 1 - } - Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Consolidated ODBC Artifacts' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - - task: PowerShell@2 - displayName: 'List and Verify Downloaded Wheels' - inputs: - targetType: 'inline' - script: | - $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" - if (-not (Test-Path $wheelsPath)) { - Write-Error "Wheel directory not found at: $wheelsPath" - exit 1 - } - $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" - Write-Host "Total wheel files found: $($wheels.Count)" - - $allValid = $true - Add-Type -AssemblyName System.IO.Compression.FileSystem - foreach ($wheel in $wheels) { - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) - $entryCount = $zip.Entries.Count - $zip.Dispose() - Write-Host " OK $($wheel.Name) ($entryCount entries)" - } catch { - Write-Error " BAD $($wheel.Name): $_" - $allValid = $false - } - } - if (-not $allValid) { - Write-Error "One or more wheel files are corrupted" - exit 1 - } - if ($wheels.Count -ne 7) { - Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" - exit 1 - } - Write-Host "All wheels verified successfully!" - - # Dry-run summary (when release is disabled). - - ${{ if eq(parameters.releaseToPyPI, false) }}: - - task: PowerShell@2 - displayName: 'Dry Run - Release Skipped' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "DRY RUN MODE - No Release Performed" - Write-Host "Package: mssql-python-odbc" - Write-Host "====================================" - Write-Host "To perform an actual release:" - Write-Host "1. Set 'releaseToPyPI' parameter to true" - Write-Host "2. Re-run pipeline" - - # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). - # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. - - ${{ if eq(parameters.releaseToPyPI, true) }}: - - job: PyPIRelease - displayName: 'ESRP Release to PyPI' - dependsOn: PrepAndValidate - - templateContext: - type: releaseJob - isProduction: true - - pool: - type: windows - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Artifacts from ODBC Build Pipeline' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - # ESRP release task — identical to the mssql-python release pipeline - # (same shared ESRP identity and PyPI publish inputs). - - task: EsrpRelease@9 - displayName: 'ESRP Release to PyPI' - inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - ContentType: 'PyPI' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' - - - task: PowerShell@2 - displayName: 'Release Summary' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "ESRP Release Completed" - Write-Host "Package: mssql-python-odbc" - Write-Host "Target: PyPI" - Write-Host "====================================" - Write-Host "Next steps:" - Write-Host "1. Verify release in the ESRP portal" - Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..731df538e 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" @@ -273,7 +297,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,7 +327,7 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" @@ -313,6 +337,6 @@ extends: Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" From d85fd92bb3d7ec75e47325f30491d629f7273272 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:50:04 +0530 Subject: [PATCH 05/31] Scope consolidate downloads by package to prevent cross-contamination Def 2199 now builds mssql-python and mssql-python-odbc in one run. Both consolidate jobs used DownloadPipelineArtifact@2 with buildType: current, which downloads every artifact in the run and copies all *.whl into dist/. In the merged run this caused: (1) the main Consolidate to sweep the 7 odbc wheels into the mssql-python release (count is warning-only, so silent), and (2) ConsolidateOdbc to see 34 wheels while expecting 7 and hard-fail. Add itemPattern to scope each download to its own package's build-stage artifacts (main: drop_Win_*/drop_MacOS_*/drop_Linux_*; odbc: drop_ODBC_*), giving clean partitioning with no wheel leakage. --- .../jobs/consolidate-artifacts-job.yml | 13 ++++++++++--- .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..336c736a5 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 42db132ff..f0d8a0715 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -32,11 +32,15 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from the current build (one per platform stage). + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). - task: DownloadPipelineArtifact@2 displayName: 'Download All ODBC Platform Artifacts' inputs: buildType: 'current' + itemPattern: 'drop_ODBC_*/**' targetPath: '$(Pipeline.Workspace)/all-artifacts' - bash: | From e3cf2a9a56827bbe9891b2c67d8ff2b0c819b6b1 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:16:59 +0530 Subject: [PATCH 06/31] Correct stale mssql-python wheel count in consolidate diagnostic (27 -> 34) The matrix is 9 Windows + 5 macOS + 4 Linux stages x 5 Python versions (cp310-cp314) = 34 wheels across 18 stages. The diagnostic said 27 (7+4+16), stale from an older 3.10-3.13 / 7-Windows matrix. Count check remains warning-only (no behavior change). --- .../jobs/consolidate-artifacts-job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 336c736a5..52eb43381 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -75,12 +75,12 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' From a4b4e59fc4f9ed09ffa5e79481875339a52159e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:49:09 +0530 Subject: [PATCH 07/31] Harden odbc release path: pin guard + wheel-content assertions official-release-pipeline.yml: gate 'Published symbols' dry-run/summary text to mssql-python so odbc runs no longer claim symbol publishing that never happens; add a mssql-python-only step asserting the wheel metadata pins mssql-python-odbc==18.6.0 (fails by design if released from a pre-pin build). build-odbc-{linux,macos,windows}-stage.yml: assert each built wheel actually contains the ODBC driver binary (msodbcsql*), guarding against the .gitignore '*.so' trap / silent sync_libs() skip shipping an empty data wheel that still passes twine check. --- .../official-release-pipeline.yml | 48 ++++++++++++++++++- .../stages/build-odbc-linux-stage.yml | 22 +++++++++ .../stages/build-odbc-macos-stage.yml | 21 ++++++++ .../stages/build-odbc-windows-stage.yml | 22 +++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 731df538e..7cb529796 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -331,7 +374,8 @@ extends: Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index d0a9883cd..969c0b8b3 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -136,6 +136,28 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - script: | docker rm -f build-$(LINUX_TAG)-$(ARCH) || true displayName: 'Stop and remove build container' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index 9b47388e3..c1e7541b1 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -79,6 +79,27 @@ stages: TargetFolder: '$(ob_outputDirectory)/wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC macOS Artifact' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 57de838ed..dea809ed0 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,6 +92,28 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + # OneBranch requires artifact naming: drop__. - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Windows Artifact' From b88e169f006f16bb26f59efc13be907bd11fc93f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:24:25 +0530 Subject: [PATCH 08/31] Hard-fail mssql-python consolidate on wheel-count mismatch Flip the 34-wheel count check from WARNING-only to a hard exit 1, symmetric with the odbc consolidate job's count guard. A mismatch means a build stage silently produced a partial wheel set (e.g. a missing Python version) or the matrix changed without updating the expected count -- previously this shipped silently. --- OneBranchPipelines/jobs/consolidate-artifacts-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 52eb43381..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -77,8 +77,13 @@ jobs: echo "Total wheel count: $WHEEL_COUNT" echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else echo "SUCCESS: All 34 wheels consolidated!" fi From d1b3d467028a4986a3558ce4653a002322484fff Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 14:04:13 +0530 Subject: [PATCH 09/31] odbc: add mssql-python-odbc selector to dummy release pipeline Mirrors the official-release-pipeline odbc changes: releasePackage parameter, consolidatedArtifactName selection for both artifact downloads, and gating the mssql-py-core version validation + symbol publishing to mssql-python only. Stays on Maven ContentType so it never publishes to PyPI. --- OneBranchPipelines/dummy-release-pipeline.yml | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..28751e845 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # the mssql-python-odbc package ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" @@ -280,7 +302,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,7 +335,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" From 9f93c36073dabd47297265c91527f26efcee4bcf Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:00:10 +0530 Subject: [PATCH 10/31] odbc: align dummy release pipeline mssql-python handling with official Adds the mssql-python-odbc pin guard (gated to mssql-python, matching official-release-pipeline) and gates the 'symbols published' summary messaging on releasePackage==mssql-python so the dummy pipeline exercises the exact same mssql-python validation/messaging as the official pipeline for both packages. --- OneBranchPipelines/dummy-release-pipeline.yml | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 28751e845..c98c351ff 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -196,7 +196,7 @@ extends: Write-Host "=====================================" # Step 3: Validate mssql-py-core is a stable version (mssql-python only; - # the mssql-python-odbc package ships no mssql-py-core.version file) + # mssql-python-odbc ships no mssql-py-core.version file) - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: - task: PowerShell@2 displayName: '[TEST] Validate mssql-py-core is a stable version' @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -339,7 +382,7 @@ extends: Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -348,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" From 2d93fe07234bf24f30353fd7e56cd6f754a437f9 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:28:24 +0530 Subject: [PATCH 11/31] odbc: bump mssql-python-odbc pin to 18.6.2 in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 2 +- OneBranchPipelines/official-release-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index c98c351ff..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 7cb529796..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." From a4e39687fc1930f1c35f17367bb1fe6391e1fe93 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:17:36 +0530 Subject: [PATCH 12/31] build: add buildPackage selector to build pipeline Gate mssql-python and mssql-python-odbc stage groups behind a buildPackage parameter (mssql-python default) so only the selected package builds, halving run time and enabling concurrent per-package builds. Mirrors the releasePackage selector pattern in the release pipelines. --- .../build-release-package-pipeline.yml | 216 ++++++++++-------- 1 file changed, 121 insertions(+), 95 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 984e52bcc..29e735080 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,22 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds use this default too, so they build mssql-python only. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -384,16 +400,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -410,15 +428,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -440,15 +459,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -462,36 +482,37 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging @@ -507,32 +528,36 @@ extends: # first-party native code is introduced. # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================================================================= # CONSOLIDATE ODBC STAGE @@ -541,18 +566,19 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - stage: ConsolidateOdbc - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 From 9d3097c8331d5caed76a8ade8a2d28baded2bdba Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:21:13 +0530 Subject: [PATCH 13/31] build: nightly build always builds both packages Add effectiveBuildPackage variable that forces 'both' on scheduled (Build.Reason=Schedule) runs while manual/PR runs honour the buildPackage parameter. Mirrors the effectiveOneBranchType pattern so the nightly keeps refreshing every artifact. --- .../build-release-package-pipeline.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 29e735080..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -76,7 +76,8 @@ parameters: # mssql-python -> Windows/macOS/Linux wheels + Consolidate # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc # both -> everything (previous default behaviour) - # Scheduled (daily) builds use this default too, so they build mssql-python only. + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. - name: buildPackage displayName: 'Package to Build' type: string @@ -192,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -401,7 +410,7 @@ extends: # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning # Gate: build mssql-python stages unless an odbc-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.windowsConfigs }}: - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self parameters: @@ -428,7 +437,7 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.macosConfigs }}: - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self parameters: @@ -459,7 +468,7 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self parameters: @@ -482,7 +491,7 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -529,7 +538,7 @@ extends: # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcWindowsConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self parameters: @@ -539,7 +548,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcMacosConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self parameters: @@ -548,7 +557,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcLinuxConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self parameters: @@ -566,7 +575,7 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - stage: ConsolidateOdbc displayName: 'Consolidate All ODBC Artifacts' dependsOn: From d94debd76d8dcb14ac9b17bb982ad66736b65520 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Mon, 13 Jul 2026 14:57:47 +0530 Subject: [PATCH 14/31] FIX: forward connection timeout to bulkcopy pycore connection (#650) ### Work Item / Issue Reference > GitHub Issue: #626 ------------------------------------------------------------------- ### Summary cursor.bulkcopy() opens a separate connection through mssql-py-core, which defaulted to a hardcoded 15s connect timeout with no way to override it from Python. I forward the cursor's query timeout (set via connect(timeout=X)) into pycore's connect_timeout when it's set, so the bulk copy connection honors the same limit. timeout=0 stays a no-override, leaving pycore on its 15s default. added tests covering the positive forward, the zero case, and that the cursor snapshot (not a later live connection change) is what's used. --------- Co-authored-by: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Co-authored-by: Saurabh Singh --- mssql_python/cursor.py | 14 ++++ tests/test_008_auth.py | 1 + tests/test_020_bulkcopy_auth_cleanup.py | 93 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 85701a408..bc2956d7a 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -2955,6 +2955,20 @@ def bulkcopy( # Translate parsed connection string into the dict py-core expects. pycore_context = connstr_to_pycore_params(params) + # Forward the cursor's query timeout to py-core so the bulkcopy + # connection uses the same limit instead of py-core's compiled-in 15s + # default. _timeout is the snapshot taken at cursor creation (same value + # _set_timeout uses). Accept any int the public Connection.timeout setter + # accepts (including IntEnum) but exclude bool, then normalise to a plain + # int so py-core's u32 extract is unambiguous. 0 stays a "no override": + # py-core's default connect path treats 0 as no deadline, but its TCP + # attempt path turns 0 into a 0ms timeout that fails instantly, so we + # leave 0 unset and let py-core apply its own 15s default. + connect_timeout = self._timeout + if isinstance(connect_timeout, int) and not isinstance(connect_timeout, bool): + if connect_timeout > 0: + pycore_context["connect_timeout"] = int(connect_timeout) + # Token acquisition — only thing cursor must handle (needs azure-identity SDK) if self.connection._auth_type: # Fresh token acquisition for mssql-py-core connection diff --git a/tests/test_008_auth.py b/tests/test_008_auth.py index d82ecaea4..add9495c4 100644 --- a/tests/test_008_auth.py +++ b/tests/test_008_auth.py @@ -586,6 +586,7 @@ def test_bulkcopy_path_preserves_user_assigned_msi_client_id(self): cursor = Cursor.__new__(Cursor) cursor._connection = mock_conn + cursor._timeout = 0 cursor.closed = False cursor.hstmt = None diff --git a/tests/test_020_bulkcopy_auth_cleanup.py b/tests/test_020_bulkcopy_auth_cleanup.py index 164438344..0dbe07d86 100644 --- a/tests/test_020_bulkcopy_auth_cleanup.py +++ b/tests/test_020_bulkcopy_auth_cleanup.py @@ -10,6 +10,7 @@ """ import secrets +from enum import IntEnum from unittest.mock import MagicMock, patch SAMPLE_TOKEN = secrets.token_hex(44) @@ -26,6 +27,7 @@ def _make_cursor(connection_str, auth_type): cursor = Cursor.__new__(Cursor) cursor._connection = mock_conn + cursor._timeout = 0 cursor.closed = False cursor.hstmt = None return cursor @@ -108,3 +110,94 @@ def capture_context(ctx, **kwargs): assert "access_token" not in captured_context assert captured_context.get("user_name") == "sa" assert captured_context.get("password") == "mypwd" + + +def _capture_bulkcopy_context(cursor): + """Run bulkcopy with a mocked pycore module and return the captured context.""" + captured_context = {} + + mock_pycore_cursor = MagicMock() + mock_pycore_cursor.bulkcopy.return_value = { + "rows_copied": 1, + "batch_count": 1, + "elapsed_time": 0.1, + } + mock_pycore_conn = MagicMock() + mock_pycore_conn.cursor.return_value = mock_pycore_cursor + + def capture_context(ctx, **kwargs): + captured_context.update(ctx) + return mock_pycore_conn + + mock_pycore_module = MagicMock() + mock_pycore_module.PyCoreConnection = capture_context + + with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) + + return captured_context + + +class TestBulkcopyConnectTimeout: + """Verify cursor.bulkcopy forwards the cursor timeout to pycore (issue #626).""" + + @patch("mssql_python.cursor.logger") + def test_positive_timeout_forwarded(self, mock_logger): + """cursor._timeout > 0 ⇒ connect_timeout reaches pycore, overriding 15s.""" + mock_logger.is_debug_enabled = False + cursor = _make_cursor("Server=localhost;Database=testdb;UID=sa;PWD=pwd", None) + cursor._timeout = 30 + + captured = _capture_bulkcopy_context(cursor) + + assert captured.get("connect_timeout") == 30 + + @patch("mssql_python.cursor.logger") + def test_zero_timeout_not_forwarded(self, mock_logger): + """cursor._timeout == 0 ⇒ no override, pycore keeps its default.""" + mock_logger.is_debug_enabled = False + cursor = _make_cursor("Server=localhost;Database=testdb;UID=sa;PWD=pwd", None) + cursor._timeout = 0 + + captured = _capture_bulkcopy_context(cursor) + + assert "connect_timeout" not in captured + + @patch("mssql_python.cursor.logger") + def test_uses_cursor_snapshot_not_live_connection(self, mock_logger): + """timeout is the cursor snapshot; later connection changes don't apply.""" + mock_logger.is_debug_enabled = False + cursor = _make_cursor("Server=localhost;Database=testdb;UID=sa;PWD=pwd", None) + cursor._timeout = 45 + cursor._connection.timeout = 99 # changed after cursor creation, must be ignored + + captured = _capture_bulkcopy_context(cursor) + + assert captured.get("connect_timeout") == 45 + + @patch("mssql_python.cursor.logger") + def test_intenum_timeout_forwarded_as_plain_int(self, mock_logger): + """IntEnum (accepted by the public setter) is forwarded, normalised to int.""" + mock_logger.is_debug_enabled = False + + class _T(IntEnum): + thirty = 30 + + cursor = _make_cursor("Server=localhost;Database=testdb;UID=sa;PWD=pwd", None) + cursor._timeout = _T.thirty + + captured = _capture_bulkcopy_context(cursor) + + assert captured.get("connect_timeout") == 30 + assert type(captured.get("connect_timeout")) is int + + @patch("mssql_python.cursor.logger") + def test_bool_timeout_not_forwarded(self, mock_logger): + """bool is a subclass of int but must not be treated as a timeout.""" + mock_logger.is_debug_enabled = False + cursor = _make_cursor("Server=localhost;Database=testdb;UID=sa;PWD=pwd", None) + cursor._timeout = True + + captured = _capture_bulkcopy_context(cursor) + + assert "connect_timeout" not in captured From 7d2240981a893bcfc03a1f77700277f4d8b03d88 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar <61936179+jahnvi480@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:47:36 +0530 Subject: [PATCH 15/31] CHORE: validate macOS wheel is self-contained by removing system ODBC before tests (#658) ### Work Item / Issue Reference > GitHub Issue: #656 ------------------------------------------------------------------- ### Summary on macOS the test job never removed system ODBC before running, so mssql-python could load a Homebrew unixODBC driver manager / msodbcsql18 from the default dyld path and pass even if the bundled copy was broken. every Linux job already uninstalls system ODBC before tests to force the bundled driver. this brings macOS in line and adds a static check of the bundle itself. - new step in the macOS test job uninstalls Homebrew msodbcsql18/mssql-tools18/unixodbc, deletes leftover libodbc/libodbcinst dylibs and odbcinst config from both Homebrew prefixes, then otool -L's the bundled driver to confirm it's intact. guarded so it's a no-op when nothing is installed. - new test (test_000_dependencies.py) walks the bundled driver's load chain with otool and asserts every inter-library reference is relocatable (@loader_path, not an absolute /opt/homebrew path), which is what breaks on machines without Homebrew. - brew trust on the mssql-release tap so the benchmark step can install msodbcsql18 for the pyodbc comparison (newer Homebrew refuses untrusted taps). --------- Co-authored-by: Gaurav Sharma Co-authored-by: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 32 +++++++ tests/test_000_dependencies.py | 109 +++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 8cc7ea8e9..be33ff7eb 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -543,6 +543,36 @@ jobs: parameters: platform: unix + - script: | + # Uninstall any system/Homebrew ODBC before tests so pytest exercises the + # driver and driver manager bundled in the wheel, not a system copy. Mirrors + # the Linux jobs. Every command is guarded so this is a no-op when nothing is + # installed (the hosted runner may or may not ship unixODBC). + echo "Removing Homebrew ODBC packages (if installed)..." + brew uninstall --ignore-dependencies --force msodbcsql18 mssql-tools18 unixodbc 2>/dev/null \ + || echo " no Homebrew ODBC packages to remove" + + echo "Removing leftover driver-manager dylibs and config from Homebrew prefixes..." + for prefix in /opt/homebrew /usr/local; do + rm -f "$prefix"/lib/libodbc.*.dylib "$prefix"/lib/libodbcinst.*.dylib \ + "$prefix"/lib/libodbc.dylib "$prefix"/lib/libodbcinst.dylib + rm -rf "$prefix"/opt/msodbcsql18 "$prefix"/opt/unixodbc + rm -f "$prefix"/etc/odbcinst.ini "$prefix"/etc/odbc.ini + done + rm -f /etc/odbcinst.ini /etc/odbc.ini "$HOME/.odbcinst.ini" "$HOME/.odbc.ini" + + echo "Confirming no system unixODBC driver manager remains on the default search path:" + if ls /opt/homebrew/lib/libodbc.*.dylib /usr/local/lib/libodbc.*.dylib 2>/dev/null; then + echo " WARNING: a system libodbc is still present" + else + echo " none found (good)" + fi + + echo "Verifying the bundled x86_64 driver load chain is intact:" + otool -L mssql_python/libs/macos/x86_64/lib/libmsodbcsql.18.dylib \ + || echo " bundled driver not found" + displayName: 'Uninstall system ODBC Driver before running tests on macOS' + - script: | echo "Build successful, running tests now" python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear @@ -615,6 +645,8 @@ jobs: - script: | echo "Installing ODBC Driver 18 for pyodbc..." brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release + # Newer Homebrew refuses to load formulae from third-party taps unless the tap is trusted + brew trust microsoft/mssql-release || echo "brew trust failed; attempting install anyway" HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 || echo "ODBC Driver 18 install failed — pyodbc benchmarks will be skipped" pip install pyodbc echo "Running performance benchmarks..." diff --git a/tests/test_000_dependencies.py b/tests/test_000_dependencies.py index 5c50c10ca..b8e0f55d4 100644 --- a/tests/test_000_dependencies.py +++ b/tests/test_000_dependencies.py @@ -6,6 +6,8 @@ import pytest import platform import os +import shutil +import subprocess import sys from pathlib import Path @@ -347,6 +349,113 @@ def test_macos_universal_dependencies(self): libodbcinst_path.exists() ), f"macOS {arch} ODBC installer library not found: {libodbcinst_path}" + @pytest.mark.skipif(dependency_tester.platform_name != "darwin", reason="macOS-specific test") + def test_macos_driver_load_chain_is_relocatable(self): + """Ensure the macOS driver's bundled load chain contains no absolute paths. + + Guards against GitHub issue #656. On a fresh Apple Silicon Mac (without + ``brew install unixodbc``) importing mssql_python fails because the arm64 + ``libmsodbcsql.18.dylib`` shipped in the wheel still hardcodes + ``/opt/homebrew/lib/libodbcinst.2.dylib`` instead of + ``@loader_path/libodbcinst.2.dylib``. + + Root cause: ``pybind/configure_dylibs.sh`` only rewrites the dylibs for + the *build host* architecture (``ARCH=$(uname -m)``). macOS wheels are + universal2 but built on an x86_64 runner, so only the x86_64 driver gets + relocated; the arm64 driver ships with Homebrew-absolute dependencies. + + At runtime ``ddbc_bindings`` ``dlopen``s ``libmsodbcsql.18.dylib`` + DIRECTLY (it does not go through the unixODBC driver manager), so this + test walks exactly that load graph: starting from the driver and + following only its *bundled* sibling dependencies. Any bundled sibling + reached via an absolute path would fail to load on a machine without + Homebrew. + + ``otool -L`` reads Mach-O load commands of any architecture regardless of + the host, so inspecting BOTH the arm64 and x86_64 drivers lets a single + x86_64 CI runner catch the arm64-only packaging bug. Non-bundled + dependencies (system libs under /usr/lib or /System, and the documented + external openssl prerequisite) are intentionally ignored. + """ + otool = shutil.which("otool") + if otool is None: + pytest.skip("otool not available on this system") + + def direct_bundled_deps(dylib_path, bundled_names): + """Return (absolute_hits, relocatable_hits) for bundled sibling deps. + + The first line of ``otool -L`` output is the file being inspected and + the second is the library's own install id (LC_ID_DYLIB); both are + self-references and are skipped by ignoring deps whose basename equals + the file's own name. + """ + result = subprocess.run([otool, "-L", str(dylib_path)], capture_output=True, text=True) + if result.returncode != 0: + return None, None + + absolute_hits, relocatable_hits = [], [] + for line in result.stdout.splitlines()[1:]: + dep = line.strip().split(" (compatibility")[0].strip() + if not dep: + continue + base = os.path.basename(dep) + if base == dylib_path.name or base not in bundled_names: + continue # self-reference or a non-bundled/external dependency + if dep.startswith("/"): + absolute_hits.append((base, dep)) + else: + relocatable_hits.append(base) + return absolute_hits, relocatable_hits + + problems = [] + checked_arches = [] + for arch in ["arm64", "x86_64"]: + lib_dir = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" + driver = lib_dir / "libmsodbcsql.18.dylib" + if not driver.exists(): + continue + checked_arches.append(arch) + + bundled = {p.name: p for p in lib_dir.glob("*.dylib")} + + # Depth-first walk of the driver's bundled load chain (queue.pop() is + # LIFO). Order does not matter here: we visit every reachable bundled + # dylib and collect all absolute-path problems regardless of traversal. + visited, queue = set(), [driver] + while queue: + current = queue.pop() + if current.name in visited: + continue + visited.add(current.name) + + absolute_hits, relocatable_hits = direct_bundled_deps(current, bundled) + if absolute_hits is None or relocatable_hits is None: + problems.append(f"{arch}/{current.name}: otool failed to inspect the library") + continue + + for base, dep in absolute_hits: + problems.append( + f"{arch}/{current.name} loads bundled '{base}' via absolute " + f"path '{dep}' (expected @loader_path); this breaks on " + f"machines without Homebrew" + ) + for base in relocatable_hits: + queue.append(bundled[base]) + + # Fail loudly instead of passing vacuously if no bundled driver was found + # to validate (e.g. the packaging layout changed). + assert checked_arches, ( + "no bundled macOS driver found to validate under " + "libs/macos/{arm64,x86_64}/lib/libmsodbcsql.18.dylib " + "(packaging layout may have changed)" + ) + + assert not problems, ( + "macOS driver load chain contains hardcoded absolute dependency paths " + "that break on machines without Homebrew (see GitHub issue #656):\n " + + "\n ".join(problems) + ) + @pytest.mark.skipif(dependency_tester.platform_name != "linux", reason="Linux-specific test") def test_linux_distribution_dependencies(self): """Test that Linux builds include distribution-specific dependencies.""" From 3202ebdfd2e89724a839b6c710a8277abe5f5357 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 15 Jul 2026 17:12:04 +0530 Subject: [PATCH 16/31] CHORE: reduce perf-benchmark run-to-run variance (normalized score, median, warmup) (#680) ### Work Item / Issue Reference > ADO Work Item: Fixed [AB#46419](https://sqlclientdrivers.visualstudio.com/c6d89619-62de-46a0-8b46-70b92a84d85e/_workitems/edit/46419) ------------------------------------------------------------------- ### Summary the CI perf benchmark flags false regressions on perf-neutral PRs. across 11 main runs of identical code, the same scenario's raw median swings 1.42x-1.73x (CV 11-18%) purely from shared-runner jitter, so the old 5% gate fired on ~34% of scenario comparisons (and ~30% false "improvements"). this reworks `benchmarks/perf-benchmarking.py` measurement and reporting. no pipeline changes, the step stays continueOnError: - metric is now the median, first sample discarded as warmup (no extra DB passes, so macOS timing is unchanged). - vs-main uses a normalized score instead of raw time: mssql-python's median relative to pyodbc measured on the same runner. pyodbc is pinned, so dividing by it cancels runner-to-runner speed. on the 11 historical runs this roughly halves variance on the I/O-bound scenarios (CV 18%->8%, 11%->5%) and trims the rest. - regression/highlight threshold raised to 20% (real perf changes move 30%+), gates only with >= 3 samples, and falls back to raw median for baselines that predate the normalized score. validated with deterministic unit checks, including the key case where absolute time doubles but the normalized score stays flat so nothing is flagged. --------- Co-authored-by: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> --- benchmarks/perf-benchmarking.py | 162 +++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 36 deletions(-) diff --git a/benchmarks/perf-benchmarking.py b/benchmarks/perf-benchmarking.py index 261e97942..79e28576f 100644 --- a/benchmarks/perf-benchmarking.py +++ b/benchmarks/perf-benchmarking.py @@ -11,6 +11,18 @@ Environment: DB_CONNECTION_STRING — required, e.g. Server=localhost;Database=...;Uid=sa;Pwd=...;TrustServerCertificate=yes + +Methodology (variance control): + Each scenario runs NUM_ITERATIONS timed passes and reports the median, discarding + the first WARMUP_ITERATIONS samples (cold cache / plan compile). The "vs main" + comparison uses a normalized score instead of raw time: each run expresses + mssql-python's time relative to pyodbc measured on the same runner (see + normalized_score()). Because pyodbc is a pinned build, dividing by it cancels the + runner's raw speed, so scores are comparable across CI runs even on different + hardware -- which is the dominant source of run-to-run noise. Measured on ~11 + historical main runs, normalization roughly halves variance on the I/O-bound + scenarios (CV 18%->8%, 11%->5%) and trims it on the rest; the gate threshold is + set above the residual. """ import argparse @@ -33,9 +45,18 @@ CONN_STR_PYODBC = None NUM_ITERATIONS = 10 +WARMUP_ITERATIONS = 1 # first N timed samples are discarded (no extra DB passes) +MIN_SAMPLES = 3 # need at least this many successful samples to gate INSERTMANY_ROWS = 100_000 INSERTMANY_BATCH_SIZE = 1000 +# Regression/highlight gate thresholds, applied to the normalized score (see +# normalized_score()). Because that score cancels machine-to-machine speed, the +# residual historical CI variance is only ~5-15% CV per scenario, so the threshold +# is set above that; real perf changes move numbers by 30%+, well clear of it. Tunable. +REGRESSION_THRESHOLD = 0.20 +HIGHLIGHT_THRESHOLD = 0.20 + def _init_conn_strings(): global CONN_STR, CONN_STR_PYODBC @@ -71,6 +92,10 @@ def add_time(self, elapsed: float, rows: int = 0): def avg(self) -> float: return statistics.mean(self.times) if self.times else 0.0 + @property + def median(self) -> float: + return statistics.median(self.times) if self.times else 0.0 + @property def min(self) -> float: return min(self.times) if self.times else 0.0 @@ -86,6 +111,7 @@ def stddev(self) -> float: def to_dict(self) -> dict: return { "avg": round(self.avg, 6), + "median": round(self.median, 6), "min": round(self.min, 6), "max": round(self.max, 6), "stddev": round(self.stddev, 6), @@ -100,7 +126,7 @@ def to_dict(self) -> dict: def run_fetch_pyodbc(query: str, name: str, iterations: int) -> BenchmarkResult: result = BenchmarkResult(name) - for _ in range(iterations): + for i in range(iterations): conn = None try: conn = pyodbc.connect(CONN_STR_PYODBC) @@ -109,7 +135,8 @@ def run_fetch_pyodbc(query: str, name: str, iterations: int) -> BenchmarkResult: cursor.execute(query) rows = cursor.fetchall() elapsed = time.perf_counter() - start - result.add_time(elapsed, len(rows)) + if i >= WARMUP_ITERATIONS: + result.add_time(elapsed, len(rows)) except Exception as e: print(f" pyodbc error: {e}") finally: @@ -123,7 +150,7 @@ def run_fetch_pyodbc(query: str, name: str, iterations: int) -> BenchmarkResult: def run_fetch_mssql(query: str, name: str, iterations: int) -> BenchmarkResult: result = BenchmarkResult(name) - for _ in range(iterations): + for i in range(iterations): conn = None try: conn = connect(CONN_STR) @@ -132,7 +159,8 @@ def run_fetch_mssql(query: str, name: str, iterations: int) -> BenchmarkResult: cursor.execute(query) rows = cursor.fetchall() elapsed = time.perf_counter() - start - result.add_time(elapsed, len(rows)) + if i >= WARMUP_ITERATIONS: + result.add_time(elapsed, len(rows)) except Exception as e: print(f" mssql-python error: {e}") finally: @@ -171,7 +199,7 @@ def _run_insertmany(conn_factory, conn_str, name: str, iterations: int) -> Bench flat.extend(row) batches.append(flat) - for _ in range(iterations): + for i in range(iterations): conn = None try: conn = conn_factory(conn_str) @@ -186,7 +214,8 @@ def _run_insertmany(conn_factory, conn_str, name: str, iterations: int) -> Bench cursor.execute(batch_sql, flat_params) elapsed = time.perf_counter() - start - result.add_time(elapsed, INSERTMANY_ROWS) + if i >= WARMUP_ITERATIONS: + result.add_time(elapsed, INSERTMANY_ROWS) except Exception as e: print(f" {name} error: {e}") finally: @@ -228,6 +257,18 @@ def _ratio_str(a: float, b: float) -> str: return f"{factor:.1f}x slower" +def normalized_score(mssql_median: float, pyodbc_median: float) -> Optional[float]: + """Runner-independent score for one scenario: how long mssql-python took + relative to pyodbc on the same runner (mssql-python median / pyodbc median). + + pyodbc is a pinned build, so it soaks up the runner's raw speed. Expressing + mssql-python's time as a multiple of pyodbc's cancels machine-to-machine speed + differences, so two scores are comparable across CI runs even on different + hardware. Lower is better; None when pyodbc has no valid measurement. + """ + return (mssql_median / pyodbc_median) if pyodbc_median > 0 else None + + def print_results( results: List[tuple], baseline: Optional[dict], @@ -243,6 +284,13 @@ def print_results( print("=" * 100) if has_baseline: + baseline_has_norm = any(v.get("norm") is not None for v in baseline.values()) + if baseline_has_norm: + print("(vs main = normalized score: mssql-python time relative to pyodbc on the") + print(" same runner, which cancels runner speed. metric = median)") + else: + print("(vs main = raw median vs main. baseline predates the normalized score, so") + print(" runner-speed differences are NOT cancelled this run. metric = median)") hdr = ( f"\n{'Scenario':<40} {'main':<10} {'this PR':<10} {'pyodbc':<10} " f"{'vs main':<16} {'vs pyodbc':<16}" @@ -256,53 +304,86 @@ def print_results( highlights = [] for name, mssql_result, pyodbc_result in results: - pr_avg = mssql_result.avg - py_avg = pyodbc_result.avg + pr_med = mssql_result.median + py_med = pyodbc_result.median if has_baseline and name in baseline: - main_avg = baseline[name]["avg"] - vs_main = _ratio_str(pr_avg, main_avg) - vs_pyodbc = _ratio_str(pr_avg, py_avg) + b = baseline[name] + main_med = b.get("median", b.get("avg", 0.0)) + # main's normalized score, stored in the baseline by save_json(). + main_score = b.get("norm") + # this PR's normalized score, computed from this run's own numbers. + pr_score = normalized_score(pr_med, py_med) + enough = (len(mssql_result.times) >= MIN_SAMPLES + and len(pyodbc_result.times) >= MIN_SAMPLES) + vs_pyodbc = _ratio_str(pr_med, py_med) + + if main_score is not None and pr_score is not None and enough: + # Both scores are runner-independent (each divides out its own + # runner's pyodbc), so comparing them is a fair PR-vs-main check + # even when the two runs landed on different hardware. + vs_main = _ratio_str(pr_score, main_score) + if pr_score > main_score * (1 + REGRESSION_THRESHOLD): + regressions.append((name, main_score, pr_score, True)) + if pr_score < main_score * (1 - HIGHLIGHT_THRESHOLD): + highlights.append((name, main_score, pr_score, True)) + elif main_med > 0 and pr_med > 0 and len(mssql_result.times) >= MIN_SAMPLES: + # Fallback to raw wall-clock: baseline predates the normalized + # score, or pyodbc had no valid run to normalize against. + vs_main = _ratio_str(pr_med, main_med) + if pr_med > main_med * (1 + REGRESSION_THRESHOLD): + regressions.append((name, main_med, pr_med, False)) + if pr_med < main_med * (1 - HIGHLIGHT_THRESHOLD): + highlights.append((name, main_med, pr_med, False)) + else: + # Too few samples / no usable baseline -> show, don't gate. + vs_main = "inconclusive" + print( - f"{name:<40} {main_avg:<10.4f} {pr_avg:<10.4f} {py_avg:<10.4f} " + f"{name:<40} {main_med:<10.4f} {pr_med:<10.4f} {py_med:<10.4f} " f"{vs_main:<16} {vs_pyodbc:<16}" ) - if main_avg > 0 and pr_avg > main_avg * 1.05: - regressions.append((name, main_avg, pr_avg)) - if main_avg > 0 and pr_avg < main_avg * 0.90: - highlights.append((name, main_avg, pr_avg)) else: - vs_pyodbc = _ratio_str(pr_avg, py_avg) + vs_pyodbc = _ratio_str(pr_med, py_med) if has_baseline: print( - f"{name:<40} {'N/A':<10} {pr_avg:<10.4f} {py_avg:<10.4f} " + f"{name:<40} {'N/A':<10} {pr_med:<10.4f} {py_med:<10.4f} " f"{'N/A':<16} {vs_pyodbc:<16}" ) else: - print(f"{name:<40} {pr_avg:<10.4f} {py_avg:<10.4f} {vs_pyodbc:<16}") + print(f"{name:<40} {pr_med:<10.4f} {py_med:<10.4f} {vs_pyodbc:<16}") print("-" * 100) if has_baseline: + rpct = int(REGRESSION_THRESHOLD * 100) + hpct = int(HIGHLIGHT_THRESHOLD * 100) + + def _fmt_change(old, new, normed): + # normalized entries are unitless scores; raw-median entries are seconds. + if normed: + return f"{old:.3f} -> {new:.3f} (normalized score)" + return f"{old:.4f}s -> {new:.4f}s" + print(f"\n{'='*100}") if regressions: - print("REGRESSIONS (>5% slower than main)") + print(f"REGRESSIONS (>{rpct}% slower than main)") print("=" * 100) - for name, main_avg, pr_avg in regressions: - factor = pr_avg / main_avg - print(f" {name}: {main_avg:.4f}s -> {pr_avg:.4f}s ({factor:.1f}x slower)") + for name, old, new, normed in regressions: + factor = new / old if old else 0 + print(f" {name}: {_fmt_change(old, new, normed)} ({factor:.2f}x slower)") else: - print("REGRESSIONS (>5% slower than main): None detected") + print(f"REGRESSIONS (>{rpct}% slower than main): None detected") print(f"\n{'='*100}") if highlights: - print("HIGHLIGHTS (>10% faster than main)") + print(f"HIGHLIGHTS (>{hpct}% faster than main)") print("=" * 100) - for name, main_avg, pr_avg in highlights: - factor = main_avg / pr_avg - print(f" {name}: {main_avg:.4f}s -> {pr_avg:.4f}s ({factor:.1f}x faster)") + for name, old, new, normed in highlights: + factor = old / new if new else 0 + print(f" {name}: {_fmt_change(old, new, normed)} ({factor:.2f}x faster)") else: - print("HIGHLIGHTS (>10% faster than main): None") + print(f"HIGHLIGHTS (>{hpct}% faster than main): None") print(f"\n{'='*100}\n") @@ -322,8 +403,17 @@ def save_json(results: List[tuple], path: str): "mssql_python": mssql_result.to_dict(), "pyodbc": pyodbc_result.to_dict(), } - # For baseline consumption, also store flat avg per scenario at top level - data["baseline"] = {name: mssql_result.to_dict() for name, mssql_result, _ in results} + # For baseline consumption, store per-scenario metrics plus the normalized score + # (see normalized_score()). PR runs compare their own normalized score against the + # stored `norm` so that runner-to-runner speed differences cancel out. + data["baseline"] = {} + for name, mssql_result, pyodbc_result in results: + entry = mssql_result.to_dict() + py_med = pyodbc_result.median + entry["pyodbc_median"] = round(py_med, 6) + score = normalized_score(mssql_result.median, py_med) + entry["norm"] = round(score, 6) if score is not None else None + data["baseline"][name] = entry with open(path, "w") as f: json.dump(data, f, indent=2) print(f"Results saved to {path}") @@ -439,7 +529,7 @@ def main(): else: print("PERFORMANCE BENCHMARKING: mssql-python vs pyodbc") print("=" * 100) - print(f" Iterations: {NUM_ITERATIONS}") + print(f" Iterations: {NUM_ITERATIONS} (first {WARMUP_ITERATIONS} discarded as warmup, metric: median)") if baseline: print(f" Baseline: {args.baseline}") print() @@ -452,14 +542,14 @@ def main(): print(f" pyodbc... ", end="", flush=True) py_result = run_fetch_pyodbc(query, name, NUM_ITERATIONS) if py_result.times: - print(f"OK ({py_result.avg:.4f}s)") + print(f"OK ({py_result.median:.4f}s)") else: print("FAILED") print(f" mssql-python... ", end="", flush=True) ms_result = run_fetch_mssql(query, name, NUM_ITERATIONS) if ms_result.times: - print(f"OK ({ms_result.avg:.4f}s)") + print(f"OK ({ms_result.median:.4f}s)") else: print("FAILED") @@ -471,14 +561,14 @@ def main(): print(f" pyodbc... ", end="", flush=True) py_insert = run_insertmany_pyodbc(NUM_ITERATIONS) if py_insert.times: - print(f"OK ({py_insert.avg:.4f}s)") + print(f"OK ({py_insert.median:.4f}s)") else: print("FAILED") print(f" mssql-python... ", end="", flush=True) ms_insert = run_insertmany_mssql(NUM_ITERATIONS) if ms_insert.times: - print(f"OK ({ms_insert.avg:.4f}s)") + print(f"OK ({ms_insert.median:.4f}s)") else: print("FAILED") From 0966b059e4493940e0aefc42e23cf017ddd6c5bc Mon Sep 17 00:00:00 2001 From: David Levy Date: Thu, 16 Jul 2026 06:03:10 -0500 Subject: [PATCH 17/31] DOC: add copilot-instructions.md for coding-agent guidance (#406) ### Work Item / Issue Reference > GitHub Issue: #247 ------------------------------------------------------------------- ### Summary adds `.github/copilot-instructions.md`: a from-scratch, repo-wide guide for coding agents working on mssql-python. it covers the architecture (python -> pybind `ddbc_bindings` -> odbc, with py-core/tds for bulk copy), the validated build/test/validation-gate commands, the C++ shutdown/init/arch hazards, the credential-scanning localhost rule, PR conventions, and a trust-and-validate close. it references the existing `.github/prompts/` files instead of duplicating them, and every path plus the CI-enforced title prefixes were checked against the current tree. --------- Co-authored-by: Gaurav Sharma Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..60b9a4dba --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,94 @@ +# mssql-python — Copilot instructions + +mssql-python is Microsoft's pip-installable Python driver for SQL Server, Azure SQL, and Azure Synapse. It is DB API 2.0 (PEP 249) compliant. The core is a C++ pybind11 native extension (`ddbc_bindings`) built with CMake and wrapped by a pure-Python package; bulk copy is backed by a separate Rust/TDS core (`mssql_py_core`). Platform-specific wheels ship for Windows (x64, ARM64), macOS (x64, ARM64), and Linux (x64, ARM64). Python 3.10+. The ODBC driver is bundled in the wheel — no external driver manager is required. + +## Architecture + +Call stack, top to bottom: + +1. **Pure-Python API** — `mssql_python/` (`connection.py`, `cursor.py`, `row.py`, `pooling.py`, `auth.py`, `exceptions.py`, type coercion). This is the DB API 2.0 surface. +2. **Extension loader** — `mssql_python/ddbc_bindings.py` detects platform/architecture and loads the correct native binary. +3. **pybind11 binding layer** — `mssql_python/pybind/` (`ddbc_bindings.cpp`/`.h`, `connection/`, `logger_bridge.*`, `unix_utils.*`), compiled to `ddbc_bindings.cp{ver}-{arch}.{so|pyd}` in `mssql_python/`. +4. **ODBC driver stack → SQL Server.** Bulk copy takes a different path: it uses `mssql_py_core` (TDS) directly, not ODBC. + +Paths you will touch: + +- `mssql_python/pybind/CMakeLists.txt` — all platform/architecture build conditionals live here. +- `mssql_python/pybind/build.sh` / `build.bat` — build entry points; `configure_dylibs.sh` fixes macOS dylib paths. +- `mssql_python/libs/{windows,macos,linux}/...` — prebuilt ODBC binaries. **NEVER hand-edit these.** +- `setup.py` / `pyproject.toml` — packaging and wheel/platform tagging. +- `mssql_python/mssql_python.pyi` + `mssql_python/py.typed` — PEP 561 type stubs. Update stubs when the public API changes. +- `tests/` — mostly-numbered `test_NNN_*.py` files (a live-SQL-Server integration suite plus no-DB dependency checks). + +## Development workflow + +Validated, step-by-step guides live in `.github/prompts/` — use them instead of rediscovering commands: + +- `setup-dev-env.prompt.md` — venv, dependencies, ODBC headers, `DB_CONNECTION_STRING`, SQL Server. +- `build-ddbc.prompt.md` — build the C++ extension. +- `run-tests.prompt.md` — pytest categories and markers. +- `create-pr.prompt.md` — the PR flow (title / issue-link / summary confirmations). + +Core facts: + +- **ALWAYS build the native extension before running Python tests.** Importing `mssql_python` needs the compiled `.so`/`.pyd`. Build with `cd mssql_python/pybind && ./build.sh` (or `build.bat` on Windows; pass `x64|arm64` as needed). +- **Tests need a live SQL Server** reachable through the `DB_CONNECTION_STRING` env var. `tests/test_000_dependencies.py` runs with no DB; most others require one. +- macOS wheels are universal2 (arm64 and x86_64); when touching dylib/rpath setup (`configure_dylibs.sh`), make sure both slices are handled, not just the build host. + +## Validation gate (run before you finish — this mirrors CI) + +```bash +black --check --line-length=100 mssql_python/ tests/ # BLOCKING in CI +python -m pytest -v # 'stress' marker excluded by default +``` + +- **`pr-format-check` (BLOCKING):** PR title must start with one of `FEAT: FIX: DOC: CHORE: STYLE: REFACTOR: RELEASE:`; the body must link a work item/issue and have a `### Summary` of at least 10 characters. +- `flake8`, `pylint`, `mypy`, `clang-format`, and `cpplint` run but are **informational**, not blocking. +- The authoritative cross-platform validation runs on **Azure DevOps** (broader OS / Python / arch coverage than the GitHub checks); consult the specific pipeline in `eng/pipelines/` for the exact matrix rather than assuming full coverage. A coverage bot posts a report comment on the PR. + +## Code standards + +**Python** + +- Black, line length 100. DB API 2.0 semantics take priority. +- Catch specific exceptions (`DatabaseError`, `IntegrityError`, `OperationalError`, …), **never a bare `except:`** — flake8 ignores E722, but the team enforces this in review. +- Use context managers for connections and cursors. Update `__all__` **and** `mssql_python/mssql_python.pyi` when changing the public API. Add a `tests/test_*.py` case for every fix. + +**C++ / pybind11** (`mssql_python/pybind/`) — hazards to respect: + +- **Process-shutdown ordering for Python handles.** Destructors of static `py::object` caches (e.g. the datetime/decimal class cache in `ddbc_bindings.cpp`) run after Python finalizes; an ODBC or threading call from a destructor after the GIL is gone can deadlock or crash the whole test suite at exit. Guard shutdown cleanup with `Py_IsInitialized()`, register cleanup via `atexit`, and prefer not to add new static Python handles. +- **Don't let a failed initialization escape as a half-built object.** `Connection::setAttribute` returns `SQLRETURN`; `applyAttrsBefore` checks it and raises via `ThrowStdException` before the connection is exposed. Follow that pattern — translate a failing return into an exception rather than handing back a partially-initialized object. +- **Handle every shipped architecture, not just `$(uname -m)`.** Universal2 bundles arm64 and x86_64; dylib/rpath work that only touches the host arch ships a broken slice for the other. Verify both. +- Hot paths favor the raw CPython API (with correct refcounting and `PyErr_Occurred()` checks) over pybind11 per-cell calls. + +## Testing conventions + +- Test files are mostly numbered `test_NNN_*.py`; `tests/test_000_dependencies.py` runs without a DB, most others need a live SQL Server. `-m "not stress"` is the default. +- Run segfault-prone or ODBC/pool global-state tests in an **isolated subprocess** so a crash or shared state cannot poison the rest of the suite. +- **Assert the contract, not just the output.** If a change's value is "we now call X once," assert the call/round-trip count — a correctness-only test won't catch a perf regression. +- For global type-mapping changes, add typed-NULL integration cases (VARBINARY, UNIQUEIDENTIFIER, XML, DECIMAL, stored-proc params) before applying the optimization broadly. + +## Security and credentials + +- **Committed connection strings that contain `UID`/`PWD` must use `SERVER=localhost` (or `127.0.0.1`) with dummy values.** Real remote or Azure credentials come only from secrets or the `DB_CONNECTION_STRING` env var, and are never committed. Automated credential scanning (see `.config/CredScanSuppressions.json`, `.gdn/`) can block unsafe patterns. +- Do **not** put `Driver=` in a connection string — the bundled driver is selected automatically. +- `TrustServerCertificate=yes` is local-development only; never suggest it in remote or production examples. + +## Contributing + +- Branch naming: `/` (e.g. `bewithgaurav/fix-656-macos-dylib`). +- Link exactly one reference in the PR body: maintainers use `AB#` (ADO auto-close); external contributors use plain `#` — **never `Closes #N`**. +- Stage specific files; **never `git add .`**. Never commit build artifacts (`*.so`, `*.pyd`, `*.dll`, `*.dylib`) or a virtual environment. +- Keep changes surgical — fix the requested thing, leave unrelated code alone, and do not add stray files. + +## Working effectively in this repo + +- **Reproduce before you claim.** Build and run a live repro against a real SQL Server before asserting a bug exists or that a fix works. Do not ship code-only assessments. +- Understand the linked issue and existing review threads before changing code. +- Search existing issues and PRs before opening a new one — do not create duplicates. +- Do not post PR or issue comments as part of a change unless explicitly asked. +- Verify that mutating git/gh operations actually landed (re-read the branch or PR) before reporting done. + +## Trust these instructions + +Trust the commands, paths, and conventions above — they were validated against the current repository. Only fall back to searching the codebase when something here is missing or proves incorrect, and when you find a gap, update these instructions. From fdb02cb62f91c3280f2d2cbf834ddb97793de7ee Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:27:59 +0530 Subject: [PATCH 18/31] Add OneBranch release pipelines for mssql-python-odbc Adds the build and release pipelines for the standalone mssql-python-odbc package: per-platform build stages (Windows x64/arm64, macOS universal2, Linux manylinux_2_28 + musllinux_1_2 for x86_64/aarch64), a consolidate job that gathers all 7 py3-none wheels, the build orchestration pipeline, and the ESRP-based official release pipeline. The build stages run no compilation (data-only package); Linux builds run in tdslibrs ACR PyPA containers and the musllinux tag is auto-detected via libc_ver. Infra follow-ups (ESRP onboarding of the mssql-python-odbc name, registering the ADO build definition to set odbcBuildDefinitionId, and sharing the ESRP variable group) are marked with TODO(infra) comments. --- .../build-release-odbc-pipeline.yml | 210 ++++++++++++++++ .../jobs/consolidate-odbc-artifacts-job.yml | 68 ++++++ .../official-release-odbc-pipeline.yml | 227 ++++++++++++++++++ .../stages/build-odbc-linux-stage.yml | 163 +++++++++++++ .../stages/build-odbc-macos-stage.yml | 107 +++++++++ .../stages/build-odbc-windows-stage.yml | 125 ++++++++++ 6 files changed, 900 insertions(+) create mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml create mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/stages/build-odbc-linux-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-macos-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-windows-stage.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml new file mode 100644 index 000000000..260fd1c3c --- /dev/null +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -0,0 +1,210 @@ +# ========================================================================================= +# OneBranch Build Pipeline for mssql-python-odbc +# ========================================================================================= +# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. +# +# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is +# NO compiled Python extension, so every wheel is tagged `py3-none-` and a +# SINGLE build per platform covers all supported Python versions (3.10+). That is why +# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces +# only 7 wheels total: +# - Windows: x64, arm64 +# - macOS: universal2 +# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) +# +# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml +# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). +# +# INFRA PREREQUISITES (see companion release pipeline for the definition ID): +# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). +# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. +# - Ensure the reused pools/images are available: Python-1ES-pool +# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the +# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. +# ========================================================================================= + +# Build number format: YYDDD.r +name: $(Year:YY)$(DayOfYear)$(Rev:.r) + +# Only trigger when ODBC-package inputs change (path filters are literal prefixes, +# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. +trigger: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +pr: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + + - name: runSdlTasks + displayName: 'Run SDL Security Tasks' + type: boolean + default: true + + # Windows: one wheel per architecture (NO per-Python matrix). + - name: odbcWindowsConfigs + type: object + default: + - arch: 'x64' + - arch: 'arm64' + + # macOS: single universal2 wheel. + - name: odbcMacosConfigs + type: object + default: + - name: 'universal2' + + # Linux: one wheel per libc/arch combo. + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + +variables: + # Scheduled builds run as Official; manual/PR builds use the parameter. + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the + # per-stage EsrpMalwareScanning tasks. + - template: /OneBranchPipelines/variables/signing-variables.yml@self + - group: 'ESRP Federated Creds (AME)' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + + # This pipeline compiles NO first-party native code — it only packages the + # pre-built ODBC driver binaries, which are signed and governed by the Microsoft + # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; + # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). + binskim: + enabled: false + justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' + apiscan: + enabled: false + justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' + armory: + enabled: false + codeql: + enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' + + credscan: + enabled: ${{ parameters.runSdlTasks }} + policheck: + enabled: ${{ parameters.runSdlTasks }} + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: ${{ parameters.runSdlTasks }} + sbom: + enabled: ${{ parameters.runSdlTasks }} + packageName: 'mssql-python-odbc' + tsa: + enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + # WINDOWS BUILD STAGES (2: x64, arm64) + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # MACOS BUILD STAGE (1: universal2) + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # CONSOLIDATE STAGE (waits for all 7 platform stages) + - stage: Consolidate + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..8ac829631 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,68 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download ALL artifacts from the current build (one per platform stage). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" + else + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + fi + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_Consolidate_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml new file mode 100644 index 000000000..3a9898c25 --- /dev/null +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -0,0 +1,227 @@ +# ========================================================================================= +# OneBranch Official Release Pipeline for mssql-python-odbc +# ========================================================================================= +# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to +# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group +# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python +# release (same inputs and FolderLocation); the only differences are the build pipeline +# source and that this package has no debug symbols to publish. +# +# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the +# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# ========================================================================================= + +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC + +# Manual trigger only — releases should be deliberate. +trigger: none +pr: none + +parameters: + - name: releaseToPyPI + displayName: 'Release to PyPI (Production)' + type: boolean + default: false # Safety: default false to prevent accidental releases. + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Shared ESRP identity — same group used by the mssql-python release pipeline. + - group: 'ESRP Federated Creds (AME)' + # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # once it is registered (the mssql-python build definition is 2199). Left as 0 so the + # pipeline compiles; the download tasks will fail at runtime until this is set. + - name: odbcBuildDefinitionId + value: '0' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + + # Reference to the ODBC build pipeline (select a run in the UI when releasing). + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + + # Artifacts were already scanned during the build; this pipeline only republishes. + binskim: + enabled: false + justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: ReleasePackages + displayName: 'Release mssql-python-odbc to PyPI' + + jobs: + # Job 1: Download + validate the consolidated wheels (custom pool). + - job: PrepAndValidate + displayName: 'Download and Validate Artifacts' + + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Consolidated ODBC Artifacts' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + - task: PowerShell@2 + displayName: 'List and Verify Downloaded Wheels' + inputs: + targetType: 'inline' + script: | + $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" + if (-not (Test-Path $wheelsPath)) { + Write-Error "Wheel directory not found at: $wheelsPath" + exit 1 + } + $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" + Write-Host "Total wheel files found: $($wheels.Count)" + + $allValid = $true + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($wheel in $wheels) { + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) + $entryCount = $zip.Entries.Count + $zip.Dispose() + Write-Host " OK $($wheel.Name) ($entryCount entries)" + } catch { + Write-Error " BAD $($wheel.Name): $_" + $allValid = $false + } + } + if (-not $allValid) { + Write-Error "One or more wheel files are corrupted" + exit 1 + } + if ($wheels.Count -ne 7) { + Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + } + Write-Host "All wheels verified successfully!" + + # Dry-run summary (when release is disabled). + - ${{ if eq(parameters.releaseToPyPI, false) }}: + - task: PowerShell@2 + displayName: 'Dry Run - Release Skipped' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "DRY RUN MODE - No Release Performed" + Write-Host "Package: mssql-python-odbc" + Write-Host "====================================" + Write-Host "To perform an actual release:" + Write-Host "1. Set 'releaseToPyPI' parameter to true" + Write-Host "2. Re-run pipeline" + + # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). + # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. + - ${{ if eq(parameters.releaseToPyPI, true) }}: + - job: PyPIRelease + displayName: 'ESRP Release to PyPI' + dependsOn: PrepAndValidate + + templateContext: + type: releaseJob + isProduction: true + + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Artifacts from ODBC Build Pipeline' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + # ESRP release task — identical to the mssql-python release pipeline + # (same shared ESRP identity and PyPI publish inputs). + - task: EsrpRelease@9 + displayName: 'ESRP Release to PyPI' + inputs: + connectedservicename: '$(ESRPConnectedServiceName)' + usemanagedidentity: true + keyvaultname: '$(AuthAKVName)' + signcertname: '$(AuthSignCertName)' + clientid: '$(EsrpClientId)' + Intent: 'PackageDistribution' + ContentType: 'PyPI' + ContentSource: 'Folder' + FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' + WaitForReleaseCompletion: true + Owners: '$(owner)' + Approvers: '$(approver)' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMAN' + DomainTenantId: '$(DomainTenantId)' + + - task: PowerShell@2 + displayName: 'Release Summary' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "ESRP Release Completed" + Write-Host "Package: mssql-python-odbc" + Write-Host "Target: PyPI" + Write-Host "====================================" + Write-Host "Next steps:" + Write-Host "1. Verify release in the ESRP portal" + Write-Host "2. Wait for approval workflow completion" + Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..798b3c341 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,163 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel + cd /workspace + $PY setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..9b47388e3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,107 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..7a812de45 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,125 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). + targetArch: ${{ parameters.architecture }} + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 From 79d60ba3a693fc43c887e3d78fbc9fb7754e93b3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:43:11 +0530 Subject: [PATCH 19/31] Address PR review: fail-fast on wheel count, armory justification, Linux twine check + container cleanup, reword infra TODOs --- OneBranchPipelines/build-release-odbc-pipeline.yml | 1 + .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++--- OneBranchPipelines/official-release-odbc-pipeline.yml | 9 +++++---- OneBranchPipelines/stages/build-odbc-linux-stage.yml | 8 +++++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml index 260fd1c3c..b64dc936b 100644 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -144,6 +144,7 @@ extends: justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' armory: enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' codeql: enabled: false justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 8ac829631..0225d34a2 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -54,10 +54,10 @@ jobs: WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then - echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" - else - echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." displayName: 'Consolidate ODBC wheels' - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 3a9898c25..90eaedddb 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -8,7 +8,7 @@ # source and that this package has no debug symbols to publish. # # This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# `mssql-python-odbc` project name lands and the two infra values below are set. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC @@ -28,7 +28,7 @@ variables: - template: /OneBranchPipelines/variables/onebranch-variables.yml@self # Shared ESRP identity — same group used by the mssql-python release pipeline. - group: 'ESRP Federated Creds (AME)' - # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' # once it is registered (the mssql-python build definition is 2199). Left as 0 so the # pipeline compiles; the download tasks will fail at runtime until this is set. - name: odbcBuildDefinitionId @@ -44,7 +44,7 @@ resources: # Reference to the ODBC build pipeline (select a run in the UI when releasing). pipelines: - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. trigger: none extends: @@ -141,7 +141,8 @@ extends: exit 1 } if ($wheels.Count -ne 7) { - Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" + exit 1 } Write-Host "All wheels verified successfully!" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 798b3c341..d0a9883cd 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -123,9 +123,10 @@ stages: # Pick any available CPython from the PyPA image. PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) echo "Using: $($PY --version)" - $PY -m pip install -q -U pip setuptools wheel + $PY -m pip install -q -U pip setuptools wheel twine cd /workspace $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl echo "Produced wheels:"; ls -lh /workspace/dist/*.whl ' displayName: 'Build ODBC wheel in container' @@ -135,6 +136,11 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Linux Artifact' inputs: From d4b82a613f8d51966c1e40a8449e82b37aa51f39 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:58:20 +0530 Subject: [PATCH 20/31] Address review: preflight guard for odbcBuildDefinitionId placeholder; drop unused targetArch var on Windows stage --- OneBranchPipelines/official-release-odbc-pipeline.yml | 11 +++++++++++ .../stages/build-odbc-windows-stage.yml | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 90eaedddb..ef06b0895 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -99,6 +99,17 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: + - task: PowerShell@2 + displayName: 'Preflight: verify odbcBuildDefinitionId is configured' + inputs: + targetType: 'inline' + script: | + if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { + Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." + exit 1 + } + Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" + - task: DownloadPipelineArtifact@2 displayName: 'Download Consolidated ODBC Artifacts' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 7a812de45..57de838ed 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -48,8 +48,6 @@ stages: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' # OneBranch-required variable (unused on Windows stages). LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' - # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). - targetArch: ${{ parameters.architecture }} steps: # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. From 68b7ba00d98ec1cd30ef7ae6ba4925d25edc8a80 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:33:46 +0530 Subject: [PATCH 21/31] Fold mssql-python-odbc build+release into existing pipelines Per review feedback, build and release the standalone mssql-python-odbc package from the existing Build-Release-Package-Pipeline (def 2199) and official release pipeline instead of separate ADO definitions. Build pipeline: add odbcWindows/Macos/LinuxConfigs params (2+1+4=7 data-only wheels), reference the 3 odbc stage templates alongside the mssql-python stages, and add a separate ConsolidateOdbc stage publishing drop_ConsolidateOdbc_ConsolidateArtifacts (distinct from mssql-python's drop_Consolidate_ConsolidateArtifacts). Release pipeline: add releasePackage parameter (mssql-python | mssql-python-odbc); select the consolidated artifact accordingly and skip symbol publishing + mssql-py-core version validation for odbc. Delete the redundant standalone orchestrators (build-release-odbc-pipeline.yml, official-release-odbc-pipeline.yml). This removes the need to register a new ADO build definition and the odbcBuildDefinitionId placeholder. Both packages now build from def 2199; the odbc stages consume setup_odbc.py from PR #663, so #663 must merge before these stages can build. --- .../build-release-odbc-pipeline.yml | 211 ---------------- .../build-release-package-pipeline.yml | 94 ++++++- .../jobs/consolidate-odbc-artifacts-job.yml | 5 +- .../official-release-odbc-pipeline.yml | 239 ------------------ .../official-release-pipeline.yml | 56 ++-- 5 files changed, 136 insertions(+), 469 deletions(-) delete mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml delete mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml deleted file mode 100644 index b64dc936b..000000000 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ /dev/null @@ -1,211 +0,0 @@ -# ========================================================================================= -# OneBranch Build Pipeline for mssql-python-odbc -# ========================================================================================= -# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. -# -# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is -# NO compiled Python extension, so every wheel is tagged `py3-none-` and a -# SINGLE build per platform covers all supported Python versions (3.10+). That is why -# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces -# only 7 wheels total: -# - Windows: x64, arm64 -# - macOS: universal2 -# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) -# -# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml -# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). -# -# INFRA PREREQUISITES (see companion release pipeline for the definition ID): -# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). -# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. -# - Ensure the reused pools/images are available: Python-1ES-pool -# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the -# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. -# ========================================================================================= - -# Build number format: YYDDD.r -name: $(Year:YY)$(DayOfYear)$(Rev:.r) - -# Only trigger when ODBC-package inputs change (path filters are literal prefixes, -# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. -trigger: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -pr: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -parameters: - - name: oneBranchType - displayName: 'OneBranch Template Type' - type: string - values: - - 'Official' - - 'NonOfficial' - default: 'NonOfficial' - - - name: runSdlTasks - displayName: 'Run SDL Security Tasks' - type: boolean - default: true - - # Windows: one wheel per architecture (NO per-Python matrix). - - name: odbcWindowsConfigs - type: object - default: - - arch: 'x64' - - arch: 'arm64' - - # macOS: single universal2 wheel. - - name: odbcMacosConfigs - type: object - default: - - name: 'universal2' - - # Linux: one wheel per libc/arch combo. - - name: odbcLinuxConfigs - type: object - default: - - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } - - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } - -variables: - # Scheduled builds run as Official; manual/PR builds use the parameter. - - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: 'Official' - ${{ else }}: - value: '${{ parameters.oneBranchType }}' - - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the - # per-stage EsrpMalwareScanning tasks. - - template: /OneBranchPipelines/variables/signing-variables.yml@self - - group: 'ESRP Federated Creds (AME)' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - -extends: - template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress - suppressionSet: default - - # This pipeline compiles NO first-party native code — it only packages the - # pre-built ODBC driver binaries, which are signed and governed by the Microsoft - # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; - # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). - binskim: - enabled: false - justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' - apiscan: - enabled: false - justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' - armory: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' - codeql: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' - - credscan: - enabled: ${{ parameters.runSdlTasks }} - policheck: - enabled: ${{ parameters.runSdlTasks }} - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: ${{ parameters.runSdlTasks }} - sbom: - enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python-odbc' - tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - # WINDOWS BUILD STAGES (2: x64, arm64) - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # MACOS BUILD STAGE (1: universal2) - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # CONSOLIDATE STAGE (waits for all 7 platform stages) - - stage: Consolidate - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..984e52bcc 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -138,6 +138,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -338,8 +365,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -466,3 +495,64 @@ extends: # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 0225d34a2..42db132ff 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -64,5 +64,8 @@ jobs: displayName: 'Publish Consolidated ODBC Artifacts' inputs: targetPath: '$(ob_outputDirectory)' - artifact: 'drop_Consolidate_ConsolidateArtifacts' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml deleted file mode 100644 index ef06b0895..000000000 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -# ========================================================================================= -# OneBranch Official Release Pipeline for mssql-python-odbc -# ========================================================================================= -# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to -# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group -# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python -# release (same inputs and FolderLocation); the only differences are the build pipeline -# source and that this package has no debug symbols to publish. -# -# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two infra values below are set. -# ========================================================================================= - -name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC - -# Manual trigger only — releases should be deliberate. -trigger: none -pr: none - -parameters: - - name: releaseToPyPI - displayName: 'Release to PyPI (Production)' - type: boolean - default: false # Safety: default false to prevent accidental releases. - -variables: - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Shared ESRP identity — same group used by the mssql-python release pipeline. - - group: 'ESRP Federated Creds (AME)' - # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' - # once it is registered (the mssql-python build definition is 2199). Left as 0 so the - # pipeline compiles; the download tasks will fail at runtime until this is set. - - name: odbcBuildDefinitionId - value: '0' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - - # Reference to the ODBC build pipeline (select a run in the UI when releasing). - pipelines: - - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. - trigger: none - -extends: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - - # Artifacts were already scanned during the build; this pipeline only republishes. - binskim: - enabled: false - justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' - credscan: - enabled: true - policheck: - enabled: true - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: true - tsa: - enabled: true - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - - stage: ReleasePackages - displayName: 'Release mssql-python-odbc to PyPI' - - jobs: - # Job 1: Download + validate the consolidated wheels (custom pool). - - job: PrepAndValidate - displayName: 'Download and Validate Artifacts' - - pool: - type: windows - isCustom: true - name: Python-1ES-pool - demands: - - imageOverride -equals PYTHON-1ES-MMS2022 - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - - steps: - - task: PowerShell@2 - displayName: 'Preflight: verify odbcBuildDefinitionId is configured' - inputs: - targetType: 'inline' - script: | - if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { - Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." - exit 1 - } - Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Consolidated ODBC Artifacts' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - - task: PowerShell@2 - displayName: 'List and Verify Downloaded Wheels' - inputs: - targetType: 'inline' - script: | - $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" - if (-not (Test-Path $wheelsPath)) { - Write-Error "Wheel directory not found at: $wheelsPath" - exit 1 - } - $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" - Write-Host "Total wheel files found: $($wheels.Count)" - - $allValid = $true - Add-Type -AssemblyName System.IO.Compression.FileSystem - foreach ($wheel in $wheels) { - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) - $entryCount = $zip.Entries.Count - $zip.Dispose() - Write-Host " OK $($wheel.Name) ($entryCount entries)" - } catch { - Write-Error " BAD $($wheel.Name): $_" - $allValid = $false - } - } - if (-not $allValid) { - Write-Error "One or more wheel files are corrupted" - exit 1 - } - if ($wheels.Count -ne 7) { - Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" - exit 1 - } - Write-Host "All wheels verified successfully!" - - # Dry-run summary (when release is disabled). - - ${{ if eq(parameters.releaseToPyPI, false) }}: - - task: PowerShell@2 - displayName: 'Dry Run - Release Skipped' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "DRY RUN MODE - No Release Performed" - Write-Host "Package: mssql-python-odbc" - Write-Host "====================================" - Write-Host "To perform an actual release:" - Write-Host "1. Set 'releaseToPyPI' parameter to true" - Write-Host "2. Re-run pipeline" - - # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). - # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. - - ${{ if eq(parameters.releaseToPyPI, true) }}: - - job: PyPIRelease - displayName: 'ESRP Release to PyPI' - dependsOn: PrepAndValidate - - templateContext: - type: releaseJob - isProduction: true - - pool: - type: windows - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Artifacts from ODBC Build Pipeline' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - # ESRP release task — identical to the mssql-python release pipeline - # (same shared ESRP identity and PyPI publish inputs). - - task: EsrpRelease@9 - displayName: 'ESRP Release to PyPI' - inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - ContentType: 'PyPI' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' - - - task: PowerShell@2 - displayName: 'Release Summary' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "ESRP Release Completed" - Write-Host "Package: mssql-python-odbc" - Write-Host "Target: PyPI" - Write-Host "====================================" - Write-Host "Next steps:" - Write-Host "1. Verify release in the ESRP portal" - Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..731df538e 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" @@ -273,7 +297,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,7 +327,7 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" @@ -313,6 +337,6 @@ extends: Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" From 7a1d351c9a402c40e6c6d68e2d942770f2a20df2 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:50:04 +0530 Subject: [PATCH 22/31] Scope consolidate downloads by package to prevent cross-contamination Def 2199 now builds mssql-python and mssql-python-odbc in one run. Both consolidate jobs used DownloadPipelineArtifact@2 with buildType: current, which downloads every artifact in the run and copies all *.whl into dist/. In the merged run this caused: (1) the main Consolidate to sweep the 7 odbc wheels into the mssql-python release (count is warning-only, so silent), and (2) ConsolidateOdbc to see 34 wheels while expecting 7 and hard-fail. Add itemPattern to scope each download to its own package's build-stage artifacts (main: drop_Win_*/drop_MacOS_*/drop_Linux_*; odbc: drop_ODBC_*), giving clean partitioning with no wheel leakage. --- .../jobs/consolidate-artifacts-job.yml | 13 ++++++++++--- .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..336c736a5 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 42db132ff..f0d8a0715 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -32,11 +32,15 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from the current build (one per platform stage). + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). - task: DownloadPipelineArtifact@2 displayName: 'Download All ODBC Platform Artifacts' inputs: buildType: 'current' + itemPattern: 'drop_ODBC_*/**' targetPath: '$(Pipeline.Workspace)/all-artifacts' - bash: | From 7e5947ea5d97c3dfeeb349813c5540e8e6ec48ce Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:16:59 +0530 Subject: [PATCH 23/31] Correct stale mssql-python wheel count in consolidate diagnostic (27 -> 34) The matrix is 9 Windows + 5 macOS + 4 Linux stages x 5 Python versions (cp310-cp314) = 34 wheels across 18 stages. The diagnostic said 27 (7+4+16), stale from an older 3.10-3.13 / 7-Windows matrix. Count check remains warning-only (no behavior change). --- .../jobs/consolidate-artifacts-job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 336c736a5..52eb43381 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -75,12 +75,12 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' From 68cba53970bc35a82bab3c28786fe36b88162ade Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:49:09 +0530 Subject: [PATCH 24/31] Harden odbc release path: pin guard + wheel-content assertions official-release-pipeline.yml: gate 'Published symbols' dry-run/summary text to mssql-python so odbc runs no longer claim symbol publishing that never happens; add a mssql-python-only step asserting the wheel metadata pins mssql-python-odbc==18.6.0 (fails by design if released from a pre-pin build). build-odbc-{linux,macos,windows}-stage.yml: assert each built wheel actually contains the ODBC driver binary (msodbcsql*), guarding against the .gitignore '*.so' trap / silent sync_libs() skip shipping an empty data wheel that still passes twine check. --- .../official-release-pipeline.yml | 48 ++++++++++++++++++- .../stages/build-odbc-linux-stage.yml | 22 +++++++++ .../stages/build-odbc-macos-stage.yml | 21 ++++++++ .../stages/build-odbc-windows-stage.yml | 22 +++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 731df538e..7cb529796 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -331,7 +374,8 @@ extends: Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index d0a9883cd..969c0b8b3 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -136,6 +136,28 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - script: | docker rm -f build-$(LINUX_TAG)-$(ARCH) || true displayName: 'Stop and remove build container' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index 9b47388e3..c1e7541b1 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -79,6 +79,27 @@ stages: TargetFolder: '$(ob_outputDirectory)/wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC macOS Artifact' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 57de838ed..dea809ed0 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,6 +92,28 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + # OneBranch requires artifact naming: drop__. - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Windows Artifact' From 0772cc0683c3624e93599ce94afb75d1438cd1c8 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:24:25 +0530 Subject: [PATCH 25/31] Hard-fail mssql-python consolidate on wheel-count mismatch Flip the 34-wheel count check from WARNING-only to a hard exit 1, symmetric with the odbc consolidate job's count guard. A mismatch means a build stage silently produced a partial wheel set (e.g. a missing Python version) or the matrix changed without updating the expected count -- previously this shipped silently. --- OneBranchPipelines/jobs/consolidate-artifacts-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 52eb43381..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -77,8 +77,13 @@ jobs: echo "Total wheel count: $WHEEL_COUNT" echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else echo "SUCCESS: All 34 wheels consolidated!" fi From 52dbd7bea53f3e9be6e95e1be909e05126d943d4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 14:04:13 +0530 Subject: [PATCH 26/31] odbc: add mssql-python-odbc selector to dummy release pipeline Mirrors the official-release-pipeline odbc changes: releasePackage parameter, consolidatedArtifactName selection for both artifact downloads, and gating the mssql-py-core version validation + symbol publishing to mssql-python only. Stays on Maven ContentType so it never publishes to PyPI. --- OneBranchPipelines/dummy-release-pipeline.yml | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..28751e845 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # the mssql-python-odbc package ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" @@ -280,7 +302,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,7 +335,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" From 19f1db171d59a9f8b49833ccb33c9ebbdfd20eaa Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:00:10 +0530 Subject: [PATCH 27/31] odbc: align dummy release pipeline mssql-python handling with official Adds the mssql-python-odbc pin guard (gated to mssql-python, matching official-release-pipeline) and gates the 'symbols published' summary messaging on releasePackage==mssql-python so the dummy pipeline exercises the exact same mssql-python validation/messaging as the official pipeline for both packages. --- OneBranchPipelines/dummy-release-pipeline.yml | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 28751e845..c98c351ff 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -196,7 +196,7 @@ extends: Write-Host "=====================================" # Step 3: Validate mssql-py-core is a stable version (mssql-python only; - # the mssql-python-odbc package ships no mssql-py-core.version file) + # mssql-python-odbc ships no mssql-py-core.version file) - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: - task: PowerShell@2 displayName: '[TEST] Validate mssql-py-core is a stable version' @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -339,7 +382,7 @@ extends: Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -348,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" From 2038ca63469e2bb17e83de6fefb6e1313be4e6b7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:28:24 +0530 Subject: [PATCH 28/31] odbc: bump mssql-python-odbc pin to 18.6.2 in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 2 +- OneBranchPipelines/official-release-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index c98c351ff..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 7cb529796..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." From 926ba53e3ee266ed14b4e48acd45e48937032da2 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:17:36 +0530 Subject: [PATCH 29/31] build: add buildPackage selector to build pipeline Gate mssql-python and mssql-python-odbc stage groups behind a buildPackage parameter (mssql-python default) so only the selected package builds, halving run time and enabling concurrent per-package builds. Mirrors the releasePackage selector pattern in the release pipelines. --- .../build-release-package-pipeline.yml | 216 ++++++++++-------- 1 file changed, 121 insertions(+), 95 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 984e52bcc..29e735080 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,22 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds use this default too, so they build mssql-python only. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -384,16 +400,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -410,15 +428,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -440,15 +459,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -462,36 +482,37 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging @@ -507,32 +528,36 @@ extends: # first-party native code is introduced. # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================================================================= # CONSOLIDATE ODBC STAGE @@ -541,18 +566,19 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - stage: ConsolidateOdbc - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 From 72c3c884eae523fa79d71aea7ab24883113bbfba Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:21:13 +0530 Subject: [PATCH 30/31] build: nightly build always builds both packages Add effectiveBuildPackage variable that forces 'both' on scheduled (Build.Reason=Schedule) runs while manual/PR runs honour the buildPackage parameter. Mirrors the effectiveOneBranchType pattern so the nightly keeps refreshing every artifact. --- .../build-release-package-pipeline.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 29e735080..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -76,7 +76,8 @@ parameters: # mssql-python -> Windows/macOS/Linux wheels + Consolidate # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc # both -> everything (previous default behaviour) - # Scheduled (daily) builds use this default too, so they build mssql-python only. + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. - name: buildPackage displayName: 'Package to Build' type: string @@ -192,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -401,7 +410,7 @@ extends: # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning # Gate: build mssql-python stages unless an odbc-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.windowsConfigs }}: - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self parameters: @@ -428,7 +437,7 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.macosConfigs }}: - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self parameters: @@ -459,7 +468,7 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self parameters: @@ -482,7 +491,7 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -529,7 +538,7 @@ extends: # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcWindowsConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self parameters: @@ -539,7 +548,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcMacosConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self parameters: @@ -548,7 +557,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcLinuxConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self parameters: @@ -566,7 +575,7 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - stage: ConsolidateOdbc displayName: 'Consolidate All ODBC Artifacts' dependsOn: From b15c8ad2ff7656d8e9c0a4b7528b13e85c94f49c Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 16:58:48 +0530 Subject: [PATCH 31/31] CHORE: reconcile GH pipeline Option A both-wheels test with validated ADO design Add hardened wheel-installation-test-stage.yml (positive+negative driver proof, source-tree shadowing guards, TESTED==0 fails stage) and wire two 'both'-gated TestBothWheels_* stages (manylinux_2_28 + musllinux x86_64) after ConsolidateOdbc, replacing the old single 3.12 invocation. --- .../build-release-package-pipeline.yml | 27 ++ .../stages/wheel-installation-test-stage.yml | 316 ++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 OneBranchPipelines/stages/wheel-installation-test-stage.yml diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index c485cc8ee..9c505f009 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -591,3 +591,30 @@ extends: parameters: oneBranchType: '${{ variables.effectiveOneBranchType }}' expectedWheelCount: 7 + + # ========================================================================= + # WHEEL INSTALLATION TEST STAGE(S) + # ========================================================================= + # Post-consolidation validation of the package split from an end user's + # perspective: install the ACTUAL published mssql-python + mssql-python-odbc + # wheels together, remove the bundled ODBC driver so the external package is + # the ONLY driver source, then run the full pytest suite. Runs only when BOTH + # packages are built in the same run (buildPackage = 'both'), because it needs + # both consolidated artifacts (drop_Consolidate_* and drop_ConsolidateOdbc_*). + - ${{ if eq(variables.effectiveBuildPackage, 'both') }}: + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_manylinux_2_28_x86_64 + jobName: InstallAndTest + linuxTag: manylinux_2_28 + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_musllinux_x86_64 + jobName: InstallAndTest + linuxTag: musllinux + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml new file mode 100644 index 000000000..7534f67f7 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -0,0 +1,316 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test Stage +# ========================================================================================= +# Validates the mssql-python / mssql-python-odbc PACKAGE SPLIT from an end-user's +# perspective, using the ACTUAL published wheel artifacts (not raw build binaries): +# +# pip install mssql_python---.whl +# pip install mssql_python_odbc--py3-none-.whl +# +# The proof is both POSITIVE and NEGATIVE: +# * POSITIVE: before touching anything we assert that the native loader WILL pick +# the external package - GetDriverPathCpp(dir(mssql_python_odbc)) points at an +# existing driver file, which is exactly the condition GetOdbcLibsBaseDir uses +# to treat the external package as authoritative. +# * NEGATIVE: we then DELETE the ODBC driver binaries that mssql-python still +# bundles under `mssql_python/libs/`. With the bundled fallback removed, a +# successful connection + passing test suite proves the driver was resolved +# from the separately-installed `mssql_python_odbc` package. +# (see GetOdbcLibsBaseDir in ddbc_bindings.cpp, which prefers the external package +# and falls back to bundled libs). Without the deletion the bundled fallback would +# mask a broken split and the test would pass even if the ODBC package were ignored. +# +# Scope: Linux manylinux_2_28 + musllinux (x86_64), all supported Python versions. +# This stage tests the INSTALL/RESOLUTION path, not per-arch compilation (already +# covered by the build stages). It runs only when BOTH packages are built in the +# same run (buildPackage = 'both'), because it needs both consolidated artifacts. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_manylinux_2_28_x86_64') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Linux distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc) + - name: linuxTag + type: string + # CPU architecture (test stage targets x86_64 only) + - name: arch + type: string + default: 'x86_64' + # Docker platform for the build/test container + - name: dockerPlatform + type: string + default: 'linux/amd64' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 90 + + variables: + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run: + # drop_Consolidate_ConsolidateArtifacts -> mssql-python wheels (all platforms/pyvers) + # drop_ConsolidateOdbc_ConsolidateArtifacts -> mssql-python-odbc wheels (7 platforms) + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + - bash: | + set -euxo pipefail + if ! docker info > /dev/null 2>&1; then + echo "Docker daemon not running, starting it..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Ensure Docker daemon is running' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'" >&2 + exit 1 + fi + + # Mount the repo and downloaded wheels read-only. Read-only /workspace is + # deliberate: this stage must exercise the INSTALLED wheels, never the + # source checkout, so nothing here should ever write into the repo tree. + docker run -d --name test-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace:ro \ + -v $(Pipeline.Workspace)/mssql-python-wheels:/wheels/mssql-python:ro \ + -v $(Pipeline.Workspace)/mssql-python-odbc-wheels:/wheels/mssql-python-odbc:ro \ + -w /workspace \ + $IMAGE \ + tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) test container' + + # Start a SQL Server 2022 container on the host so pytest has a live server. + - script: | + set -euxo pipefail + docker run -d --name sqlserver-$(LINUX_TAG)-$(ARCH) \ + --platform linux/amd64 \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ + -p 1433:1433 \ + mcr.microsoft.com/mssql/server:2022-latest + + echo "Waiting for SQL Server to be ready..." + for i in {1..30}; do + if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready" + break + fi + sleep 2 + done + + SQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sqlserver-$(LINUX_TAG)-$(ARCH)) + echo "SQL Server IP: $SQL_IP" + echo "##vso[task.setvariable variable=SQL_IP]$SQL_IP" + displayName: 'Start SQL Server container for testing' + env: + DB_PASSWORD: $(DB_PASSWORD) + + # For each Python version: install BOTH wheels, remove the bundled ODBC + # driver from the installed mssql_python package, then run pytest. A pass + # proves the driver was loaded from the separate mssql_python_odbc package. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + + # Count how many Python versions actually ran the full test so the stage + # cannot go green having silently skipped every version (e.g. if none of + # the /opt/python/ interpreters exist in the image). + TESTED=0 + # Inner script uses exit code 3 to signal "interpreter absent - skipped". + SKIP_RC=3 + + for PYBIN in cp310 cp311 cp312 cp313 cp314; do + echo "" + echo "=====================================================" + echo "End-user both-wheels test: $PYBIN on $(LINUX_TAG)/$(ARCH)" + echo "=====================================================" + + set +e + docker exec -e PYBIN=$PYBIN -e LINUX_TAG="$(LINUX_TAG)" -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" \ + test-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -euo pipefail + + PY=/opt/python/${PYBIN}-${PYBIN}/bin/python + test -x "$PY" || { echo "Python $PY missing - skipping ${PYBIN}"; exit 3; } + echo "Using: $($PY --version)" + + # Fresh isolated venv = clean end-user machine + TEST_DIR="/enduser_${PYBIN}" + rm -rf "$TEST_DIR" + $PY -m venv "$TEST_DIR" + VENV_PY="$TEST_DIR/bin/python" + + # Run all Python invocations from the venv dir (never /workspace) and + # with PYTHONSAFEPATH so the repo checkout at /workspace/mssql_python + # can NEVER shadow the freshly installed wheel. Without this, cwd + # (/workspace) lands on sys.path and "import mssql_python" would resolve + # to the source tree instead of the package under test. + export PYTHONSAFEPATH=1 + cd "$TEST_DIR" + + $VENV_PY -m pip install -q -U pip + + # Platform wheel tag: manylinux_2_28 vs musllinux_1_2, x86_64 + if [ "$LINUX_TAG" = "manylinux_2_28" ]; then PLAT="manylinux"; else PLAT="musllinux"; fi + + # Select the mssql-python wheel matching THIS Python version + libc + MP_WHEEL=$(ls /wheels/mssql-python/dist/mssql_python-*${PYBIN}-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$MP_WHEEL" ]; then + echo "ERROR: no mssql-python wheel for ${PYBIN}/${PLAT}"; exit 1 + fi + + # ODBC wheel is data-only (py3-none-), one per platform + ODBC_WHEEL=$(ls /wheels/mssql-python-odbc/dist/mssql_python_odbc-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$ODBC_WHEEL" ]; then + echo "ERROR: no mssql-python-odbc wheel for ${PLAT}"; exit 1 + fi + + echo "Installing mssql-python: $MP_WHEEL" + $VENV_PY -m pip install -q "$MP_WHEEL" + echo "Installing mssql-python-odbc: $ODBC_WHEEL" + $VENV_PY -m pip install -q "$ODBC_WHEEL" + + # Show where the external ODBC package lives (the split target) + $VENV_PY -c "import mssql_python_odbc, os; print(\"mssql_python_odbc libs:\", mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" + + # POSITIVE assertion: prove the native loader WILL resolve the driver + # from the external mssql_python_odbc package. This mirrors exactly what + # GetOdbcLibsBaseDir() does in ddbc_bindings.cpp: it takes the parent dir + # of mssql_python_odbc.__file__ as the base and treats the external + # package as authoritative iff GetDriverPathCpp(base) exists. If this + # driver file is present, the loader is guaranteed to pick the external + # package (non-Windows only checks the driver; Windows also needs + # mssql-auth.dll, which this platform test does not cover). + $VENV_PY -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print(\"external driver path:\", drv); assert os.path.exists(drv), \"external ODBC driver not found - loader would NOT select external package: \"+drv; print(\"POSITIVE: loader will resolve the driver from the external mssql_python_odbc package\")" + + # DECISIVE (negative) STEP: remove the ODBC driver bundled inside + # mssql_python. With the fallback gone, any successful connection must + # be using the driver from the separate mssql_python_odbc package. + MP_LIBS=$($VENV_PY -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), \"libs\"))") + case "$MP_LIBS" in + /workspace/*) + echo "ERROR: resolved mssql_python to the source tree ($MP_LIBS), not the installed wheel"; exit 1 ;; + esac + if [ -d "$MP_LIBS" ]; then + echo "Removing bundled mssql_python/libs to force use of external ODBC package: $MP_LIBS" + rm -rf "$MP_LIBS" + else + echo "Note: mssql_python has no bundled libs/ (already split at build time)" + fi + + # Smoke test: import + live connection using ONLY the external driver + $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK\")" + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK:\", cur.fetchone()); c.close()" + + # Copy the suite OUT of /workspace so pytest collection cannot import + # the source-tree package either, then run against the installed wheels. + cp -r /workspace/tests "$TEST_DIR/tests" + cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" 2>/dev/null || true + $VENV_PY -m pip install -q pytest + if [ -f /workspace/requirements.txt ]; then + $VENV_PY -m pip install -q -r /workspace/requirements.txt + fi + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 + echo "All tests passed for ${PYBIN} using the EXTERNAL mssql_python_odbc driver" + ' + rc=$? + set -e + + if [ "$rc" -eq "$SKIP_RC" ]; then + echo "Skipped $PYBIN (interpreter not present in image)" + elif [ "$rc" -ne 0 ]; then + echo "ERROR: both-wheels test failed for $PYBIN (exit $rc)" + exit "$rc" + else + TESTED=$((TESTED + 1)) + echo "Both-wheels test complete for $PYBIN" + fi + done + + if [ "$TESTED" -eq 0 ]; then + echo "ERROR: no Python versions were tested - every interpreter was missing. Failing the stage." + exit 1 + fi + + echo "=====================================================" + echo "All available Python versions ($TESTED) passed the both-wheels end-user test" + echo "=====================================================" + displayName: 'Install both wheels + pytest (Python 3.10-3.14)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + docker stop test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + docker rm test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Cleanup containers' + condition: always()