From 96e6a5e53dfb66d65ec4ad8d3c21f5d15a9c96bd Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Sun, 6 Sep 2026 18:05:50 -0300 Subject: [PATCH 1/6] ci: replace Marketplace PAT with Entra OIDC publishing Refs #489. Keep GitHub RELEASE_PAT and Python Trusted Publishing unchanged. Permanent identity setup and authorized production rollout remain pending. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/marketplace-login/action.yml | 36 ++ .github/skills/release-management/SKILL.md | 118 ++++- .github/workflows/release.yml | 98 +++-- .github/workflows/staging.yml | 63 +-- CHANGELOG.md | 7 + docs/release-process.md | 385 +++++++++++------ plugins/agentops/package.json | 4 +- scripts/marketplace.py | 252 +++++++++++ scripts/release.ps1 | 29 +- scripts/release.sh | 29 +- scripts/staging.ps1 | 42 +- scripts/staging.sh | 16 +- tests/unit/test_marketplace_publishing.py | 432 +++++++++++++++++++ 13 files changed, 1262 insertions(+), 249 deletions(-) create mode 100644 .github/actions/marketplace-login/action.yml create mode 100644 scripts/marketplace.py create mode 100644 tests/unit/test_marketplace_publishing.py diff --git a/.github/actions/marketplace-login/action.yml b/.github/actions/marketplace-login/action.yml new file mode 100644 index 00000000..b32acd5c --- /dev/null +++ b/.github/actions/marketplace-login/action.yml @@ -0,0 +1,36 @@ +name: Marketplace OIDC login +description: Authenticate the dedicated publishing identity without Azure roles or stored secrets. +inputs: + client-id: + description: Dedicated Marketplace managed identity client ID. + required: true + tenant-id: + description: Approved publishing identity tenant ID. + required: true + profile-id: + description: Marketplace profiles/me ID (not the Entra principal ID). + required: true +runs: + using: composite + steps: + - name: Validate Marketplace identity configuration + shell: bash + env: + CLIENT_ID: ${{ inputs.client-id }} + TENANT_ID: ${{ inputs.tenant-id }} + PROFILE_ID: ${{ inputs.profile-id }} + run: | + GUID='^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + for NAME in CLIENT_ID TENANT_ID PROFILE_ID; do + if [[ ! "${!NAME}" =~ $GUID ]]; then + echo "::error::Missing or invalid Marketplace $NAME. Configure the protected Marketplace environment variables; see docs/release-process.md." + exit 1 + fi + done + + - name: Sign in with Marketplace workload identity + uses: azure/login@v3 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + allow-no-subscriptions: true diff --git a/.github/skills/release-management/SKILL.md b/.github/skills/release-management/SKILL.md index 226e0a58..480d0719 100644 --- a/.github/skills/release-management/SKILL.md +++ b/.github/skills/release-management/SKILL.md @@ -66,7 +66,9 @@ Examples: `release/v2.4.2`, `release/v0.2.0` - Updates `CHANGELOG.md` (adds versioned section `[0.2.0] - YYYY-MM-DD`) - Pushes the branch (triggers staging pipeline automatically) - Opens a PR: `release/v0.2.0` → `main` -4. Wait for staging pipeline to pass (build → TestPyPI → verify). +4. Wait for staging pipeline to pass (build → TestPyPI → verify), and review + the separately protected `marketplace-staging` deployment for the legitimate + Marketplace pre-release. The branch push is a real publication attempt. 5. Get the PR reviewed and merge into `main`. 6. Tag the release on `main` **and sync `develop` in the same sitting**. Tagging publishes to PyPI immediately; there is no approval prompt. Leaving `develop` @@ -90,8 +92,9 @@ Examples: `release/v2.4.2`, `release/v0.2.0` `## [0.2.0]` heading above develop's unreleased entries, nesting new work inside a shipped version. 7. Watch the Release workflow finish (build → TestPyPI → verify → publish-pypi → - github-release). The `release` environment has no protection rules, so nothing - pauses for review. + github-release). The Python `release` environment has no protection rules, + so PyPI does not pause for review. The separate `marketplace-release` + deployment requires review for the stable extension publication. 8. Delete the release branch: ```bash git push origin --delete release/v0.2.0 @@ -217,7 +220,7 @@ docs: update changelog for 2.4.2 chore: prepare release 2.4.2 ``` -## Required Secrets +## Publishing Authentication Both `staging.yml` and `release.yml` publish through [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/) using @@ -225,7 +228,6 @@ Both `staging.yml` and `release.yml` publish through | Secret | Scope | Purpose | |---|---|---| -| `VSCE_PAT` | repository | VS Code Marketplace PAT with **Marketplace: Manage**, used by the extension publish jobs in both `staging.yml` and `release.yml` | | `RELEASE_PAT` | repository | Used by `cut-release.yml` to open the release PR | Neither the `staging` nor the `release` environment holds any secret. Confirm with @@ -235,6 +237,108 @@ Trusted Publishing is configured on the index side (pypi.org and test.pypi.org **Manage → Publishing**) and must match the repository, workflow filename, and environment name exactly. A mismatch fails with `403` at upload time. +### Marketplace: dedicated Entra OIDC identity + +Marketplace jobs use a **dedicated user-assigned managed identity (UAMI)**, +`azure/login@v3` with `allow-no-subscriptions: true`, and `id-token: write`. +No Azure RBAC grant is needed solely to publish. Publisher membership supplies +that permission. Do not change Python Trusted Publishing, shared `AZURE_*` E2E +variables, repository-wide OIDC configuration, or the GitHub `RELEASE_PAT`. + +Use two **new, separate** GitHub environments, not Python `staging`/`release`. +Configure required reviewers and selected branch/tag deployment policies +**before setting variables**: + +| Environment | Allowed deployment refs | +| --- | --- | +| `marketplace-staging` | Branches `release/*` for legitimate `release/vX.Y.Z` candidates | +| `marketplace-release` | Tags `v*`; optionally protected `main` for manual dispatch with tag input | + +Stable manual job guards allow only `main` or the same release tag as the input. +An environment subject alone does not restrict branches; protections are essential. +For an authorized pre-migration-tag retry, explicitly dispatch the **new migrated +workflow on protected `main`** with a valid release tag. The Marketplace job +checks out tooling from `github.workflow_sha` at the workspace root and extension +source from `refs/tags/` into `release-source/`: old extension source uses +the new OIDC action/helper. Tag-based dispatch must match the tag input. +Re-running a historical old workflow still executes its old PAT code, not the +migrated workflow. A retry is a real publication attempt. + +Each new environment needs these variables: + +- `MARKETPLACE_AZURE_CLIENT_ID`: UAMI client GUID. +- `MARKETPLACE_AZURE_TENANT_ID`: approved identity tenant GUID. +- `MARKETPLACE_PROFILE_ID`: Marketplace `profiles/me` profile `id`, + **not** the Entra principal/object ID. + +Verify customization read-only with +`gh api repos/Azure/agentops/actions/oidc/customization/sub`. The verified ordered +claim keys are `repository_owner_id`, `repository_id`, `context` with +`use_default: false`. Federation must use: + +- Issuer: `https://token.actions.githubusercontent.com` +- Audience: `api://AzureADTokenExchange` +- Staging subject: `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` +- Release subject: `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-release` + +Resolve the UAMI Marketplace profile using a CLI token for resource +`499b84ac-1321-427f-aa17-267ca6975798` and a read-only request to +`https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1`. +Record only the profile `id`; never print or store token values. A publisher +Owner must grant that profile **Contributor, not Owner**, on `AgentOpsAccelerator`. + +### Shared helper and local publishing + +- Requires Python 3.11+ (stdlib), Azure CLI, and Node 22 with `vsce >=3.9.2` + for publishing. Workflows pin `npm install -g @vscode/vsce@3.9.2`. +- `python scripts/marketplace.py check` is read-only: it selects + `MARKETPLACE_AZURE_TENANT_ID`, pins the self profile to + `MARKETPLACE_PROFILE_ID`, and checks explicit Contributor/Owner/Creator + membership with zero deny permissions. A generic HTTP 200 is not sufficient. +- `python scripts/marketplace.py publish --package-path PATH [--pre-release] [--allow-already-exists]` + preflights before uploading. CI opts into already-existing-version handling; + local defaults fail. The flag maps to native `vsce --skip-duplicate` + (existing-version/409 only), not output substring matching; other errors + always propagate. The child environment clears inherited PAT and + EnvironmentCredential variables and selects the tenant. **No PAT fallback.** +- Local staging/release scripts preflight before side effects when `vsce` + exists, but then publish. Their prior missing-`vsce` extension-packaging skip + remains unchanged and does not prove Marketplace access. + Use `check` alone for no-upload validation. First run + `az login --tenant --allow-no-subscriptions` and set + `MARKETPLACE_AZURE_TENANT_ID` and `MARKETPLACE_PROFILE_ID` for **your interactive + publishing identity**, not the UAMI. Your identity needs publisher Contributor + or Owner membership. Profile pinning prevents wrong account/tenant publishing. +- Extension `npm run publish` / `npm run publish:prerelease` package a VSIX + before calling the shared helper, which preflights before upload. They require + Python 3.11+ and the helper from the repository checkout. +- CLI profile and role preflight success is **not proof of actual upload**. + +### Staged rollout (not completed by merging code) + +1. Obtain approval for permanent ownership and production tenant/subscription + placement outside code rollout. The earlier non-production + personal-subscription probe is feasibility evidence, not policy approval. +2. Configure the dedicated UAMI, exact federation, publisher Contributor + membership, environment reviewers and deployment policies, then variables. +3. Run an authorized permission-only preflight (`check`, no publishing scripts). + No standalone read-only workflow is added by this change. Arrange approved + OIDC permission validation before the first release; a local interactive + `check` alone does not validate CI federation. +4. Explicitly authorize and verify a **legitimate** Marketplace pre-release. +5. Explicitly authorize and verify a **legitimate** stable publication. +6. **Only then** remove the legacy GitHub `VSCE_PAT`. Its owner must confirm no + other consumers before revoking the underlying Azure DevOps PAT. + Include historical workflow re-runs in that review; retire old PAT paths + and use the migrated workflow on `main` for authorized old-tag retries. + **Never touch `RELEASE_PAT`.** + +[Prior successful no-upload proof](https://github.com/Azure/agentops/actions/runs/34046045685/attempts/3) +used temporary resources, all since deleted. Do not claim permanent resources, +actual publication, or legacy-secret removal are complete based on that proof. +See [the release guide](../../../docs/release-process.md#104-marketplace-entra-oidc-identity-setup) +for setup and rollout details. + ## Default Decision Logic | Situation | Action | @@ -254,3 +358,7 @@ environment name exactly. A mismatch fails with `403` at upload time. - Never end a release without running `git log --oneline origin/develop..origin/main` and seeing empty output. - Never publish without running `python -m pytest tests/ -x -q` first. +- Never treat release workflows as dry runs or create dummy release branches, + tags, or production versions to test them. Staging includes a real Marketplace + pre-release attempt. Use local packaging and standalone read-only preflight + instead; Marketplace approvals do not pause Python publishing. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86474c82..3e362665 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,13 +34,14 @@ # Authentication is handled via OpenID Connect (OIDC) between GitHub Actions # and PyPI/TestPyPI. # -# Required GitHub secrets: -# VSCE_PAT — VS Code Marketplace PAT (repository secret, not an environment secret). -# MUST have "Marketplace: Manage" scope on the AgentOpsAccelerator publisher. +# Marketplace uses Entra OIDC, not a PAT. The separate marketplace-release +# environment requires protected deployment refs and dedicated identity variables: +# MARKETPLACE_AZURE_CLIENT_ID, MARKETPLACE_AZURE_TENANT_ID, MARKETPLACE_PROFILE_ID. +# See docs/release-process.md for approval, federation and rollout prerequisites. # # Required GitHub environments: # staging — TestPyPI publish -# release — PyPI + VSIX stable publish +# release — PyPI publish (unchanged) # Both exist only to scope Trusted Publishing. Neither has protection rules # today, so neither gates anything. Check with: # gh api repos/Azure/agentops/environments \ @@ -51,9 +52,9 @@ # → Add publisher: GitHub, owner=Azure, repo=agentops, workflow=release.yml, environment=staging # 2. https://pypi.org/manage/project/agentops-accelerator/settings/publishing/ # → Add publisher: GitHub, owner=Azure, repo=agentops, workflow=release.yml, environment=release -# 3. https://dev.azure.com/ → PAT with Marketplace scope on the AzDO account that -# owns the AgentOpsAccelerator publisher → Create VSCE_PAT. -# Verify scope with: vsce ls-publishers -p $VSCE_PAT (must list AgentOpsAccelerator). +# 3. Configure marketplace-release, federate the dedicated managed identity, +# and grant it Contributor on AgentOpsAccelerator (not Azure Contributor). +# Python Trusted Publishing and the GitHub RELEASE_PAT are unchanged. # 4. GitHub repo → Settings → Environments → Create "release" (the name must match # the publisher config in step 2). Required reviewers are NOT configured today, # so the PyPI publish runs unattended. @@ -200,26 +201,53 @@ jobs: # Runs after PyPI publish so PyPI and Marketplace identity stay consistent on every release. publish-vsix: needs: [build, publish-pypi] # gate on successful lint + test (build) and on PyPI publish + if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') runs-on: ubuntu-latest - environment: release # same environment as the PyPI publish; no protection rules, does not gate + environment: marketplace-release + permissions: + contents: read + id-token: write env: VSIX_FILE: agentops-skills.vsix + MARKETPLACE_AZURE_TENANT_ID: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + MARKETPLACE_PROFILE_ID: ${{ vars.MARKETPLACE_PROFILE_ID }} steps: - - uses: actions/checkout@v7 + - name: Validate Marketplace release ref + env: + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + run: | + if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Marketplace stable publishing requires a vX.Y.Z release tag." + exit 1 + fi + if [[ "$GITHUB_REF" == refs/tags/* && "$GITHUB_REF_NAME" != "$RELEASE_TAG" ]]; then + echo "::error::The requested release tag must match the workflow tag ref." + exit 1 + fi + + - name: Check out trusted publishing tooling + uses: actions/checkout@v7 + with: + ref: ${{ github.workflow_sha }} + + - name: Check out release sources + uses: actions/checkout@v7 with: fetch-depth: 0 # Full history for version derivation + ref: refs/tags/${{ inputs.tag || github.ref_name }} + path: release-source - name: Sync VSIX version from git tag + working-directory: release-source + env: + RELEASE_TAG: ${{ inputs.tag || github.ref_name }} run: | # Derive version from the tag name directly (most reliable for releases) - TAG="${{ inputs.tag || github.ref_name }}" - TAG=${TAG:-$(git tag -l 'v*' --sort=-v:refname | head -1)} - TAG=${TAG:-v0.0.0} - VERSION=${TAG#v} + VERSION=${RELEASE_TAG#v} jq --arg v "$VERSION" '.version = $v' \ plugins/agentops/package.json > plugins/agentops/package.json.tmp mv plugins/agentops/package.json.tmp plugins/agentops/package.json - echo "VSIX version set to $VERSION (from tag $TAG)" + echo "VSIX version set to $VERSION (from tag $RELEASE_TAG)" - name: Set up Node.js uses: actions/setup-node@v7 @@ -227,44 +255,42 @@ jobs: node-version: "22" - name: Install vsce - run: npm install -g @vscode/vsce + run: npm install -g @vscode/vsce@3.9.2 + + - name: Set up Python for Marketplace preflight + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Authenticate Marketplace identity + uses: ./.github/actions/marketplace-login + with: + client-id: ${{ vars.MARKETPLACE_AZURE_CLIENT_ID }} + tenant-id: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + profile-id: ${{ vars.MARKETPLACE_PROFILE_ID }} - name: Copy root assets for VSIX + working-directory: release-source run: | cp CHANGELOG.md plugins/agentops/CHANGELOG.md cp icon.png plugins/agentops/icon.png - name: Package VSIX - working-directory: plugins/agentops + working-directory: release-source/plugins/agentops run: vsce package -o "${VSIX_FILE}" - name: Publish stable to VS Code Marketplace - # Tolerate ONLY the "already exists" case (staging pre-release may have - # published the same version first). Any other failure must fail the job. - working-directory: plugins/agentops - env: - VSCE_PAT: ${{ secrets.VSCE_PAT }} + # Preflight requires the exact profile and an explicit publishing role. + # Only a duplicate-version publish error is tolerated on reruns. run: | - set -o pipefail - if OUTPUT=$(vsce publish --packagePath "${VSIX_FILE}" -p "${VSCE_PAT}" 2>&1); then - RC=0 - else - RC=$? - fi - echo "$OUTPUT" - if [ "$RC" -ne 0 ]; then - if echo "$OUTPUT" | grep -qi "already exists"; then - echo "::warning::VSIX version already published (likely by staging pre-release). Treating as success." - exit 0 - fi - exit "$RC" - fi + python scripts/marketplace.py publish \ + --package-path "release-source/plugins/agentops/${VSIX_FILE}" --allow-already-exists - name: Upload VSIX artifact uses: actions/upload-artifact@v7 with: name: vsix - path: plugins/agentops/${{ env.VSIX_FILE }} + path: release-source/plugins/agentops/${{ env.VSIX_FILE }} # Create GitHub Release with built artifacts (Python dist + VSIX). # Gating: publish-pypi + publish-vsix MUST succeed. diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 582c5dc4..1bf264bb 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -27,19 +27,19 @@ # Authentication is handled via OpenID Connect (OIDC) between GitHub Actions # and TestPyPI. # -# Required GitHub secrets: -# VSCE_PAT — VS Code Marketplace PAT (repository secret, not an environment secret). -# MUST have "Marketplace: Manage" scope on the AgentOpsAccelerator publisher. +# Marketplace uses Entra OIDC, not a PAT. The separate marketplace-staging +# environment requires protected deployment branches and dedicated identity +# variables: MARKETPLACE_AZURE_CLIENT_ID, MARKETPLACE_AZURE_TENANT_ID, +# MARKETPLACE_PROFILE_ID. See docs/release-process.md for custom OIDC subjects. # # Setup (Trusted Publishing + VSCE): # 1. https://test.pypi.org/manage/project/agentops-accelerator/settings/publishing/ # → Add publisher: GitHub, owner=Azure, repo=agentops, workflow=staging.yml, environment=staging # 2. GitHub repo → Settings → Environments → Create "staging" (the name must match # the publisher config in step 1). No protection rules are configured today. -# 3. https://dev.azure.com/ → PAT with Marketplace scope on the AzDO account that -# owns the AgentOpsAccelerator publisher → Create VSCE_PAT. -# Verify scope with: vsce ls-publishers -p $VSCE_PAT (must list AgentOpsAccelerator). -# 4. Add VSCE_PAT as a repository secret (Settings → Secrets and variables → Actions) +# 3. Configure marketplace-staging, federate the dedicated managed identity, +# and grant it Contributor on AgentOpsAccelerator (not Azure Contributor). +# Python Trusted Publishing and the GitHub RELEASE_PAT are unchanged. name: Staging @@ -139,8 +139,15 @@ jobs: # Runs in parallel with the TestPyPI flow (only needs source checkout). publish-vsix-prerelease: needs: build # gate on successful lint + test + if: startsWith(github.ref, 'refs/heads/release/v') runs-on: ubuntu-latest - environment: staging + environment: marketplace-staging + permissions: + contents: read + id-token: write + env: + MARKETPLACE_AZURE_TENANT_ID: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + MARKETPLACE_PROFILE_ID: ${{ vars.MARKETPLACE_PROFILE_ID }} steps: - uses: actions/checkout@v7 @@ -165,7 +172,19 @@ jobs: node-version: "22" - name: Install vsce - run: npm install -g @vscode/vsce + run: npm install -g @vscode/vsce@3.9.2 + + - name: Set up Python for Marketplace preflight + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Authenticate Marketplace identity + uses: ./.github/actions/marketplace-login + with: + client-id: ${{ vars.MARKETPLACE_AZURE_CLIENT_ID }} + tenant-id: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + profile-id: ${{ vars.MARKETPLACE_PROFILE_ID }} - name: Copy root assets for VSIX run: | @@ -177,27 +196,12 @@ jobs: run: vsce package --pre-release -o agentops-skills.vsix - name: Publish pre-release to VS Code Marketplace - # Tolerate ONLY the "already exists" case (re-run of the same pre-release - # version). Any other failure (auth, network, validation) must fail the job - # so staging surfaces real publish problems instead of silently green-lighting. - working-directory: plugins/agentops - env: - VSCE_PAT: ${{ secrets.VSCE_PAT }} + # Preflight requires the exact profile and an explicit publishing role. + # Only a duplicate-version publish error is tolerated on reruns. run: | - set -o pipefail - if OUTPUT=$(vsce publish --pre-release --packagePath agentops-skills.vsix -p "${VSCE_PAT}" 2>&1); then - RC=0 - else - RC=$? - fi - echo "$OUTPUT" - if [ "$RC" -ne 0 ]; then - if echo "$OUTPUT" | grep -qi "already exists"; then - echo "::warning::VSIX pre-release version already published. Treating as success." - exit 0 - fi - exit "$RC" - fi + python scripts/marketplace.py publish \ + --package-path plugins/agentops/agentops-skills.vsix \ + --pre-release --allow-already-exists - name: Show VSIX info working-directory: plugins/agentops @@ -210,4 +214,3 @@ jobs: with: name: vsix path: plugins/agentops/agentops-skills.vsix - diff --git a/CHANGELOG.md b/CHANGELOG.md index 27232c85..f96a2b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Changed +- **Marketplace publishing moves from PATs to Microsoft Entra OIDC.** Dedicated + managed-identity environments and a shared tenant/profile-pinned publishing + helper replace Marketplace PAT authentication. Permission-only preflight and + staged rollout are documented; GitHub `RELEASE_PAT` and PyPI/TestPyPI Trusted + Publishing are unchanged. + ## [0.15.0] - 2026-09-06 ### Added diff --git a/docs/release-process.md b/docs/release-process.md index 026593eb..eb077faa 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -315,7 +315,10 @@ python -c "from agentops import __version__; print(__version__)" ## 7. Staging Pipeline (TestPyPI) -The staging pipeline validates a release candidate by publishing to TestPyPI and verifying the installed package works. +The staging pipeline validates a release candidate by publishing to TestPyPI and +verifying the installed package works. It also attempts a **real Marketplace +pre-release** through the separate `marketplace-staging` environment. It is not +a dry run; use only legitimate, authorized release candidates. **Workflow file**: `.github/workflows/staging.yml` @@ -329,8 +332,10 @@ flowchart TD build["_build
tests + package
Version: 0.2.1.dev3 (setuptools-scm)"] publish["publish-testpypi
Upload to TestPyPI (staging environment)
Trusted Publishing (OIDC, no token)"] verify["verify-testpypi
Install from TestPyPI in fresh environment
agentops --version / --help / init"] + vsix["publish-vsix-prerelease
Real Marketplace pre-release
marketplace-staging: review + Entra OIDC"] push --> build --> publish --> verify + push --> vsix ``` ### What Gets Validated @@ -380,99 +385,53 @@ ls .agentops/ ## 8. End-to-End Pipeline Testing -Before cutting a real release, you can validate the entire pipeline end-to-end using a disposable test branch and tag. This is especially useful when: - -- You've modified any workflow file (`_build.yml`, `staging.yml`, `release.yml`) -- You've changed `pyproject.toml` build configuration -- You've updated setuptools-scm settings -- A new engineer wants to understand the release process hands-on +**Release workflows are not dry runs.** Pushing `release/*` triggers TestPyPI +and a real Marketplace pre-release attempt; pushing `v*` triggers production +publishing. Never create dummy release branches, tags, or Marketplace versions +to test workflow changes. Deleting a ref does not undo an upload. ### 8.1 Test the Staging Pipeline -#### Step 1: Create a Test Release Branch - -From the branch that contains your workflow changes (or from `develop`): - -```bash -git checkout develop # or your feature branch with workflow changes -git pull origin develop -git checkout -b release/v0.0.0-test -git push origin release/v0.0.0-test -``` - -This triggers the `staging.yml` workflow automatically. - -#### Step 2: Monitor the Pipeline - -1. Go to **Actions** tab → find the **Staging** workflow run for `release/v0.0.0-test` -2. Watch all 3 jobs: - -``` -Job 1: build / build → Should tests pass? Package build? -Job 2: publish-testpypi → Does TestPyPI upload succeed? -Job 3: verify-testpypi → Can the package install and run? -``` - -3. Click into each job to inspect step-level output -4. If a job fails, read the logs, fix the issue, push again: - -```bash -# Fix and re-push -git add . -git commit -m "fix: correct workflow issue" -git push origin release/v0.0.0-test -# Pipeline re-runs automatically -``` - -#### Step 3: Verify on TestPyPI (Optional) - -Confirm the test package appeared on TestPyPI: +Before authorizing a real candidate, run the existing tests and package locally +without publishing: -```bash -# Check the version that was published -python -m setuptools_scm - -# Install and test manually -pip install "agentops-accelerator==$(python -m setuptools_scm)" \ - --index-url https://test.pypi.org/simple/ \ - --extra-index-url https://pypi.org/simple/ - -agentops --version -agentops --help - -# Test init -cd $(mktemp -d) -agentops init -ls .agentops/ +```powershell +python -m pytest tests/ -x -q +uv build +# With Node 22 installed: +npm install -g @vscode/vsce@3.9.2 +Copy-Item CHANGELOG.md,icon.png -Destination plugins\agentops +Push-Location plugins\agentops +npm run package +Pop-Location ``` -#### Step 4: Clean Up the Test Branch - -```bash -# Delete remote branch -git push origin --delete release/v0.0.0-test +For identity and publisher permissions, use the standalone read-only +`python scripts/marketplace.py check` after the +[local identity setup](#local-publishing). Do not invoke a staging or release +script merely to test credentials: those scripts publish after preflight. -# Switch back and delete local branch -git checkout develop -git branch -d release/v0.0.0-test -``` +When a legitimate release candidate is explicitly authorized, push its +`release/vX.Y.Z` branch and monitor **Staging** in Actions. Review the TestPyPI +upload and install verification, and approve the separately protected +`marketplace-staging` deployment only for the intended Marketplace pre-release. +Re-pushing the branch is another publication attempt, not a test-only run. ### 8.2 Test the Full Release Pipeline > **There is no safe dry run.** The `publish-pypi` job does not pause, so pushing > any `v*` tag publishes that version to real PyPI. There is no reject button to -> catch it. PyPI versions cannot be deleted, only yanked, so a throwaway -> `v0.0.0-test.1` tag leaves a permanent artifact on the project page. +> catch it. PyPI versions cannot be deleted, only yanked; a throwaway tag can +> leave permanent published artifacts. -Test everything except the final publish by pushing a `release/v*` branch, which -exercises build → TestPyPI → verify (see [8.1](#81-test-the-staging-pipeline)). -That covers every job the release pipeline runs before `publish-pypi`, using the -same build and the same `pypa/gh-action-pypi-publish` action. +Use packaging and the read-only Marketplace preflight in [8.1](#81-test-the-staging-pipeline) +for no-upload validation. A full end-to-end publish requires an explicitly +authorized, legitimate release. The `marketplace-release` approval gate is +separate from PyPI and does not pause `publish-pypi`. -If you genuinely need to validate `publish-pypi` end to end, add required -reviewers to the `release` environment first (see -[Enabling a real approval gate](#enabling-a-real-approval-gate)). With reviewers -attached, the job pauses and you can reject it. +If a PyPI approval gate is required, configure reviewers on `release` first +(see [Enabling a real approval gate](#enabling-a-real-approval-gate)). +Rejecting a deployment proves neither successful authentication nor upload. #### Verifying the publish path without publishing @@ -491,43 +450,27 @@ tag has already been pushed. ### 8.3 Quick E2E Test Summary -| What to test | Command | What to watch | -| ---------------------- | -------------------------------------------------------------------- | ------------------------------------ | -| Staging only | `git push origin release/v0.0.0-test` | 3 jobs: build → TestPyPI → verify | -| Full release | `git push origin v0.0.0-test.1` | Publishes to PyPI. No undo. Avoid. | -| Cleanup (branch) | `git push origin --delete release/v0.0.0-test` | Branch removed | -| Cleanup (tag) | `git push origin --delete v0.0.0-test.1 && git tag -d v0.0.0-test.1` | Tag removed, PyPI version remains | +| What to validate | Method | What it proves | +| --- | --- | --- | +| Tests and packaging | Existing tests, `uv build`, extension `npm run package` | Build correctness; no upload | +| Marketplace identity and access | `python scripts/marketplace.py check` | Profile and explicit publisher role; no upload | +| Real pre-release | Authorized `release/vX.Y.Z` candidate and Marketplace approval | TestPyPI and real Marketplace pre-release publication | +| Real stable release | Authorized `vX.Y.Z` tag and Marketplace approval | Production publication; not reversible by deleting the tag | ### 8.4 Testing Workflow Changes on a Feature Branch -If you're modifying the workflow files on a feature branch (not yet merged to `develop`), you can still test them: - -```bash -# Your workflow changes are on feature/my-ci-changes -git checkout feature/my-ci-changes - -# Create a test release branch directly from your feature branch -git checkout -b release/v0.0.0-test -git push origin release/v0.0.0-test - -# GitHub Actions uses the workflow files from the pushed branch, -# so your modifications are what actually runs -``` - -This is useful because GitHub Actions reads workflow files from the branch being pushed, not from `main` or `develop`. Your modified workflows execute immediately without needing to merge first. - -After testing: - -```bash -# Clean up -git push origin --delete release/v0.0.0-test -git checkout feature/my-ci-changes -git branch -d release/v0.0.0-test -``` +Run the existing targeted workflow/helper tests and packaging on the feature +branch. Inspect workflow diffs and environment protections without dispatching +release workflows. Do not bypass deployment policies or create a dummy +`release/*` branch to obtain a publishing identity. A standalone, explicitly +approved read-only preflight can validate federation and publisher access, but +must not call `publish` or the staging/release scripts. ## 9. Production Release Pipeline (PyPI) -The production pipeline publishes a final release to PyPI and creates a GitHub Release. +The production pipeline publishes a final release to PyPI and creates a GitHub +Release. Its Marketplace stable publish uses the separate protected +`marketplace-release` environment; this does not change Python publishing. **Workflow file**: `.github/workflows/release.yml` @@ -542,9 +485,10 @@ flowchart TD publishTest["publish-testpypi
Final TestPyPI upload (clean version)"] verifyTest["verify-testpypi
Smoke test from TestPyPI"] publishPypi["publish-pypi
Publishes to PyPI immediately
Trusted Publishing (OIDC, no token)
environment: release (no protection rules)"] + vsix["publish-vsix
Stable Marketplace publish
marketplace-release: review + Entra OIDC"] ghRelease["github-release
Creates GitHub Release with artifacts
Auto-generated release notes"] - tag --> build --> publishTest --> verifyTest --> publishPypi --> ghRelease + tag --> build --> publishTest --> verifyTest --> publishPypi --> vsix --> ghRelease classDef gate fill:#fff3cd,stroke:#856404,color:#000; class tag gate; @@ -552,7 +496,8 @@ flowchart TD > **Pushing the tag is the point of no return.** The `publish-pypi` job declares > `environment: release`, but that environment currently has **no protection -> rules**, so nothing pauses for review. Verify for yourself: +> rules**, so Python publishing does not pause for review. The Marketplace +> environment is separate. Verify for yourself: > > ```bash > gh api repos/Azure/agentops/environments --jq '.environments[] | {name, protection_rules}' @@ -594,10 +539,12 @@ The branch push triggers the staging pipeline automatically. Wait for it to pass #### Step 3: Monitor Staging 1. Go to **Actions** tab → find the **Staging** workflow run -2. Verify all 3 jobs pass: +2. Verify the Python jobs pass: - ✅ `build / build` - tests pass, package builds - ✅ `publish-testpypi` - uploaded to TestPyPI - ✅ `verify-testpypi` - installed and smoke-tested +3. Review and approve the legitimate Marketplace pre-release deployment in + `marketplace-staging`, then verify that publication succeeded. If any job fails, fix the issue on the release branch and push. The pipeline re-runs automatically. @@ -710,7 +657,9 @@ This section covers one-time setup required before the pipelines can run. ### 10.1 GitHub Environments -Create two environments in **Settings → Environments → New environment**: +Keep the existing Python environments unchanged. Create **separate Marketplace +environments** in **Settings → Environments → New environment** as described +below; do not reuse `staging` or `release` for Marketplace authentication. #### `staging` Environment @@ -726,17 +675,57 @@ Create two environments in **Settings → Environments → New environment**: real gate, add required reviewers (see [Enabling a real approval gate](#enabling-a-real-approval-gate)). - **Deployment branches**: Optionally restrict to `main` branch and `v*` tags -- **Secrets**: None. `VSCE_PAT` is a **repository** secret, not an environment secret, - so it resolves in both `staging.yml` and `release.yml` without being attached here. +- **Secrets**: None. Python uploads continue to use Trusted Publishing. + +#### `marketplace-staging` and `marketplace-release` Environments + +Before setting any variables, configure **required reviewers** and **selected +branch/tag deployment policies**: + +| Environment | Allowed deployment refs | Purpose | +| --- | --- | --- | +| `marketplace-staging` | Branches `release/*` (legitimate `release/vX.Y.Z` candidates) | Real Marketplace pre-release publishing | +| `marketplace-release` | Tags `v*`; optionally the protected `main` branch for manual dispatch with a tag input | Stable Marketplace publishing | + +For manual stable releases, the job guards allow only `main` or the same release +tag as the input. Do not allow feature branches. An environment-based federated +subject **does not restrict branches by itself**: these environment protections +are essential and must exist before enabling the identity. + +For an authorized retry of a pre-migration tag, explicitly dispatch the **new +migrated workflow on protected `main`**, supplying the valid release tag. +The Marketplace job checks out trusted tooling from `github.workflow_sha` at +the workspace root and extension source from `refs/tags/` into +`release-source/`. This packages the old extension source using the new OIDC +action/helper. A tag-based dispatch must match the tag input. Re-running a +historical old workflow run still executes its old PAT code; it does **not** +adopt the migrated workflow automatically. This is a real release retry, not +a read-only check. + +Set these **environment variables**, not secrets, in each new environment: + +| Variable | Value | +| --- | --- | +| `MARKETPLACE_AZURE_CLIENT_ID` | Dedicated user-assigned managed identity (UAMI) client GUID | +| `MARKETPLACE_AZURE_TENANT_ID` | Approved identity tenant GUID | +| `MARKETPLACE_PROFILE_ID` | Marketplace `profiles/me` profile `id`, **not** the Entra principal/object ID | + +The jobs request `id-token: write` and use `azure/login@v3` with +`allow-no-subscriptions: true`. No Azure RBAC grant is needed solely to publish +an extension: Marketplace publisher membership supplies that permission. +Do not change shared `AZURE_*` E2E variables or repository-wide OIDC settings. #### Repository secrets | Secret | Value | How to get it | | ------------ | ---------------------------------------------------- | ------------------------------------------------------------------------------- | -| `VSCE_PAT` | VS Code Marketplace PAT with **Marketplace: Manage** | [dev.azure.com](https://dev.azure.com) → User settings → Personal access tokens | | `RELEASE_PAT`| PAT used by `cut-release.yml` to open the release PR | GitHub → Settings → Developer settings → Personal access tokens | -No PyPI API token is stored. Check the current rules and secret locations at any time: +`RELEASE_PAT` is a **GitHub** PAT and is unchanged by this migration. Marketplace +publishing has no PAT fallback. The legacy repository `VSCE_PAT` must be retained +until the staged rollout is validated; do not interpret this documentation as +confirmation it has been removed. No PyPI API token is stored. Check the current +rules and secret names (never secret values) at any time: ```bash gh api repos/Azure/agentops/environments/release --jq '.protection_rules' @@ -779,6 +768,128 @@ new project name, either upload once manually with a temporary API token, or use flow to reserve the name for the workflow. `agentops-accelerator` is already registered on both indexes, so this only matters if the package is renamed. +### 10.4 Marketplace Entra OIDC Identity Setup + +**Permanent ownership and the approved production tenant/subscription must be +decided outside the code rollout.** Use a dedicated UAMI, not an E2E identity. +The earlier non-production personal-subscription probe established feasibility, +not policy approval or a permanent hosting location. + +Verify the existing subject customization read-only before creating federation: + +```powershell +gh api repos/Azure/agentops/actions/oidc/customization/sub +gh api repos/Azure/agentops --jq '{repository_id: .id, repository_owner_id: .owner.id}' +``` + +Verified for this migration: `use_default: false`, with ordered claim keys +`repository_owner_id`, `repository_id`, `context`; owner ID `6844498` and +repository ID `1161883340`. The UAMI needs two federated credentials: + +| Field | Value | +| --- | --- | +| Issuer | `https://token.actions.githubusercontent.com` | +| Audience | `api://AzureADTokenExchange` | +| Staging subject | `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` | +| Release subject | `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-release` | + +Do not replace the repository customization with GitHub's default `repo:...` +subject: that can break other federated consumers. If verification differs, +stop and reconcile the identity configuration with repository owners. + +After authenticating as the UAMI in the approved tenant, obtain a CLI token for +resource `499b84ac-1321-427f-aa17-267ca6975798` and use it only in the Authorization +header of a read-only request to +`https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1`. +Record only the returned profile `id`. Never print, persist, or paste the token. +Have an **Owner** of publisher `AgentOpsAccelerator` grant that profile +**Contributor**, not Owner. Set `MARKETPLACE_PROFILE_ID` to this Marketplace +profile ID, not the UAMI's Entra principal ID. + +### 10.5 Shared Publishing Helper and Local Use + +`scripts/marketplace.py` requires Python 3.11+ (standard library only) and the +Azure CLI. Publishing also requires Node 22 and `vsce` 3.9.2 or newer; workflows +pin `npm install -g @vscode/vsce@3.9.2`. + +```text +python scripts/marketplace.py check +python scripts/marketplace.py publish --package-path PATH [--pre-release] [--allow-already-exists] +``` + +- `check` is **read-only** and never publishes. It selects + `MARKETPLACE_AZURE_TENANT_ID`, checks the CLI identity's self profile against + `MARKETPLACE_PROFILE_ID`, and requires an explicit publisher + Contributor/Owner/Creator role with **zero deny permissions**. A generic HTTP + 200 response is not sufficient proof of publishing permission. +- `publish` runs that preflight before uploading the specified package. + The child environment clears inherited PAT and EnvironmentCredential variables + and selects the tenant so `vsce` uses the intended CLI credential. There is + **no PAT fallback**, and token values are never printed. +- CI passes `--allow-already-exists` to preserve re-run handling. Local defaults + fail on an existing version; opt in only when that behavior is intended. + This maps to `vsce`'s native `--skip-duplicate` for existing-version/409 + handling, not output substring matching. Other errors always propagate. +- Profile pinning prevents a wrong account or tenant from publishing. + A successful CLI profile/role preflight is **not proof of an actual upload**. + +#### Local Publishing + +The local `scripts/staging.ps1`, `scripts/staging.sh`, `scripts/release.ps1`, +and `scripts/release.sh` run permission preflight before side effects when +`vsce` is available. Their existing behavior of skipping extension packaging +when `vsce` is missing is unchanged; that skip is not a successful Marketplace +validation. These are publication scripts, not credential tests. + +The extension's `npm run publish` and `npm run publish:prerelease` scripts also +package a VSIX and then invoke the shared helper. They require Python 3.11+ and +the helper from a repository checkout, not just a standalone extension folder. +The helper preflights before upload; these npm scripts package before preflight. + +Log in with `az login --tenant --allow-no-subscriptions`. +Set `MARKETPLACE_AZURE_TENANT_ID` and `MARKETPLACE_PROFILE_ID` for **your interactive +publishing identity**, not the CI UAMI. Resolve your profile using the same +`profiles/me` endpoint above; your identity needs publisher Contributor or Owner +membership. Then run `python scripts/marketplace.py check` alone first. +Only invoke a publishing script or `publish` for an explicitly authorized release. + +### 10.6 Staged Rollout Checklist + +This is a deployment checklist, **not a claim that permanent resources are +configured or publication has been tested**. + +- [ ] Approve permanent identity ownership, production tenant/subscription + placement, and operational responsibility outside the code rollout. +- [ ] Create the dedicated UAMI and the two exact federated credentials; resolve + its Marketplace profile and have a publisher Owner grant Contributor. +- [ ] Configure required reviewers and selected branch/tag deployment policies + on both new environments **before setting their three variables**. Preserve + Python `staging`/`release`, repository-wide OIDC, shared E2E variables, and + `RELEASE_PAT`. +- [ ] Run a permission-only preflight under the intended federated identity: + `python scripts/marketplace.py check`. Verify the pinned profile and explicit + role with no deny permissions. Do not call publishing scripts. + This change does not add a standalone read-only workflow: arrange an + explicitly approved OIDC permission-validation run before the first release. + A local interactive `check` validates that local identity, not CI federation. +- [ ] Obtain explicit authorization for a legitimate pre-release, approve its + `marketplace-staging` deployment, publish it, and verify the Marketplace result. +- [ ] Obtain explicit authorization for a legitimate stable release, approve its + `marketplace-release` deployment, publish it, and verify the Marketplace result. + Do not create dummy production versions for validation. +- [ ] **Only after both real publication paths succeed**, remove the GitHub + `VSCE_PAT` secret. Have its owner revoke the underlying Azure DevOps PAT after + confirming there are no other consumers. **Never remove or revoke `RELEASE_PAT`.** + Include historical workflow re-runs in that consumer review: retire old PAT + execution paths and use the migrated workflow on `main` for authorized old-tag + retries rather than re-running historical workflows. + +Prior evidence: +[successful no-upload preflight, attempt 3](https://github.com/Azure/agentops/actions/runs/34046045685/attempts/3). +All temporary probe resources were deleted. This proves feasibility only, not +permanent configuration, policy approval, actual publication, or legacy-secret +retirement. + ## 11. Workflow File Reference All workflow files are in `.github/workflows/`: @@ -809,6 +920,7 @@ Key detail: Uses `fetch-depth: 0` to ensure setuptools-scm has full git history ``` Trigger: push to release/* branches, or workflow_dispatch Flow: _build → publish-testpypi → verify-testpypi + + parallel Marketplace pre-release (marketplace-staging) Purpose: Validate release candidates before production ``` @@ -816,13 +928,16 @@ Key details: - `skip-existing: true` allows re-pushes without upload failures - Verify step uses a retry loop (5 attempts, 30s apart) for TestPyPI index propagation - Smoke tests cover `--version`, `--help`, and `agentops init` +- The extension job separately uses `marketplace-staging`, Entra OIDC, and the + shared helper with `--pre-release --allow-already-exists`. Staging is a real + Marketplace pre-release attempt, not a safe disposable-branch test. ### `release.yml` - Production Release ``` Trigger: push v* tags, or workflow_dispatch -Flow: _build → publish-testpypi → verify-testpypi → publish-pypi → github-release -Purpose: Publish to PyPI and create GitHub Release +Flow: _build → publish-testpypi → verify-testpypi → publish-pypi → publish-vsix → github-release +Purpose: Publish to PyPI and Marketplace, then create GitHub Release ``` Key details: @@ -830,6 +945,12 @@ Key details: - PyPI upload uses Trusted Publishing (`id-token: write`), not an API token - `github-release` uses `gh release create` with `--generate-notes` for automatic release notes - Built artifacts (.whl, .tar.gz) are attached to the GitHub Release +- The extension job separately uses `marketplace-release`, Entra OIDC, and the + shared helper with `--allow-already-exists`. Its reviewers do not gate PyPI. +- Marketplace tooling is checked out from `github.workflow_sha` at the root; + extension source comes from the requested release tag under `release-source/`. + Retry pre-migration tags by dispatching the migrated workflow on protected + `main`, not by re-running historical PAT-based workflows. ### `cut-release.yml` - Cut Release (Manual Dispatch) @@ -847,7 +968,9 @@ Key details: - The branch push triggers `staging.yml` automatically - Fails safely if the branch already exists - Refuses to run when `## [Unreleased]` is empty, because this workflow only inserts a versioned heading beneath that one and would otherwise publish an empty release section -- Does NOT auto-tag or auto-publish - tagging remains a manual, intentional step +- Does NOT auto-tag; stable tagging remains a manual, intentional step. The + release-branch push does trigger staging publication, including a real + Marketplace pre-release attempt. ## 12. Release Checklist @@ -858,11 +981,14 @@ Use this checklist when cutting a release: - [ ] `CHANGELOG.md` has entries under `## [Unreleased]` for all user-visible changes, including anything Dependabot merged (Cut Release aborts if the section is empty) - [ ] Tests pass locally: `uv run pytest tests/ -x -q` - [ ] Version from setuptools-scm looks correct: `python -m setuptools_scm` +- [ ] Marketplace identity, reviewer and deployment protections are ready; + read-only preflight passed and a legitimate publication is authorized **Staging** - [ ] Release branch created via **Cut Release** workflow (or manually) - [ ] CHANGELOG automatically updated with version and date -- [ ] Staging pipeline passes: build + TestPyPI + verify (all 3 green) +- [ ] Staging Python jobs pass: build + TestPyPI + verify +- [ ] Marketplace pre-release deployment approved and legitimate publication verified - [ ] PR opened: `release/v0.X.Y` → `main` **Production (tag + sync, do these together)** @@ -870,6 +996,7 @@ Use this checklist when cutting a release: - [ ] PR merged to `main` - [ ] Version tag created and pushed: `v0.X.Y` (this publishes to PyPI immediately) - [ ] Release pipeline runs: build + TestPyPI + verify + publish-pypi all green +- [ ] Marketplace stable deployment approved and legitimate publication verified - [ ] **`main` merged back into `develop` and pushed** - [ ] **`git log --oneline origin/develop..origin/main` prints nothing** - [ ] `CHANGELOG.md` on `develop` shows only genuinely unreleased work under `## [Unreleased]` @@ -921,8 +1048,10 @@ Use this checklist when cutting a release: | Problem | Cause | Solution | | --------------------------------- | ----------------------------------- | ---------------------------------------------------------------------- | -| "Environment not found" error | GitHub Environment not created | Create `staging` and `release` environments in Settings → Environments | -| "Secret not found" error | Secret not added to the environment | Add secrets to the specific environment, not repository-level secrets | +| "Environment not found" error | GitHub Environment not created | Preserve Python `staging`/`release`; create separate protected `marketplace-staging`/`marketplace-release` environments | +| Marketplace variable missing | Dedicated environment setup incomplete | Configure reviewers and deployment policies first, then all three `MARKETPLACE_*` variables | +| Marketplace federation fails | Issuer, audience, or customized subject mismatch | Verify repository customization and exact environment subjects; do not change repo-wide OIDC | +| Marketplace profile/role preflight fails | Wrong tenant/account/profile or missing publisher role/deny permissions | Select the correct tenant, resolve `profiles/me`, and ask the publisher Owner to review Contributor membership; never fall back to a PAT | | No one was asked to approve the publish | `release` has no required reviewers | Confirm with `gh api repos/Azure/agentops/environments/release --jq '.protection_rules'` | | Reviewer can't approve deployment | Not listed as required reviewer | Update the environment's required reviewers list | @@ -938,6 +1067,7 @@ flowchart TD rel --> stagingBuild["_build
test + build"] stagingBuild --> stagingTest["TestPyPI publish"] stagingTest --> stagingVerify["Verify install"] + rel --> stagingVsix["Marketplace pre-release
marketplace-staging: review + Entra OIDC"] rel -->|PR| main(["main"]) main -->|tag| tag(["v0.2.0"]) @@ -946,7 +1076,8 @@ flowchart TD relBuild --> relTest["TestPyPI"] relTest --> relVerify["Verify"] relVerify --> relPypi["PyPI
(no approval gate)"] - relPypi --> relGh["GitHub Release"] + relPypi --> relVsix["Marketplace stable
marketplace-release: review + Entra OIDC"] + relVsix --> relGh["GitHub Release"] main -->|merge back, REQUIRED| develop @@ -954,6 +1085,7 @@ flowchart TD stagingBuild stagingTest stagingVerify + stagingVsix end subgraph Release["Release (release.yml)"] @@ -961,6 +1093,7 @@ flowchart TD relTest relVerify relPypi + relVsix relGh end diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index de687bbb..e414fbb2 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -60,7 +60,7 @@ "vscode:prepublish": "echo 'Declarative extension — no build step required'", "package": "vsce package", "package:prerelease": "vsce package --pre-release", - "publish": "vsce publish", - "publish:prerelease": "vsce publish --pre-release" + "publish": "vsce package -o agentops-skills.vsix && python ../../scripts/marketplace.py publish --package-path agentops-skills.vsix", + "publish:prerelease": "vsce package --pre-release -o agentops-skills.vsix && python ../../scripts/marketplace.py publish --pre-release --package-path agentops-skills.vsix" } } diff --git a/scripts/marketplace.py b/scripts/marketplace.py new file mode 100644 index 00000000..a8ab8975 --- /dev/null +++ b/scripts/marketplace.py @@ -0,0 +1,252 @@ +"""Check Marketplace publishing permission and publish with Entra (never a PAT). + +Requires an existing Azure CLI login, MARKETPLACE_AZURE_TENANT_ID, and +MARKETPLACE_PROFILE_ID. No cloud resources or publisher memberships are mutated. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import HTTPRedirectHandler, Request, build_opener +from uuid import UUID +from zipfile import BadZipFile, ZipFile + +PUBLISHER = "AgentOpsAccelerator" +EXTENSION = "agentops-accelerator" +RESOURCE = "499b84ac-1321-427f-aa17-267ca6975798" +MIN_VSCE = (3, 9, 2) +PROFILE_URL = "https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1" +ROLES_URL = ( + "https://marketplace.visualstudio.com/_apis/securityroles/scopes/" + f"gallery.publisher/roleassignments/resources/{PUBLISHER}?api-version=7.1-preview.1" +) +PUBLISHING_ROLES = {"Contributor", "Owner", "Creator"} + + +class MarketplaceError(Exception): + """An actionable publishing error, safe to print without credentials.""" + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # Never forward an Authorization header to a redirected endpoint. + return None + + +def required_guid(name: str) -> str: + value = os.environ.get(name, "").strip() + try: + return str(UUID(value)) + except ValueError: + raise MarketplaceError( + f"Set {name} to the approved identity's GUID. See docs/release-process.md." + ) from None + + +def publishing_environment(tenant: str) -> dict[str, str]: + env = os.environ.copy() + # vsce tries EnvironmentCredential before AzureCliCredential. Prevent a + # developer's unrelated app credentials (or old PAT) from winning that chain. + for name in ( + "VSCE_PAT", + "AZURE_CLIENT_SECRET", + "AZURE_CLIENT_CERTIFICATE_PATH", + "AZURE_CLIENT_CERTIFICATE_PASSWORD", + "AZURE_USERNAME", + "AZURE_PASSWORD", + "AZURE_CLIENT_ID", + ): + env.pop(name, None) + env["AZURE_TENANT_ID"] = tenant + return env + + +def executable(name: str) -> str: + path = shutil.which(name) + if not path: + raise MarketplaceError(f"{name} is required for Marketplace publishing; install it first.") + return path + + +def mask(value: str) -> None: + if os.environ.get("GITHUB_ACTIONS") == "true": + escaped = value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::add-mask::{escaped}", flush=True) + + +def get_token(tenant: str, env: dict[str, str]) -> str: + try: + result = subprocess.run( + [ + executable("az"), "account", "get-access-token", + "--tenant", tenant, "--resource", RESOURCE, + "--query", "accessToken", "--output", "tsv", "--only-show-errors", + ], + env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=60, check=False, + ) + except (OSError, subprocess.TimeoutExpired): + raise MarketplaceError("Azure CLI token acquisition failed or timed out.") from None + if result.returncode or not result.stdout.strip(): + raise MarketplaceError( + "Azure CLI could not obtain a Marketplace token. Check the OIDC login and " + "tenant, or run az login --tenant --allow-no-subscriptions." + ) + token = result.stdout.strip() + mask(token) + return token + + +def get_json(url: str, authorization: str) -> dict: + request = Request(url, headers={"Authorization": authorization, "Accept": "application/json"}) + try: + with build_opener(NoRedirect()).open(request, timeout=30) as response: + payload = json.load(response) + except HTTPError as error: + raise MarketplaceError( + f"Marketplace permission check returned HTTP {error.code}. " + "Check the identity tenant and ask a publisher Owner to grant Contributor access." + ) from None + except (URLError, TimeoutError, OSError): + raise MarketplaceError("Marketplace permission check failed: network error or timeout.") from None + except (ValueError, UnicodeError): + raise MarketplaceError("Marketplace permission check returned invalid JSON.") from None + if not isinstance(payload, dict): + raise MarketplaceError("Marketplace permission check returned an unexpected response.") + return payload + + +def publishing_role(profile: dict, assignments: dict, expected_profile: str) -> str: + profile_id = profile.get("id") + if not isinstance(profile_id, str) or profile_id.lower() != expected_profile.lower(): + raise MarketplaceError( + "Marketplace identity mismatch. The Azure CLI login does not match " + "MARKETPLACE_PROFILE_ID; check the selected account and tenant." + ) + values = assignments.get("value") + if not isinstance(values, list): + raise MarketplaceError("Marketplace returned an invalid role-assignment list.") + roles = set() + for assignment in values: + if not isinstance(assignment, dict): + raise MarketplaceError("Marketplace returned an invalid role assignment.") + user = assignment.get("user") + if not isinstance(user, dict): + raise MarketplaceError("Marketplace returned a role assignment without a user.") + user_id = user.get("id") + if not isinstance(user_id, str) or user_id.lower() != profile_id.lower(): + continue + role = assignment.get("role") + if not isinstance(role, dict) or type(role.get("denyPermissions")) is not int: + raise MarketplaceError("Marketplace returned invalid permission details for this identity.") + if role["denyPermissions"] != 0: + raise MarketplaceError("The Marketplace identity has denied permissions; ask a publisher Owner.") + name = role.get("name") + if isinstance(name, str) and name in PUBLISHING_ROLES: + roles.add(name) + if not roles: + raise MarketplaceError( + f"No explicit publishing role for this identity on {PUBLISHER}. " + "Contributor (or Owner/Creator) is required; Reader access is not sufficient." + ) + return min(roles) + + +def preflight(tenant: str, expected_profile: str, env: dict[str, str]) -> tuple[str, str]: + token = get_token(tenant, env) + # Match vsce's Basic OAuth convention, rather than testing only Bearer access. + basic = base64.b64encode(f"OAuth:{token}".encode()).decode("ascii") + mask(basic) + authorization = f"Basic {basic}" + profile = get_json(PROFILE_URL, authorization) + assignments = get_json(ROLES_URL, authorization) + role = publishing_role(profile, assignments, expected_profile) + print(f"Marketplace preflight passed: {PUBLISHER}, explicit {role} role. No upload performed.") + return token, basic + + +def validate_package(path: Path) -> None: + try: + with ZipFile(path) as archive: + manifest = json.loads(archive.read("extension/package.json")) + except (OSError, BadZipFile, KeyError, ValueError, UnicodeError): + raise MarketplaceError("Cannot read extension/package.json from the VSIX package.") from None + if ( + not isinstance(manifest, dict) + or manifest.get("publisher") != PUBLISHER + or manifest.get("name") != EXTENSION + ): + raise MarketplaceError(f"The VSIX must contain {PUBLISHER}.{EXTENSION}.") + + +def publish( + path: Path, pre_release: bool, allow_already_exists: bool, + tenant: str, expected_profile: str, env: dict[str, str], +) -> int: + validate_package(path) + vsce = executable("vsce") + try: + version = subprocess.run( + [vsce, "--version"], env=env, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired): + raise MarketplaceError("Could not determine vsce version.") from None + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", version.stdout.strip()) + if version.returncode or not match or tuple(map(int, match.groups())) < MIN_VSCE: + raise MarketplaceError("Install @vscode/vsce@3.9.2 or newer for Entra publishing.") + token, basic = preflight(tenant, expected_profile, env) + command = [vsce, "publish", "--azure-credential", "--packagePath", str(path)] + if pre_release: + command.append("--pre-release") + if allow_already_exists: + # vsce handles existing versions and HTTP 409 specifically. Do not + # suppress authentication/network failures by matching arbitrary output. + command.append("--skip-duplicate") + try: + result = subprocess.run( + command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace", check=False, + ) + except OSError: + raise MarketplaceError("Could not start vsce publishing.") from None + output = result.stdout.replace(token, "***").replace(basic, "***") + print(output, end="" if output.endswith("\n") else "\n") + return result.returncode + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("check", help="Check the selected identity and publishing role; never upload.") + upload = commands.add_parser("publish", help="Check permission, then publish an existing VSIX.") + upload.add_argument("--package-path", type=Path, required=True) + upload.add_argument("--pre-release", action="store_true") + upload.add_argument("--allow-already-exists", action="store_true") + args = parser.parse_args(argv) + try: + tenant = required_guid("MARKETPLACE_AZURE_TENANT_ID") + profile = required_guid("MARKETPLACE_PROFILE_ID") + env = publishing_environment(tenant) + if args.command == "check": + preflight(tenant, profile, env) + return 0 + return publish( + args.package_path, args.pre_release, args.allow_already_exists, tenant, profile, env, + ) + except MarketplaceError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release.ps1 b/scripts/release.ps1 index ff783227..c80dc17e 100644 --- a/scripts/release.ps1 +++ b/scripts/release.ps1 @@ -19,14 +19,20 @@ # - twine: pip install twine # - TESTPYPI_TOKEN env var # - PYPI_TOKEN env var (API token from pypi.org) -# - VSCE_PAT env var (VS Code Marketplace PAT) -# - npm + vsce: npm install -g @vscode/vsce +# - Python 3.11+, Azure CLI login in the publisher identity's tenant +# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) +# - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # ───────────────────────────────────────────────────────────────────── Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $skipVSIX = $false +$marketplaceScript = Join-Path $PSScriptRoot "marketplace.py" +if (Get-Command vsce -ErrorAction SilentlyContinue) { + python $marketplaceScript check + if ($LASTEXITCODE -ne 0) { throw "Marketplace preflight failed; no release actions were started." } +} # ── Step 1: Prompt for version ────────────────────────────────────── $version = Read-Host "Enter release version to publish (e.g. 0.1.6) — no 'v' prefix" @@ -125,23 +131,22 @@ if (-not $vsceAvailable) { Copy-Item icon.png plugins/agentops/icon.png -Force -ErrorAction SilentlyContinue Push-Location plugins/agentops - vsce package -o agentops-skills.vsix - Write-Host ">>> VSIX built: agentops-skills.vsix (v$version)" -ForegroundColor Green + try { + vsce package -o agentops-skills.vsix + if ($LASTEXITCODE -ne 0) { throw "VSIX packaging failed; publication aborted." } + Write-Host ">>> VSIX built: agentops-skills.vsix (v$version)" -ForegroundColor Green - if (-not $env:VSCE_PAT) { - Write-Host ">>> VSCE_PAT not set — skipping Marketplace publish" -ForegroundColor DarkYellow - } else { # Verify the VSIX package.json matches the release version $vsixPkg = Get-Content package.json -Raw | ConvertFrom-Json if ($vsixPkg.version -ne $version) { - Write-Error "VSIX version mismatch! package.json=$($vsixPkg.version), expected=$version. Aborting publish." - Pop-Location - exit 1 + throw "VSIX version mismatch! package.json=$($vsixPkg.version), expected=$version. Aborting publish." } - vsce publish --packagePath agentops-skills.vsix -p $env:VSCE_PAT + python $marketplaceScript publish --package-path agentops-skills.vsix + if ($LASTEXITCODE -ne 0) { throw "Marketplace stable publication failed." } Write-Host ">>> VSIX stable published to Marketplace (v$version)" -ForegroundColor Green + } finally { + Pop-Location } - Pop-Location } # ── Step 8: Create GitHub Release ─────────────────────────────────── diff --git a/scripts/release.sh b/scripts/release.sh index ed4a4d43..0fb1f07d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -20,14 +20,19 @@ # - twine: pip install twine # - TESTPYPI_TOKEN env var # - PYPI_TOKEN env var (API token from pypi.org) -# - VSCE_PAT env var (VS Code Marketplace PAT) -# - npm + vsce: npm install -g @vscode/vsce +# - Python 3.11+, Azure CLI login in the publisher identity's tenant +# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) +# - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # - jq installed # ───────────────────────────────────────────────────────────────────── set -euo pipefail skip_vsix=false +marketplace_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/marketplace.py" +if command -v vsce &>/dev/null; then + python "$marketplace_script" check +fi # ── Step 1: Prompt for version ────────────────────────────────────── read -rp "Enter release version to publish (e.g. 0.1.6) — no 'v' prefix: " version @@ -126,19 +131,15 @@ else vsce package -o agentops-skills.vsix echo ">>> VSIX built: agentops-skills.vsix (v$version)" - if [ -z "${VSCE_PAT:-}" ]; then - echo ">>> VSCE_PAT not set — skipping Marketplace publish" - else - # Verify the VSIX package.json matches the release version - vsix_version=$(jq -r '.version' package.json) - if [ "$vsix_version" != "$version" ]; then - echo "ERROR: VSIX version mismatch! package.json=$vsix_version, expected=$version. Aborting publish." >&2 - popd >/dev/null - exit 1 - fi - vsce publish --packagePath agentops-skills.vsix -p "$VSCE_PAT" - echo ">>> VSIX stable published to Marketplace (v$version)" + # Verify the VSIX package.json matches the release version + vsix_version=$(jq -r '.version' package.json) + if [ "$vsix_version" != "$version" ]; then + echo "ERROR: VSIX version mismatch! package.json=$vsix_version, expected=$version. Aborting publish." >&2 + popd >/dev/null + exit 1 fi + python "$marketplace_script" publish --package-path agentops-skills.vsix + echo ">>> VSIX stable published to Marketplace (v$version)" popd >/dev/null fi diff --git a/scripts/staging.ps1 b/scripts/staging.ps1 index f2219137..8b18ccd1 100644 --- a/scripts/staging.ps1 +++ b/scripts/staging.ps1 @@ -15,9 +15,10 @@ # Prereqs: # - uv installed # - twine: pip install twine (for TestPyPI upload) -# - npm + vsce: npm install -g @vscode/vsce +# - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # - TESTPYPI_TOKEN env var (API token from test.pypi.org) -# - VSCE_PAT env var (VS Code Marketplace PAT) +# - Python 3.11+, Azure CLI login in the publisher identity's tenant +# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) # ───────────────────────────────────────────────────────────────────── Set-StrictMode -Version Latest @@ -25,6 +26,11 @@ $ErrorActionPreference = "Stop" $skipTestPyPI = $false $skipVSIX = $false +$marketplaceScript = Join-Path $PSScriptRoot "marketplace.py" +if (Get-Command vsce -ErrorAction SilentlyContinue) { + python $marketplaceScript check + if ($LASTEXITCODE -ne 0) { throw "Marketplace preflight failed; no staging actions were started." } +} # ── Step 1: Lint ──────────────────────────────────────────────────── Write-Host "`n>>> [1/7] Linting with ruff..." -ForegroundColor Yellow @@ -92,7 +98,7 @@ Write-Host "`n>>> [6/7] Building VSIX pre-release..." -ForegroundColor Yellow $vsceAvailable = Get-Command vsce -ErrorAction SilentlyContinue if (-not $vsceAvailable) { Write-Host ">>> vsce not found — skipping VSIX build" -ForegroundColor DarkYellow - Write-Host " Install with: npm install -g @vscode/vsce" -ForegroundColor DarkGray + Write-Host " Install with: npm install -g @vscode/vsce@3.9.2" -ForegroundColor DarkGray $skipVSIX = $true } else { # Sync version from latest git tag @@ -118,12 +124,15 @@ if (-not $vsceAvailable) { Copy-Item icon.png plugins/agentops/icon.png -Force -ErrorAction SilentlyContinue Push-Location plugins/agentops - vsce package --pre-release -o agentops-skills.vsix - Write-Host ">>> VSIX built: agentops-skills.vsix (v$baseVersion)" -ForegroundColor Green - Pop-Location - - # Restore original package.json to prevent version drift - Set-Content $pkgPath -Value $pkgOriginal -NoNewline + try { + vsce package --pre-release -o agentops-skills.vsix + if ($LASTEXITCODE -ne 0) { throw "VSIX packaging failed; publication aborted." } + Write-Host ">>> VSIX built: agentops-skills.vsix (v$baseVersion)" -ForegroundColor Green + } finally { + Pop-Location + # Restore original package.json even if packaging failed. + Set-Content $pkgPath -Value $pkgOriginal -NoNewline + } Write-Host ">>> package.json restored to committed version" -ForegroundColor DarkGray } @@ -131,16 +140,15 @@ if (-not $vsceAvailable) { Write-Host "`n>>> [7/7] Publishing VSIX pre-release..." -ForegroundColor Yellow if ($skipVSIX) { Write-Host ">>> Skipped (vsce not available)" -ForegroundColor DarkYellow -} elseif (-not $env:VSCE_PAT) { - Write-Host ">>> VSCE_PAT not set — skipping Marketplace publish" -ForegroundColor DarkYellow - Write-Host " Set it with: `$env:VSCE_PAT = 'your-pat'" -ForegroundColor DarkGray } else { Push-Location plugins/agentops - # Verify the VSIX contains the expected version before publishing - $vsixPkg = Get-Content package.json -Raw | ConvertFrom-Json - Write-Host " VSIX will publish from packagePath (version in VSIX: $baseVersion)" -ForegroundColor DarkGray - vsce publish --pre-release --packagePath agentops-skills.vsix -p $env:VSCE_PAT - Pop-Location + try { + Write-Host " VSIX will publish from packagePath (version in VSIX: $baseVersion)" -ForegroundColor DarkGray + python $marketplaceScript publish --pre-release --package-path agentops-skills.vsix + if ($LASTEXITCODE -ne 0) { throw "Marketplace pre-release publication failed." } + } finally { + Pop-Location + } Write-Host ">>> VSIX pre-release published to Marketplace" -ForegroundColor Green } diff --git a/scripts/staging.sh b/scripts/staging.sh index 9ed4264a..cd210472 100755 --- a/scripts/staging.sh +++ b/scripts/staging.sh @@ -16,15 +16,20 @@ # Prereqs: # - uv installed # - twine: pip install twine (for TestPyPI upload) -# - npm + vsce: npm install -g @vscode/vsce +# - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # - TESTPYPI_TOKEN env var (API token from test.pypi.org) -# - VSCE_PAT env var (VS Code Marketplace PAT) +# - Python 3.11+, Azure CLI login in the publisher identity's tenant +# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) # ───────────────────────────────────────────────────────────────────── set -euo pipefail skip_testpypi=false skip_vsix=false +marketplace_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/marketplace.py" +if command -v vsce &>/dev/null; then + python "$marketplace_script" check +fi # ── Step 1: Lint ──────────────────────────────────────────────────── echo -e "\n>>> [1/7] Linting with ruff..." @@ -89,7 +94,7 @@ fi echo -e "\n>>> [6/7] Building VSIX pre-release..." if ! command -v vsce &>/dev/null; then echo ">>> vsce not found — skipping VSIX build" - echo " Install with: npm install -g @vscode/vsce" + echo " Install with: npm install -g @vscode/vsce@3.9.2" skip_vsix=true else # Sync version from latest git tag @@ -126,13 +131,10 @@ fi echo -e "\n>>> [7/7] Publishing VSIX pre-release..." if $skip_vsix; then echo ">>> Skipped (vsce not available)" -elif [ -z "${VSCE_PAT:-}" ]; then - echo ">>> VSCE_PAT not set — skipping Marketplace publish" - echo ' Set it with: export VSCE_PAT="your-pat"' else pushd plugins/agentops >/dev/null echo " VSIX will publish from packagePath (version in VSIX: $base_version)" - vsce publish --pre-release --packagePath agentops-skills.vsix -p "$VSCE_PAT" + python "$marketplace_script" publish --pre-release --package-path agentops-skills.vsix popd >/dev/null echo ">>> VSIX pre-release published to Marketplace" fi diff --git a/tests/unit/test_marketplace_publishing.py b/tests/unit/test_marketplace_publishing.py new file mode 100644 index 00000000..6c29fdc0 --- /dev/null +++ b/tests/unit/test_marketplace_publishing.py @@ -0,0 +1,432 @@ +"""Marketplace publishing contracts; no Azure credentials or uploads required.""" + +from __future__ import annotations + +import base64 +import importlib.util +import io +import json +import os +import shutil +import subprocess +from pathlib import Path +from urllib.error import HTTPError, URLError +from zipfile import ZipFile + +import pytest +from ruamel.yaml import YAML + +ROOT = Path(__file__).resolve().parents[2] +TENANT = "11111111-1111-1111-1111-111111111111" +PROFILE = "22222222-2222-2222-2222-222222222222" +OTHER = "33333333-3333-3333-3333-333333333333" +TOKEN = "dummy-access-token" + + +@pytest.fixture +def marketplace(monkeypatch): + spec = importlib.util.spec_from_file_location("marketplace", ROOT / "scripts" / "marketplace.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setenv("MARKETPLACE_AZURE_TENANT_ID", TENANT) + monkeypatch.setenv("MARKETPLACE_PROFILE_ID", PROFILE) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + return module + + +def assignment(role="Contributor", profile=PROFILE, deny=0): + return {"user": {"id": profile}, "role": {"name": role, "denyPermissions": deny}} + + +@pytest.mark.parametrize("role", ["Contributor", "Owner", "Creator"]) +def test_requires_explicit_publishing_role(marketplace, role): + assert marketplace.publishing_role( + {"id": PROFILE}, {"value": [assignment(role)]}, PROFILE, + ) == role + + +@pytest.mark.parametrize("values", [ + [], [assignment("Reader")], [assignment(profile=OTHER)], + [assignment(deny=1)], [assignment(deny=None)], [assignment(deny="0")], + [assignment(), assignment("Reader", deny=1)], + [{}], ["invalid"], None, +]) +def test_read_access_and_malformed_or_denied_roles_fail_closed(marketplace, values): + with pytest.raises(marketplace.MarketplaceError): + marketplace.publishing_role({"id": PROFILE}, {"value": values}, PROFILE) + + +def test_profile_mismatch_rejects_even_an_owner(marketplace): + with pytest.raises(marketplace.MarketplaceError, match="identity mismatch"): + marketplace.publishing_role( + {"id": OTHER}, {"value": [assignment("Owner", OTHER)]}, PROFILE, + ) + + +@pytest.mark.parametrize("name", ["MARKETPLACE_AZURE_TENANT_ID", "MARKETPLACE_PROFILE_ID"]) +@pytest.mark.parametrize("value", ["", "not-a-guid"]) +def test_missing_configuration_fails_before_any_authentication( + marketplace, monkeypatch, capsys, name, value, +): + monkeypatch.setenv(name, value) + monkeypatch.setattr(marketplace.subprocess, "run", lambda *a, **k: pytest.fail("must not run")) + assert marketplace.main(["check"]) == 1 + assert name in capsys.readouterr().err + + +def test_child_environment_cannot_use_pat_or_environment_credentials(marketplace, monkeypatch): + forbidden = [ + "VSCE_PAT", "AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", + "AZURE_CLIENT_CERTIFICATE_PATH", "AZURE_CLIENT_CERTIFICATE_PASSWORD", + "AZURE_USERNAME", "AZURE_PASSWORD", + ] + for name in forbidden: + monkeypatch.setenv(name, "unrelated-credential") + monkeypatch.setenv("RELEASE_PAT", "github-credential") + monkeypatch.setenv("AZURE_TENANT_ID", OTHER) + env = marketplace.publishing_environment(TENANT) + assert env["AZURE_TENANT_ID"] == TENANT + assert env["RELEASE_PAT"] == "github-credential" + assert all(name not in env for name in forbidden) + assert marketplace.os.environ["VSCE_PAT"] == "unrelated-credential" + + +def mock_identity(marketplace, monkeypatch, *, role="Contributor", profile=PROFILE): + requests = [] + monkeypatch.setattr(marketplace.shutil, "which", lambda name: f"/tools/{name}") + + def get_json(url, authorization): + requests.append((url, authorization)) + if url == marketplace.PROFILE_URL: + return {"id": profile} + assert url == marketplace.ROLES_URL + return {"value": [assignment(role, profile)]} + + monkeypatch.setattr(marketplace, "get_json", get_json) + return requests + + +def test_read_only_preflight_matches_vsce_auth_without_upload(marketplace, monkeypatch, capsys): + requests = mock_identity(marketplace, monkeypatch) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert command == [ + "/tools/az", "account", "get-access-token", "--tenant", TENANT, + "--resource", marketplace.RESOURCE, "--query", "accessToken", + "--output", "tsv", "--only-show-errors", + ] + assert kwargs["capture_output"] is True + assert kwargs["timeout"] == 60 + assert kwargs["env"]["AZURE_TENANT_ID"] == TENANT + return subprocess.CompletedProcess(command, 0, TOKEN + "\n", "") + + monkeypatch.setattr(marketplace.subprocess, "run", run) + assert marketplace.main(["check"]) == 0 + assert len(calls) == 1 + basic = base64.b64encode(f"OAuth:{TOKEN}".encode()).decode() + assert requests == [ + (marketplace.PROFILE_URL, f"Basic {basic}"), + (marketplace.ROLES_URL, f"Basic {basic}"), + ] + output = capsys.readouterr().out + assert TOKEN not in output and basic not in output + assert "explicit Contributor" in output and "No upload" in output + + +@pytest.mark.parametrize("failure", ["exit", "empty", "timeout", "oserror"]) +def test_az_errors_do_not_print_credentials(marketplace, monkeypatch, capsys, failure): + monkeypatch.setattr(marketplace.shutil, "which", lambda name: name) + + def run(command, **kwargs): + if failure == "timeout": + raise subprocess.TimeoutExpired(command, 60, output=TOKEN, stderr=TOKEN) + if failure == "oserror": + raise OSError(TOKEN) + return subprocess.CompletedProcess(command, 1 if failure == "exit" else 0, "", TOKEN) + + monkeypatch.setattr(marketplace.subprocess, "run", run) + assert marketplace.main(["check"]) == 1 + output = capsys.readouterr() + assert TOKEN not in output.out + output.err + assert "ERROR:" in output.err + + +@pytest.mark.parametrize("failure", [ + HTTPError("https://example.invalid", 403, TOKEN, {}, None), + HTTPError("https://example.invalid", 302, TOKEN, {}, None), + URLError(TOKEN), TimeoutError(TOKEN), +]) +def test_http_failures_are_safe(marketplace, monkeypatch, failure): + class Opener: + def open(self, request, timeout): + assert timeout == 30 + raise failure + + monkeypatch.setattr(marketplace, "build_opener", lambda *a: Opener()) + with pytest.raises(marketplace.MarketplaceError) as error: + marketplace.get_json(marketplace.PROFILE_URL, f"Basic {TOKEN}") + assert TOKEN not in str(error.value) + + +@pytest.mark.parametrize("body", [b"not json", b"[]", b"null", b"\xff"]) +def test_malformed_http_payload_fails(marketplace, monkeypatch, body): + class Opener: + def open(self, request, timeout): + return io.BytesIO(body) + + monkeypatch.setattr(marketplace, "build_opener", lambda *a: Opener()) + with pytest.raises(marketplace.MarketplaceError): + marketplace.get_json(marketplace.PROFILE_URL, "Basic dummy") + + +def test_authorization_is_never_redirected(marketplace): + assert marketplace.NoRedirect().redirect_request( + None, None, 302, "", {}, "https://other.invalid", + ) is None + + +@pytest.fixture +def vsix(tmp_path): + path = tmp_path / "extension with spaces.vsix" + with ZipFile(path, "w") as archive: + archive.writestr("extension/package.json", json.dumps({ + "publisher": "AgentOpsAccelerator", "name": "agentops-accelerator", "version": "1.2.3", + })) + return path + + +@pytest.mark.parametrize("pre_release", [True, False]) +@pytest.mark.parametrize("allow_duplicate", [True, False]) +def test_publish_uses_entra_and_native_duplicate_handling( + marketplace, monkeypatch, capsys, vsix, pre_release, allow_duplicate, +): + mock_identity(marketplace, monkeypatch) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert "VSCE_PAT" not in kwargs["env"] + if command[1] == "--version": + return subprocess.CompletedProcess(command, 0, "3.9.2\n", "") + if command[1] == "account": + return subprocess.CompletedProcess(command, 0, TOKEN, "") + basic = base64.b64encode(f"OAuth:{TOKEN}".encode()).decode() + return subprocess.CompletedProcess(command, 0, f"Published {TOKEN} {basic}\n") + + monkeypatch.setattr(marketplace.subprocess, "run", run) + args = ["publish", "--package-path", str(vsix)] + if pre_release: + args.append("--pre-release") + if allow_duplicate: + args.append("--allow-already-exists") + assert marketplace.main(args) == 0 + assert [cmd[1] for cmd in calls] == ["--version", "account", "publish"] + expected = ["/tools/vsce", "publish", "--azure-credential", "--packagePath", str(vsix)] + if pre_release: + expected.append("--pre-release") + if allow_duplicate: + expected.append("--skip-duplicate") + assert calls[-1] == expected + output = capsys.readouterr().out + assert TOKEN not in output + assert "***" in output + + +@pytest.mark.parametrize("role,profile", [("Reader", PROFILE), ("Owner", OTHER)]) +def test_failed_preflight_never_reaches_publish(marketplace, monkeypatch, vsix, role, profile): + mock_identity(marketplace, monkeypatch, role=role, profile=profile) + + def run(command, **kwargs): + assert command[1] != "publish" + return subprocess.CompletedProcess( + command, 0, "3.9.2" if command[1] == "--version" else TOKEN, "", + ) + + monkeypatch.setattr(marketplace.subprocess, "run", run) + assert marketplace.main(["publish", "--package-path", str(vsix)]) == 1 + + +def test_publish_error_with_already_exists_text_is_not_swallowed(marketplace, monkeypatch, vsix): + mock_identity(marketplace, monkeypatch) + + def run(command, **kwargs): + if command[1] == "--version": + return subprocess.CompletedProcess(command, 0, "3.9.2", "") + if command[1] == "account": + return subprocess.CompletedProcess(command, 0, TOKEN, "") + return subprocess.CompletedProcess(command, 7, "Auth failed: identity already exists\n") + + monkeypatch.setattr(marketplace.subprocess, "run", run) + assert marketplace.main([ + "publish", "--package-path", str(vsix), "--allow-already-exists", + ]) == 7 + + +@pytest.mark.parametrize("version", ["2.26.0", "3.9.1", "garbage", "3.9.2-dev.1"]) +def test_old_or_unknown_vsce_rejected(marketplace, monkeypatch, vsix, version): + monkeypatch.setattr(marketplace.shutil, "which", lambda name: name) + monkeypatch.setattr( + marketplace.subprocess, "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 0, version, ""), + ) + monkeypatch.setattr(marketplace, "preflight", lambda *a: pytest.fail("must fail before auth")) + assert marketplace.main(["publish", "--package-path", str(vsix)]) == 1 + + +@pytest.mark.parametrize("manifest", [ + {}, {"publisher": "other", "name": "agentops-accelerator"}, + {"publisher": "AgentOpsAccelerator", "name": "other"}, [], +]) +def test_wrong_extension_never_authenticates(marketplace, monkeypatch, vsix, manifest): + with ZipFile(vsix, "w") as archive: + archive.writestr("extension/package.json", json.dumps(manifest)) + monkeypatch.setattr(marketplace.subprocess, "run", lambda *a, **k: pytest.fail("must not run")) + assert marketplace.main(["publish", "--package-path", str(vsix)]) == 1 + + +def load_workflow(name): + return YAML(typ="safe").load((ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("file,job_name,environment,pre_release", [ + ("staging.yml", "publish-vsix-prerelease", "marketplace-staging", True), + ("release.yml", "publish-vsix", "marketplace-release", False), +]) +def test_workflows_use_dedicated_oidc_environments(file, job_name, environment, pre_release): + job = load_workflow(file)["jobs"][job_name] + assert job["environment"] == environment + assert job["permissions"] == {"contents": "read", "id-token": "write"} + assert job["if"] + assert job["env"]["MARKETPLACE_PROFILE_ID"] == "${{ vars.MARKETPLACE_PROFILE_ID }}" + steps = job["steps"] + login = next(step for step in steps if step.get("uses") == "./.github/actions/marketplace-login") + assert login["with"] == { + "client-id": "${{ vars.MARKETPLACE_AZURE_CLIENT_ID }}", + "tenant-id": "${{ vars.MARKETPLACE_AZURE_TENANT_ID }}", + "profile-id": "${{ vars.MARKETPLACE_PROFILE_ID }}", + } + assert any(step.get("run") == "npm install -g @vscode/vsce@3.9.2" for step in steps) + publish = next(step for step in steps if "python scripts/marketplace.py publish" in step.get("run", "")) + assert ("--pre-release" in publish["run"]) == pre_release + assert "--allow-already-exists" in publish["run"] + assert steps.index(login) < steps.index(publish) + assert "VSCE_PAT" not in json.dumps(job) + assert "continue-on-error" not in json.dumps(job) + + +def test_login_does_not_require_subscription_or_shared_e2e_identity(): + action = YAML(typ="safe").load( + (ROOT / ".github" / "actions" / "marketplace-login" / "action.yml").read_text(), + ) + steps = action["runs"]["steps"] + assert "PROFILE_ID" in steps[0]["run"] + login = steps[1] + assert login["uses"] == "azure/login@v3" + assert login["with"]["allow-no-subscriptions"] is True + assert "subscription-id" not in login["with"] + + +def test_release_retry_keeps_tooling_separate_from_older_release_sources(): + steps = load_workflow("release.yml")["jobs"]["publish-vsix"]["steps"] + checkouts = [step for step in steps if step.get("uses") == "actions/checkout@v7"] + assert checkouts[0]["with"] == {"ref": "${{ github.workflow_sha }}"} + assert checkouts[1]["with"]["ref"] == "refs/tags/${{ inputs.tag || github.ref_name }}" + assert checkouts[1]["with"]["path"] == "release-source" + for name in ("Sync VSIX version from git tag", "Copy root assets for VSIX"): + assert next(step for step in steps if step.get("name") == name)["working-directory"] == "release-source" + package = next(step for step in steps if step.get("name") == "Package VSIX") + assert package["working-directory"] == "release-source/plugins/agentops" + artifact = next(step for step in steps if step.get("uses") == "actions/upload-artifact@v7") + assert artifact["with"]["path"] == "release-source/plugins/agentops/${{ env.VSIX_FILE }}" + + +def test_python_publishing_and_github_release_contracts_stay_intact(): + staging = load_workflow("staging.yml")["jobs"]["publish-testpypi"] + release = load_workflow("release.yml")["jobs"] + for job, environment in [ + (staging, "staging"), (release["publish-testpypi"], "staging"), + (release["publish-pypi"], "release"), + ]: + assert job["environment"] == environment + assert job["permissions"]["id-token"] == "write" + assert any(step.get("uses") == "pypa/gh-action-pypi-publish@release/v1" for step in job["steps"]) + assert release["publish-vsix"]["needs"] == ["build", "publish-pypi"] + assert release["github-release"]["needs"] == ["publish-pypi", "publish-vsix"] + cut_release = (ROOT / ".github" / "workflows" / "cut-release.yml").read_text(encoding="utf-8") + assert "secrets.RELEASE_PAT" in cut_release + + +@pytest.mark.parametrize("name", ["release.ps1", "release.sh", "staging.ps1", "staging.sh"]) +def test_local_scripts_preflight_before_release_actions(name): + text = (ROOT / "scripts" / name).read_text(encoding="utf-8") + assert "VSCE_PAT" not in text + assert "marketplace.py" in text + assert " publish --" in text + assert text.index(" check") < text.index("uv build") + if name.startswith("release."): + assert text.index(" check") < text.index('git push origin') + if name.endswith(".ps1"): + assert 'throw "Marketplace' in text + assert "finally {" in text + + +def test_npm_publishing_also_uses_shared_permission_preflight(): + scripts = json.loads((ROOT / "plugins" / "agentops" / "package.json").read_text())["scripts"] + for name in ("publish", "publish:prerelease"): + assert "scripts/marketplace.py publish" in scripts[name] + assert "vsce publish" not in scripts[name] + assert scripts["package"] == "vsce package" + assert scripts["package:prerelease"] == "vsce package --pre-release" + + +@pytest.mark.parametrize("ref,tag,expected", [ + ("refs/tags/v1.2.3", "v1.2.3", 0), + ("refs/heads/main", "v1.2.3", 0), + ("refs/heads/main", "main", 1), + ("refs/tags/v1.2.3", "v4.5.6", 1), + ("refs/tags/v1.2.3", "v1.2.3-rc1", 1), + ("refs/heads/main", "v1.2.3; echo unexpected", 1), +]) +def test_stable_release_ref_guard(ref, tag, expected): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is not installed") + step = load_workflow("release.yml")["jobs"]["publish-vsix"]["steps"][0] + assert step["name"] == "Validate Marketplace release ref" + result = subprocess.run( + [bash, "-c", step["run"]], + env={**os.environ, "GITHUB_REF": ref, "GITHUB_REF_NAME": ref.rsplit("/", 1)[-1], "RELEASE_TAG": tag}, + capture_output=True, text=True, timeout=15, check=False, + ) + assert result.returncode == expected + assert "unexpected" not in result.stdout + + +@pytest.mark.parametrize("client,tenant,profile,expected", [ + (OTHER, TENANT, PROFILE, 0), ("", TENANT, PROFILE, 1), + (OTHER, "", PROFILE, 1), (OTHER, TENANT, "", 1), + ("$(echo unexpected)", TENANT, PROFILE, 1), +]) +def test_login_configuration_guard(client, tenant, profile, expected): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is not installed") + action = YAML(typ="safe").load( + (ROOT / ".github" / "actions" / "marketplace-login" / "action.yml").read_text(), + ) + script = action["runs"]["steps"][0]["run"] + result = subprocess.run( + [bash, "-c", script], + env={**os.environ, "CLIENT_ID": client, "TENANT_ID": tenant, "PROFILE_ID": profile}, + capture_output=True, text=True, timeout=15, check=False, + ) + assert result.returncode == expected + assert "unexpected" not in result.stdout + + +def test_github_masks_escape_workflow_command_delimiters(marketplace, monkeypatch, capsys): + monkeypatch.setenv("GITHUB_ACTIONS", "true") + marketplace.mask("dummy%\r\nsecret") + assert capsys.readouterr().out == "::add-mask::dummy%25%0D%0Asecret\n" From e9127e355d62acacfe0ebcb1e08d5a1ae11affa7 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 7 Sep 2026 10:05:40 -0300 Subject: [PATCH 2/6] ci: add permission-only Marketplace identity validation Refs #489. Bootstrap the profile before publisher authorization and keep protected discover/check runs separate from publishing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/marketplace-login/action.yml | 17 +++- .github/skills/release-management/SKILL.md | 14 ++- .github/workflows/marketplace-preflight.yml | 100 +++++++++++++++++++ CHANGELOG.md | 3 + docs/release-process.md | 29 +++++- scripts/marketplace.py | 34 ++++++- tests/unit/test_marketplace_publishing.py | 62 ++++++++++++ 7 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/marketplace-preflight.yml diff --git a/.github/actions/marketplace-login/action.yml b/.github/actions/marketplace-login/action.yml index b32acd5c..e3c2c4bb 100644 --- a/.github/actions/marketplace-login/action.yml +++ b/.github/actions/marketplace-login/action.yml @@ -9,7 +9,10 @@ inputs: required: true profile-id: description: Marketplace profiles/me ID (not the Entra principal ID). - required: true + required: false + discover-profile: + description: Skip only the profile configuration check when bootstrapping a profile ID. + default: "false" runs: using: composite steps: @@ -19,9 +22,19 @@ runs: CLIENT_ID: ${{ inputs.client-id }} TENANT_ID: ${{ inputs.tenant-id }} PROFILE_ID: ${{ inputs.profile-id }} + DISCOVER_PROFILE: ${{ inputs.discover-profile }} run: | GUID='^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' - for NAME in CLIENT_ID TENANT_ID PROFILE_ID; do + DISCOVER_PROFILE="${DISCOVER_PROFILE:-false}" + if [[ "$DISCOVER_PROFILE" != "true" && "$DISCOVER_PROFILE" != "false" ]]; then + echo "::error::discover-profile must be true or false." + exit 1 + fi + NAMES=(CLIENT_ID TENANT_ID) + if [[ "$DISCOVER_PROFILE" == "false" ]]; then + NAMES+=(PROFILE_ID) + fi + for NAME in "${NAMES[@]}"; do if [[ ! "${!NAME}" =~ $GUID ]]; then echo "::error::Missing or invalid Marketplace $NAME. Configure the protected Marketplace environment variables; see docs/release-process.md." exit 1 diff --git a/.github/skills/release-management/SKILL.md b/.github/skills/release-management/SKILL.md index 480d0719..3be71edd 100644 --- a/.github/skills/release-management/SKILL.md +++ b/.github/skills/release-management/SKILL.md @@ -295,6 +295,14 @@ Owner must grant that profile **Contributor, not Owner**, on `AgentOpsAccelerato `MARKETPLACE_AZURE_TENANT_ID`, pins the self profile to `MARKETPLACE_PROFILE_ID`, and checks explicit Contributor/Owner/Creator membership with zero deny permissions. A generic HTTP 200 is not sufficient. +- `python scripts/marketplace.py discover --out marketplace-profile.json` + bootstraps the Marketplace profile without an expected ID or publisher access. + Only tenant/profile IDs are written, never tokens. +- `marketplace-preflight.yml` provides discover/check via manual dispatch or a + reviewed reusable-workflow caller. It never publishes. Respect the selected + environment's exact ref policy and human review. Pre-merge bootstrap uses a + separate protected validation environment/branch, not dummy release refs. + A validation-context result is not proof of staging/release authentication. - `python scripts/marketplace.py publish --package-path PATH [--pre-release] [--allow-already-exists]` preflights before uploading. CI opts into already-existing-version handling; local defaults fail. The flag maps to native `vsce --skip-duplicate` @@ -322,9 +330,9 @@ Owner must grant that profile **Contributor, not Owner**, on `AgentOpsAccelerato 2. Configure the dedicated UAMI, exact federation, publisher Contributor membership, environment reviewers and deployment policies, then variables. 3. Run an authorized permission-only preflight (`check`, no publishing scripts). - No standalone read-only workflow is added by this change. Arrange approved - OIDC permission validation before the first release; a local interactive - `check` alone does not validate CI federation. + Use `marketplace-preflight.yml`: discover the profile, grant Contributor, + then check its explicit publishing role. A local interactive `check` alone + does not validate CI federation. Do not bypass required human approvals. 4. Explicitly authorize and verify a **legitimate** Marketplace pre-release. 5. Explicitly authorize and verify a **legitimate** stable publication. 6. **Only then** remove the legacy GitHub `VSCE_PAT`. Its owner must confirm no diff --git a/.github/workflows/marketplace-preflight.yml b/.github/workflows/marketplace-preflight.yml new file mode 100644 index 00000000..c18801a4 --- /dev/null +++ b/.github/workflows/marketplace-preflight.yml @@ -0,0 +1,100 @@ +name: Marketplace permission check +run-name: Marketplace ${{ inputs.mode }} (${{ inputs.environment }}) - no upload + +on: + workflow_dispatch: + inputs: + mode: + description: Discover the profile ID or check its publishing role (never uploads). + type: choice + options: [discover, check] + default: check + required: true + environment: + description: Protected identity environment to validate. + type: choice + options: [marketplace-validation, marketplace-staging, marketplace-release] + default: marketplace-validation + required: true + workflow_call: + inputs: + mode: + type: string + required: true + environment: + type: string + required: true + +permissions: + contents: read + +jobs: + preflight: + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: ${{ inputs.environment }} + permissions: + contents: read + id-token: write + env: + MARKETPLACE_AZURE_TENANT_ID: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + MARKETPLACE_PROFILE_ID: ${{ vars.MARKETPLACE_PROFILE_ID }} + PREFLIGHT_MODE: ${{ inputs.mode }} + PREFLIGHT_ENVIRONMENT: ${{ inputs.environment }} + steps: + - name: Validate read-only request + shell: bash + run: | + case "$PREFLIGHT_MODE" in discover|check) ;; *) echo "::error::Invalid preflight mode"; exit 1 ;; esac + case "$PREFLIGHT_ENVIRONMENT" in + marketplace-validation|marketplace-staging|marketplace-release) ;; + *) echo "::error::Invalid Marketplace environment"; exit 1 ;; + esac + + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Authenticate the selected identity + uses: ./.github/actions/marketplace-login + with: + client-id: ${{ vars.MARKETPLACE_AZURE_CLIENT_ID }} + tenant-id: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} + profile-id: ${{ vars.MARKETPLACE_PROFILE_ID }} + discover-profile: ${{ inputs.mode == 'discover' && 'true' || 'false' }} + + - name: Discover profile without publisher access + if: inputs.mode == 'discover' + run: python scripts/marketplace.py discover --out marketplace-profile.json + + - name: Verify profile and explicit publishing role + if: inputs.mode == 'check' + run: python scripts/marketplace.py check + + - name: Save non-secret profile IDs + if: inputs.mode == 'discover' + uses: actions/upload-artifact@v7 + with: + name: marketplace-profile + path: marketplace-profile.json + retention-days: 7 + + - name: Record validation scope + shell: bash + run: | + { + echo "## Marketplace permission validation" + echo "Environment: \`$PREFLIGHT_ENVIRONMENT\`" + echo "Mode: \`$PREFLIGHT_MODE\`" + echo "This run did not upload, publish, or change publisher permissions." + if [[ "$PREFLIGHT_MODE" == "discover" ]]; then + echo "Profile discovered; publishing authorization has not been checked." + else + echo "Expected profile and explicit publishing role confirmed." + fi + echo "This proves only the selected environment, not other contexts or production-policy approval." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index f96a2b5a..c717b4dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres helper replace Marketplace PAT authentication. Permission-only preflight and staged rollout are documented; GitHub `RELEASE_PAT` and PyPI/TestPyPI Trusted Publishing are unchanged. +- **Marketplace permissions can be validated without publishing.** A dedicated + discover/check workflow bootstraps the identity's profile ID and verifies its + explicit publisher role behind environment approvals, without release uploads. ## [0.15.0] - 2026-09-06 diff --git a/docs/release-process.md b/docs/release-process.md index eb077faa..2617a10d 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -814,6 +814,7 @@ pin `npm install -g @vscode/vsce@3.9.2`. ```text python scripts/marketplace.py check +python scripts/marketplace.py discover --out marketplace-profile.json python scripts/marketplace.py publish --package-path PATH [--pre-release] [--allow-already-exists] ``` @@ -833,6 +834,30 @@ python scripts/marketplace.py publish --package-path PATH [--pre-release] [--all - Profile pinning prevents a wrong account or tenant from publishing. A successful CLI profile/role preflight is **not proof of an actual upload**. +#### Permission-only GitHub workflow + +`marketplace-preflight.yml` is separate from the release workflows and never +publishes. Its `workflow_dispatch` inputs select `discover` or `check` and one +of `marketplace-validation`, `marketplace-staging`, or `marketplace-release`. +It also supports `workflow_call` for a reviewed bootstrap workflow. + +Use `discover` first: only client/tenant configuration is required. It reads the +new identity's Marketplace profile without requiring publisher membership and +uploads only tenant/profile IDs in the seven-day `marketplace-profile` artifact. +After granting Contributor, set `MARKETPLACE_PROFILE_ID` and run `check`. +Discovery alone is not proof of publisher access. + +Configure a separate `marketplace-validation` environment for pre-merge tests, +with an exact approved validation branch, required reviewers, and its own +environment-subject federated credential. Before the workflow is available on +the default branch, an isolated no-upload push wrapper can call it from a +reviewed validation branch. Do not change the existing staging/release branch +policies to admit test branches, create dummy release refs, or bypass reviewers. +An approval pause is a real human handoff, not a reason to use another identity. +Only the selected context is validated; staging/release need their own checks +on allowed refs. Remove the isolated wrapper/branch and its federation when +validation is retired; never remove shared resources. + #### Local Publishing The local `scripts/staging.ps1`, `scripts/staging.sh`, `scripts/release.ps1`, @@ -869,8 +894,8 @@ configured or publication has been tested**. - [ ] Run a permission-only preflight under the intended federated identity: `python scripts/marketplace.py check`. Verify the pinned profile and explicit role with no deny permissions. Do not call publishing scripts. - This change does not add a standalone read-only workflow: arrange an - explicitly approved OIDC permission-validation run before the first release. + Use the dedicated `marketplace-preflight.yml` workflow, not staging/release. + Its discovery mode resolves the profile before checking publisher membership. A local interactive `check` validates that local identity, not CI federation. - [ ] Obtain explicit authorization for a legitimate pre-release, approve its `marketplace-staging` deployment, publish it, and verify the Marketplace result. diff --git a/scripts/marketplace.py b/scripts/marketplace.py index a8ab8975..6968110a 100644 --- a/scripts/marketplace.py +++ b/scripts/marketplace.py @@ -1,7 +1,8 @@ """Check Marketplace publishing permission and publish with Entra (never a PAT). -Requires an existing Azure CLI login, MARKETPLACE_AZURE_TENANT_ID, and -MARKETPLACE_PROFILE_ID. No cloud resources or publisher memberships are mutated. +Requires an existing Azure CLI login and MARKETPLACE_AZURE_TENANT_ID. +Check/publish also require MARKETPLACE_PROFILE_ID; discover bootstraps that ID. +No cloud resources or publisher memberships are mutated. """ from __future__ import annotations @@ -161,14 +162,23 @@ def publishing_role(profile: dict, assignments: dict, expected_profile: str) -> return min(roles) -def preflight(tenant: str, expected_profile: str, env: dict[str, str]) -> tuple[str, str]: +def get_profile(tenant: str, env: dict[str, str]) -> tuple[dict, str, str]: token = get_token(tenant, env) # Match vsce's Basic OAuth convention, rather than testing only Bearer access. basic = base64.b64encode(f"OAuth:{token}".encode()).decode("ascii") mask(basic) authorization = f"Basic {basic}" profile = get_json(PROFILE_URL, authorization) - assignments = get_json(ROLES_URL, authorization) + try: + profile["id"] = str(UUID(profile["id"])) + except (KeyError, ValueError, TypeError, AttributeError): + raise MarketplaceError("Marketplace returned an invalid profile ID.") from None + return profile, token, basic + + +def preflight(tenant: str, expected_profile: str, env: dict[str, str]) -> tuple[str, str]: + profile, token, basic = get_profile(tenant, env) + assignments = get_json(ROLES_URL, f"Basic {basic}") role = publishing_role(profile, assignments, expected_profile) print(f"Marketplace preflight passed: {PUBLISHER}, explicit {role} role. No upload performed.") return token, basic @@ -227,6 +237,8 @@ def publish( def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) + discover = commands.add_parser("discover", help="Read the profile ID; no publisher access required.") + discover.add_argument("--out", type=Path, help="Write only the tenant and profile IDs as JSON.") commands.add_parser("check", help="Check the selected identity and publishing role; never upload.") upload = commands.add_parser("publish", help="Check permission, then publish an existing VSIX.") upload.add_argument("--package-path", type=Path, required=True) @@ -235,8 +247,20 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: tenant = required_guid("MARKETPLACE_AZURE_TENANT_ID") - profile = required_guid("MARKETPLACE_PROFILE_ID") env = publishing_environment(tenant) + if args.command == "discover": + discovered, _, _ = get_profile(tenant, env) + if args.out: + try: + args.out.write_text( + json.dumps({"tenant_id": tenant, "profile_id": discovered["id"]}, indent=2) + "\n", + encoding="utf-8", + ) + except OSError: + raise MarketplaceError("Cannot write the Marketplace profile output file.") from None + print(f"Marketplace profile ID: {discovered['id']}. No publisher authorization or upload tested.") + return 0 + profile = required_guid("MARKETPLACE_PROFILE_ID") if args.command == "check": preflight(tenant, profile, env) return 0 diff --git a/tests/unit/test_marketplace_publishing.py b/tests/unit/test_marketplace_publishing.py index 6c29fdc0..24eb035c 100644 --- a/tests/unit/test_marketplace_publishing.py +++ b/tests/unit/test_marketplace_publishing.py @@ -430,3 +430,65 @@ def test_github_masks_escape_workflow_command_delimiters(marketplace, monkeypatc monkeypatch.setenv("GITHUB_ACTIONS", "true") marketplace.mask("dummy%\r\nsecret") assert capsys.readouterr().out == "::add-mask::dummy%25%0D%0Asecret\n" + + +def test_discovery_bootstraps_without_profile_or_publisher_access(marketplace, monkeypatch, tmp_path, capsys): + monkeypatch.delenv("MARKETPLACE_PROFILE_ID", raising=False) + monkeypatch.setattr(marketplace, "get_token", lambda *a: TOKEN) + requests = [] + + def get_json(url, authorization): + requests.append(url) + assert url == marketplace.PROFILE_URL + return {"id": PROFILE, "emailAddress": "not-for-artifacts@example.invalid"} + + monkeypatch.setattr(marketplace, "get_json", get_json) + output = tmp_path / "profile.json" + assert marketplace.main(["discover", "--out", str(output)]) == 0 + assert json.loads(output.read_text()) == {"tenant_id": TENANT, "profile_id": PROFILE} + assert requests == [marketplace.PROFILE_URL] + assert TOKEN not in output.read_text() + capsys.readouterr().out + + +@pytest.mark.parametrize("profile", [{}, {"id": "invalid"}, {"id": None}, {"id": 42}]) +def test_discovery_rejects_invalid_profile(marketplace, monkeypatch, profile): + monkeypatch.setattr(marketplace, "get_token", lambda *a: TOKEN) + monkeypatch.setattr(marketplace, "get_json", lambda *a: profile) + assert marketplace.main(["discover"]) == 1 + + +def test_preflight_workflow_has_no_upload_or_mutation_commands(): + workflow = load_workflow("marketplace-preflight.yml") + assert set(workflow["on"]) == {"workflow_dispatch", "workflow_call"} + modes = workflow["on"]["workflow_dispatch"]["inputs"]["mode"]["options"] + assert modes == ["discover", "check"] + job = workflow["jobs"]["preflight"] + assert job["permissions"] == {"contents": "read", "id-token": "write"} + assert job["environment"] == "${{ inputs.environment }}" + steps = job["steps"] + runs = "\n".join(step.get("run", "") for step in steps) + for forbidden in ("vsce ", "marketplace.py publish", "gh release", "az role", "curl ", "git push"): + assert forbidden not in runs + assert "python scripts/marketplace.py discover --out marketplace-profile.json" in runs + assert "python scripts/marketplace.py check" in runs + artifact = next(step for step in steps if step.get("uses") == "actions/upload-artifact@v7") + assert artifact["with"]["path"] == "marketplace-profile.json" + assert artifact["if"] == "inputs.mode == 'discover'" + login = next(step for step in steps if step.get("uses") == "./.github/actions/marketplace-login") + assert login["with"]["discover-profile"] == "${{ inputs.mode == 'discover' && 'true' || 'false' }}" + + +@pytest.mark.parametrize("mode,expected", [("true", 0), ("false", 1), ("wrong", 1)]) +def test_profile_bypass_is_explicit_and_only_for_discovery(mode, expected): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is not installed") + action = YAML(typ="safe").load( + (ROOT / ".github" / "actions" / "marketplace-login" / "action.yml").read_text(), + ) + result = subprocess.run( + [bash, "-c", action["runs"]["steps"][0]["run"]], + env={**os.environ, "CLIENT_ID": OTHER, "TENANT_ID": TENANT, "PROFILE_ID": "", "DISCOVER_PROFILE": mode}, + capture_output=True, text=True, timeout=15, check=False, + ) + assert result.returncode == expected From 78f6caff34571875626145b92d9b96097c595db1 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 7 Sep 2026 10:17:23 -0300 Subject: [PATCH 3/6] fix: use profile-service auth and Marketplace role response schema Use standard Entra authentication for profile discovery and identity.id from the SecurityRoles API. Keep Basic OAuth for the publisher-role check. Refs #489. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- scripts/marketplace.py | 17 +++++++++-------- tests/unit/test_marketplace_publishing.py | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/marketplace.py b/scripts/marketplace.py index 6968110a..99354c15 100644 --- a/scripts/marketplace.py +++ b/scripts/marketplace.py @@ -113,8 +113,9 @@ def get_json(url: str, authorization: str) -> dict: with build_opener(NoRedirect()).open(request, timeout=30) as response: payload = json.load(response) except HTTPError as error: + operation = "profile lookup" if url == PROFILE_URL else "publisher role lookup" raise MarketplaceError( - f"Marketplace permission check returned HTTP {error.code}. " + f"Marketplace {operation} returned HTTP {error.code}. " "Check the identity tenant and ask a publisher Owner to grant Contributor access." ) from None except (URLError, TimeoutError, OSError): @@ -140,11 +141,11 @@ def publishing_role(profile: dict, assignments: dict, expected_profile: str) -> for assignment in values: if not isinstance(assignment, dict): raise MarketplaceError("Marketplace returned an invalid role assignment.") - user = assignment.get("user") - if not isinstance(user, dict): - raise MarketplaceError("Marketplace returned a role assignment without a user.") - user_id = user.get("id") - if not isinstance(user_id, str) or user_id.lower() != profile_id.lower(): + identity = assignment.get("identity") + if not isinstance(identity, dict): + raise MarketplaceError("Marketplace returned a role assignment without an identity.") + identity_id = identity.get("id") + if not isinstance(identity_id, str) or identity_id.lower() != profile_id.lower(): continue role = assignment.get("role") if not isinstance(role, dict) or type(role.get("denyPermissions")) is not int: @@ -167,8 +168,8 @@ def get_profile(tenant: str, env: dict[str, str]) -> tuple[dict, str, str]: # Match vsce's Basic OAuth convention, rather than testing only Bearer access. basic = base64.b64encode(f"OAuth:{token}".encode()).decode("ascii") mask(basic) - authorization = f"Basic {basic}" - profile = get_json(PROFILE_URL, authorization) + # Profile and Marketplace services use different authentication conventions. + profile = get_json(PROFILE_URL, "Bearer " + token) try: profile["id"] = str(UUID(profile["id"])) except (KeyError, ValueError, TypeError, AttributeError): diff --git a/tests/unit/test_marketplace_publishing.py b/tests/unit/test_marketplace_publishing.py index 24eb035c..962b7bd3 100644 --- a/tests/unit/test_marketplace_publishing.py +++ b/tests/unit/test_marketplace_publishing.py @@ -35,7 +35,7 @@ def marketplace(monkeypatch): def assignment(role="Contributor", profile=PROFILE, deny=0): - return {"user": {"id": profile}, "role": {"name": role, "denyPermissions": deny}} + return {"identity": {"id": profile}, "role": {"name": role, "denyPermissions": deny}} @pytest.mark.parametrize("role", ["Contributor", "Owner", "Creator"]) @@ -127,7 +127,7 @@ def run(command, **kwargs): assert len(calls) == 1 basic = base64.b64encode(f"OAuth:{TOKEN}".encode()).decode() assert requests == [ - (marketplace.PROFILE_URL, f"Basic {basic}"), + (marketplace.PROFILE_URL, "Bearer " + TOKEN), (marketplace.ROLES_URL, f"Basic {basic}"), ] output = capsys.readouterr().out From 219b3e66dd2747da070dd4cf28c54744b328f0d9 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 7 Sep 2026 11:42:58 -0300 Subject: [PATCH 4/6] fix: reserve Marketplace publication for stable releases Stage VSIX artifacts without authentication or upload; remove local and npm prerelease publishing paths to avoid reserving stable versions. Refs #489. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/_build.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/cut-release.yml | 6 +-- .github/workflows/release.yml | 2 +- .github/workflows/staging.yml | 53 +++++------------------ CHANGELOG.md | 3 ++ plugins/agentops/package.json | 3 +- scripts/staging.ps1 | 37 +++------------- scripts/staging.sh | 32 +++----------- tests/unit/test_marketplace_publishing.py | 34 ++++++++++++--- 10 files changed, 64 insertions(+), 110 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index 508b17bc..7b3846d5 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -3,7 +3,7 @@ # Workflows: # 1. ci.yml — Lint + test on every push/PR; build VSIX validation # 2. _build.yml — Reusable Python build (test + package), called by staging and release -# 3. staging.yml — Staging: release/* → TestPyPI + VSIX pre-release +# 3. staging.yml — Staging: release/* → TestPyPI + VSIX artifact only # 4. release.yml — Production: v* tag → PyPI + VSIX stable + GitHub Release # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bd29653..237252ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ # Workflows: # 1. ci.yml — Lint + changelog guard + test on every push/PR; publish dev builds to TestPyPI on develop; VSIX build validation # 2. _build.yml — Reusable build (test + package), called by staging and release -# 3. staging.yml — Staging: release/* branch → TestPyPI → verify; VSIX pre-release → Marketplace +# 3. staging.yml — Staging: release/* branch → TestPyPI → verify; VSIX artifact only # 4. release.yml — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop # diff --git a/.github/workflows/cut-release.yml b/.github/workflows/cut-release.yml index 8089e518..999ab1a8 100644 --- a/.github/workflows/cut-release.yml +++ b/.github/workflows/cut-release.yml @@ -3,7 +3,7 @@ # Workflows: # 1. ci.yml — Lint + test on every push/PR; VSIX build validation # 2. _build.yml — Reusable build (test + package), called by staging and release -# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace +# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX artifact only # 4. release.yml — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop # @@ -134,7 +134,7 @@ jobs: - Branch \`release/v${{ env.version }}\` created from \`develop\` - \`CHANGELOG.md\` updated: versioned section \`[${{ env.version }}]\` added - Plugin versions synced to \`${{ env.version }}\` (package.json, plugin.json, marketplace.json) - - Staging pipeline triggered automatically (build → TestPyPI + VSIX pre-release → verify) + - Staging pipeline triggered automatically (build → TestPyPI + VSIX artifact → verify; no Marketplace upload) ### Next steps 1. Wait for the **Staging** pipeline to pass. This is the only verification that runs before PyPI. @@ -145,7 +145,7 @@ jobs: 6. Verify the sync: \`git fetch origin && git log --oneline origin/develop..origin/main\` must print nothing ### Checklist - - [ ] Staging pipeline passes (build + TestPyPI + VSIX pre-release + verify) + - [ ] Staging pipeline passes (build + TestPyPI + VSIX artifact + verify) - [ ] CHANGELOG entries reviewed - [ ] PR approved and merged to main - [ ] Tag \`v${{ env.version }}\` pushed (publishes to PyPI, irreversible) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e362665..4216c79d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ # Workflows: # 1. ci.yml — Lint + test on every push/PR; VSIX build validation # 2. _build.yml — Reusable build (test + package), called by staging and release -# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace +# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX artifact only # 4. release.yml — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop # diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 1bf264bb..ed9c23cc 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1,21 +1,21 @@ -# AgentOps Toolkit — Staging (TestPyPI + VSIX Pre-release) +# AgentOps Toolkit — Staging (TestPyPI + VSIX Artifact) # # Workflows: # 1. ci.yml — Lint + test on every push/PR; VSIX build validation # 2. _build.yml — Reusable build (test + package), called by staging and release -# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX pre-release → Marketplace +# 3. staging.yml — Staging: release/* → TestPyPI → verify; VSIX artifact only # 4. release.yml — Production: v* tag → TestPyPI → verify → PyPI → GH Release; VSIX stable → Marketplace # 5. cut-release.yml — Manual dispatch: create release branch + PR from develop # # Triggered by pushes to release/* branches. # Calls the reusable _build.yml, publishes to TestPyPI, verifies the -# package installs correctly with a CLI smoke test, and publishes the -# VS Code extension as a pre-release to the Marketplace. +# package installs correctly with a CLI smoke test, and packages the +# VS Code extension as a pre-release artifact without publishing to Marketplace. # # Branch flow: # develop → release/v0.2.0 → push → this workflow # → build → TestPyPI → verify install → ✅ ready to merge and tag -# → VSIX pre-release → Marketplace (early access channel) +# → VSIX pre-release artifact (download and install manually) # # Versioning: # Uses setuptools-scm — on a release branch 5 commits after the last tag, @@ -27,19 +27,15 @@ # Authentication is handled via OpenID Connect (OIDC) between GitHub Actions # and TestPyPI. # -# Marketplace uses Entra OIDC, not a PAT. The separate marketplace-staging -# environment requires protected deployment branches and dedicated identity -# variables: MARKETPLACE_AZURE_CLIENT_ID, MARKETPLACE_AZURE_TENANT_ID, -# MARKETPLACE_PROFILE_ID. See docs/release-process.md for custom OIDC subjects. +# Staging does not authenticate to or publish to Marketplace. Only the stable +# release workflow publishes the extension, avoiding a version collision. # # Setup (Trusted Publishing + VSCE): # 1. https://test.pypi.org/manage/project/agentops-accelerator/settings/publishing/ # → Add publisher: GitHub, owner=Azure, repo=agentops, workflow=staging.yml, environment=staging # 2. GitHub repo → Settings → Environments → Create "staging" (the name must match # the publisher config in step 1). No protection rules are configured today. -# 3. Configure marketplace-staging, federate the dedicated managed identity, -# and grant it Contributor on AgentOpsAccelerator (not Azure Contributor). -# Python Trusted Publishing and the GitHub RELEASE_PAT are unchanged. +# No Marketplace identity or PAT is required for VSIX packaging. name: Staging @@ -134,20 +130,13 @@ jobs: test -f .azure/config.json echo "✅ agentops init succeeded" - # ── VSIX Pre-release ───────────────────────────────────────────────── - # Publish the VS Code extension as a pre-release to the Marketplace. - # Runs in parallel with the TestPyPI flow (only needs source checkout). - publish-vsix-prerelease: + # Package a downloadable candidate; reserve Marketplace versions for stable tags. + build-vsix: needs: build # gate on successful lint + test if: startsWith(github.ref, 'refs/heads/release/v') runs-on: ubuntu-latest - environment: marketplace-staging permissions: contents: read - id-token: write - env: - MARKETPLACE_AZURE_TENANT_ID: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} - MARKETPLACE_PROFILE_ID: ${{ vars.MARKETPLACE_PROFILE_ID }} steps: - uses: actions/checkout@v7 @@ -174,18 +163,6 @@ jobs: - name: Install vsce run: npm install -g @vscode/vsce@3.9.2 - - name: Set up Python for Marketplace preflight - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - - name: Authenticate Marketplace identity - uses: ./.github/actions/marketplace-login - with: - client-id: ${{ vars.MARKETPLACE_AZURE_CLIENT_ID }} - tenant-id: ${{ vars.MARKETPLACE_AZURE_TENANT_ID }} - profile-id: ${{ vars.MARKETPLACE_PROFILE_ID }} - - name: Copy root assets for VSIX run: | cp CHANGELOG.md plugins/agentops/CHANGELOG.md @@ -195,19 +172,11 @@ jobs: working-directory: plugins/agentops run: vsce package --pre-release -o agentops-skills.vsix - - name: Publish pre-release to VS Code Marketplace - # Preflight requires the exact profile and an explicit publishing role. - # Only a duplicate-version publish error is tolerated on reruns. - run: | - python scripts/marketplace.py publish \ - --package-path plugins/agentops/agentops-skills.vsix \ - --pre-release --allow-already-exists - - name: Show VSIX info working-directory: plugins/agentops run: | ls -lh agentops-skills.vsix - echo "✅ VSIX pre-release published to Marketplace" + echo "VSIX candidate packaged only; Marketplace publication is reserved for the stable release." - name: Upload VSIX artifact uses: actions/upload-artifact@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index c717b4dd..e4ded10f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] ### Changed +- **Extension publication is stable-only.** Staging produces a downloadable + VSIX candidate without Marketplace authentication or upload. Only stable + releases publish, preventing pre-releases from reserving the stable version. - **Marketplace publishing moves from PATs to Microsoft Entra OIDC.** Dedicated managed-identity environments and a shared tenant/profile-pinned publishing helper replace Marketplace PAT authentication. Permission-only preflight and diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index e414fbb2..3624bc53 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -60,7 +60,6 @@ "vscode:prepublish": "echo 'Declarative extension — no build step required'", "package": "vsce package", "package:prerelease": "vsce package --pre-release", - "publish": "vsce package -o agentops-skills.vsix && python ../../scripts/marketplace.py publish --package-path agentops-skills.vsix", - "publish:prerelease": "vsce package --pre-release -o agentops-skills.vsix && python ../../scripts/marketplace.py publish --pre-release --package-path agentops-skills.vsix" + "publish": "vsce package -o agentops-skills.vsix && python ../../scripts/marketplace.py publish --package-path agentops-skills.vsix" } } diff --git a/scripts/staging.ps1 b/scripts/staging.ps1 index 8b18ccd1..c70aa9f4 100644 --- a/scripts/staging.ps1 +++ b/scripts/staging.ps1 @@ -9,7 +9,7 @@ # 4. Publish to TestPyPI # 5. Verify install from TestPyPI + smoke test # 6. Build VSIX pre-release -# 7. Publish VSIX pre-release to Marketplace +# Marketplace publication happens only in the stable release. # # Usage: .\scripts\staging.ps1 # Prereqs: @@ -17,8 +17,6 @@ # - twine: pip install twine (for TestPyPI upload) # - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # - TESTPYPI_TOKEN env var (API token from test.pypi.org) -# - Python 3.11+, Azure CLI login in the publisher identity's tenant -# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) # ───────────────────────────────────────────────────────────────────── Set-StrictMode -Version Latest @@ -26,31 +24,26 @@ $ErrorActionPreference = "Stop" $skipTestPyPI = $false $skipVSIX = $false -$marketplaceScript = Join-Path $PSScriptRoot "marketplace.py" -if (Get-Command vsce -ErrorAction SilentlyContinue) { - python $marketplaceScript check - if ($LASTEXITCODE -ne 0) { throw "Marketplace preflight failed; no staging actions were started." } -} # ── Step 1: Lint ──────────────────────────────────────────────────── -Write-Host "`n>>> [1/7] Linting with ruff..." -ForegroundColor Yellow +Write-Host "`n>>> [1/6] Linting with ruff..." -ForegroundColor Yellow uv run ruff check src/ tests/ Write-Host ">>> Lint passed" -ForegroundColor Green # ── Step 2: Test ──────────────────────────────────────────────────── -Write-Host "`n>>> [2/7] Running tests..." -ForegroundColor Yellow +Write-Host "`n>>> [2/6] Running tests..." -ForegroundColor Yellow uv run pytest tests/ -v --tb=short Write-Host ">>> Tests passed" -ForegroundColor Green # ── Step 3: Build ─────────────────────────────────────────────────── -Write-Host "`n>>> [3/7] Building package..." -ForegroundColor Yellow +Write-Host "`n>>> [3/6] Building package..." -ForegroundColor Yellow if (Test-Path dist) { Remove-Item dist -Recurse -Force } uv build Write-Host ">>> Build artifacts:" -ForegroundColor Green Get-ChildItem dist/ | ForEach-Object { Write-Host " $_" } # ── Step 4: Publish to TestPyPI ───────────────────────────────────── -Write-Host "`n>>> [4/7] Publishing to TestPyPI..." -ForegroundColor Yellow +Write-Host "`n>>> [4/6] Publishing to TestPyPI..." -ForegroundColor Yellow if (-not $env:TESTPYPI_TOKEN) { Write-Host ">>> TESTPYPI_TOKEN not set — skipping TestPyPI publish" -ForegroundColor DarkYellow Write-Host " Set it with: `$env:TESTPYPI_TOKEN = 'pypi-...'" -ForegroundColor DarkGray @@ -61,7 +54,7 @@ if (-not $env:TESTPYPI_TOKEN) { } # ── Step 5: Verify TestPyPI install ───────────────────────────────── -Write-Host "`n>>> [5/7] Verifying TestPyPI install..." -ForegroundColor Yellow +Write-Host "`n>>> [5/6] Verifying TestPyPI install..." -ForegroundColor Yellow if ($skipTestPyPI) { Write-Host ">>> Skipped (no TestPyPI publish)" -ForegroundColor DarkYellow } else { @@ -94,7 +87,7 @@ if ($skipTestPyPI) { } # ── Step 6: Build VSIX ────────────────────────────────────────────── -Write-Host "`n>>> [6/7] Building VSIX pre-release..." -ForegroundColor Yellow +Write-Host "`n>>> [6/6] Building VSIX pre-release artifact (no Marketplace upload)..." -ForegroundColor Yellow $vsceAvailable = Get-Command vsce -ErrorAction SilentlyContinue if (-not $vsceAvailable) { Write-Host ">>> vsce not found — skipping VSIX build" -ForegroundColor DarkYellow @@ -136,22 +129,6 @@ if (-not $vsceAvailable) { Write-Host ">>> package.json restored to committed version" -ForegroundColor DarkGray } -# ── Step 7: Publish VSIX pre-release ──────────────────────────────── -Write-Host "`n>>> [7/7] Publishing VSIX pre-release..." -ForegroundColor Yellow -if ($skipVSIX) { - Write-Host ">>> Skipped (vsce not available)" -ForegroundColor DarkYellow -} else { - Push-Location plugins/agentops - try { - Write-Host " VSIX will publish from packagePath (version in VSIX: $baseVersion)" -ForegroundColor DarkGray - python $marketplaceScript publish --pre-release --package-path agentops-skills.vsix - if ($LASTEXITCODE -ne 0) { throw "Marketplace pre-release publication failed." } - } finally { - Pop-Location - } - Write-Host ">>> VSIX pre-release published to Marketplace" -ForegroundColor Green -} - # ── Summary ───────────────────────────────────────────────────────── Write-Host "`n✅ Staging complete!" -ForegroundColor Green Write-Host " Lint: passed" -ForegroundColor Cyan diff --git a/scripts/staging.sh b/scripts/staging.sh index cd210472..5f37cc88 100755 --- a/scripts/staging.sh +++ b/scripts/staging.sh @@ -10,7 +10,7 @@ # 4. Publish to TestPyPI # 5. Verify install from TestPyPI + smoke test # 6. Build VSIX pre-release -# 7. Publish VSIX pre-release to Marketplace +# Marketplace publication happens only in the stable release. # # Usage: ./scripts/staging.sh # Prereqs: @@ -18,38 +18,32 @@ # - twine: pip install twine (for TestPyPI upload) # - Node.js 22 + vsce: npm install -g @vscode/vsce@3.9.2 # - TESTPYPI_TOKEN env var (API token from test.pypi.org) -# - Python 3.11+, Azure CLI login in the publisher identity's tenant -# - MARKETPLACE_AZURE_TENANT_ID and MARKETPLACE_PROFILE_ID (see docs/release-process.md) # ───────────────────────────────────────────────────────────────────── set -euo pipefail skip_testpypi=false skip_vsix=false -marketplace_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/marketplace.py" -if command -v vsce &>/dev/null; then - python "$marketplace_script" check -fi # ── Step 1: Lint ──────────────────────────────────────────────────── -echo -e "\n>>> [1/7] Linting with ruff..." +echo -e "\n>>> [1/6] Linting with ruff..." uv run ruff check src/ tests/ echo ">>> Lint passed" # ── Step 2: Test ──────────────────────────────────────────────────── -echo -e "\n>>> [2/7] Running tests..." +echo -e "\n>>> [2/6] Running tests..." uv run pytest tests/ -v --tb=short echo ">>> Tests passed" # ── Step 3: Build ─────────────────────────────────────────────────── -echo -e "\n>>> [3/7] Building package..." +echo -e "\n>>> [3/6] Building package..." rm -rf dist/ uv build echo ">>> Build artifacts:" ls -lh dist/ # ── Step 4: Publish to TestPyPI ───────────────────────────────────── -echo -e "\n>>> [4/7] Publishing to TestPyPI..." +echo -e "\n>>> [4/6] Publishing to TestPyPI..." if [ -z "${TESTPYPI_TOKEN:-}" ]; then echo ">>> TESTPYPI_TOKEN not set — skipping TestPyPI publish" echo ' Set it with: export TESTPYPI_TOKEN="pypi-..."' @@ -60,7 +54,7 @@ else fi # ── Step 5: Verify TestPyPI install ───────────────────────────────── -echo -e "\n>>> [5/7] Verifying TestPyPI install..." +echo -e "\n>>> [5/6] Verifying TestPyPI install..." if $skip_testpypi; then echo ">>> Skipped (no TestPyPI publish)" else @@ -91,7 +85,7 @@ else fi # ── Step 6: Build VSIX ────────────────────────────────────────────── -echo -e "\n>>> [6/7] Building VSIX pre-release..." +echo -e "\n>>> [6/6] Building VSIX pre-release artifact (no Marketplace upload)..." if ! command -v vsce &>/dev/null; then echo ">>> vsce not found — skipping VSIX build" echo " Install with: npm install -g @vscode/vsce@3.9.2" @@ -127,18 +121,6 @@ else echo ">>> package.json restored to committed version" fi -# ── Step 7: Publish VSIX pre-release ──────────────────────────────── -echo -e "\n>>> [7/7] Publishing VSIX pre-release..." -if $skip_vsix; then - echo ">>> Skipped (vsce not available)" -else - pushd plugins/agentops >/dev/null - echo " VSIX will publish from packagePath (version in VSIX: $base_version)" - python "$marketplace_script" publish --pre-release --package-path agentops-skills.vsix - popd >/dev/null - echo ">>> VSIX pre-release published to Marketplace" -fi - # ── Summary ───────────────────────────────────────────────────────── echo -e "\n✅ Staging complete!" echo " Lint: passed" diff --git a/tests/unit/test_marketplace_publishing.py b/tests/unit/test_marketplace_publishing.py index 962b7bd3..f598182b 100644 --- a/tests/unit/test_marketplace_publishing.py +++ b/tests/unit/test_marketplace_publishing.py @@ -291,7 +291,6 @@ def load_workflow(name): @pytest.mark.parametrize("file,job_name,environment,pre_release", [ - ("staging.yml", "publish-vsix-prerelease", "marketplace-staging", True), ("release.yml", "publish-vsix", "marketplace-release", False), ]) def test_workflows_use_dedicated_oidc_environments(file, job_name, environment, pre_release): @@ -316,6 +315,22 @@ def test_workflows_use_dedicated_oidc_environments(file, job_name, environment, assert "continue-on-error" not in json.dumps(job) +def test_staging_only_packages_candidate_without_marketplace_credentials(): + jobs = load_workflow("staging.yml")["jobs"] + assert "publish-vsix-prerelease" not in jobs + job = jobs["build-vsix"] + assert job["permissions"] == {"contents": "read"} + assert "environment" not in job + assert "env" not in job + serialized = json.dumps(job) + for forbidden in ("marketplace-login", "marketplace.py", "id-token", "VSCE_PAT", "MARKETPLACE_"): + assert forbidden not in serialized + steps = job["steps"] + assert any(step.get("run") == "vsce package --pre-release -o agentops-skills.vsix" for step in steps) + artifact = next(step for step in steps if step.get("uses") == "actions/upload-artifact@v7") + assert artifact["with"]["name"] == "vsix" + + def test_login_does_not_require_subscription_or_shared_e2e_identity(): action = YAML(typ="safe").load( (ROOT / ".github" / "actions" / "marketplace-login" / "action.yml").read_text(), @@ -358,7 +373,7 @@ def test_python_publishing_and_github_release_contracts_stay_intact(): assert "secrets.RELEASE_PAT" in cut_release -@pytest.mark.parametrize("name", ["release.ps1", "release.sh", "staging.ps1", "staging.sh"]) +@pytest.mark.parametrize("name", ["release.ps1", "release.sh"]) def test_local_scripts_preflight_before_release_actions(name): text = (ROOT / "scripts" / name).read_text(encoding="utf-8") assert "VSCE_PAT" not in text @@ -374,13 +389,22 @@ def test_local_scripts_preflight_before_release_actions(name): def test_npm_publishing_also_uses_shared_permission_preflight(): scripts = json.loads((ROOT / "plugins" / "agentops" / "package.json").read_text())["scripts"] - for name in ("publish", "publish:prerelease"): - assert "scripts/marketplace.py publish" in scripts[name] - assert "vsce publish" not in scripts[name] + assert "scripts/marketplace.py publish" in scripts["publish"] + assert "vsce publish" not in scripts["publish"] + assert "publish:prerelease" not in scripts assert scripts["package"] == "vsce package" assert scripts["package:prerelease"] == "vsce package --pre-release" +@pytest.mark.parametrize("name", ["staging.ps1", "staging.sh"]) +def test_local_staging_does_not_require_or_use_marketplace_identity(name): + text = (ROOT / "scripts" / name).read_text(encoding="utf-8") + for forbidden in ("marketplace.py", "MARKETPLACE_", "VSCE_PAT", "vsce publish"): + assert forbidden not in text + assert "vsce package --pre-release -o agentops-skills.vsix" in text + assert "twine upload --repository testpypi" in text + + @pytest.mark.parametrize("ref,tag,expected", [ ("refs/tags/v1.2.3", "v1.2.3", 0), ("refs/heads/main", "v1.2.3", 0), From 0f351269362d14ed9ecd2541532bd54d82824097 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 7 Sep 2026 11:43:52 -0300 Subject: [PATCH 5/6] docs: describe stable-only Marketplace release flow Refs #489. Keep staging artifact-only and retire the Marketplace PAT after verified stable publishing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/skills/release-management/SKILL.md | 94 ++++++---- docs/release-process.md | 197 ++++++++++++++------- 2 files changed, 191 insertions(+), 100 deletions(-) diff --git a/.github/skills/release-management/SKILL.md b/.github/skills/release-management/SKILL.md index 3be71edd..fd1eda52 100644 --- a/.github/skills/release-management/SKILL.md +++ b/.github/skills/release-management/SKILL.md @@ -66,9 +66,10 @@ Examples: `release/v2.4.2`, `release/v0.2.0` - Updates `CHANGELOG.md` (adds versioned section `[0.2.0] - YYYY-MM-DD`) - Pushes the branch (triggers staging pipeline automatically) - Opens a PR: `release/v0.2.0` → `main` -4. Wait for staging pipeline to pass (build → TestPyPI → verify), and review - the separately protected `marketplace-staging` deployment for the legitimate - Marketplace pre-release. The branch push is a real publication attempt. +4. Wait for staging to pass (build → TestPyPI → verify, plus parallel `build-vsix`). + Download and manually evaluate the pre-release VSIX artifact. Staging uploads + Python to TestPyPI but never uploads to Marketplace or reserves a Marketplace + version. No Marketplace environment or approval is required during staging. 5. Get the PR reviewed and merge into `main`. 6. Tag the release on `main` **and sync `develop` in the same sitting**. Tagging publishes to PyPI immediately; there is no approval prompt. Leaving `develop` @@ -92,9 +93,10 @@ Examples: `release/v2.4.2`, `release/v0.2.0` `## [0.2.0]` heading above develop's unreleased entries, nesting new work inside a shipped version. 7. Watch the Release workflow finish (build → TestPyPI → verify → publish-pypi → - github-release). The Python `release` environment has no protection rules, - so PyPI does not pause for review. The separate `marketplace-release` - deployment requires review for the stable extension publication. + publish-vsix → github-release). The Python `release` environment has no + protection rules: Python publishes **before Marketplace approval**. A human + must approve `marketplace-release` for stable extension publication; do not + bypass that gate. GitHub Release waits for successful Marketplace publication. 8. Delete the release branch: ```bash git push origin --delete release/v0.2.0 @@ -239,20 +241,31 @@ environment name exactly. A mismatch fails with `403` at upload time. ### Marketplace: dedicated Entra OIDC identity -Marketplace jobs use a **dedicated user-assigned managed identity (UAMI)**, +Marketplace publishing jobs use a **dedicated user-assigned managed identity (UAMI)**, `azure/login@v3` with `allow-no-subscriptions: true`, and `id-token: write`. No Azure RBAC grant is needed solely to publish. Publisher membership supplies that permission. Do not change Python Trusted Publishing, shared `AZURE_*` E2E variables, repository-wide OIDC configuration, or the GitHub `RELEASE_PAT`. -Use two **new, separate** GitHub environments, not Python `staging`/`release`. -Configure required reviewers and selected branch/tag deployment policies -**before setting variables**: +Marketplace publication is **stable-only**, through the production `vX.Y.Z` +tag flow. Previously staging could publish pre-release `X.Y.Z`, causing stable +publication of the same version to skip as a duplicate. Artifact-only staging +eliminates that collision; `package --pre-release` reserves no cloud version. + +`marketplace-release` is the only required publishing environment, separate +from Python `staging`/`release`. Retain its required human reviewers and selected +branch/tag deployment policies **before enabling identity variables**: | Environment | Allowed deployment refs | | --- | --- | -| `marketplace-staging` | Branches `release/*` for legitimate `release/vX.Y.Z` candidates | | `marketplace-release` | Tags `v*`; optionally protected `main` for manual dispatch with tag input | +| `marketplace-staging` | Existing policy; optional legacy/read-only preflight context | +| `marketplace-validation` | Existing approved validation-ref policy; optional read-only preflight context | + +The staging and validation Marketplace environments were created earlier, not +deleted by this change. Neither is required to stage/package or adds a staging +approval. `build-vsix` has no environment, `id-token`, Azure login, or profile +variables; it only produces a VSIX artifact for manual evaluation. Stable manual job guards allow only `main` or the same release tag as the input. An environment subject alone does not restrict branches; protections are essential. @@ -264,7 +277,8 @@ the new OIDC action/helper. Tag-based dispatch must match the tag input. Re-running a historical old workflow still executes its old PAT code, not the migrated workflow. A retry is a real publication attempt. -Each new environment needs these variables: +The publishing environment needs these variables (optional diagnostic +environments need them only when their preflight is used): - `MARKETPLACE_AZURE_CLIENT_ID`: UAMI client GUID. - `MARKETPLACE_AZURE_TENANT_ID`: approved identity tenant GUID. @@ -278,8 +292,8 @@ claim keys are `repository_owner_id`, `repository_id`, `context` with - Issuer: `https://token.actions.githubusercontent.com` - Audience: `api://AzureADTokenExchange` -- Staging subject: `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` - Release subject: `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-release` +- Optional legacy staging preflight subject: `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` Resolve the UAMI Marketplace profile using a CLI token for resource `499b84ac-1321-427f-aa17-267ca6975798` and a read-only request to @@ -299,43 +313,58 @@ Owner must grant that profile **Contributor, not Owner**, on `AgentOpsAccelerato bootstraps the Marketplace profile without an expected ID or publisher access. Only tenant/profile IDs are written, never tokens. - `marketplace-preflight.yml` provides discover/check via manual dispatch or a - reviewed reusable-workflow caller. It never publishes. Respect the selected - environment's exact ref policy and human review. Pre-merge bootstrap uses a - separate protected validation environment/branch, not dummy release refs. - A validation-context result is not proof of staging/release authentication. + reviewed reusable-workflow caller. It never publishes and retains + validation/staging/release environment choices for diagnostics. Respect the + selected environment's existing ref policy and human review; no new + diagnostic environment or approval is required by staging. Never create + dummy release refs. A validation/staging-context result does not prove + release-context authentication or actual stable publication. - `python scripts/marketplace.py publish --package-path PATH [--pre-release] [--allow-already-exists]` preflights before uploading. CI opts into already-existing-version handling; local defaults fail. The flag maps to native `vsce --skip-duplicate` (existing-version/409 only), not output substring matching; other errors always propagate. The child environment clears inherited PAT and EnvironmentCredential variables and selects the tenant. **No PAT fallback.** -- Local staging/release scripts preflight before side effects when `vsce` - exists, but then publish. Their prior missing-`vsce` extension-packaging skip - remains unchanged and does not prove Marketplace access. + Low-level `--pre-release` support is not part of the normal release flow. +- Local `scripts/staging.sh` and `scripts/staging.ps1` run lint/tests, Python + build, TestPyPI upload, smoke verification, and VSIX artifact packaging only. + They perform no Azure preflight or Marketplace publication. For pure + no-upload packaging use `uv build` and extension `npm run package:prerelease`; + the staging scripts still upload Python to TestPyPI. +- Local release scripts preflight before side effects when `vsce` exists, then + publish stable. Their prior missing-`vsce` extension-packaging skip remains + unchanged and does not prove Marketplace access. Use `check` alone for no-upload validation. First run `az login --tenant --allow-no-subscriptions` and set `MARKETPLACE_AZURE_TENANT_ID` and `MARKETPLACE_PROFILE_ID` for **your interactive publishing identity**, not the UAMI. Your identity needs publisher Contributor or Owner membership. Profile pinning prevents wrong account/tenant publishing. -- Extension `npm run publish` / `npm run publish:prerelease` package a VSIX - before calling the shared helper, which preflights before upload. They require +- Extension `npm run publish` packages a stable VSIX before calling the shared + helper, which preflights before upload. It requires Python 3.11+ and the helper from the repository checkout. +- `npm run publish:prerelease` is removed. `npm run package:prerelease` remains + artifact-only with `package --pre-release`, not a Marketplace upload. - CLI profile and role preflight success is **not proof of actual upload**. ### Staged rollout (not completed by merging code) -1. Obtain approval for permanent ownership and production tenant/subscription - placement outside code rollout. The earlier non-production +1. Confirm permanent ownership, approved production tenant/subscription + placement, and operational responsibility under existing policy outside code + rollout. The earlier non-production personal-subscription probe is feasibility evidence, not policy approval. -2. Configure the dedicated UAMI, exact federation, publisher Contributor - membership, environment reviewers and deployment policies, then variables. +2. Verify the dedicated UAMI, exact release federation, publisher Contributor + membership, `marketplace-release` reviewers and deployment policies, then + variables. Legacy diagnostic environments are not staging prerequisites. 3. Run an authorized permission-only preflight (`check`, no publishing scripts). Use `marketplace-preflight.yml`: discover the profile, grant Contributor, then check its explicit publishing role. A local interactive `check` alone does not validate CI federation. Do not bypass required human approvals. -4. Explicitly authorize and verify a **legitimate** Marketplace pre-release. -5. Explicitly authorize and verify a **legitimate** stable publication. -6. **Only then** remove the legacy GitHub `VSCE_PAT`. Its owner must confirm no +4. Verify a legitimate staging candidate's TestPyPI upload and artifact-only + VSIX for manual evaluation. No Marketplace pre-release publication is needed. +5. Explicitly authorize a **legitimate** stable release, obtain human + `marketplace-release` approval, and verify actual Marketplace publication. +6. **Only after actual stable publication succeeds and remaining consumers are + checked**, remove the legacy GitHub `VSCE_PAT`. Its owner must confirm no other consumers before revoking the underlying Azure DevOps PAT. Include historical workflow re-runs in that review; retire old PAT paths and use the migrated workflow on `main` for authorized old-tag retries. @@ -367,6 +396,7 @@ for setup and rollout details. and seeing empty output. - Never publish without running `python -m pytest tests/ -x -q` first. - Never treat release workflows as dry runs or create dummy release branches, - tags, or production versions to test them. Staging includes a real Marketplace - pre-release attempt. Use local packaging and standalone read-only preflight - instead; Marketplace approvals do not pause Python publishing. + tags, or production versions to test them. Staging uploads to TestPyPI but + packages the VSIX only. Use pure local packaging and standalone read-only + preflight for no-upload validation. Marketplace approval does not pause Python + publishing; GitHub Release waits for successful stable Marketplace publication. diff --git a/docs/release-process.md b/docs/release-process.md index 2617a10d..4f64dc20 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -316,10 +316,16 @@ python -c "from agentops import __version__; print(__version__)" ## 7. Staging Pipeline (TestPyPI) The staging pipeline validates a release candidate by publishing to TestPyPI and -verifying the installed package works. It also attempts a **real Marketplace -pre-release** through the separate `marketplace-staging` environment. It is not +verifying the installed package works. Its `build-vsix` job produces a VSIX +artifact for manual evaluation only; it never uploads to Marketplace or reserves +a Marketplace version. TestPyPI uploads are real, so staging as a whole is not a dry run; use only legitimate, authorized release candidates. +**Marketplace publication is stable-only:** the production `vX.Y.Z` tag flow +publishes the extension after human `marketplace-release` approval. Previously, +staging could publish pre-release `X.Y.Z`, causing stable publication of the same +version to skip as a duplicate. Artifact-only staging eliminates that collision. + **Workflow file**: `.github/workflows/staging.yml` **Trigger**: Push to any `release/*` branch @@ -332,7 +338,7 @@ flowchart TD build["_build
tests + package
Version: 0.2.1.dev3 (setuptools-scm)"] publish["publish-testpypi
Upload to TestPyPI (staging environment)
Trusted Publishing (OIDC, no token)"] verify["verify-testpypi
Install from TestPyPI in fresh environment
agentops --version / --help / init"] - vsix["publish-vsix-prerelease
Real Marketplace pre-release
marketplace-staging: review + Entra OIDC"] + vsix["build-vsix
Package pre-release VSIX artifact for manual evaluation
No Marketplace upload or identity"] push --> build --> publish --> verify push --> vsix @@ -346,6 +352,10 @@ flowchart TD 4. **Package installs** - `pip install` from TestPyPI resolves all dependencies 5. **CLI works** - `agentops --version` and `--help` run without errors 6. **Init works** - `agentops init` creates the expected workspace files +7. **Extension packages** - `build-vsix` produces an installable pre-release VSIX + artifact. Download and install it manually in VS Code to evaluate the candidate. + Packaging needs no GitHub environment, `id-token`, Azure login, or Marketplace + profile variables, and has no cloud reservation or Marketplace approval. ### Iterating on a Release Branch @@ -385,8 +395,8 @@ ls .agentops/ ## 8. End-to-End Pipeline Testing -**Release workflows are not dry runs.** Pushing `release/*` triggers TestPyPI -and a real Marketplace pre-release attempt; pushing `v*` triggers production +**Release workflows are not dry runs.** Pushing `release/*` triggers a real TestPyPI +upload and artifact-only VSIX packaging; pushing `v*` triggers production publishing. Never create dummy release branches, tags, or Marketplace versions to test workflow changes. Deleting a ref does not undo an upload. @@ -402,20 +412,24 @@ uv build npm install -g @vscode/vsce@3.9.2 Copy-Item CHANGELOG.md,icon.png -Destination plugins\agentops Push-Location plugins\agentops -npm run package +npm run package:prerelease Pop-Location ``` For identity and publisher permissions, use the standalone read-only `python scripts/marketplace.py check` after the -[local identity setup](#local-publishing). Do not invoke a staging or release -script merely to test credentials: those scripts publish after preflight. +[local identity setup](#local-publishing). Do not invoke a release script merely +to test credentials: it publishes after preflight. Local `scripts/staging.sh` +and `scripts/staging.ps1` run lint/tests, Python build, TestPyPI upload and smoke +verification, then VSIX artifact packaging. They perform no Azure preflight or +Marketplace publish, but their TestPyPI upload still makes them unsuitable for +pure no-upload validation. When a legitimate release candidate is explicitly authorized, push its `release/vX.Y.Z` branch and monitor **Staging** in Actions. Review the TestPyPI -upload and install verification, and approve the separately protected -`marketplace-staging` deployment only for the intended Marketplace pre-release. -Re-pushing the branch is another publication attempt, not a test-only run. +upload and install verification, and download the `build-vsix` artifact for +manual extension evaluation. Staging needs no Marketplace environment or +approval. Re-pushing the branch can upload to TestPyPI again, not Marketplace. ### 8.2 Test the Full Release Pipeline @@ -452,9 +466,9 @@ tag has already been pushed. | What to validate | Method | What it proves | | --- | --- | --- | -| Tests and packaging | Existing tests, `uv build`, extension `npm run package` | Build correctness; no upload | +| Tests and packaging | Existing tests, `uv build`, extension `npm run package:prerelease` | Build correctness; no upload or Marketplace version reservation | | Marketplace identity and access | `python scripts/marketplace.py check` | Profile and explicit publisher role; no upload | -| Real pre-release | Authorized `release/vX.Y.Z` candidate and Marketplace approval | TestPyPI and real Marketplace pre-release publication | +| Real staging candidate | Authorized `release/vX.Y.Z` candidate | TestPyPI upload/install and VSIX artifact for manual evaluation; no Marketplace upload | | Real stable release | Authorized `vX.Y.Z` tag and Marketplace approval | Production publication; not reversible by deleting the tag | ### 8.4 Testing Workflow Changes on a Feature Branch @@ -471,6 +485,9 @@ must not call `publish` or the staging/release scripts. The production pipeline publishes a final release to PyPI and creates a GitHub Release. Its Marketplace stable publish uses the separate protected `marketplace-release` environment; this does not change Python publishing. +The tag publishes Python **before Marketplace approval**. A human must approve +`marketplace-release`; do not bypass that gate. GitHub Release creation waits +for successful Marketplace publication. **Workflow file**: `.github/workflows/release.yml` @@ -543,8 +560,8 @@ The branch push triggers the staging pipeline automatically. Wait for it to pass - ✅ `build / build` - tests pass, package builds - ✅ `publish-testpypi` - uploaded to TestPyPI - ✅ `verify-testpypi` - installed and smoke-tested -3. Review and approve the legitimate Marketplace pre-release deployment in - `marketplace-staging`, then verify that publication succeeded. +3. Verify `build-vsix` passes, then download its pre-release VSIX artifact for + manual evaluation. There is no Marketplace upload or approval during staging. If any job fails, fix the issue on the release branch and push. The pipeline re-runs automatically. @@ -603,11 +620,15 @@ review. Open `CHANGELOG.md` after the merge and confirm that everything under #### Step 6: Watch the release pipeline 1. Go to **Actions** tab → find the **Release** workflow run for `v0.2.0` -2. The pipeline runs build → TestPyPI → verify → **publish-pypi** → github-release +2. The pipeline runs build → TestPyPI → verify → **publish-pypi** → + **publish-vsix** → github-release 3. `publish-pypi` does not pause. It publishes to PyPI via [Trusted Publishing](https://docs.pypi.org/trusted-publishers/) using the workflow's OIDC identity, so there is no API token to rotate -4. `github-release` then creates a GitHub Release with the built artifacts and +4. After Python publishes, a human approves the `marketplace-release` deployment + for stable extension publication. Do not bypass the required approval. +5. Only after successful Marketplace publication, `github-release` creates a + GitHub Release with the built artifacts and auto-generated release notes If the run fails after `publish-pypi` succeeded, the package is already on PyPI. @@ -657,9 +678,9 @@ This section covers one-time setup required before the pipelines can run. ### 10.1 GitHub Environments -Keep the existing Python environments unchanged. Create **separate Marketplace -environments** in **Settings → Environments → New environment** as described -below; do not reuse `staging` or `release` for Marketplace authentication. +Keep the existing Python environments unchanged. Use the **separate +`marketplace-release` environment** for publishing as described below; do not +reuse `staging` or `release` for Marketplace authentication. #### `staging` Environment @@ -677,15 +698,23 @@ below; do not reuse `staging` or `release` for Marketplace authentication. - **Deployment branches**: Optionally restrict to `main` branch and `v*` tags - **Secrets**: None. Python uploads continue to use Trusted Publishing. -#### `marketplace-staging` and `marketplace-release` Environments +#### Marketplace Environments -Before setting any variables, configure **required reviewers** and **selected -branch/tag deployment policies**: +`marketplace-release` is the only required Marketplace publishing environment. +Retain its **required human reviewers** and **selected branch/tag deployment +policies** before enabling its identity variables: | Environment | Allowed deployment refs | Purpose | | --- | --- | --- | -| `marketplace-staging` | Branches `release/*` (legitimate `release/vX.Y.Z` candidates) | Real Marketplace pre-release publishing | | `marketplace-release` | Tags `v*`; optionally the protected `main` branch for manual dispatch with a tag input | Stable Marketplace publishing | +| `marketplace-staging` | Existing selected-ref policy | Optional legacy/read-only preflight diagnostics; not used by staging or packaging | +| `marketplace-validation` | Existing approved validation-ref policy | Optional read-only preflight diagnostics | + +`marketplace-staging` and `marketplace-validation` were created earlier; the +stable-only flow does not delete them. Neither is required to stage or package +the extension, and neither introduces an approval into the staging pipeline. +Manual preflight still offers validation/staging/release contexts and respects +the selected environment's existing protections. For manual stable releases, the job guards allow only `main` or the same release tag as the input. Do not allow feature branches. An environment-based federated @@ -702,7 +731,8 @@ historical old workflow run still executes its old PAT code; it does **not** adopt the migrated workflow automatically. This is a real release retry, not a read-only check. -Set these **environment variables**, not secrets, in each new environment: +Set these **environment variables**, not secrets, in `marketplace-release` +(and in an optional diagnostic environment only when using its preflight): | Variable | Value | | --- | --- | @@ -710,10 +740,12 @@ Set these **environment variables**, not secrets, in each new environment: | `MARKETPLACE_AZURE_TENANT_ID` | Approved identity tenant GUID | | `MARKETPLACE_PROFILE_ID` | Marketplace `profiles/me` profile `id`, **not** the Entra principal/object ID | -The jobs request `id-token: write` and use `azure/login@v3` with +Publishing and identity-preflight jobs request `id-token: write` and use `azure/login@v3` with `allow-no-subscriptions: true`. No Azure RBAC grant is needed solely to publish an extension: Marketplace publisher membership supplies that permission. Do not change shared `AZURE_*` E2E variables or repository-wide OIDC settings. +The staging `build-vsix` job has no environment, OIDC permission, Azure login, +or Marketplace profile variables. #### Repository secrets @@ -723,7 +755,8 @@ Do not change shared `AZURE_*` E2E variables or repository-wide OIDC settings. `RELEASE_PAT` is a **GitHub** PAT and is unchanged by this migration. Marketplace publishing has no PAT fallback. The legacy repository `VSCE_PAT` must be retained -until the staged rollout is validated; do not interpret this documentation as +until an actual stable Marketplace publication succeeds and remaining consumers +are checked; do not interpret this documentation as confirmation it has been removed. No PyPI API token is stored. Check the current rules and secret names (never secret values) at any time: @@ -784,14 +817,15 @@ gh api repos/Azure/agentops --jq '{repository_id: .id, repository_owner_id: .own Verified for this migration: `use_default: false`, with ordered claim keys `repository_owner_id`, `repository_id`, `context`; owner ID `6844498` and -repository ID `1161883340`. The UAMI needs two federated credentials: +repository ID `1161883340`. Stable publication requires the release federated +credential; any legacy staging credential is optional for read-only diagnostics: | Field | Value | | --- | --- | | Issuer | `https://token.actions.githubusercontent.com` | | Audience | `api://AzureADTokenExchange` | -| Staging subject | `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` | | Release subject | `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-release` | +| Optional legacy staging subject | `repository_owner_id:6844498:repository_id:1161883340:environment:marketplace-staging` | Do not replace the repository customization with GitHub's default `repo:...` subject: that can break other federated consumers. If verification differs, @@ -833,6 +867,8 @@ python scripts/marketplace.py publish --package-path PATH [--pre-release] [--all handling, not output substring matching. Other errors always propagate. - Profile pinning prevents a wrong account or tenant from publishing. A successful CLI profile/role preflight is **not proof of an actual upload**. +- Low-level `--pre-release` support may remain in the helper, but it is not part + of the normal release flow. Staging only packages; production publishes stable. #### Permission-only GitHub workflow @@ -847,29 +883,35 @@ uploads only tenant/profile IDs in the seven-day `marketplace-profile` artifact. After granting Contributor, set `MARKETPLACE_PROFILE_ID` and run `check`. Discovery alone is not proof of publisher access. -Configure a separate `marketplace-validation` environment for pre-merge tests, -with an exact approved validation branch, required reviewers, and its own -environment-subject federated credential. Before the workflow is available on -the default branch, an isolated no-upload push wrapper can call it from a -reviewed validation branch. Do not change the existing staging/release branch -policies to admit test branches, create dummy release refs, or bypass reviewers. +The existing optional `marketplace-validation` and legacy `marketplace-staging` +environments can be used for read-only diagnostics under their configured ref, +reviewer, and federation policies. Neither is required by `build-vsix` or the +local staging scripts. Do not change the existing environment policies to admit +test branches, create dummy release refs, or bypass reviewers. An approval pause is a real human handoff, not a reason to use another identity. -Only the selected context is validated; staging/release need their own checks -on allowed refs. Remove the isolated wrapper/branch and its federation when -validation is retired; never remove shared resources. +Only the selected context is validated; validation/staging diagnostic success +does not prove release-context authentication or actual stable publication. +This flow requires no new diagnostic environment or approval. #### Local Publishing -The local `scripts/staging.ps1`, `scripts/staging.sh`, `scripts/release.ps1`, -and `scripts/release.sh` run permission preflight before side effects when -`vsce` is available. Their existing behavior of skipping extension packaging -when `vsce` is missing is unchanged; that skip is not a successful Marketplace -validation. These are publication scripts, not credential tests. - -The extension's `npm run publish` and `npm run publish:prerelease` scripts also -package a VSIX and then invoke the shared helper. They require Python 3.11+ and +The local `scripts/staging.ps1` and `scripts/staging.sh` run lint/tests, Python +build, TestPyPI upload, smoke verification, and VSIX artifact packaging only. +They need no Azure preflight, Marketplace identity, or Marketplace approval. +They never upload the extension. For pure artifact packaging without even a +TestPyPI upload, use `uv build` and extension `npm run package:prerelease`. + +The local `scripts/release.ps1` and `scripts/release.sh` retain stable publishing +and permission preflight before side effects when `vsce` is available. The +existing missing-`vsce` extension-packaging skip is not proof of Marketplace +access. Release scripts are publication scripts, not credential tests. + +The extension's `npm run publish` packages a stable VSIX and invokes the shared +helper. `npm run publish:prerelease` is removed; `npm run package:prerelease` +remains artifact-only, preserving `package --pre-release` without reserving a +cloud version. Stable publishing requires Python 3.11+ and the helper from a repository checkout, not just a standalone extension folder. -The helper preflights before upload; these npm scripts package before preflight. +The helper preflights before upload; `npm run publish` packages before preflight. Log in with `az login --tenant --allow-no-subscriptions`. Set `MARKETPLACE_AZURE_TENANT_ID` and `MARKETPLACE_PROFILE_ID` for **your interactive @@ -883,12 +925,14 @@ Only invoke a publishing script or `publish` for an explicitly authorized releas This is a deployment checklist, **not a claim that permanent resources are configured or publication has been tested**. -- [ ] Approve permanent identity ownership, production tenant/subscription - placement, and operational responsibility outside the code rollout. -- [ ] Create the dedicated UAMI and the two exact federated credentials; resolve +- [ ] Confirm permanent identity ownership, approved production tenant/subscription + placement, and operational responsibility under existing policy; code rollout + and earlier feasibility probes do not settle these decisions. +- [ ] Verify the dedicated UAMI and exact release federated credential; resolve its Marketplace profile and have a publisher Owner grant Contributor. -- [ ] Configure required reviewers and selected branch/tag deployment policies - on both new environments **before setting their three variables**. Preserve +- [ ] Verify required reviewers and selected branch/tag deployment policies + on `marketplace-release` **before setting its three variables**. Optional + legacy diagnostic environments are not staging prerequisites. Preserve Python `staging`/`release`, repository-wide OIDC, shared E2E variables, and `RELEASE_PAT`. - [ ] Run a permission-only preflight under the intended federated identity: @@ -897,14 +941,16 @@ configured or publication has been tested**. Use the dedicated `marketplace-preflight.yml` workflow, not staging/release. Its discovery mode resolves the profile before checking publisher membership. A local interactive `check` validates that local identity, not CI federation. -- [ ] Obtain explicit authorization for a legitimate pre-release, approve its - `marketplace-staging` deployment, publish it, and verify the Marketplace result. +- [ ] Verify the legitimate staging candidate uploads to TestPyPI and produces a + VSIX artifact for manual evaluation without any Marketplace upload. - [ ] Obtain explicit authorization for a legitimate stable release, approve its `marketplace-release` deployment, publish it, and verify the Marketplace result. Do not create dummy production versions for validation. -- [ ] **Only after both real publication paths succeed**, remove the GitHub - `VSCE_PAT` secret. Have its owner revoke the underlying Azure DevOps PAT after - confirming there are no other consumers. **Never remove or revoke `RELEASE_PAT`.** +- [ ] **Only after actual stable Marketplace publication succeeds and remaining + consumers are checked**, remove the GitHub `VSCE_PAT` secret. Have its owner + revoke the underlying Azure DevOps PAT only after confirming there are no other + consumers. No pre-release publication is required for acceptance. + **Never remove or revoke `RELEASE_PAT`.** Include historical workflow re-runs in that consumer review: retire old PAT execution paths and use the migrated workflow on `main` for authorized old-tag retries rather than re-running historical workflows. @@ -945,7 +991,7 @@ Key detail: Uses `fetch-depth: 0` to ensure setuptools-scm has full git history ``` Trigger: push to release/* branches, or workflow_dispatch Flow: _build → publish-testpypi → verify-testpypi - + parallel Marketplace pre-release (marketplace-staging) + + parallel build-vsix (artifact-only pre-release package) Purpose: Validate release candidates before production ``` @@ -953,9 +999,9 @@ Key details: - `skip-existing: true` allows re-pushes without upload failures - Verify step uses a retry loop (5 attempts, 30s apart) for TestPyPI index propagation - Smoke tests cover `--version`, `--help`, and `agentops init` -- The extension job separately uses `marketplace-staging`, Entra OIDC, and the - shared helper with `--pre-release --allow-already-exists`. Staging is a real - Marketplace pre-release attempt, not a safe disposable-branch test. +- `build-vsix` packages with `--pre-release` for manual evaluation only. It has + no environment, `id-token`, Azure login, profile variables, or Marketplace + upload. TestPyPI still uploads; staging is not a disposable-branch dry run. ### `release.yml` - Production Release @@ -971,7 +1017,8 @@ Key details: - `github-release` uses `gh release create` with `--generate-notes` for automatic release notes - Built artifacts (.whl, .tar.gz) are attached to the GitHub Release - The extension job separately uses `marketplace-release`, Entra OIDC, and the - shared helper with `--allow-already-exists`. Its reviewers do not gate PyPI. + shared helper with `--allow-already-exists`. Python publishes before its human + approval; GitHub Release waits for successful stable Marketplace publication. - Marketplace tooling is checked out from `github.workflow_sha` at the root; extension source comes from the requested release tag under `release-source/`. Retry pre-migration tags by dispatching the migrated workflow on protected @@ -994,8 +1041,22 @@ Key details: - Fails safely if the branch already exists - Refuses to run when `## [Unreleased]` is empty, because this workflow only inserts a versioned heading beneath that one and would otherwise publish an empty release section - Does NOT auto-tag; stable tagging remains a manual, intentional step. The - release-branch push does trigger staging publication, including a real - Marketplace pre-release attempt. + release-branch push triggers TestPyPI publication and VSIX artifact packaging, + never Marketplace publication. + +### `marketplace-preflight.yml` - Read-Only Diagnostics + +``` +Trigger: workflow_dispatch or reviewed workflow_call +Inputs: discover/check; marketplace-validation/marketplace-staging/marketplace-release +Flow: selected environment → Entra OIDC → profile discovery or permission check +Purpose: Diagnose the selected identity context without publishing +``` + +Key detail: Existing environment protections apply to the selected diagnostic +context only. Validation and legacy staging contexts are optional, not staging +or packaging prerequisites. No preflight result substitutes for verifying an +actual stable publication. ## 12. Release Checklist @@ -1013,7 +1074,7 @@ Use this checklist when cutting a release: - [ ] Release branch created via **Cut Release** workflow (or manually) - [ ] CHANGELOG automatically updated with version and date - [ ] Staging Python jobs pass: build + TestPyPI + verify -- [ ] Marketplace pre-release deployment approved and legitimate publication verified +- [ ] `build-vsix` artifact built and manually evaluated; no Marketplace upload - [ ] PR opened: `release/v0.X.Y` → `main` **Production (tag + sync, do these together)** @@ -1038,7 +1099,7 @@ Use this checklist when cutting a release: | Problem | Cause | Solution | | ---------------------------------------- | ----------------------------------- | --------------------------------------------- | | `setuptools_scm` can't determine version | Shallow clone (missing git history) | Ensure `fetch-depth: 0` in checkout step | -| Version shows `0.0.0` locally | Not in a git repo or no tags exist | Run `git tag v0.0.1` to create an initial tag | +| Version shows `0.0.0` locally | Not in a git repo or no tags exist | Verify the checkout and existing tag history; never create a dummy release tag to test versioning | | `ModuleNotFoundError` in tests | Dependencies not installed | Run `uv sync --group dev` | | Tests fail on Windows but pass on Linux | Path separator issues | Use `pathlib.Path`, not string concatenation | @@ -1073,7 +1134,7 @@ Use this checklist when cutting a release: | Problem | Cause | Solution | | --------------------------------- | ----------------------------------- | ---------------------------------------------------------------------- | -| "Environment not found" error | GitHub Environment not created | Preserve Python `staging`/`release`; create separate protected `marketplace-staging`/`marketplace-release` environments | +| "Environment not found" error | Required publishing or selected diagnostic environment unavailable | Preserve Python `staging`/`release`; stable Marketplace publishing requires protected `marketplace-release` only. Staging VSIX packaging requires no Marketplace environment | | Marketplace variable missing | Dedicated environment setup incomplete | Configure reviewers and deployment policies first, then all three `MARKETPLACE_*` variables | | Marketplace federation fails | Issuer, audience, or customized subject mismatch | Verify repository customization and exact environment subjects; do not change repo-wide OIDC | | Marketplace profile/role preflight fails | Wrong tenant/account/profile or missing publisher role/deny permissions | Select the correct tenant, resolve `profiles/me`, and ask the publisher Owner to review Contributor membership; never fall back to a PAT | @@ -1092,7 +1153,7 @@ flowchart TD rel --> stagingBuild["_build
test + build"] stagingBuild --> stagingTest["TestPyPI publish"] stagingTest --> stagingVerify["Verify install"] - rel --> stagingVsix["Marketplace pre-release
marketplace-staging: review + Entra OIDC"] + rel --> stagingVsix["build-vsix
Pre-release VSIX artifact for manual evaluation
No Marketplace upload or identity"] rel -->|PR| main(["main"]) main -->|tag| tag(["v0.2.0"]) From 747f07fd245f196090409976184e523344d8ccf8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:53:49 +0000 Subject: [PATCH 6/6] chore: prepare release 0.15.1 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- CHANGELOG.md | 2 ++ plugins/agentops/package.json | 2 +- plugins/agentops/plugin.json | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index cb48b3a2..2876981a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.15.0", + "version": "0.15.1", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index cb48b3a2..2876981a 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.15.0", + "version": "0.15.1", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ded10f..0a1d91a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +## [0.15.1] - 2026-09-07 + ### Changed - **Extension publication is stable-only.** Staging produces a downloadable VSIX candidate without Marketplace authentication or upload. Only stable diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 3624bc53..a45b473c 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.15.0", + "version": "0.15.1", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index 2887fa4c..d6d58156 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.15.0", + "version": "0.15.1", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops"