From eb21fe760fb3d34183b64b46f3f2e844a3fef255 Mon Sep 17 00:00:00 2001 From: Tommaso Bolis Date: Mon, 12 Oct 2026 11:15:06 +0200 Subject: [PATCH] ci: test each new framework release in isolation against the PR lock framework-release-watch.yml runs daily (and on dispatch, optionally for one package). scripts/check_framework_releases.py lists every framework extra's direct requirement whose newest stable PyPI release is ahead of its adapter-contract lock pin; each release is then installed with only that pin moved (or unconstrained, "needs co-bump") and run through the adapter-contract checks. The alert job keeps one framework-release issue per release and closes it once the lock catches up. PRs that change the workflow or script get a dry run into the job summary. Closes #1066 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/framework-release-watch.yml | 284 ++++++++ CONTRIBUTING.md | 9 +- docs/ci.md | 49 +- python/scripts/check_framework_releases.py | 661 ++++++++++++++++++ .../unit/test_check_framework_releases.py | 407 +++++++++++ python/tests/unit/test_ci_workflows.py | 37 +- 6 files changed, 1442 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/framework-release-watch.yml create mode 100644 python/scripts/check_framework_releases.py create mode 100644 python/tests/unit/test_check_framework_releases.py diff --git a/.github/workflows/framework-release-watch.yml b/.github/workflows/framework-release-watch.yml new file mode 100644 index 00000000..58f8b1a9 --- /dev/null +++ b/.github/workflows/framework-release-watch.yml @@ -0,0 +1,284 @@ +# Framework release watch (#1066). +# +# PR CI installs each framework from its committed lock (ADR 0007 rule 3) and +# the nightly resolves everything fresh at once (rule 4), so neither says which +# upstream release breaks which adapter. This workflow tests each new release +# on its own: everything else stays at the PR lock and only that package moves. +# +# `detect` — scripts/check_framework_releases.py lists every framework +# extra's direct requirement whose newest stable PyPI release is +# ahead of python/constraints/adapter-contract--py3.12.txt. +# `evaluate` — one leg per release: install from the lock with only that pin +# moved (or, when that does not resolve, unconstrained: "needs +# co-bump"), then the adapter-contract checks: verify_frameworks.py +# --require-installed, the whole suite with the skip guard armed, +# and the example smoke. Each check is continue-on-error, so a +# red check is a result, recorded in result.json, not a red run. +# `alert` — the only job that writes. It opens or updates ONE issue per +# release, labelled `framework-release`, and closes the ones the +# lock has caught up with. A rerun with unchanged results edits +# the issue but adds no comment. +# +# On a pull request touching this workflow or the script it is a dry run: the +# planned issue bodies go to the job summary and nothing is written. +# +# Alerting (docs/ci.md convention 4): a failed run (PyPI down, a broken plan) +# opens or updates ONE open issue labelled `framework-release-watch-failure`. +# +# Security: matrix values and inputs reach run: bodies through env:, never +# interpolated. The scheduled and dispatch runs check out the default branch, +# so they watch develop's lock. + +name: Framework release watch + +on: + schedule: + - cron: "30 5 * * *" # 05:30 UTC daily, before the 06:00 nightly + workflow_dispatch: + inputs: + package: + description: "Check only this watched package (e.g. crewai); empty checks all" + type: string + default: "" + pull_request: + paths: + - .github/workflows/framework-release-watch.yml + - python/scripts/check_framework_releases.py + +permissions: + contents: read + +concurrency: + group: framework-release-watch-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + working-directory: python + +jobs: + detect: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + count: ${{ steps.detect.outputs.count }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + # Standard library only: nothing to install. + - name: List releases ahead of the PR lock + id: detect + env: + PACKAGE: ${{ inputs.package }} + run: | + set -euo pipefail + args=(--json) + if [ -n "${PACKAGE:-}" ]; then args+=(--package "$PACKAGE"); fi + matrix=$(python scripts/check_framework_releases.py "${args[@]}") + echo "$matrix" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "count=$(jq length <<<"$matrix")" >> "$GITHUB_OUTPUT" + + evaluate: + needs: [detect] + if: needs.detect.outputs.count != '0' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.detect.outputs.matrix) }} + name: evaluate (${{ matrix.package }} ${{ matrix.latest }}) + env: + FRAMEWORK: ${{ matrix.framework }} + EXTRA: ${{ matrix.extra }} + PACKAGE: ${{ matrix.package }} + LATEST: ${{ matrix.latest }} + RELEASE: ${{ toJSON(matrix) }} + # tests/conftest.py's skip guard: an importorskip of this framework fails. + DONKEY_CONTRACT_EXTRA: ${{ matrix.extra }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Write the PR lock with only this package moved + run: >- + python scripts/check_framework_releases.py + --constraints-for "$PACKAGE==$LATEST" --extra "$EXTRA" + --out "$RUNNER_TEMP/constraints.txt" + # The adapter-contract job's install line exactly (ci.yml), so every + # other package is held at the pins compiled for it. + - name: Does the release resolve against the lock? + id: mode + run: | + set -euo pipefail + python -m pip install --upgrade pip + if pip install --dry-run -e ".[llm,local,$EXTRA]" --group dev \ + -c "$RUNNER_TEMP/constraints.txt" > "$RUNNER_TEMP/resolve.log" 2>&1; then + echo "mode=locked" >> "$GITHUB_OUTPUT" + else + tail -n 20 "$RUNNER_TEMP/resolve.log" + echo "mode=co-bump" >> "$GITHUB_OUTPUT" + fi + - name: Install + id: install + continue-on-error: true + env: + MODE: ${{ steps.mode.outputs.mode }} + run: | + set -euo pipefail + if [ "$MODE" = locked ]; then + pip install -e ".[llm,local,$EXTRA]" --group dev -c "$RUNNER_TEMP/constraints.txt" + else + pip install -e ".[llm,local,$EXTRA]" --group dev "$PACKAGE==$LATEST" + fi + - name: Signature verification (§8), fail if not installed + id: verify + if: steps.install.outcome == 'success' + continue-on-error: true + run: | + set -euo pipefail + python scripts/verify_frameworks.py --only "$FRAMEWORK" --require-installed --json \ + | tee "$RUNNER_TEMP/verify.json" + - name: Whole suite, skip guard armed + id: pytest + if: steps.install.outcome == 'success' + continue-on-error: true + run: python -m pytest -q + - name: Strict example smoke + id: smoke + if: steps.install.outcome == 'success' + continue-on-error: true + run: timeout 120 python scripts/smoke_example.py "$FRAMEWORK" + - name: Record the result + if: always() + env: + MODE: ${{ steps.mode.outputs.mode }} + INSTALL: ${{ steps.install.outcome }} + VERIFY: ${{ steps.verify.outcome }} + PYTEST: ${{ steps.pytest.outcome }} + SMOKE: ${{ steps.smoke.outcome }} + run: >- + python scripts/check_framework_releases.py + --record-result "$RUNNER_TEMP/result/result.json" + --release "$RELEASE" --mode "${MODE:-co-bump}" + --verify-json "$RUNNER_TEMP/verify.json" + --outcome "install=$INSTALL" "verify=$VERIFY" "pytest=$PYTEST" "smoke=$SMOKE" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: result-${{ matrix.extra }}-${{ matrix.package }} + path: ${{ runner.temp }}/result/result.json + retention-days: 14 + + alert: + # `needs:` lists every other job (tests/unit/test_ci_workflows.py asserts + # it); `always()` so stale issues are closed even when nothing is new. + if: always() + needs: [detect, evaluate] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The close comment links the commit that last moved each lock. + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + if: needs.detect.outputs.count != '0' && needs.detect.outputs.count != '' + with: + pattern: result-* + path: ${{ runner.temp }}/results + - name: List the open framework-release issues + if: github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # `gh issue create --label` fails when a label does not exist. + gh label create framework-release --repo "$REPO" --force \ + --color 1D76DB --description "A framework release is ahead of the PR lock (#1066)" + # `dependencies` may already exist with its own colour: never overwrite it. + gh label create dependencies --repo "$REPO" --color 0366D6 \ + --description "Pull requests that update a dependency file" 2>/dev/null || true + gh issue list --repo "$REPO" --label framework-release --state open \ + --limit 200 --json number,body > "$RUNNER_TEMP/open-issues.json" + - name: Plan the issue writes + env: + EVENT: ${{ github.event_name }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + run: | + set -euo pipefail + open="$RUNNER_TEMP/open-issues.json" + [ -f "$open" ] || echo '[]' > "$open" + dry=() + if [ "$EVENT" = pull_request ]; then dry=(--dry-run); fi + python scripts/check_framework_releases.py --plan-issues \ + --results "$RUNNER_TEMP/results" --open-issues "$open" \ + --run-url "$RUN_URL" --repo-url "$REPO_URL" \ + --out "$RUNNER_TEMP/plan.json" "${dry[@]}" >> "$GITHUB_STEP_SUMMARY" + - name: Open, update and close the framework-release issues + if: github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + while read -r op; do + kind=$(jq -r .op <<<"$op") + number=$(jq -r '.number // empty' <<<"$op") + case "$kind" in + create) + jq -r .body <<<"$op" | gh issue create --repo "$REPO" \ + --title "$(jq -r .title <<<"$op")" \ + --label framework-release --label dependencies --body-file - ;; + edit) + jq -r .body <<<"$op" | gh issue edit "$number" --repo "$REPO" \ + --title "$(jq -r .title <<<"$op")" --body-file - ;; + comment) + jq -r .body <<<"$op" | gh issue comment "$number" --repo "$REPO" --body-file - ;; + close) + jq -r .comment <<<"$op" | gh issue comment "$number" --repo "$REPO" --body-file - + gh issue close "$number" --repo "$REPO" \ + --reason "$(jq -r .reason <<<"$op")" < /dev/null ;; + esac + done < <(jq -c '.[]' "$RUNNER_TEMP/plan.json") + - name: Open or update the framework-release-watch-failure tracking issue + if: >- + github.event_name != 'pull_request' && + (failure() || contains(needs.*.result, 'failure')) + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh label create framework-release-watch-failure --repo "$REPO" --force \ + --color B60205 --description "The framework release watch failed (#1066)" + existing=$(gh issue list --repo "$REPO" --label framework-release-watch-failure \ + --state open --json number --jq '.[0].number // empty') + body="The framework release watch failed: ${RUN_URL} + + New framework releases are not being tested against the PR lock until this is fixed (docs/ci.md, \`framework-release-watch.yml\`)." + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$REPO" --body "$body" + else + gh issue create --repo "$REPO" --title "Framework release watch failing" \ + --label framework-release-watch-failure --body "$body" + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c67b126a..ee7e5707 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -751,8 +751,13 @@ build plan has the rationale behind each rule: `.github/workflows/lock-refresh.yml` recompiles every file and opens or updates one `chore(deps): refresh the PR CI lock` PR ([ADR 0011](docs/adr/0011-nightly-lock-refresh-pr.md), - [`docs/ci.md`](docs/ci.md#lock-refreshyml-the-lock-bump)). **Refresh the - lock by hand** in the PR that changes a dependency: + [`docs/ci.md`](docs/ci.md#lock-refreshyml-the-lock-bump)). An open + `framework-release` issue is the other trigger: when + `.github/workflows/framework-release-watch.yml` marks a framework release + safe to bump and no bump PR carries it yet, refresh the lock for it + ([`docs/ci.md`](docs/ci.md#framework-release-watchyml-one-release-at-a-time), + #1066). **Refresh the lock by hand** in the PR that changes a dependency, or + for a `framework-release` issue: ```bash cd python python scripts/compile_constraints.py # needs `uv` on PATH (dev-time only; CI still uses plain pip) diff --git a/docs/ci.md b/docs/ci.md index 5abac769..5eb1f39a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -20,6 +20,7 @@ policy behind the locks and the nightly run, see - [ci.yml: the PR gate](#ciyml-the-pr-gate) - [nightly-matrix.yml: the canary](#nightly-matrixyml-the-canary) - [lock-refresh.yml: the lock bump](#lock-refreshyml-the-lock-bump) + - [framework-release-watch.yml: one release at a time](#framework-release-watchyml-one-release-at-a-time) - [live-contract-check.yml: the gateway contract](#live-contract-checkyml-the-gateway-contract) - [Scenario maps](#scenario-maps) - [Feature or fix (normal path)](#feature-or-fix-normal-path) @@ -50,6 +51,7 @@ should be split. | Did this change break something? | `ci.yml` (PR-gating jobs, resolving from the committed lock) | Every PR; pushes to `main` and `develop` | | Did an upstream release break us? | `nightly-matrix.yml`, plus the fresh-resolve jobs in `ci.yml` (`all-extra-resolves`, `anthropic-stacks`, `adk-stacks`) | Daily 06:00 UTC; every PR for the `ci.yml` jobs | | Can the lock move to the newest releases? | `lock-refresh.yml` (opens one bump PR; its CI answers) | After each green nightly on `develop`; manual | +| Which new framework release breaks which adapter? | `framework-release-watch.yml` (one `framework-release` issue per release ahead of the lock) | Daily 05:30 UTC; manual, optionally for one package; a dry run on PRs that change it | | Did the live gateway's contract drift? | `live-contract-check.yml` (the `openai-model-routing` proxy only) | Mondays 07:00 UTC | | Did a secret get committed? | `secret-scan.yml` (gitleaks over the full history) | Every PR; pushes to `main` and `develop` | | Is the published site current? | `docs.yml` (build and deploy to GitHub Pages) | Pushes to `main` under `website/**`; every 6 hours; manual | @@ -142,6 +144,47 @@ Without either, the PR step fails and the run files the `lock-refresh-failure` issue. Check the setup with `gh workflow run lock-refresh.yml --ref develop`. +### `framework-release-watch.yml`: one release at a time + +The nightly resolves everything fresh at once, so a red nightly doesn't say +which upstream release broke which adapter. This workflow tests each new +release on its own (#1066). `detect` runs +`python scripts/check_framework_releases.py`, which lists every direct +requirement of each framework extra whose newest stable PyPI release is ahead +of its pin in `python/constraints/adapter-contract--py3.12.txt`. +Pre-releases and yanked releases don't count. `evaluate` runs one leg per +release. It installs the `adapter-contract` job's line with that one pin moved. +If that doesn't resolve, it installs the release unconstrained and marks it +"needs co-bump". Then it runs the `adapter-contract` checks: +`verify_frameworks.py --require-installed`, the whole suite with the skip guard +armed, and the example smoke. A red check is a result, not a red run. + +`alert` keeps one open issue per release, titled +`framework-release: → ` and labelled +`framework-release` and `dependencies`. The issue lists the versions, the bump +kind, the release date and notes, the result of each check, the run link and +the next action. A rerun refreshes the body and comments only when the results +changed. A newer release of the same package closes the older issue as +superseded. Once the lock reaches the release, the next run closes the issue +with a link to the lock commit. + +What to do with an issue: + +- **All green: safe to bump.** The next `lock-refresh.yml` PR may already + carry it. If not, from `python/` run `python scripts/compile_constraints.py`, + then `python scripts/generate_docs.py`, then + `python scripts/verify_frameworks.py --only --emit-verified`, and + then the pre-PR gate. +- **Anything red.** Adapt the adapter to the release, or record the + incompatibility in [`verified-apis.md` §8.1](verified-apis.md). Never add a + ceiling (`§8.4`). + +Run it for one package with +`gh workflow run framework-release-watch.yml --ref develop -f package=crewai`. +A PR that changes the workflow or the script gets a dry run: the planned issue +bodies go to the job summary and nothing is written. A failed run opens or +updates one issue labelled `framework-release-watch-failure`. + ### `live-contract-check.yml`: the gateway contract Every Monday this workflow runs the `sandbox` suite (`tests/sandbox/`) against @@ -363,8 +406,10 @@ Each cron workflow ends with an `if: failure()` job or step that opens or updates one labelled tracking issue (never one issue per run), using `issues: write` on that job only. `live-contract-check.yml` does this with the `live-contract-drift` label (#753), `nightly-matrix.yml` with the -`nightly-failure` label (#755), and `lock-refresh.yml`, which follows the -nightly, with the `lock-refresh-failure` label (#763). `docs.yml`'s cron still needs the same step. +`nightly-failure` label (#755), `lock-refresh.yml`, which follows the +nightly, with the `lock-refresh-failure` label (#763), and +`framework-release-watch.yml` with the `framework-release-watch-failure` label +(#1066). `docs.yml`'s cron still needs the same step. ### 5. One task runner mirrors CI. **In place** (#765) diff --git a/python/scripts/check_framework_releases.py b/python/scripts/check_framework_releases.py new file mode 100644 index 00000000..7ebe4c1d --- /dev/null +++ b/python/scripts/check_framework_releases.py @@ -0,0 +1,661 @@ +"""Find framework releases newer than the PR CI lock and plan their issues (#1066). + +PR CI installs each framework from its committed lock, +``constraints/adapter-contract--py3.12.txt`` (ADR 0007 rule 3), and the +nightly resolves everything fresh at once (rule 4). Neither says which upstream +release broke what. ``.github/workflows/framework-release-watch.yml`` runs this +script to find each watched package whose newest stable PyPI release is ahead +of its pin, test that one release against an otherwise unchanged lock, and +keep one issue per release open until the lock catches up. + +The watched set is every framework extra (one per adapter-contract lock file, +which ``tests/unit/test_check_framework_releases.py`` holds equal to +``ADAPTERS``) times that extra's direct requirements in ``pyproject.toml``. +Pre-releases, dev releases and fully yanked releases are never "latest". + + python scripts/check_framework_releases.py [--json] [--package NAME] + # releases ahead of the lock: a table, or the workflow's matrix as JSON + python scripts/check_framework_releases.py --constraints-for PKG==VER \\ + --extra EXTRA --out PATH + # the extra's lock with only PKG's pin moved to VER + python scripts/check_framework_releases.py --record-result PATH \\ + --release JSON --mode locked|co-bump --outcome STEP=OUTCOME ... + # one evaluate leg's result.json, from inside its venv + python scripts/check_framework_releases.py --plan-issues --results DIR \\ + --open-issues FILE --run-url URL --repo-url URL --out PLAN + # the create/edit/comment/close operations for the alert job; prints + # the rendered plan as Markdown for the job summary + +Uses only the standard library (plus ``check_new_dependencies.py`` beside it), +so the detect and alert jobs run it without installing the package. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +from collections.abc import Callable, Iterable, Mapping +from dataclasses import asdict, dataclass +from importlib import metadata +from pathlib import Path +from typing import Any + +from check_new_dependencies import Fetch, canonicalize, fetch_project, requirement_name + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - 3.10 backfill + import tomli as tomllib + +_PYTHON_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PYPROJECT = _PYTHON_ROOT / "pyproject.toml" +DEFAULT_CONSTRAINTS = _PYTHON_ROOT / "constraints" +_LOCK_PYTHON = "3.12" +_LOCK = re.compile(rf"^adapter-contract-(?P.+)-py{re.escape(_LOCK_PYTHON)}\.txt$") +_PIN = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9._-]*)==(?P\S+?)\s*(?:#.*)?$") +# PEP 440's public version grammar, case-insensitive, with its optional separators. +_VERSION = re.compile( + r"""^v?(?:(?P\d+)!)?(?P\d+(?:\.\d+)*) + (?P
[-_.]?(?:a|b|c|rc|alpha|beta|pre|preview)[-_.]?\d*)?
+    (?P-\d+|[-_.]?(?:post|rev|r)[-_.]?\d*)?
+    (?P[-_.]?dev[-_.]?\d*)?
+    (?:\+[a-z0-9]+(?:[-_.][a-z0-9]+)*)?$""",
+    re.IGNORECASE | re.VERBOSE,
+)
+_CHECKS = (
+    ("install", "install"),
+    ("verify", "`verify_frameworks.py --require-installed`"),
+    ("pytest", "`pytest -q` (skip guard armed)"),
+    ("smoke", "`smoke_example.py`"),
+)
+_ICONS = {"success": "✅ pass", "failure": "❌ fail", "skipped": "⏭️ skipped"}
+_MARKER = re.compile(r"")
+_IMPACT = re.compile(r".*?", re.DOTALL)
+_IMPACT_PLACEHOLDER = (
+    "\n"
+    "_No changelog impact review yet (#1067). Read the release notes above._\n"
+    ""
+)
+
+
+@dataclass(frozen=True, order=True)
+class Version:
+    """The parts of a PEP 440 version this script compares and classifies."""
+
+    epoch: int
+    release: tuple[int, ...]
+    post: int  # -1 when there is no post-release segment
+    prerelease: bool  # a pre- or dev release
+
+    @classmethod
+    def parse(cls, text: str) -> Version | None:
+        """``text`` as a version, or ``None`` if it is not PEP 440."""
+        match = _VERSION.match(text.strip())
+        if match is None:
+            return None
+        release = tuple(int(part) for part in match["release"].split("."))
+        while len(release) > 1 and release[-1] == 0:
+            release = release[:-1]  # 1.2.0 == 1.2
+        post = match["post"]
+        return cls(
+            epoch=int(match["epoch"] or 0),
+            release=release,
+            post=int(re.sub(r"\D", "", post) or 0) if post else -1,
+            prerelease=bool(match["pre"] or match["dev"]),
+        )
+
+
+def bump_kind(locked: str, latest: str) -> str:
+    """``major``, ``minor`` or ``patch``: the first release component that moved.
+
+    An epoch change counts as major, and a post-release of the same version as a
+    patch."""
+    old, new = Version.parse(locked), Version.parse(latest)
+    if old is None or new is None:
+        raise ValueError(f"not a PEP 440 version: {locked!r} or {latest!r}")
+    if old.epoch != new.epoch:
+        return "major"
+    width = max(len(old.release), len(new.release), 3)
+    a = old.release + (0,) * (width - len(old.release))
+    b = new.release + (0,) * (width - len(new.release))
+    for index, kind in ((0, "major"), (1, "minor")):
+        if a[index] != b[index]:
+            return kind
+    return "patch"
+
+
+@dataclass(frozen=True)
+class Release:
+    """A project's newest stable release on PyPI."""
+
+    version: str
+    released_at: str  # date of the earliest upload, YYYY-MM-DD
+    release_notes_url: str
+
+
+def latest_release(project: Mapping[str, Any]) -> Release | None:
+    """The newest release that is not a pre/dev release, has files, and is not
+    entirely yanked. ``None`` if the project has none."""
+    best: tuple[Version, str, list[Mapping[str, Any]]] | None = None
+    for text, files in project.get("releases", {}).items():
+        version = Version.parse(text)
+        if version is None or version.prerelease:
+            continue
+        live = [f for f in files if not f.get("yanked")]
+        if not live:
+            continue
+        if best is None or version > best[0]:
+            best = (version, text, live)
+    if best is None:
+        return None
+    _, text, files = best
+    uploaded = min(f["upload_time_iso_8601"] for f in files)
+    name = project.get("info", {}).get("name", "")
+    return Release(text, uploaded[:10], release_notes_url(project, name, text))
+
+
+def release_notes_url(project: Mapping[str, Any], name: str, version: str) -> str:
+    """The project's changelog link if it declares one, else its source or home
+    page, else the version's PyPI page."""
+    urls = {k.lower(): v for k, v in (project.get("info", {}).get("project_urls") or {}).items()}
+    for words in (
+        ("changelog", "change log", "changes", "release notes", "releases", "history"),
+        ("source", "repository", "github", "code"),
+        ("homepage", "home"),
+    ):
+        for key, url in urls.items():
+            if any(word in key for word in words):
+                return str(url)
+    return f"https://pypi.org/project/{name}/{version}/"
+
+
+def read_pins(text: str) -> dict[str, str]:
+    """``{canonical name: version}`` for every ``name==version`` line of a lock."""
+    pins: dict[str, str] = {}
+    for line in text.splitlines():
+        match = _PIN.match(line.strip())
+        if match:
+            pins[canonicalize(match["name"])] = match["version"]
+    return pins
+
+
+def rewrite_pin(text: str, package: str, version: str) -> str:
+    """The lock ``text`` with exactly ``package``'s pin moved to ``version``."""
+    lines = text.splitlines(keepends=True)
+    hits = [
+        (i, m["name"])
+        for i, line in enumerate(lines)
+        if (m := _PIN.match(line.strip())) and canonicalize(m["name"]) == canonicalize(package)
+    ]
+    if len(hits) != 1:
+        raise ValueError(f"expected one pin for {package!r} in the lock, found {len(hits)}")
+    index, name = hits[0]
+    lines[index] = f"{name}=={version}\n"
+    return "".join(lines)
+
+
+@dataclass(frozen=True)
+class Watch:
+    """One watched package: a direct requirement of one framework extra."""
+
+    framework: str  # verify_frameworks.py's key and the examples/ directory
+    extra: str
+    package: str
+    locked: str
+
+
+def lock_path(extra: str, constraints: Path = DEFAULT_CONSTRAINTS) -> Path:
+    """The adapter-contract lock file for ``extra``."""
+    return constraints / f"adapter-contract-{extra}-py{_LOCK_PYTHON}.txt"
+
+
+def watched(pyproject: Mapping[str, Any], constraints: Path) -> list[Watch]:
+    """Every framework extra's direct requirements, with their locked versions.
+
+    The extras are the adapter-contract lock files', and the framework key is
+    the extra with ``-`` as ``_``, as the adapter-contract nox session does."""
+    extras = pyproject.get("project", {}).get("optional-dependencies", {})
+    found: list[Watch] = []
+    for path in sorted(constraints.glob(f"adapter-contract-*-py{_LOCK_PYTHON}.txt")):
+        match = _LOCK.match(path.name)
+        if match is None:
+            continue
+        extra = match["extra"]
+        if extra not in extras:
+            raise ValueError(f"{path.name}: pyproject.toml has no extra {extra!r}")
+        pins = read_pins(path.read_text(encoding="utf-8"))
+        for raw in extras[extra]:
+            package = requirement_name(raw)
+            if package not in pins:
+                raise ValueError(f"{path.name}: no pin for {extra!r}'s requirement {package!r}")
+            found.append(Watch(extra.replace("-", "_"), extra, package, pins[package]))
+    return found
+
+
+def find_releases(watches: Iterable[Watch], fetch: Fetch) -> list[dict[str, str]]:
+    """One matrix entry per watched package whose newest stable release is ahead
+    of its pin."""
+    entries: list[dict[str, str]] = []
+    for watch in watches:
+        project = fetch(watch.package)
+        if project is None:
+            raise LookupError(f"{watch.package}: not found on PyPI")
+        latest = latest_release(project)
+        newest = Version.parse(latest.version) if latest else None
+        locked = Version.parse(watch.locked)
+        if latest is None or newest is None or locked is None or newest <= locked:
+            continue
+        entries.append(
+            {
+                **asdict(watch),
+                "latest": latest.version,
+                "bump": bump_kind(watch.locked, latest.version),
+                "released_at": latest.released_at,
+                "release_notes_url": latest.release_notes_url,
+            }
+        )
+    return entries
+
+
+def record_result(
+    release: Mapping[str, str],
+    mode: str,
+    outcomes: Mapping[str, str],
+    lock: Mapping[str, str],
+    installed: Callable[[str], str | None],
+    verify: list[Mapping[str, Any]] | None = None,
+) -> dict[str, Any]:
+    """One evaluate leg's result: which lock pins its venv moved, and how each
+    check went. ``installed`` reads a distribution's version (``None`` if absent)."""
+    package = release["package"]
+    moved = {
+        name: [pinned, now]
+        for name, pinned in sorted(lock.items())
+        if name != package and (now := installed(name)) is not None and now != pinned
+    }
+    failures = [
+        f"{r.get('framework')}: {r.get('verdict')} {r.get('detail', '')}".strip()
+        for r in verify or []
+        if not str(r.get("verdict", "")).startswith(("UNVERIFIED", "VERIFIED"))
+    ]
+    return {
+        "release": dict(release),
+        "mode": mode,
+        "installed": installed(package),
+        "moved": moved,
+        "outcomes": {step: outcomes.get(step, "skipped") for step, _ in _CHECKS},
+        "verify_failures": failures,
+    }
+
+
+def _installed_version(name: str) -> str | None:
+    try:
+        return metadata.version(name)
+    except metadata.PackageNotFoundError:
+        return None
+
+
+def _digest(result: Mapping[str, Any]) -> str:
+    """What makes two runs' results the same, so an unchanged rerun adds no comment."""
+    keep = {k: result[k] for k in ("mode", "installed", "moved", "outcomes", "verify_failures")}
+    keep["locked"] = result["release"]["locked"]
+    raw = json.dumps(keep, sort_keys=True).encode()
+    return hashlib.sha256(raw).hexdigest()[:16]
+
+
+def _green(result: Mapping[str, Any]) -> bool:
+    return all(outcome == "success" for outcome in result["outcomes"].values())
+
+
+def issue_title(release: Mapping[str, str]) -> str:
+    """The tracking issue's title."""
+    return f"framework-release: {release['package']} {release['locked']} → {release['latest']}"
+
+
+def render_results(result: Mapping[str, Any], run_url: str) -> str:
+    """The results section: install mode, a row per check, the run link."""
+    release = result["release"]
+    if result["mode"] == "locked":
+        mode = f"Installed against the PR lock with only `{release['package']}` moved."
+    else:
+        mode = (
+            f"`{release['package']}=={release['latest']}` does not resolve against the "
+            "PR lock, so it was installed **unconstrained (needs co-bump)**."
+        )
+    lines = [mode]
+    if result["moved"]:
+        moved = ", ".join(f"`{n}` {a} → {b}" for n, (a, b) in result["moved"].items())
+        lines.append(f"Other locked packages that moved: {moved}.")
+    lines += ["", "| Check | Result |", "| --- | --- |"]
+    for step, label in _CHECKS:
+        outcome = result["outcomes"][step]
+        lines.append(f"| {label} | {_ICONS.get(outcome, outcome)} |")
+    for failure in result["verify_failures"]:
+        lines.append(f"\n`verify_frameworks.py`: {failure}")
+    lines.append(f"\nRun: {run_url}")
+    return "\n".join(lines)
+
+
+def _next_action(result: Mapping[str, Any]) -> str:
+    release = result["release"]
+    if not _green(result):
+        return (
+            "**Not safe to bump yet.** Adapt the adapter to this release, or record the "
+            "incompatibility in `docs/verified-apis.md` §8.1. Never add a version "
+            "ceiling (§8.4)."
+        )
+    co_bump = (
+        " It needs the co-bumps listed above, which the recompile picks up."
+        if result["mode"] != "locked"
+        else ""
+    )
+    return (
+        f"**Safe to bump.**{co_bump} The nightly lock-refresh PR (`lock-refresh.yml`) "
+        "may already carry it. Otherwise, from `python/`:\n\n"
+        "1. `python scripts/compile_constraints.py`\n"
+        "2. `python scripts/generate_docs.py`\n"
+        f"3. `python scripts/verify_frameworks.py --only {release['framework']} "
+        "--emit-verified`, and record the rows in `docs/verified-apis.md` §8\n"
+        "4. the pre-PR gate (CONTRIBUTING.md §1)"
+    )
+
+
+def render_issue(result: Mapping[str, Any], run_url: str, impact: str | None = None) -> str:
+    """The whole issue body. ``impact`` keeps an existing impact section (#1067)."""
+    release = result["release"]
+    marker = {
+        "extra": release["extra"],
+        "package": release["package"],
+        "version": release["latest"],
+        "digest": _digest(result),
+    }
+    lock = f"python/constraints/{lock_path(release['extra']).name}"
+    return "\n".join(
+        [
+            f"",
+            f"`{release['package']}` {release['latest']} is out; the PR lock pins "
+            f"{release['locked']}.",
+            "",
+            "| | |",
+            "| --- | --- |",
+            f"| Framework | `{release['framework']}` (extra `{release['extra']}`) |",
+            f"| Locked | `{release['locked']}` in `{lock}` |",
+            f"| Latest | `{release['latest']}` ({release['bump']} bump), "
+            f"released {release['released_at']} |",
+            f"| Release notes | {release['release_notes_url']} |",
+            "",
+            "## Results against the PR lock",
+            "",
+            render_results(result, run_url),
+            "",
+            "## Impact",
+            "",
+            impact or _IMPACT_PLACEHOLDER,
+            "",
+            "## Next action",
+            "",
+            _next_action(result),
+            "",
+            "_Opened and kept current by `framework-release-watch.yml` (docs/ci.md). "
+            "It closes this issue once the lock reaches this version._",
+        ]
+    )
+
+
+@dataclass(frozen=True)
+class OpenIssue:
+    """An open ``framework-release`` issue this workflow wrote."""
+
+    number: int
+    extra: str
+    package: str
+    version: str
+    digest: str
+    body: str
+
+
+def parse_open_issues(raw: Iterable[Mapping[str, Any]]) -> list[OpenIssue]:
+    """The open issues that carry this workflow's marker; any other is left alone."""
+    found: list[OpenIssue] = []
+    for issue in raw:
+        body = issue.get("body") or ""
+        match = _MARKER.search(body)
+        if match is None:
+            continue
+        data = json.loads(match["data"])
+        found.append(
+            OpenIssue(
+                int(issue["number"]),
+                data["extra"],
+                data["package"],
+                data["version"],
+                data.get("digest", ""),
+                body,
+            )
+        )
+    return found
+
+
+def plan_issues(
+    results: Iterable[Mapping[str, Any]],
+    open_issues: Iterable[OpenIssue],
+    watches: Iterable[Watch],
+    run_url: str,
+    lock_commit_url: Callable[[str], str],
+) -> list[dict[str, Any]]:
+    """The issue operations for one run, in order.
+
+    * a release with no open issue gets one (``create``);
+    * an open issue for the same release gets its body refreshed (``edit``),
+      plus a ``comment`` only when the results changed;
+    * an open issue for an older release of a package that now has a newer one
+      is closed as superseded;
+    * an open issue whose release the lock has reached, or whose package is no
+      longer watched, is closed with a comment linking the lock commit.
+    """
+    pins = {(w.extra, w.package): w.locked for w in watches}
+    by_key: dict[tuple[str, str], list[OpenIssue]] = {}
+    for issue in open_issues:
+        by_key.setdefault((issue.extra, issue.package), []).append(issue)
+    ops: list[dict[str, Any]] = []
+    seen: set[tuple[str, str]] = set()
+    for result in sorted(results, key=lambda r: (r["release"]["extra"], r["release"]["package"])):
+        release = result["release"]
+        key = (release["extra"], release["package"])
+        seen.add(key)
+        current = [i for i in by_key.get(key, []) if i.version == release["latest"]]
+        for older in (i for i in by_key.get(key, []) if i.version != release["latest"]):
+            ops.append(
+                {
+                    "op": "close",
+                    "number": older.number,
+                    "reason": "not planned",
+                    "comment": f"Superseded by `{release['package']}` {release['latest']}, "
+                    f"tracked in its own issue: {run_url}",
+                }
+            )
+        if not current:
+            ops.append(
+                {
+                    "op": "create",
+                    "title": issue_title(release),
+                    "body": render_issue(result, run_url),
+                }
+            )
+            continue
+        issue, *duplicates = current
+        impact = _IMPACT.search(issue.body)
+        ops.append(
+            {
+                "op": "edit",
+                "number": issue.number,
+                "title": issue_title(release),
+                "body": render_issue(result, run_url, impact[0] if impact else None),
+            }
+        )
+        if issue.digest != _digest(result):
+            ops.append(
+                {
+                    "op": "comment",
+                    "number": issue.number,
+                    "body": "The results changed.\n\n" + render_results(result, run_url),
+                }
+            )
+        for duplicate in duplicates:
+            ops.append(
+                {
+                    "op": "close",
+                    "number": duplicate.number,
+                    "reason": "not planned",
+                    "comment": f"Duplicate of #{issue.number}.",
+                }
+            )
+    for key, issues in sorted(by_key.items()):
+        if key in seen:
+            continue
+        extra, package = key
+        for issue in issues:
+            locked = pins.get(key)
+            if locked is None:
+                comment = f"`{package}` is no longer a direct requirement of a watched extra."
+            else:
+                old, new = Version.parse(issue.version), Version.parse(locked)
+                if old is None or new is None or new < old:
+                    continue  # still ahead of the lock; this run did not evaluate it
+                comment = (
+                    f"The lock now pins `{package}=={locked}` "
+                    f"({lock_commit_url(extra)}), which covers {issue.version}."
+                )
+            ops.append(
+                {"op": "close", "number": issue.number, "reason": "completed", "comment": comment}
+            )
+    return ops
+
+
+def render_plan(ops: list[dict[str, Any]], dry_run: bool) -> str:
+    """The job summary: every planned operation, with the rendered bodies."""
+    lines = ["# Framework release watch", ""]
+    if dry_run:
+        lines += ["Dry run (pull request): no issue was written. The plan:", ""]
+    if not ops:
+        lines.append("Nothing to do: every watched package is at its newest release.")
+    for op in ops:
+        target = op.get("title") or f"#{op['number']}"
+        lines.append(f"## {op['op']}: {target}")
+        text = op.get("body") or op.get("comment") or ""
+        lines += ["", "
Body", "", text, "", "
", ""] + return "\n".join(lines) + + +def _git_lock_commit_url(repo_url: str) -> Callable[[str], str]: + def url(extra: str) -> str: + path = lock_path(extra) + sha = subprocess.run( + ["git", "log", "-1", "--format=%H", "--", str(path)], + cwd=_PYTHON_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return f"{repo_url}/commit/{sha}" if sha else f"`{path.name}`" + + return url + + +def _load_pyproject(path: Path) -> dict[str, Any]: + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--pyproject", type=Path, default=DEFAULT_PYPROJECT) + parser.add_argument("--constraints", type=Path, default=DEFAULT_CONSTRAINTS) + parser.add_argument("--json", action="store_true", help="print the matrix as JSON") + parser.add_argument("--package", help="check only this watched package") + parser.add_argument("--constraints-for", metavar="PKG==VER", help="write a rewritten lock") + parser.add_argument("--extra", help="the lock --constraints-for rewrites") + parser.add_argument("--out", type=Path, help="output file") + parser.add_argument("--record-result", type=Path, metavar="PATH", help="write result.json") + parser.add_argument("--release", help="the leg's matrix entry, as JSON") + parser.add_argument("--mode", choices=("locked", "co-bump")) + parser.add_argument("--outcome", nargs="*", default=[], metavar="STEP=OUTCOME") + parser.add_argument("--verify-json", type=Path, help="verify_frameworks.py --json output") + parser.add_argument("--plan-issues", action="store_true", help="plan the alert job's writes") + parser.add_argument("--results", type=Path, help="directory of result.json files") + parser.add_argument("--open-issues", type=Path, help="gh issue list --json number,body") + parser.add_argument("--run-url", default="") + parser.add_argument("--repo-url", default="") + parser.add_argument("--dry-run", action="store_true", help="label the summary a dry run") + return parser + + +def main(argv: list[str], fetch: Fetch = fetch_project) -> int: + args = _parser().parse_args(argv) + watches = watched(_load_pyproject(args.pyproject), args.constraints) + + if args.constraints_for: + package, _, version = args.constraints_for.partition("==") + if not (version and args.extra and args.out): + print("::error::--constraints-for needs PKG==VER, --extra and --out") + return 2 + lock = lock_path(args.extra, args.constraints) + text = rewrite_pin(lock.read_text(encoding="utf-8"), package, version) + args.out.write_text(text, encoding="utf-8") + return 0 + + if args.record_result: + release = json.loads(args.release) + outcomes = dict(item.partition("=")[::2] for item in args.outcome) + lock = read_pins(lock_path(release["extra"], args.constraints).read_text(encoding="utf-8")) + verify = None + if args.verify_json and args.verify_json.is_file() and args.verify_json.stat().st_size: + verify = json.loads(args.verify_json.read_text(encoding="utf-8")) + result = record_result(release, args.mode, outcomes, lock, _installed_version, verify) + args.record_result.parent.mkdir(parents=True, exist_ok=True) + args.record_result.write_text(json.dumps(result, indent=2), encoding="utf-8") + return 0 + + if args.plan_issues: + files = ( + sorted(args.results.rglob("*.json")) if args.results and args.results.is_dir() else [] + ) + results = [json.loads(f.read_text(encoding="utf-8")) for f in files] + raw = json.loads(args.open_issues.read_text(encoding="utf-8")) if args.open_issues else [] + ops = plan_issues( + results, + parse_open_issues(raw), + watches, + args.run_url, + _git_lock_commit_url(args.repo_url), + ) + if args.out: + args.out.write_text(json.dumps(ops, indent=2), encoding="utf-8") + print(render_plan(ops, args.dry_run)) + return 0 + + if args.package: + wanted = canonicalize(args.package) + watches = [w for w in watches if w.package == wanted] + if not watches: + print(f"::error::{args.package} is not a watched package") + return 2 + entries = find_releases(watches, fetch) + if args.json: + print(json.dumps(entries)) + elif not entries: + print("Every watched package is at its newest stable release.") + else: + for e in entries: + print( + f"{e['package']:<32} {e['locked']:>12} -> {e['latest']:<12} " + f"{e['bump']:<6} {e['released_at']} ({e['extra']})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/python/tests/unit/test_check_framework_releases.py b/python/tests/unit/test_check_framework_releases.py new file mode 100644 index 00000000..c91ea635 --- /dev/null +++ b/python/tests/unit/test_check_framework_releases.py @@ -0,0 +1,407 @@ +"""``scripts/check_framework_releases.py``, the framework release watch (#1066). + +The script finds each framework extra's direct requirement whose newest stable +PyPI release is ahead of the adapter-contract lock, and plans the +``framework-release`` issues for ``framework-release-watch.yml``. These tests +never touch the network: the PyPI lookup is injected. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from donkey_kit.integrations import ADAPTERS + +if sys.version_info >= (3, 11): + import tomllib +else: # pragma: no cover - the 3.10 floor + import tomli as tomllib + +_PYTHON = Path(__file__).resolve().parents[2] +_SCRIPTS = _PYTHON / "scripts" + + +def _load() -> ModuleType: + name = "ddk_check_framework_releases" + spec = importlib.util.spec_from_file_location(name, _SCRIPTS / "check_framework_releases.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # dataclasses resolve annotations through it + sys.path.insert(0, str(_SCRIPTS)) # its sibling import of check_new_dependencies + try: + spec.loader.exec_module(module) + finally: + sys.path.remove(str(_SCRIPTS)) + return module + + +cfr = _load() + +_LOCK_HEADER = "# GENERATED FILE — do not edit by hand.\n#\n" + + +def _file(day: str, *, yanked: bool = False) -> dict[str, Any]: + return {"upload_time_iso_8601": f"{day}T12:00:00.000000Z", "yanked": yanked} + + +def _project(releases: dict[str, list[dict[str, Any]]], **urls: str) -> dict[str, Any]: + return {"info": {"name": "pkg", "project_urls": urls or None}, "releases": releases} + + +# --- versions ----------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("locked", "latest", "kind"), + [ + ("1.15.23", "1.15.27", "patch"), + ("1.6.7", "1.7.0", "minor"), + ("0.23.1", "1.0", "major"), + ("1.2", "1.2.1", "patch"), + ("1.2.0", "1.2.0.post1", "patch"), + ("2.0", "1!0.1", "major"), + ], +) +def test_bump_kind_is_the_first_release_component_that_moved( + locked: str, latest: str, kind: str +) -> None: + assert cfr.bump_kind(locked, latest) == kind + + +@pytest.mark.parametrize( + ("older", "newer"), + [("1.9", "1.10"), ("1.2", "1.2.post1"), ("1.2.0", "1.2.1"), ("9.0", "1!0.1")], +) +def test_versions_order_numerically(older: str, newer: str) -> None: + assert cfr.Version.parse(older) < cfr.Version.parse(newer) + + +def test_trailing_zeros_do_not_change_a_version() -> None: + assert cfr.Version.parse("1.2.0") == cfr.Version.parse("1.2") + + +@pytest.mark.parametrize("text", ["1.0a1", "2.0.0rc3", "1.5.dev2", "1.5.post1.dev0", "3.0b2"]) +def test_pre_and_dev_releases_are_flagged(text: str) -> None: + version = cfr.Version.parse(text) + assert version is not None and version.prerelease + + +# --- the newest stable release -------------------------------------------------- + + +def test_latest_skips_pre_releases_yanked_and_empty_releases() -> None: + project = _project( + { + "1.0": [_file("2026-01-01")], + "1.1": [_file("2026-02-01")], + "1.2": [_file("2026-03-01", yanked=True)], # every file yanked + "1.3": [], # no files at all + "2.0rc1": [_file("2026-04-01")], + "2.0.dev1": [_file("2026-04-02")], + "not-a-version": [_file("2026-05-01")], + } + ) + latest = cfr.latest_release(project) + assert latest is not None and latest.version == "1.1" + + +def test_a_release_with_one_live_file_still_counts() -> None: + project = _project( + { + "1.0": [_file("2026-01-01")], + "1.1": [_file("2026-02-03", yanked=True), _file("2026-02-02")], + } + ) + assert cfr.latest_release(project) == cfr.Release( + "1.1", "2026-02-02", "https://pypi.org/project/pkg/1.1/" + ) + + +def test_a_project_with_only_pre_releases_has_no_latest() -> None: + assert cfr.latest_release(_project({"0.1a1": [_file("2026-01-01")]})) is None + + +def test_release_notes_prefer_the_changelog_then_source_then_home() -> None: + releases = {"1.0": [_file("2026-01-01")]} + project = _project(releases, Homepage="h", Source="s", Changelog="c") + assert cfr.latest_release(project).release_notes_url == "c" + assert cfr.latest_release(_project(releases, Homepage="h", Source="s")).release_notes_url == "s" + assert cfr.latest_release(_project(releases, Homepage="h")).release_notes_url == "h" + + +# --- the lock ------------------------------------------------------------------- + + +def test_read_pins_canonicalizes_names_and_skips_comments() -> None: + text = _LOCK_HEADER + "Google_ADK==2.11.0\nlitellm==1.104.0\n\n# a comment\n" + assert cfr.read_pins(text) == {"google-adk": "2.11.0", "litellm": "1.104.0"} + + +def test_rewrite_pin_moves_only_that_package() -> None: + text = _LOCK_HEADER + "anyio==4.0\nlangchain-core==1.6.7\nlangchain-openai==1.6.7\n" + rewritten = cfr.rewrite_pin(text, "langchain_core", "1.6.9") + assert ( + rewritten == _LOCK_HEADER + "anyio==4.0\nlangchain-core==1.6.9\nlangchain-openai==1.6.7\n" + ) + + +def test_rewrite_pin_refuses_a_package_the_lock_does_not_pin() -> None: + with pytest.raises(ValueError, match="found 0"): + cfr.rewrite_pin(_LOCK_HEADER + "anyio==4.0\n", "crewai", "2.0") + + +def _tree(tmp_path: Path, locks: dict[str, str]) -> Path: + constraints = tmp_path / "constraints" + constraints.mkdir() + for extra, body in locks.items(): + (constraints / f"adapter-contract-{extra}-py3.12.txt").write_text(_LOCK_HEADER + body) + (constraints / "test-py3.12.txt").write_text(_LOCK_HEADER + "unrelated==1.0\n") + return constraints + + +def test_watched_is_each_lock_extras_direct_requirements(tmp_path: Path) -> None: + extras = {"openai-agents": ["openai-agents>=0.20", "openai>=2.45"], "llm": ["httpx>=0.27"]} + locks = {"openai-agents": "openai==3.24.0\nopenai-agents==0.23.1\nanyio==4\n"} + constraints = _tree(tmp_path, locks) + pyproject = {"project": {"optional-dependencies": extras}} + assert cfr.watched(pyproject, constraints) == [ + cfr.Watch("openai_agents", "openai-agents", "openai-agents", "0.23.1"), + cfr.Watch("openai_agents", "openai-agents", "openai", "3.24.0"), + ] + + +def test_watched_fails_when_the_lock_misses_a_direct_requirement(tmp_path: Path) -> None: + constraints = _tree(tmp_path, {"crewai": "anyio==4\n"}) + pyproject = {"project": {"optional-dependencies": {"crewai": ["crewai>=1.15.3"]}}} + with pytest.raises(ValueError, match="no pin"): + cfr.watched(pyproject, constraints) + + +def test_watched_set_is_adapters_times_their_extras_direct_requirements() -> None: + # The roster-drift check (#1066): the script derives the watched set from + # the lock files, without importing donkey_kit, so it must equal ADAPTERS, + # pair for pair, like test_nightly_framework_legs_match_adapters. + pyproject = tomllib.loads((_PYTHON / "pyproject.toml").read_text(encoding="utf-8")) + extras = pyproject["project"]["optional-dependencies"] + expected = { + (key, spec.extra, cfr.requirement_name(raw)) + for key, spec in ADAPTERS.items() + for raw in extras[spec.extra] + } + actual = { + (w.framework, w.extra, w.package) for w in cfr.watched(pyproject, _PYTHON / "constraints") + } + assert actual == expected + + +# --- detection ------------------------------------------------------------------ + + +def test_find_releases_lists_only_packages_ahead_of_the_lock() -> None: + watches = [ + cfr.Watch("crewai", "crewai", "crewai", "1.15.23"), + cfr.Watch("anthropic", "anthropic", "anthropic", "1.11.0"), + ] + projects = { + "crewai": _project({"1.15.23": [_file("2026-09-01")], "1.15.27": [_file("2026-10-09")]}), + # Only a pre-release is newer than the pin: nothing to report. + "anthropic": _project( + {"1.11.0": [_file("2026-09-01")], "1.12.0rc1": [_file("2026-10-01")]} + ), + } + assert cfr.find_releases(watches, projects.get) == [ + { + "framework": "crewai", + "extra": "crewai", + "package": "crewai", + "locked": "1.15.23", + "latest": "1.15.27", + "bump": "patch", + "released_at": "2026-10-09", + "release_notes_url": "https://pypi.org/project/pkg/1.15.27/", + } + ] + + +def test_find_releases_fails_on_a_package_pypi_does_not_know() -> None: + with pytest.raises(LookupError): + cfr.find_releases([cfr.Watch("x", "x", "gone", "1.0")], lambda _name: None) + + +# --- one leg's result ------------------------------------------------------------- + +_RELEASE = { + "framework": "crewai", + "extra": "crewai", + "package": "crewai", + "locked": "1.15.23", + "latest": "1.15.27", + "bump": "patch", + "released_at": "2026-10-09", + "release_notes_url": "https://example.test/notes", +} +_GREEN = {"install": "success", "verify": "success", "pytest": "success", "smoke": "success"} + + +def _result(**outcomes: str) -> dict[str, Any]: + installed = {"crewai": "1.15.27", "anyio": "4.0"} + return cfr.record_result( + _RELEASE, "locked", _GREEN | outcomes, {"crewai": "1.15.23", "anyio": "4.0"}, installed.get + ) + + +def test_record_result_lists_the_lock_pins_the_venv_moved() -> None: + installed = {"crewai": "1.15.27", "openai": "3.29.0", "anyio": "4.0"} + lock = {"crewai": "1.15.23", "openai": "3.24.0", "anyio": "4.0", "absent": "1.0"} + result = cfr.record_result(_RELEASE, "co-bump", {"install": "success"}, lock, installed.get) + assert result["installed"] == "1.15.27" + assert result["moved"] == {"openai": ["3.24.0", "3.29.0"]} + assert result["outcomes"] == { + "install": "success", + "verify": "skipped", + "pytest": "skipped", + "smoke": "skipped", + } + + +def test_record_result_keeps_only_failing_verify_rows() -> None: + verify = [ + {"framework": "crewai", "verdict": "UNVERIFIED (signature confirmed)", "detail": ""}, + {"framework": "crewai.x", "verdict": "SIGNATURE FAIL", "detail": "TypeError: base_url"}, + ] + result = cfr.record_result(_RELEASE, "locked", _GREEN, {}, lambda _n: None, verify) + assert result["verify_failures"] == ["crewai.x: SIGNATURE FAIL TypeError: base_url"] + + +# --- issues ---------------------------------------------------------------------- + +_WATCHES = [cfr.Watch("crewai", "crewai", "crewai", "1.15.23")] + + +def _open(body: str, number: int = 7) -> list[Any]: + return cfr.parse_open_issues([{"number": number, "body": body}]) + + +def _plan(results: list[dict[str, Any]], issues: list[Any], watches: Any = None) -> list[Any]: + return cfr.plan_issues( + results, issues, _WATCHES if watches is None else watches, "RUN", lambda e: f"COMMIT-{e}" + ) + + +def test_a_new_release_opens_one_issue() -> None: + (op,) = _plan([_result()], []) + assert op["op"] == "create" + assert op["title"] == "framework-release: crewai 1.15.23 → 1.15.27" + assert "**Safe to bump.**" in op["body"] + assert "--only crewai --emit-verified" in op["body"] + assert "RUN" in op["body"] + + +def test_a_red_result_says_adapt_or_record_never_a_ceiling() -> None: + (op,) = _plan([_result(pytest="failure")], []) + assert "Not safe to bump yet" in op["body"] + assert "never add a version ceiling" in op["body"].lower() + assert "| `pytest -q` (skip guard armed) | ❌ fail |" in op["body"] + + +def test_a_rerun_with_unchanged_results_edits_but_does_not_comment() -> None: + body = cfr.render_issue(_result(), "OLD-RUN") + ops = _plan([_result()], _open(body)) + assert [op["op"] for op in ops] == ["edit"] + assert ops[0]["number"] == 7 and "RUN" in ops[0]["body"] + + +def test_a_rerun_with_changed_results_comments_once() -> None: + body = cfr.render_issue(_result(), "OLD-RUN") + ops = _plan([_result(smoke="failure")], _open(body)) + assert [op["op"] for op in ops] == ["edit", "comment"] + assert ops[1]["body"].startswith("The results changed.") + + +def test_an_edit_keeps_the_impact_section() -> None: + impact = "\nreviewed: no breaking change\n" + body = cfr.render_issue(_result(), "OLD-RUN", impact) + (op,) = _plan([_result()], _open(body)) + assert impact in op["body"] + + +def test_a_newer_release_supersedes_the_older_issue() -> None: + older = dict(_RELEASE, latest="1.15.25") + body = cfr.render_issue(cfr.record_result(older, "locked", _GREEN, {}, lambda _n: None), "R") + ops = _plan([_result()], _open(body)) + assert [(op["op"], op.get("number")) for op in ops] == [("close", 7), ("create", None)] + assert ops[0]["reason"] == "not planned" + + +def test_an_issue_the_lock_caught_up_with_is_closed_with_the_lock_commit() -> None: + body = cfr.render_issue(_result(), "R") + bumped = [cfr.Watch("crewai", "crewai", "crewai", "1.15.27")] + (op,) = _plan([], _open(body), bumped) + assert op == { + "op": "close", + "number": 7, + "reason": "completed", + "comment": "The lock now pins `crewai==1.15.27` (COMMIT-crewai), which covers 1.15.27.", + } + + +def test_an_issue_still_ahead_of_the_lock_but_not_evaluated_stays_open() -> None: + # A dispatch run for another package evaluates nothing for crewai. + assert _plan([], _open(cfr.render_issue(_result(), "R"))) == [] + + +def test_an_issue_for_an_unwatched_package_is_closed() -> None: + (op,) = _plan([], _open(cfr.render_issue(_result(), "R")), []) + assert op["op"] == "close" and "no longer a direct requirement" in op["comment"] + + +def test_issues_without_the_marker_are_left_alone() -> None: + assert _open("a hand-written issue that happens to carry the label") == [] + + +def test_the_marker_round_trips() -> None: + (issue,) = _open(cfr.render_issue(_result(), "R"), number=3) + assert (issue.number, issue.extra, issue.package, issue.version) == ( + 3, + "crewai", + "crewai", + "1.15.27", + ) + + +def test_the_plan_summary_flags_a_dry_run() -> None: + summary = cfr.render_plan(_plan([_result()], []), dry_run=True) + assert "Dry run" in summary and "## create: framework-release: crewai" in summary + + +# --- the CLI ----------------------------------------------------------------------- + + +def test_cli_json_matrix_is_empty_when_nothing_is_new( + capsys: pytest.CaptureFixture[str], +) -> None: + # Every package's only release is a pre-release far ahead of any pin. + project = _project({"999.0rc1": [_file("2026-10-01")]}) + assert cfr.main(["--json"], fetch=lambda _name: project) == 0 + assert json.loads(capsys.readouterr().out) == [] + + +def test_cli_rejects_an_unwatched_package(capsys: pytest.CaptureFixture[str]) -> None: + assert cfr.main(["--package", "requests"], fetch=lambda _n: None) == 2 + assert "not a watched package" in capsys.readouterr().out + + +def test_cli_constraints_for_writes_the_rewritten_lock(tmp_path: Path) -> None: + out = tmp_path / "lock.txt" + argv = ["--constraints-for", "crewai==99.0", "--extra", "crewai", "--out", str(out)] + assert cfr.main(argv) == 0 + lock = cfr.lock_path("crewai").read_text(encoding="utf-8") + pins = cfr.read_pins(lock) + assert cfr.read_pins(out.read_text(encoding="utf-8")) == pins | {"crewai": "99.0"} diff --git a/python/tests/unit/test_ci_workflows.py b/python/tests/unit/test_ci_workflows.py index 343469fe..94416efd 100644 --- a/python/tests/unit/test_ci_workflows.py +++ b/python/tests/unit/test_ci_workflows.py @@ -33,7 +33,12 @@ def _needs(job: dict[str, Any]) -> set[str]: @pytest.mark.parametrize( ("workflow", "aggregate"), - [("ci.yml", "ci-ok"), ("nightly-matrix.yml", "alert"), ("lock-refresh.yml", "alert")], + [ + ("ci.yml", "ci-ok"), + ("nightly-matrix.yml", "alert"), + ("lock-refresh.yml", "alert"), + ("framework-release-watch.yml", "alert"), + ], ) def test_aggregate_job_needs_every_other_job(workflow: str, aggregate: str) -> None: jobs = _workflow(workflow)["jobs"] @@ -88,3 +93,33 @@ def test_lock_refresh_write_scopes_stay_on_their_own_jobs() -> None: assert jobs["alert"]["if"] == "failure()" assert jobs["alert"]["permissions"] == {"issues": "write"} assert set(jobs) == {"refresh", "alert"} + + +def test_framework_release_watch_alert_is_the_only_job_that_writes_issues() -> None: + # #1066: `alert` closes stale issues even when nothing is new, so it runs + # always(), and it is the one job with a write scope. + workflow = _workflow("framework-release-watch.yml") + assert workflow["permissions"] == {"contents": "read"} + jobs = workflow["jobs"] + assert jobs["alert"]["if"] == "always()" + assert jobs["alert"]["permissions"] == {"contents": "read", "issues": "write"} + assert not [k for k, v in jobs.items() if k != "alert" and "permissions" in v] + + +def test_framework_release_watch_dry_run_on_pull_requests_writes_nothing() -> None: + # Every step that calls `gh` with the write token is skipped on a PR (#1066). + steps = _workflow("framework-release-watch.yml")["jobs"]["alert"]["steps"] + writers = [s for s in steps if "GH_TOKEN" in s.get("env", {})] + assert writers + for step in writers: + assert "github.event_name != 'pull_request'" in step["if"], step["name"] + + +def test_framework_release_watch_matches_the_script_paths_it_runs() -> None: + triggers = _workflow("framework-release-watch.yml")[True] + assert set(triggers["pull_request"]["paths"]) == { + ".github/workflows/framework-release-watch.yml", + "python/scripts/check_framework_releases.py", + } + assert "package" in triggers["workflow_dispatch"]["inputs"] + assert triggers["schedule"] == [{"cron": "30 5 * * *"}]