diff --git a/.claude/rules/ci.md b/.claude/rules/ci.md index 924a56f62b..8215631cbc 100644 --- a/.claude/rules/ci.md +++ b/.claude/rules/ci.md @@ -22,9 +22,9 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ | 23959123 "release tags are created by admins" | tags `v*`, `libs-*`, `plugin-*` | creation | admins | | 25308049 "release branches are created by admins" | `release/*` | creation | admins | | 23957966 "main and release branches require green gates" | default branch, `release/*` | the five required checks, `do_not_enforce_on_create` | none | -| 25129799 "branches keep their history" | default branch, `release/*`, `feeds` | deletion, non-fast-forward | none | +| 25129799 "branches keep their history" | default branch, `release/*`, `gh-pages` | deletion, non-fast-forward | none | -- `feeds` takes no required checks: CI pushes it with `GITHUB_TOKEN`, and the signed feed proves its content. +- `gh-pages` takes no required checks (`RELEASING.md`, Keys and environments). - There are no gate jobs. A test job runs on `!cancelled() && needs.changes.outputs.run != 'false'`. A job its `if:` skips reports success, and a failed changes job runs the suite rather than skipping it. - `Changes (macOS)`, `Changes (iOS)` and `Changes (Release)` are not required checks. - A PR need not be up to date with main to merge (`strict_required_status_checks_policy: false`). diff --git a/.claude/rules/plugin-system.md b/.claude/rules/plugin-system.md index 7b7d2090bf..4db3fb7ba3 100644 --- a/.claude/rules/plugin-system.md +++ b/.claude/rules/plugin-system.md @@ -65,7 +65,7 @@ TableProPluginKit is built with Library Evolution, so an already-built plugin ke ## Structure - PluginKit-free logic a plugin shares with the iOS app lives in a TableProCore product, such as `TableProMySQLCore`, `TableProPostgreSQLCore` or `TableProRedisCore`. -- A new driver needs: its `project.yml` target and a `RegistryPlugins` entry (`BundledPlugins` and the app's `copy: { destination: plugins }` dependency for a bundled one), a `DatabaseType` constant, for a registry one an entry in `.github/plugin-registry.json` keyed by its target, with `name` and `iconName` (a bundled one stays out of it), a curated snapshot in `PluginMetadataRegistry` (without it the type never appears in the New Connection picker), its `TableProProvides*` keys (below), a row in `../tablepro-docs/databases/index.mdx` through a paired docs PR, and a changelog fragment. +- A new driver needs: its `project.yml` target and a `RegistryPlugins` entry (`BundledPlugins` and the app's `copy: { destination: plugins }` dependency for a bundled one), a `DatabaseType` constant, for a registry one an entry in `.github/plugin-registry.json` keyed by its target, with `name` (a bundled one stays out of it), a curated snapshot in `PluginMetadataRegistry` (without it the type never appears in the New Connection picker), its `TableProProvides*` keys (below), a row in `../tablepro-docs/databases/index.mdx` through a paired docs PR, and a changelog fragment. - A new engine ships as a registry plugin (`AGENTS.md`, Scope). - `PluginManager` registers a plugin from the `TableProProvidesDatabaseTypeIds`, `TableProProvidesExportFormatIds` and `TableProProvidesImportFormatIds` arrays in its `Info.plist` and loads its code on first use. A plugin with none of them loads at launch. The first database type id is the primary one. - `scripts/ci/check-plugin-provides.py` checks those arrays against the class's `databaseTypeId`, `additionalDatabaseTypeIds` and `formatId` in Repo Hygiene; `--write` stamps them. diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md index 57e10f2cf8..a387f3135f 100644 --- a/.claude/skills/release/SKILL.md +++ b/.claude/skills/release/SKILL.md @@ -26,7 +26,7 @@ Before any decision, gather: - the tags, newest first: `git -c versionsort.suffix=-beta tag -l 'v*' --sort=-v:refname | head`; - the live lines: `git ls-remote --heads origin 'release/*'`; - open regressions: `gh issue list --label regression --state open`; -- open Dependabot alerts: `gh api repos/TableProApp/TablePro/dependabot/alerts -q '.[] | select(.state == "open") | .security_advisory.summary'`; +- open Dependabot alerts: `gh api --paginate 'repos/TableProApp/TablePro/dependabot/alerts?state=open&per_page=100' -q '.[].security_advisory.summary'`; - main's push runs: `gh run list --branch main --event push --limit 10`, with macOS Tests, iOS and Package Tests and Release Build green on the commit you would cut; - whether `main` has the last stable's `## [X.Y.Z]` heading. If not, the forward-port is missing and the next tag will refuse to build: do that first (section 9). @@ -70,7 +70,7 @@ Run: `RELEASING.md`, Notes, then Cutting a stable or patch. - Follow the run with `gh run list --workflow release.yml --limit 3` and `gh run watch `. - Sort a failure into infrastructure (runner, network, notary service, an expired certificate) or code (a build, test or check the tag's commit fails). Infrastructure is a re-run; code is a fix on `main`, a pick, and the next number. -- Confirm the result from the source, not the cached site: `gh api -H 'Accept: application/vnd.github.raw' 'repos/TableProApp/TablePro/contents/appcast.xml?ref=feeds' | grep -c ''`, and `gh release view v --json assets -q '.assets | length'`. +- Confirm the result from the source, not the cached site: `gh api -H 'Accept: application/vnd.github.raw' 'repos/TableProApp/TablePro/contents/appcast.xml?ref=gh-pages' | grep -c ''` prints 1, and `gh release view v --json assets -q '.assets | length'`. Run: `RELEASING.md`, What a tag runs, and When a run fails. diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 98e1a758f6..51208dcd97 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,8 @@ self-hosted-runner: # GitHub's Xcode 27 preview image, newer than the label list in actionlint 1.7.12. labels: - xcode-27 +paths: + .github/workflows/*.yml: + ignore: + # actionlint 1.7.12 does not know concurrency.queue yet (rhysd/actionlint#657). + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/actions/setup-xcodegen/action.yml b/.github/actions/setup-xcodegen/action.yml index 96b40c8aea..b803ce9134 100644 --- a/.github/actions/setup-xcodegen/action.yml +++ b/.github/actions/setup-xcodegen/action.yml @@ -19,7 +19,7 @@ runs: echo "XcodeGen $XCODEGEN_VERSION already installed" exit 0 fi - curl -sSLo "$RUNNER_TEMP/xcodegen.zip" \ + curl -fsSLo "$RUNNER_TEMP/xcodegen.zip" \ "https://github.com/yonaskolb/XcodeGen/releases/download/${XCODEGEN_VERSION}/xcodegen.zip" echo "${XCODEGEN_SHA256} $RUNNER_TEMP/xcodegen.zip" | shasum -a 256 -c - unzip -q "$RUNNER_TEMP/xcodegen.zip" -d "$RUNNER_TEMP/xcodegen-dist" diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 5455fb8da1..a9eb89b33c 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -2,108 +2,82 @@ "retainFromPluginKit": 39, "plugins": { "BeancountDriver": { - "name": "Beancount Driver", - "iconName": "beancount-icon" + "name": "Beancount Driver" }, "BigQueryDriverPlugin": { - "name": "BigQuery Driver", - "iconName": "bigquery-icon" + "name": "BigQuery Driver" }, "BigtableDriverPlugin": { - "name": "Bigtable Driver", - "iconName": "bigtable-icon" + "name": "Bigtable Driver" }, "CassandraDriver": { - "name": "Cassandra Driver", - "iconName": "cassandra-icon" + "name": "Cassandra Driver" }, "CloudflareD1DriverPlugin": { - "name": "Cloudflare D1 Driver", - "iconName": "cloudflare-d1-icon" + "name": "Cloudflare D1 Driver" }, "CloudflareR2SQLDriverPlugin": { - "name": "Cloudflare R2 SQL Driver", - "iconName": "cloudflare-r2-sql-icon" + "name": "Cloudflare R2 SQL Driver" }, "DamengDriver": { - "name": "Dameng Driver", - "iconName": "cylinder" + "name": "Dameng Driver" }, "DuckDBDriver": { - "name": "DuckDB Driver", - "iconName": "duckdb-icon" + "name": "DuckDB Driver" }, "DynamoDBDriverPlugin": { - "name": "DynamoDB Driver", - "iconName": "dynamodb-icon" + "name": "DynamoDB Driver" }, "ElasticsearchDriverPlugin": { - "name": "Elasticsearch Driver", - "iconName": "elasticsearch-icon" + "name": "Elasticsearch Driver" }, "EtcdDriverPlugin": { - "name": "etcd Driver", - "iconName": "etcd-icon" + "name": "etcd Driver" }, "FirebaseDriverPlugin": { - "name": "Firebase Driver", - "iconName": "firebase-icon" + "name": "Firebase Driver" }, "HanaDriver": { - "name": "SAP HANA Driver", - "iconName": "cylinder" + "name": "SAP HANA Driver" }, "KafkaDriverPlugin": { - "name": "Kafka Driver", - "iconName": "kafka-icon" + "name": "Kafka Driver" }, "LibSQLDriverPlugin": { - "name": "libSQL / Turso Driver", - "iconName": "libsql-icon" + "name": "libSQL / Turso Driver" }, "MSSQLDriver": { - "name": "MSSQL Driver", - "iconName": "mssql-icon" + "name": "MSSQL Driver" }, "MongoDBDriver": { - "name": "MongoDB Driver", - "iconName": "mongodb-icon" + "name": "MongoDB Driver" }, "OracleDriver": { - "name": "Oracle Driver", - "iconName": "oracle-icon" + "name": "Oracle Driver" }, "ParquetExport": { - "name": "Parquet Export", - "iconName": "square.grid.3x3" + "name": "Parquet Export" }, "SnowflakeDriverPlugin": { - "name": "Snowflake Driver", - "iconName": "snowflake-icon" + "name": "Snowflake Driver" }, "SpannerDriverPlugin": { - "name": "Spanner Driver", - "iconName": "spanner-icon" + "name": "Spanner Driver" }, "SurrealDBDriverPlugin": { - "name": "SurrealDB Driver", - "iconName": "surrealdb-icon" + "name": "SurrealDB Driver" }, "TeradataDriver": { - "name": "Teradata Driver", - "iconName": "teradata-icon" + "name": "Teradata Driver" }, "TrinoDriverPlugin": { - "name": "Trino Driver", - "iconName": "trino-icon" + "name": "Trino Driver" }, "TypesenseDriverPlugin": { - "name": "Typesense Driver", - "iconName": "typesense-icon" + "name": "Typesense Driver" }, "WeaviateDriverPlugin": { - "name": "Weaviate Driver", - "iconName": "weaviate-icon" + "name": "Weaviate Driver" } } } diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index f247191350..a4c0c8d175 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,6 +1,6 @@ name: Pages -# Deploys the feeds branch to updates.tablepro.app. A branch-source Pages site would not do: a push +# Deploys the gh-pages branch to updates.tablepro.app. A branch-source Pages site would not do: a push # made with GITHUB_TOKEN starts no Pages build. Dispatch it at a v* tag, which github-pages requires. on: @@ -18,10 +18,11 @@ jobs: environment: name: github-pages url: ${{ steps.deploy.outputs.page_url }} - # Never cancelled. A newer pending deploy replaces an older pending one, and both deploy the feeds head. + # Deploys wait in turn instead of replacing each other, so a caller's later jobs (release.yml notify) always run. concurrency: group: pages cancel-in-progress: false + queue: max permissions: contents: read pages: write @@ -29,7 +30,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: feeds + ref: gh-pages persist-credentials: false - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1400bacf6b..87b29eb650 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,18 +1,12 @@ name: Release # A v* tag builds, signs and notarizes the app and every registry plugin, publishes the GitHub -# release, then adds it to the feeds branch and deploys that to updates.tablepro.app. +# release, then adds it to the gh-pages branch and deploys that to updates.tablepro.app. on: push: tags: ['v*'] -# Never cancelled: the build jobs hold notary submissions, and every step after them is idempotent, -# so a re-run of the same tag finishes what the first run left. -concurrency: - group: release-${{ github.ref }} - cancel-in-progress: false - permissions: contents: read @@ -212,6 +206,8 @@ jobs: attestations: write env: TAG: ${{ needs.verify.outputs.tag }} + BUILD: ${{ needs.verify.outputs.build }} + CHANNEL: ${{ needs.verify.outputs.channel }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -231,19 +227,43 @@ jobs: name: notes path: notes - # A published release is immutable, so every asset goes on the draft and is checked there first. - - name: Draft the release - id: draft - run: python3 scripts/release/publish.py draft "$TAG" --notes notes/body.md dist/* >> "$GITHUB_OUTPUT" + - name: Check for a published release + id: release + run: | + if gh api "repos/{owner}/{repo}/releases/tags/$TAG" --silent 2> /dev/null; then + echo published=true >> "$GITHUB_OUTPUT" + fi - name: Attest the assets - if: steps.draft.outputs.state == 'draft' - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + if: steps.release.outputs.published != 'true' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 with: subject-path: dist/* + # gh release create drafts, attaches every asset, then publishes: the order an immutable release needs. - name: Publish the release - run: python3 scripts/release/publish.py release "$TAG" + if: steps.release.outputs.published != 'true' + run: | + set -euo pipefail + # A draft a killed attempt left would shadow the release in gh's lookups by tag. + if [ "$(gh release view "$TAG" --json isDraft --jq .isDraft 2> /dev/null)" = true ]; then + gh release delete "$TAG" --yes + fi + flags=(--prerelease --latest=false) + if [ "$CHANNEL" = stable ]; then + latest="$(gh release view --json tagName --jq .tagName)" + latest_build="$(python3 scripts/release/version.py parse "$latest" 2> /dev/null | sed -n 's/^build=//p' || true)" + flags=(--latest="$([ "$BUILD" -gt "${latest_build:-0}" ] && echo true || echo false)") + fi + gh release create "$TAG" --verify-tag --title "$TAG" --notes-file notes/body.md "${flags[@]}" dist/* + + - name: Verify the release + run: | + set -euo pipefail + gh release verify "$TAG" + for asset in dist/*; do + gh release verify-asset "$TAG" "$asset" + done feed: name: Feed @@ -251,15 +271,24 @@ jobs: runs-on: macos-latest environment: release timeout-minutes: 20 + concurrency: + group: gh-pages + queue: max permissions: contents: write + attestations: read env: TAG: ${{ needs.verify.outputs.tag }} - BUILD: ${{ needs.verify.outputs.build }} - CHANNEL: ${{ needs.verify.outputs.channel }} CRITICAL: ${{ needs.verify.outputs.critical }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: gh-pages + path: build/gh-pages - uses: ./.github/actions/setup-sparkle @@ -276,8 +305,17 @@ jobs: set -euo pipefail flags=() [ "$CRITICAL" != true ] || flags+=(--critical) - python3 scripts/release/feed.py add "$TAG" --build "$BUILD" --channel "$CHANNEL" \ - --notes notes/feed.md ${flags[@]+"${flags[@]}"} + python3 scripts/release/feed.py add "$TAG" --site build/gh-pages --notes notes/feed.md \ + ${flags[@]+"${flags[@]}"} + + - name: Push gh-pages + working-directory: build/gh-pages + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add appcast.xml plugins/v3.json + git diff --cached --quiet || { git commit -m "release: $TAG" && git push origin HEAD:gh-pages; } pages: name: Pages @@ -312,7 +350,6 @@ jobs: env: TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} - TELEGRAM_TOPIC_ID: ${{ secrets.TELEGRAM_TOPIC_ID }} run: | set -euo pipefail { @@ -323,8 +360,5 @@ jobs: fi printf '\n%s/%s/releases/tag/%s\n' "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$TAG" } > msg.txt - topic=() - [ -z "$TELEGRAM_TOPIC_ID" ] || topic=(--data-urlencode "message_thread_id=$TELEGRAM_TOPIC_ID") curl -fsS --retry 3 -o /dev/null --data-urlencode "chat_id=$TELEGRAM_CHAT_ID" \ - --data-urlencode text@msg.txt ${topic[@]+"${topic[@]}"} \ - "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" + --data-urlencode text@msg.txt "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 2d021b5f93..eb6da250eb 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -83,7 +83,7 @@ jobs: - name: Check every script sources the libraries it calls into run: python3 scripts/ci/check-lib-sourcing.py - - name: Check the plugin manifest against the plugin classes + - name: Check plugin-registry.json against the project.yml plugin aggregates run: python3 scripts/ci/check-plugin-manifest.py # The app registers a plugin from these keys without loading it, and loads one without them at launch. diff --git a/.github/workflows/withdraw.yml b/.github/workflows/withdraw.yml index a728e5100b..7a70c579d8 100644 --- a/.github/workflows/withdraw.yml +++ b/.github/workflows/withdraw.yml @@ -21,6 +21,9 @@ jobs: runs-on: macos-latest environment: release timeout-minutes: 20 + concurrency: + group: gh-pages + queue: max permissions: contents: write env: @@ -28,16 +31,35 @@ jobs: GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: gh-pages + path: build/gh-pages - uses: ./.github/actions/setup-sparkle - name: Remove the version from the feed and the plugin manifest env: SPARKLE_PRIVATE_KEY: ${{ secrets.SPARKLE_PRIVATE_KEY }} - run: python3 scripts/release/feed.py withdraw "$VERSION" + run: python3 scripts/release/feed.py withdraw "$VERSION" --site build/gh-pages + + - name: Push gh-pages + working-directory: build/gh-pages + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add appcast.xml plugins/v3.json + git diff --cached --quiet || { git commit -m "withdraw: v$VERSION" && git push origin HEAD:gh-pages; } - name: Mark the highest stable as Latest - run: python3 scripts/release/publish.py latest + run: | + set -euo pipefail + latest="$(python3 scripts/release/feed.py latest --site build/gh-pages)" + gh release edit "v$latest" --latest pages: name: Pages diff --git a/RELEASING.md b/RELEASING.md index f138a6543f..542b2c9479 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,7 +9,7 @@ This runbook covers the macOS app, its registry plugins and the static libraries - Sparkle never downgrades: with the toggle off, a beta install keeps its beta until a stable passes it. So a beta never writes storage the previous stable cannot read (`.claude/rules/data-sync-storage.md`). - A stable rolls out in phases of 21600 s. A beta is not phased, and on GitHub it is a prerelease. Latest goes only to the highest stable, so a patch on an older line publishes with `--latest=false`. - Critical means a security fix or data loss, nothing else. It is ` [CRITICAL]` on the version's `CHANGELOG.md` heading, never on a beta. Every later stable item carries the highest critical build forward (`sparkle:criticalUpdate sparkle:version`). -- CI writes the orphan `feeds` branch, which holds `appcast.xml` and the v3 plugin manifest (`https://updates.tablepro.app/plugins/v3.json`), and `pages.yml` deploys it to `updates.tablepro.app`. The appcast is EdDSA-signed, and the app requires that (`SURequireSignedFeed`, `SUVerifyUpdateBeforeExtraction`). +- CI writes the orphan `gh-pages` branch, which holds `appcast.xml` and the v3 plugin manifest (`https://updates.tablepro.app/plugins/v3.json`), and `pages.yml` deploys it to `updates.tablepro.app`. The appcast is EdDSA-signed, and the app requires that (`SURequireSignedFeed`, `SUVerifyUpdateBeforeExtraction`). - The legacy `appcast.xml` on `main` is the `SUFeedURL` of apps up to 0.81 (`https://raw.githubusercontent.com/TableProApp/TablePro/main/appcast.xml`), and `plugins.json` in TableProApp/plugins is their registry. No tag writes either. The tablepro.app download button and the Homebrew cask `tablepro` read the same appcast. v0.82.0 is the one bridge. Once it publishes, TableProApp/web moves to the new feed. Then a PR copies the v0.82.0 feed item verbatim into the legacy `appcast.xml`, a PR in TableProApp/plugins points `plugins.json` at its plugin zips, and a Homebrew/homebrew-cask PR moves `tablepro` to `TablePro-#{version}.dmg`, with livecheck on the new feed's items that have no channel. Both files then freeze. ## Versions @@ -84,18 +84,18 @@ python3 scripts/release/notes.py docs X.Y.Z --ref vX.Y.Z --mdx ../tablepro-docs/ A `v*` tag runs `release.yml` and nothing else. A run is never cancelled, and every job reads the version, build and channel from `verify`. -- `verify`: `version.py check` refuses a tag that breaks the grammar or the bounds; that `origin/release/` does not contain; whose previous tag on the line is not its ancestor; a stable X.Y.0 with no published beta, or a beta after its X.Y.0; a `CHANGELOG.md` missing the stable's heading, or a published stable of this or the previous line below it. Then it needs macOS Tests, iOS and Package Tests and Release Build green on the tag commit or the nearest first-parent commit that ran them, waiting on a pending run. Last, it writes the notes. +- `verify`: `version.py check` refuses a tag that breaks the grammar or the bounds; that `origin/release/` does not contain; whose previous tag on the line is not its ancestor; a stable X.Y.0 with no published beta, or a beta after its X.Y.0; a `CHANGELOG.md` missing the stable's heading, or a published stable of this or the previous line below it; a stable whose tag still holds fragments in `changelog.d/`. Then it needs macOS Tests, iOS and Package Tests and Release Build green on the tag commit or the nearest first-parent commit that ran them, waiting on a pending run. Last, it writes the notes. - `app` (`signing`): a GA Xcode for a stable (`setup-xcode` `require-release`), the CloudKit Production schema check, `build.sh app` (archive, export with `scripts/release/ExportOptions.plist`, universal and plugin load checks), `notarize.sh` on the app, `package.sh app`, `notarize.sh` on the DMG. - `plugins` (`signing`): the Dameng and HANA bridges, `build.sh plugins` (each target `registry.py targets` prints, universal, signed with Developer ID, the hardened runtime and a timestamp), one notary submission for all, `package.sh plugins` (the zips, described as the manifest will read them). -- `github-release`: `publish.py draft` puts every asset on a draft and checks each digest, `attest-build-provenance` attests them, `publish.py release` publishes. -- `feed` (`release`): `feed.py add` downloads the archive and the zips from the release, checks their digests, runs `generate_appcast` and `registry.py add`, runs `feed.py check`, and pushes `feeds`, retrying from a fresh fetch up to 3 times. -- `pages` deploys `feeds`. `notify`, stable only and allowed to fail, posts the lead to Telegram. +- `github-release`: `actions/attest` attests `dist/*`; `gh release create` drafts, uploads and publishes; `gh release verify` and `verify-asset` check the published release. +- `feed` (`release`): `feed.py add` checks the published appcast's signature, downloads the archive and the zips from the release, checks each with `gh release verify-asset`, runs `generate_appcast` and `registry.add`, checks the new item, and the job pushes `gh-pages`. The feed and withdraw jobs share the concurrency group `gh-pages` with `queue: max`. +- `pages` deploys `gh-pages`. `notify`, stable only and allowed to fail, posts the lead to Telegram. - Assets: `TablePro-.dmg`, `TablePro-.tar.xz` (the Sparkle archive), `TablePro--dSYMs.zip`, `TablePro--Plugins-dSYMs.zip` and one `-.zip` per registry plugin. ## When a run fails - Tags never move. Before `github-release` publishes, an infrastructure failure is a re-run of the failed jobs; a code failure is fixed on `main`, picked to the line, and ships as the next number. -- After it publishes: **Re-run failed jobs**. Every later step skips what is done: a published release, or a feed item or manifest release with the same build and sha256. +- After it publishes: **Re-run failed jobs**. Every later step skips what is done: a published release, a feed item with the same version and build, or a manifest release with the same build and sha256. - A stable that never published: the next release PR runs `git revert ` before `notes.py assemble`, and the leftover draft goes (`gh release delete vX.Y.Z`). - No GA Xcode on the runner image fails a stable in `app`. The fix is a PR pointing `.github/actions/setup-xcode` at a GA Xcode on an image that has one. Betas still ship. @@ -113,19 +113,20 @@ gh workflow run withdraw.yml --ref v -f version=X.Y.Z | Environment | Refs | Secrets | |---|---|---| | `signing` | tags `v*` | `CERTIFICATES_P12`, `CERTIFICATES_PASSWORD`, `PROVISIONING_PROFILE`, `ASC_KEY_ID`, `ASC_ISSUER_ID`, `ASC_KEY_P8`, `ANALYTICS_HMAC_SECRET`, `CLOUDKIT_MANAGEMENT_TOKEN` | -| `release` | tags `v*` | `SPARKLE_PRIVATE_KEY`, `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID`, `TELEGRAM_TOPIC_ID` | +| `release` | tags `v*` | `SPARKLE_PRIVATE_KEY`, `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID` | | `github-pages` | tags `v*` | none | - No repo-level secret or deploy key is left, and no environment has required reviewers: admin-only, immutable tags are the gate. Rulesets: `.claude/rules/ci.md`, Rulesets. - Notarization uses the App Store Connect API key, which `.github/actions/apple-signing` stores as the `TablePro` notarytool profile. The action warns when the certificate expires within 60 days. - An offline backup of the Sparkle EdDSA key exists and the maintainer holds it. The key never rotates without a signed transition, since every app from 0.82 verifies the feed with it. -- `feeds` takes pushes from `github-actions[bot]` with `GITHUB_TOKEN` and has no required checks; the signed feed proves its content. It was seeded once with the legacy `appcast.xml` and an empty v3 manifest. +- `gh-pages` takes pushes from `github-actions[bot]` with `GITHUB_TOKEN` and has no required checks. `feed.py add` and `withdraw` refuse an `appcast.xml` whose signature does not verify, so a hand edit is re-signed with `sign_update` before it is pushed. +- `updates.tablepro.app` enforces HTTPS (Settings > Pages). ## Plugins - One tag builds the app and every registry plugin, and the plugin zips are assets of the app's release. Each plugin's version and build are the app's. There are no plugin tags. -- `.github/plugin-registry.json` holds `retainFromPluginKit` and the publish set, keyed by target, with each plugin's `name` and `iconName`. Bundled plugins are not in it. -- The v3 manifest lists each plugin's releases: version, build, channel, PluginKit version, `minimumSystemVersion`, URL and sha256. The bundle id and `databaseTypeIds` come from the built bundle. +- `.github/plugin-registry.json` holds `retainFromPluginKit` and the publish set, keyed by target, with each plugin's `name`. Bundled plugins are not in it. +- The v3 manifest lists each plugin's releases: version, build, channel, PluginKit version, `minimumSystemVersion`, URL and sha256. The bundle id, `databaseTypeIds` and `exportFormatIds` come from the built bundle. - Selection: a release in the running build's channel (a `-beta.N` or local build takes beta and stable, any other stable only), with its PluginKit version in the app's range and its `minimumSystemVersion` at or below the running macOS. The highest build wins. Installed registry plugins update after the app does. - Retention per channel, kit and `minimumSystemVersion`: the two highest builds stay; a beta goes once a stable with the same key has a higher build; a kit below `retainFromPluginKit` goes. A release pruned on arrival is a logged no-op. - `retainFromPluginKit` is the kit of the oldest app version still on at least 1% of the machines seen in the last 30 days (license dashboard, App Versions). Check it before each stable and raise it with a normal PR. The manifest keeps the higher of its own value and the tag's, so an older line never lowers it. diff --git a/scripts/ci/check-plugin-manifest.py b/scripts/ci/check-plugin-manifest.py index 971802dce9..38113e0404 100755 --- a/scripts/ci/check-plugin-manifest.py +++ b/scripts/ci/check-plugin-manifest.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -"""Fails when .github/plugin-registry.json, the plugin classes and the project.yml plugin aggregates disagree.""" +"""Fails when .github/plugin-registry.json and the project.yml plugin aggregates disagree.""" import pathlib -import re import sys +import yaml + REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT / "scripts" / "release")) @@ -12,18 +13,6 @@ MANIFEST = REPO_ROOT / ".github" / "plugin-registry.json" PROJECT = REPO_ROOT / "project.yml" -PLUGINS_DIR = REPO_ROOT / "Plugins" - -ICON = re.compile(r"^\s*(?:public |internal |private )?static (?:let|var) iconName(?:\s*:\s*String)?\s*=\s*\"([^\"]+)\"", re.M) - -TOP_LEVEL_KEY = re.compile(r"^([A-Za-z_][\w-]*):") -ENTRY = re.compile(r"^ ([A-Za-z_][\w-]*):\s*$") -INLINE_LIST = re.compile(r"^ (templates|targets):\s*\[(.*)\]\s*$") -BLOCK_LIST = re.compile(r"^ (templates|targets|dependencies):\s*$") -ITEM = re.compile(r"^ - (.*?)\s*$") -ITEM_FIELD = re.compile(r"^ (.*?)\s*$") -DEPENDENCY_TARGET = re.compile(r"^target:\s*(\S+)") -COPY_TO_PLUGINS = re.compile(r"\bcopy:\s*\{?\s*destination:\s*plugins\b") BUNDLED = "BundledPlugins" REGISTRY = "RegistryPlugins" @@ -32,52 +21,13 @@ APP_TARGET = "TablePro" -def parse_project(text): - sections = {"targets": {}, "aggregateTargets": {}} - section = entry = field = None - for raw in text.splitlines(): - line = re.sub(r"\s+#.*$", "", raw) - if not line.strip() or line.lstrip().startswith("#"): - continue - if match := TOP_LEVEL_KEY.match(line): - section, entry, field = match.group(1), None, None - continue - if section not in sections: - continue - if match := ENTRY.match(line): - entry = sections[section].setdefault(match.group(1), {"templates": [], "targets": [], "dependencies": []}) - field = None - elif entry is None: - continue - elif match := INLINE_LIST.match(line): - entry[match.group(1)] += [name.strip() for name in match.group(2).split(",") if name.strip()] - field = None - elif match := BLOCK_LIST.match(line): - field = match.group(1) - elif field and (match := ITEM.match(line)): - entry[field].append([match.group(1)] if field == "dependencies" else match.group(1)) - elif field == "dependencies" and entry["dependencies"] and (match := ITEM_FIELD.match(line)): - entry["dependencies"][-1].append(match.group(1)) - elif not line.startswith(" "): - field = None - return sections - - def embedded_plugins(app): - found = set() - for item in app["dependencies"]: - target = DEPENDENCY_TARGET.match(item[0]) - if target and COPY_TO_PLUGINS.search(" ".join(item[1:])): - found.add(target.group(1)) - return found - - -def publish_set(plugins): - return set(plugins) + return {dependency["target"] for dependency in app.get("dependencies") or [] + if "target" in dependency and (dependency.get("copy") or {}).get("destination") == "plugins"} def aggregate_problems(project, plugins): - targets, aggregates = project["targets"], project["aggregateTargets"] + targets, aggregates = project.get("targets") or {}, project.get("aggregateTargets") or {} problems = [] for name in (BUNDLED, REGISTRY, ALL): if name not in aggregates: @@ -90,19 +40,19 @@ def aggregate_problems(project, plugins): registry = set(aggregates[REGISTRY]["targets"]) bundled = set(aggregates[BUNDLED]["targets"]) - published = publish_set(plugins) + published = set(plugins) for name in sorted(published - registry): problems.append(f"{MANIFEST.name} publishes {name} but {REGISTRY} does not build it") for name in sorted(registry - published): problems.append(f"{REGISTRY} builds {name} but {MANIFEST.name} does not publish it") - embedded = embedded_plugins(targets.get(APP_TARGET, {"dependencies": []})) + embedded = embedded_plugins(targets.get(APP_TARGET) or {}) for name in sorted(embedded - bundled): problems.append(f"the {APP_TARGET} target embeds {name} but {BUNDLED} does not list it") for name in sorted(bundled - embedded): problems.append(f"{BUNDLED} lists {name} but the {APP_TARGET} target does not embed it") - drivers = {name for name, entry in targets.items() if DRIVER_TEMPLATE in entry["templates"]} + drivers = {name for name, entry in targets.items() if DRIVER_TEMPLATE in (entry.get("templates") or [])} for name in sorted(drivers): count = (name in bundled) + (name in registry) if count != 1: @@ -112,55 +62,20 @@ def aggregate_problems(project, plugins): return problems -def plugin_directory(target): - def normalise(name): - return name.lower().replace("driverplugin", "").replace("driver", "").replace("plugin", "") - - wanted = normalise(target) - for path in sorted(PLUGINS_DIR.iterdir()): - if path.is_dir() and normalise(path.name) == wanted: - return path - return None - - -def declared_icon(directory): - for source in sorted(directory.rglob("*.swift")): - found = ICON.search(source.read_text(encoding="utf-8")) - if found: - return found.group(1) - return None - - -def declaration_problems(plugins): - problems = [] - for target, entry in sorted(plugins.items()): - directory = plugin_directory(target) - if directory is None: - problems.append(f"{target}: no plugin directory matches it") - continue - icon = declared_icon(directory) - if icon is None: - problems.append(f"{target}: could not read iconName from {directory.name}") - elif icon != entry["iconName"]: - problems.append(f"{target}: {MANIFEST.name} says iconName {entry['iconName']!r} " - f"but the plugin declares {icon!r}") - return problems - - def main(): try: plugins = registry.load_metadata(MANIFEST)["plugins"] except registry.RegistryError as error: sys.exit(f"::error::{error}") - project = parse_project(PROJECT.read_text(encoding="utf-8")) + project = yaml.safe_load(PROJECT.read_text(encoding="utf-8")) - problems = declaration_problems(plugins) + aggregate_problems(project, plugins) + problems = aggregate_problems(project, plugins) if problems: for problem in problems: print(f"::error::{problem}", file=sys.stderr) - sys.exit(f"\n{len(problems)} problem(s) between {MANIFEST.name}, the plugin classes and {PROJECT.name}") + sys.exit(f"\n{len(problems)} problem(s) between {MANIFEST.name} and {PROJECT.name}") - print(f"{len(plugins)} plugins agree with {MANIFEST.name}, and {PROJECT.name}'s plugin aggregates agree with both") + print(f"{len(plugins)} plugins in {MANIFEST.name} agree with {PROJECT.name}'s plugin aggregates") if __name__ == "__main__": diff --git a/scripts/ci/test_check_plugin_manifest.py b/scripts/ci/test_check_plugin_manifest.py index 7f370e53fb..6674c0da50 100644 --- a/scripts/ci/test_check_plugin_manifest.py +++ b/scripts/ci/test_check_plugin_manifest.py @@ -2,10 +2,11 @@ """Fixture tests for check-plugin-manifest.py.""" import importlib.util -import tempfile import unittest from pathlib import Path +import yaml + SPEC = importlib.util.spec_from_file_location("check_plugin_manifest", Path(__file__).with_name("check-plugin-manifest.py")) check = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(check) @@ -69,11 +70,11 @@ App: all """ -PLUGINS = {"OracleDriver": {"name": "Oracle Driver", "iconName": "oracle-icon"}} +PLUGINS = {"OracleDriver": {"name": "Oracle Driver"}} def problems(project=PROJECT, plugins=None): - return check.aggregate_problems(check.parse_project(project), PLUGINS if plugins is None else plugins) + return check.aggregate_problems(yaml.safe_load(project), PLUGINS if plugins is None else plugins) def edited(old, new): @@ -81,31 +82,11 @@ def edited(old, new): return PROJECT.replace(old, new, 1) -class ParseProjectTests(unittest.TestCase): - def test_reads_inline_and_block_templates(self): - targets = check.parse_project(PROJECT)["targets"] - self.assertEqual(targets["MySQLDriver"]["templates"], ["DriverPlugin", "MySQLLinkage"]) - self.assertEqual(targets["RedisDriver"]["templates"], ["DriverPlugin"]) - self.assertEqual(targets["OracleDriver"]["templates"], ["DriverPlugin"]) - - def test_embedded_plugins_take_both_copy_forms(self): - app = check.parse_project(PROJECT)["targets"]["TablePro"] +class EmbeddedPluginsTests(unittest.TestCase): + def test_takes_both_copy_forms_and_only_the_plugins_destination(self): + app = yaml.safe_load(PROJECT)["targets"]["TablePro"] self.assertEqual(check.embedded_plugins(app), {"MySQLDriver", "RedisDriver"}) - def test_reads_aggregate_targets(self): - aggregates = check.parse_project(PROJECT)["aggregateTargets"] - self.assertEqual(aggregates["BundledPlugins"]["targets"], ["MySQLDriver", "RedisDriver"]) - self.assertEqual(aggregates["RegistryPlugins"]["targets"], ["OracleDriver"]) - self.assertEqual(aggregates["AllPlugins"]["targets"], ["BundledPlugins", "RegistryPlugins"]) - - def test_ignores_template_dependencies(self): - self.assertNotIn("DriverPlugin", check.parse_project(PROJECT)["targets"]) - - -class PublishSetTests(unittest.TestCase): - def test_every_target_in_the_metadata_is_published(self): - self.assertEqual(check.publish_set(PLUGINS), {"OracleDriver"}) - class AggregateProblemsTests(unittest.TestCase): def test_consistent_project_passes(self): @@ -123,7 +104,7 @@ def test_registry_aggregate_target_that_is_not_published(self): ) def test_bundled_plugin_in_the_metadata(self): - plugins = {**PLUGINS, "RedisDriver": {"name": "Redis Driver", "iconName": "redis-icon"}} + plugins = {**PLUGINS, "RedisDriver": {"name": "Redis Driver"}} self.assertEqual( problems(plugins=plugins), ["plugin-registry.json publishes RedisDriver but RegistryPlugins does not build it"], @@ -157,31 +138,5 @@ def test_missing_aggregate(self): self.assertEqual(problems(project), ["project.yml has no RegistryPlugins aggregate target"]) -class DeclarationProblemsTests(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - source = Path(self.temp.name) / "OracleDriverPlugin" / "OraclePlugin.swift" - source.parent.mkdir() - source.write_text('final class OraclePlugin {\n static let iconName = "oracle-icon"\n}\n') - self.saved = check.PLUGINS_DIR - check.PLUGINS_DIR = Path(self.temp.name) - self.addCleanup(setattr, check, "PLUGINS_DIR", self.saved) - - def test_matching_icon_passes(self): - self.assertEqual(check.declaration_problems(PLUGINS), []) - - def test_icon_that_differs_from_the_plugin(self): - plugins = {"OracleDriver": {"name": "Oracle Driver", "iconName": "cylinder"}} - self.assertEqual( - check.declaration_problems(plugins), - ["OracleDriver: plugin-registry.json says iconName 'cylinder' but the plugin declares 'oracle-icon'"], - ) - - def test_target_with_no_plugin_directory(self): - plugins = {"TeradataDriver": {"name": "Teradata Driver", "iconName": "teradata-icon"}} - self.assertEqual(check.declaration_problems(plugins), ["TeradataDriver: no plugin directory matches it"]) - - if __name__ == "__main__": unittest.main() diff --git a/scripts/release/feed.py b/scripts/release/feed.py index 701c3a1dc0..ecfd23df55 100755 --- a/scripts/release/feed.py +++ b/scripts/release/feed.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 import argparse -import hashlib -import json import lzma import os import plistlib @@ -13,28 +11,23 @@ import tarfile import tempfile import xml.etree.ElementTree as ET -from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Callable, Dict, Iterator, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple +from xml.dom import minidom +from xml.parsers.expat import ExpatError import registry import version ROOT = Path(__file__).resolve().parent.parent.parent -SPARKLE = "{http://www.andymatuschak.org/xml-namespaces/sparkle}" -BRANCH = "feeds" -REMOTE_BRANCH = f"refs/remotes/origin/{BRANCH}" +SPARKLE_NAMESPACE = "http://www.andymatuschak.org/xml-namespaces/sparkle" +SPARKLE = "{" + SPARKLE_NAMESPACE + "}" APPCAST = "appcast.xml" MANIFEST = "plugins/v3.json" METADATA = ".github/plugin-registry.json" -PROJECT = "project.yml" -PLUGINKIT_CONFIG = "Configs/PluginKit.xcconfig" +CHANGELOG_URL = "https://docs.tablepro.app/changelog" PHASED_ROLLOUT_SECONDS = 21600 -PUSH_ATTEMPTS = 3 -BOT_NAME = "github-actions[bot]" -BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" -ITEM_TEXT = re.compile(r"[ \t]*.*?[ \t]*\n?", re.DOTALL) INTEGER = re.compile(r"[0-9]+") @@ -51,25 +44,15 @@ class Item: critical_version: Optional[str] url: Optional[str] signature: Optional[str] - length: Optional[str] @property def label(self) -> str: return self.short_version or self.build or "an item" -@dataclass(frozen=True) -class Expected: - release: version.Version - url: str - - def _sparkle_value(element: ET.Element, name: str) -> Optional[str]: child = element.find(SPARKLE + name) - if child is not None: - return (child.text or "").strip() - enclosure = element.find("enclosure") - return enclosure.get(SPARKLE + name) if enclosure is not None else None + return (child.text or "").strip() if child is not None else None def parse_items(data: bytes) -> List[Item]: @@ -84,16 +67,14 @@ def parse_items(data: bytes) -> List[Item]: for element in channel.findall("item"): enclosure = element.find("enclosure") critical = element.find(f".//{SPARKLE}criticalUpdate") - name = element.find(SPARKLE + "channel") items.append(Item( build=_sparkle_value(element, "version"), short_version=_sparkle_value(element, "shortVersionString"), - channel=(name.text or "").strip() if name is not None else None, + channel=_sparkle_value(element, "channel"), is_critical=critical is not None, critical_version=critical.get(SPARKLE + "version") if critical is not None else None, url=enclosure.get("url") if enclosure is not None else None, signature=enclosure.get(SPARKLE + "edSignature") if enclosure is not None else None, - length=enclosure.get("length") if enclosure is not None else None, )) return items @@ -111,66 +92,49 @@ def carried_critical(items: List[Item]) -> Optional[int]: return max(builds) if builds else None -def feed_problems(data: bytes, previous: Optional[bytes] = None, expected: Optional[Expected] = None) -> List[str]: - items = parse_items(data) - problems = [] - if previous is not None and any(item.channel is None for item in parse_items(previous)) \ - and not any(item.channel is None for item in items): - problems.append("the feed no longer has a default-channel item") - - versions: Dict[int, set] = {} +def highest_stable(items: List[Item]) -> Optional[version.Version]: + stables = [] for item in items: - if item.url is None: - problems.append(f"{item.label} has no enclosure") - if not item.signature: - problems.append(f"{item.label}'s enclosure has no sparkle:edSignature") - if item.length is None or not INTEGER.fullmatch(item.length): - problems.append(f"{item.label}'s enclosure has no length") - if item.channel is not None and item.channel != "beta": - problems.append(f"{item.label} is on channel {item.channel!r}; the only channel is beta") - if item.channel == "beta" and item.is_critical: - problems.append(f"{item.label} is a beta marked critical") - if item.build is None or not INTEGER.fullmatch(item.build): - problems.append(f"{item.label}: sparkle:version {item.build!r} is not an integer") - continue - try: - parsed = version.parse_version(item.short_version or "") - except ValueError: - problems.append(f"build {item.build}: shortVersionString {item.short_version!r} is not a release version") + if item.channel is not None: continue - versions.setdefault(int(item.build), set()).add(parsed) + parsed = version.try_parse_tag(f"v{item.short_version}") if item.short_version else None + if parsed is not None and not parsed.is_beta: + stables.append(parsed) + return max(stables, key=lambda candidate: candidate.build) if stables else None + + +def new_item_problems(items: List[Item], release: version.Version, url: str) -> List[str]: + channel = "beta" if release.is_beta else None + found = [item for item in items if item.build == str(release.build) and item.short_version == str(release) + and item.channel == channel and item.url == url] + if len(found) != 1: + return [f"{APPCAST} has {len(found)} {release.channel} items for {release} ({release.build}) at {url}, not one"] + if not found[0].signature: + return [f"{release}'s enclosure has no sparkle:edSignature; " + "SPARKLE_PRIVATE_KEY does not match the archive's SUPublicEDKey"] + return [] - ordered = [] - for build, found in sorted(versions.items()): - if len(found) > 1: - problems.append(f"build {build} carries {', '.join(sorted(str(each) for each in found))}") - else: - ordered.append((build, found.pop())) - for (low_build, low), (high_build, high) in zip(ordered, ordered[1:]): - if not low.build < high.build: - problems.append(f"{high} has the higher build ({high_build} over {low_build}) but not the higher version") - - if expected is not None: - release = expected.release - channel = "beta" if release.is_beta else None - if not any(item.build == str(release.build) and item.short_version == str(release) - and item.channel == channel and item.url == expected.url for item in items): - problems.append(f"no {release.channel} item for {release} ({release.build}) at {expected.url}") - return problems - - -def remove_items(text: str, short_version: str) -> Tuple[str, int]: - items = parse_items(text.encode("utf-8")) - spans = list(ITEM_TEXT.finditer(text)) - if len(spans) != len(items): - raise FeedError(f"{APPCAST} has {len(items)} items but {len(spans)} blocks in its text; edit it by hand") - doomed = [span for span, item in zip(spans, items) if item.short_version == short_version] - for span in reversed(doomed): - text = text[:span.start()] + text[span.end():] - left = [item for item in items if item.short_version != short_version] - if parse_items(text.encode("utf-8")) != left: - raise FeedError(f"removing {short_version} from the text of {APPCAST} changed other items; edit it by hand") - return text, len(doomed) + +def _text(element: minidom.Element) -> str: + return "".join(node.data for node in element.childNodes if node.nodeType == node.TEXT_NODE).strip() + + +def remove_items(data: bytes, short_version: str) -> Tuple[bytes, int]: + try: + document = minidom.parseString(data) + except ExpatError as error: + raise FeedError(f"{APPCAST} is not well-formed XML: {error}") from error + channels = document.getElementsByTagName("channel") + if not channels: + raise FeedError(f"{APPCAST} has no rss channel") + channel = channels[0] + doomed = [node for node in channel.childNodes + if node.nodeType == node.ELEMENT_NODE and node.tagName == "item" + and any(_text(each) == short_version + for each in node.getElementsByTagNameNS(SPARKLE_NAMESPACE, "shortVersionString"))] + for item in doomed: + channel.removeChild(item) + return document.toxml(encoding="utf-8", standalone=True), len(doomed) def appcast_has(items: List[Item], release: version.Version) -> bool: @@ -188,7 +152,7 @@ def appcast_arguments(stage: Path, tag: str, release: version.Version, critical_ "--ed-key-file", "-", "--embed-release-notes", "--maximum-deltas", "0", "--versions", str(release.build), "--download-url-prefix", f"https://github.com/{repository}/releases/download/{tag}/", - "--full-release-notes-url", f"https://github.com/{repository}/releases/tag/{tag}", + "--full-release-notes-url", CHANGELOG_URL, ] if release.is_beta: command += ["--channel", "beta"] @@ -199,18 +163,6 @@ def appcast_arguments(stage: Path, tag: str, release: version.Version, critical_ return command + [str(stage)] -def pluginkit_version(root: Path) -> int: - path = root / PLUGINKIT_CONFIG - try: - text = path.read_text(encoding="utf-8") - except OSError as error: - raise FeedError(f"{PLUGINKIT_CONFIG}: {error}") from error - found = re.search(r"^TABLEPRO_PLUGINKIT_VERSION\s*=\s*([0-9]+)\s*$", text, re.MULTILINE) - if found is None: - raise FeedError(f"{PLUGINKIT_CONFIG} sets no TABLEPRO_PLUGINKIT_VERSION") - return int(found.group(1)) - - def check_archive(path: Path, release: version.Version) -> None: name = "TablePro.app/Contents/Info.plist" try: @@ -228,14 +180,6 @@ def check_archive(path: Path, release: version.Version) -> None: raise FeedError(f"{path.name} holds {found[0]} ({found[1]}), not {release} ({release.build})") -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - class Releases: def __init__(self, repository: str, run: Callable = subprocess.run): self.repository = repository @@ -247,78 +191,15 @@ def _gh(self, *args: str) -> str: raise FeedError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") return result.stdout - def digests(self, tag: str) -> Dict[str, str]: - release = json.loads(self._gh("release", "view", tag, "--json", "isDraft,assets")) - if release.get("isDraft"): - raise FeedError(f"{tag} is a draft; the feed only points at a published release") - return {asset["name"]: asset.get("digest") or "" for asset in release.get("assets", [])} - def download(self, tag: str, names: List[str], directory: Path) -> None: patterns = [argument for name in names for argument in ("--pattern", name)] self._gh("release", "download", tag, "--dir", str(directory), "--clobber", *patterns) + missing = [name for name in names if not (directory / name).is_file()] + if missing: + raise FeedError(f"{tag} has no {', '.join(missing)}") - -def fetch_assets(releases: Releases, tag: str, names: List[str], directory: Path) -> None: - wanted = [name for name in names if not (directory / name).is_file()] - if not wanted: - return - digests = releases.digests(tag) - missing = [name for name in wanted if name not in digests] - if missing: - raise FeedError(f"{tag} has no {', '.join(missing)}") - directory.mkdir(parents=True, exist_ok=True) - releases.download(tag, wanted, directory) - for name in wanted: - path = directory / name - if not path.is_file(): - raise FeedError(f"gh did not download {name} from {tag}") - found = f"sha256:{_sha256(path)}" - if digests[name] != found: - path.unlink() - raise FeedError(f"{name} downloaded as {found}, but {tag} lists {digests[name] or 'no digest'}") - - -def _git(root: Path, *args: str, run: Callable = subprocess.run, check: bool = True, - env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess: - result = run(["git", "-C", str(root), *args], capture_output=True, text=True, - env={**os.environ, **env} if env else None) - if check and result.returncode != 0: - raise FeedError(f"git {' '.join(args)}: {result.stderr.strip()}") - return result - - -@contextmanager -def feeds_worktree(root: Path, run: Callable = subprocess.run) -> Iterator[Path]: - _git(root, "fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{BRANCH}:{REMOTE_BRANCH}", run=run) - parent = Path(tempfile.mkdtemp(prefix="feeds-")) - tree = parent / BRANCH - _git(root, "worktree", "add", "--quiet", "--detach", str(tree), REMOTE_BRANCH, run=run) - try: - yield tree - finally: - _git(root, "worktree", "remove", "--force", str(tree), run=run, check=False) - shutil.rmtree(parent, ignore_errors=True) - - -def commit(tree: Path, message: str, run: Callable = subprocess.run) -> bool: - paths = [path for path in (APPCAST, MANIFEST) if (tree / path).exists()] - _git(tree, "add", "--", *paths, run=run) - if _git(tree, "diff", "--cached", "--quiet", run=run, check=False).returncode == 0: - return False - identity = {f"GIT_{role}_{field}": value for role in ("AUTHOR", "COMMITTER") - for field, value in (("NAME", BOT_NAME), ("EMAIL", BOT_EMAIL))} - _git(tree, "commit", "--quiet", "-m", message, run=run, env=identity) - return True - - -def push(tree: Path, run: Callable = subprocess.run) -> bool: - result = _git(tree, "push", "--quiet", "origin", f"HEAD:refs/heads/{BRANCH}", run=run, check=False) - if result.returncode == 0: - return True - # A rejected fast-forward or a lost ref lock means another run pushed first. - if "[rejected]" in result.stderr or ("[remote rejected]" in result.stderr and "cannot lock ref" in result.stderr): - return False - raise FeedError(f"git push origin {BRANCH}: {result.stderr.strip()}") + def verify(self, tag: str, path: Path) -> str: + return self._gh("release", "verify-asset", tag, str(path)).strip() def _tool(tools: Path, name: str, key: str, *args: str, run: Callable = subprocess.run) -> str: @@ -341,133 +222,104 @@ def verify_signature(path: Path, key: str, tools: Path, run: Callable = subproce return [] -def _refuse(problems: List[str]) -> None: +def _refuse(problems: List[str], reason: str = "the feed fails its checks") -> None: if problems: - raise FeedError("the feed fails its checks, so nothing was pushed:\n " + "\n ".join(problems)) + raise FeedError(f"{reason}, so nothing was written:\n " + "\n ".join(problems)) -def _publish(tree: Path, message: str, run: Callable, log: Callable[[str], None]) -> Optional[bool]: - if not commit(tree, message, run): - log(f"origin/{BRANCH} already has everything from '{message}'") - return False - if push(tree, run): - log(f"pushed '{message}' to origin/{BRANCH}") - return True - return None +def _published_appcast(site: Path, key: str, tools: Path, run: Callable) -> Path: + appcast = site / APPCAST + if not appcast.is_file(): + raise FeedError(f"{site} has no {APPCAST}") + _refuse(verify_signature(appcast, key, tools, run), + f"the published {APPCAST}'s signature does not verify; it changed outside CI") + return appcast -def add(root: Path, tag: str, build: int, channel: str, notes: Path, critical: bool, tools: Path, key: str, +def add(root: Path, site: Path, tag: str, notes: Path, critical: bool, tools: Path, key: str, releases: Releases, repository: str, run: Callable = subprocess.run, log: Callable[[str], None] = print) -> bool: release = version.parse_tag(tag) - if build != release.build or channel != release.channel: - raise FeedError(f"{tag} is build {release.build} on {release.channel}, not build {build} on {channel}") if critical and release.is_beta: raise FeedError(f"{tag} is a beta, and a beta is never critical") if not notes.is_file(): raise FeedError(f"{notes} does not exist") + appcast = _published_appcast(site, key, tools, run) metadata = registry.load_metadata(root / METADATA) - bundled = registry.bundled_targets((root / PROJECT).read_text(encoding="utf-8")) - plugins = [f"{target}-{release}.zip" for target in registry.publish_set(metadata, bundled)] - kit = pluginkit_version(root) + plugins = [f"{target}-{release}.zip" for target in registry.publish_set(metadata)] + kit = registry.pluginkit_version(root) archive = f"TablePro-{release}.tar.xz" url_base = f"https://github.com/{repository}/releases/download/{tag}/" - expected = Expected(release, url_base + archive) + items = parse_items(appcast.read_bytes()) + has_item = appcast_has(items, release) with tempfile.TemporaryDirectory(prefix="feed-") as scratch: downloads = Path(scratch) / "downloads" - entries = None - for attempt in range(1, PUSH_ATTEMPTS + 1): - wanted: Optional[Expected] = expected - with feeds_worktree(root, run) as tree: - appcast = tree / APPCAST - if not appcast.is_file(): - raise FeedError(f"origin/{BRANCH} has no {APPCAST}; seed the branch first (RELEASING.md)") - previous = appcast.read_bytes() - items = parse_items(previous) - if appcast_has(items, release): - log(f"{APPCAST} already has {release} ({release.build})") - else: - fetch_assets(releases, tag, [archive], downloads) - check_archive(downloads / archive, release) - stage = Path(scratch) / f"stage-{attempt}" - stage.mkdir() - shutil.copy2(downloads / archive, stage / archive) - shutil.copy2(notes, stage / f"TablePro-{release}.md") - shutil.copy2(appcast, stage / APPCAST) - critical_build = release.build if critical else None - if critical_build is None and not release.is_beta: - critical_build = carried_critical(items) - arguments = appcast_arguments(stage, tag, release, critical_build, repository) - log(_tool(tools, "generate_appcast", key, *arguments, run=run)) - if appcast_has(parse_items((stage / APPCAST).read_bytes()), release): - shutil.copy2(stage / APPCAST, appcast) - # generate_appcast signs the feed only once an archive sets SURequireSignedFeed. - log(sign(appcast, key, tools, run)) - else: - # The archive matched the tag, so only Sparkle's per-branch limit left it out. - log(f"generate_appcast keeps newer items on {release}'s branch, so {APPCAST} stays as it is") - wanted = None - if entries is None: - fetch_assets(releases, tag, plugins, downloads) - entries = registry.describe(downloads, str(release), release.build, kit, metadata, bundled) - registry.add(tree / MANIFEST, entries, release.channel, url_base, metadata["retainFromPluginKit"], log) - _refuse(feed_problems(appcast.read_bytes(), previous, wanted) - + verify_signature(appcast, key, tools, run)) - pushed = _publish(tree, f"release: {tag}", run, log) - if pushed is not None: - return pushed - log(f"origin/{BRANCH} moved (attempt {attempt} of {PUSH_ATTEMPTS}); starting again from a fresh fetch") - raise FeedError(f"origin/{BRANCH} moved during each of {PUSH_ATTEMPTS} attempts") - - -def withdraw(root: Path, text: str, tools: Path, key: str, run: Callable = subprocess.run, + downloads.mkdir() + names = plugins if has_item else [archive, *plugins] + releases.download(tag, names, downloads) + for name in names: + log(releases.verify(tag, downloads / name)) + entries = registry.describe(downloads, str(release), release.build, kit, metadata) + + changed = False + if has_item: + log(f"{APPCAST} already has {release} ({release.build})") + else: + check_archive(downloads / archive, release) + stage = Path(scratch) / "stage" + stage.mkdir() + (downloads / archive).rename(stage / archive) + shutil.copy2(notes, stage / f"TablePro-{release}.md") + shutil.copy2(appcast, stage / APPCAST) + critical_build = release.build if critical else None + if critical_build is None and not release.is_beta: + critical_build = carried_critical(items) + arguments = appcast_arguments(stage, tag, release, critical_build, repository) + log(_tool(tools, "generate_appcast", key, *arguments, run=run)) + staged = parse_items((stage / APPCAST).read_bytes()) + if appcast_has(staged, release): + _refuse(new_item_problems(staged, release, url_base + archive) + + verify_signature(stage / APPCAST, key, tools, run)) + shutil.copy2(stage / APPCAST, appcast) + changed = True + else: + # The archive matched the tag, so only Sparkle's per-branch limit left it out. + log(f"generate_appcast keeps newer items on {release}'s branch, so {APPCAST} stays as it is") + manifest = site / MANIFEST + if registry.add(manifest, entries, release.channel, url_base, metadata["retainFromPluginKit"], log): + changed = True + return changed + + +def withdraw(site: Path, text: str, tools: Path, key: str, run: Callable = subprocess.run, log: Callable[[str], None] = print) -> bool: release = version.parse_version(text) - for attempt in range(1, PUSH_ATTEMPTS + 1): - with feeds_worktree(root, run) as tree: - appcast = tree / APPCAST - previous = appcast.read_bytes() - edited, removed = remove_items(previous.decode("utf-8"), str(release)) - if removed: - if any(item.channel is None for item in parse_items(previous)) \ - and not any(item.channel is None for item in parse_items(edited.encode("utf-8"))): - raise FeedError(f"withdrawing {release} would leave the feed with no default-channel item") - appcast.write_text(edited, encoding="utf-8") - log(sign(appcast, key, tools, run)) - log(f"removed {removed} {release} item(s) from {APPCAST}") - else: - log(f"{APPCAST} has no {release} item") - if (tree / MANIFEST).exists(): - registry.withdraw(tree / MANIFEST, str(release), log) - _refuse(feed_problems(appcast.read_bytes(), previous) + verify_signature(appcast, key, tools, run)) - pushed = _publish(tree, f"withdraw: v{release}", run, log) - if pushed is not None: - return pushed - log(f"origin/{BRANCH} moved (attempt {attempt} of {PUSH_ATTEMPTS}); starting again from a fresh fetch") - raise FeedError(f"origin/{BRANCH} moved during each of {PUSH_ATTEMPTS} attempts") - - -def check(root: Path, tools: Path, key: str, run: Callable = subprocess.run) -> List[str]: - with feeds_worktree(root, run) as tree: - appcast = tree / APPCAST - if not appcast.is_file(): - return [f"origin/{BRANCH} has no {APPCAST}"] - problems = feed_problems(appcast.read_bytes()) + verify_signature(appcast, key, tools, run) - try: - if registry.load_manifest(tree / MANIFEST) is None: - problems.append(f"origin/{BRANCH} has no {MANIFEST}") - except registry.RegistryError as error: - problems.append(str(error)) - return problems - - -def published_items(root: Path, run: Callable = subprocess.run) -> List[Item]: - _git(root, "fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{BRANCH}:{REMOTE_BRANCH}", run=run) - result = run(["git", "-C", str(root), "show", f"{REMOTE_BRANCH}:{APPCAST}"], capture_output=True) - if result.returncode != 0: - raise FeedError(f"origin/{BRANCH} has no {APPCAST}: {result.stderr.decode(errors='replace').strip()}") - return parse_items(result.stdout) + appcast = _published_appcast(site, key, tools, run) + previous = appcast.read_bytes() + edited, removed = remove_items(previous, str(release)) + changed = False + if removed: + if any(item.channel is None for item in parse_items(previous)) \ + and not any(item.channel is None for item in parse_items(edited)): + raise FeedError(f"withdrawing {release} would leave the feed with no default-channel item") + appcast.write_bytes(edited) + log(sign(appcast, key, tools, run)) + log(f"removed {removed} {release} item(s) from {APPCAST}") + changed = True + else: + log(f"{APPCAST} has no {release} item") + manifest = site / MANIFEST + if manifest.exists() and registry.withdraw(manifest, str(release), log): + changed = True + return changed + + +def latest(site: Path) -> version.Version: + highest = highest_stable(parse_items((site / APPCAST).read_bytes())) + if highest is None: + raise FeedError(f"{site / APPCAST} has no stable item") + return highest def _environment() -> Tuple[Path, str]: @@ -477,40 +329,34 @@ def _environment() -> Tuple[Path, str]: raise FeedError("SPARKLE_BIN is not set; run the setup-sparkle action first") if not key: raise FeedError("SPARKLE_PRIVATE_KEY is not set") - for name in ("generate_appcast", "sign_update"): - if not os.access(Path(tools) / name, os.X_OK): - raise FeedError(f"{tools} has no {name}") return Path(tools), key def main(argv: Optional[List[str]] = None) -> int: - parser = argparse.ArgumentParser(description="Writes the appcast and the plugin manifest on the feeds branch.") + parser = argparse.ArgumentParser(description="Edits the appcast and the plugin manifest in a gh-pages checkout.") commands = parser.add_subparsers(dest="command", required=True) command = commands.add_parser("add", help="add a published release to the feed and the manifest") command.add_argument("tag") - command.add_argument("--build", type=int, required=True) - command.add_argument("--channel", choices=registry.CHANNELS, required=True) + command.add_argument("--site", type=Path, required=True) command.add_argument("--notes", type=Path, required=True) command.add_argument("--critical", action="store_true") - commands.add_parser("withdraw", help="remove a version from the feed and the manifest").add_argument("version") - commands.add_parser("check", help="check the published feed") + command = commands.add_parser("withdraw", help="remove a version from the feed and the manifest") + command.add_argument("version") + command.add_argument("--site", type=Path, required=True) + command = commands.add_parser("latest", help="print the highest stable version in the feed") + command.add_argument("--site", type=Path, required=True) args = parser.parse_args(argv) repository = os.environ.get("GITHUB_REPOSITORY", version.REPOSITORY) try: + if args.command == "latest": + print(latest(args.site)) + return 0 tools, key = _environment() if args.command == "add": - add(ROOT, args.tag, args.build, args.channel, args.notes, args.critical, tools, key, - Releases(repository), repository) - elif args.command == "withdraw": - withdraw(ROOT, args.version, tools, key) + add(ROOT, args.site, args.tag, args.notes, args.critical, tools, key, Releases(repository), repository) else: - problems = check(ROOT, tools, key) - for problem in problems: - print(f"error: {problem}", file=sys.stderr) - if problems: - return 1 - print(f"origin/{BRANCH} passes its checks") + withdraw(args.site, args.version, tools, key) except (FeedError, registry.RegistryError, ValueError, OSError) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/scripts/release/package.sh b/scripts/release/package.sh index ef7b33a87f..00ef203958 100755 --- a/scripts/release/package.sh +++ b/scripts/release/package.sh @@ -68,7 +68,7 @@ package_app() { } package_plugins() { - local plugin target symbols="$DIST/TablePro-$VERSION-Plugins-dSYMs.zip" count=0 kit + local plugin target symbols="$DIST/TablePro-$VERSION-Plugins-dSYMs.zip" count=0 [ -d "$OUT/plugins" ] || die "no $OUT/plugins; run scripts/release/build.sh plugins first" [ -d "$OUT/plugins-dSYMs" ] || die "no $OUT/plugins-dSYMs; run scripts/release/build.sh plugins first" case "${BUILD:-}" in '' | *[!0-9]*) die "set BUILD to an integer, got '${BUILD:-}'" ;; esac @@ -84,11 +84,7 @@ package_plugins() { [ "$count" -gt 0 ] || die "no plugins in $OUT/plugins" # The feed job describes these zips again after download; this fails the build that made them instead. - kit="$(sed -nE 's/^TABLEPRO_PLUGINKIT_VERSION[[:space:]]*=[[:space:]]*([0-9]+)[[:space:]]*$/\1/p' \ - Configs/PluginKit.xcconfig)" - [ -n "$kit" ] || die "Configs/PluginKit.xcconfig sets no TABLEPRO_PLUGINKIT_VERSION" - python3 scripts/release/registry.py describe --zips "$DIST" --version "$VERSION" --build "$BUILD" \ - --kit "$kit" --metadata .github/plugin-registry.json --out "$WORK/entries.json" + python3 scripts/release/registry.py describe --zips "$DIST" --version "$VERSION" --build "$BUILD" ditto -c -k --keepParent "$OUT/plugins-dSYMs" "$symbols" echo "Packaged $count plugins as $DIST/-$VERSION.zip" diff --git a/scripts/release/publish.py b/scripts/release/publish.py deleted file mode 100755 index 43902cc940..0000000000 --- a/scripts/release/publish.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import hashlib -import json -import os -import subprocess -import sys -import time -from pathlib import Path -from typing import Callable, Dict, List, Optional - -import feed -import version - -ROOT = Path(__file__).resolve().parent.parent.parent -FIND_ATTEMPTS = 5 -FIND_DELAY_SECONDS = 2 - - -class PublishError(Exception): - pass - - -def draft_action(is_published: bool, has_draft: bool) -> str: - if is_published: - return "published" - return "reuse" if has_draft else "create" - - -def should_be_latest(release: version.Version, latest_tag: Optional[str]) -> bool: - if release.is_beta: - return False - latest = version.try_parse_tag(latest_tag) if latest_tag else None - return latest is None or release.build > latest.build - - -def release_flags(release: version.Version, latest_tag: Optional[str]) -> List[str]: - flags = ["--draft=false"] - if release.is_beta: - return flags + ["--prerelease", "--latest=false"] - return flags + [f"--latest={'true' if should_be_latest(release, latest_tag) else 'false'}"] - - -def highest_stable(items: List[feed.Item]) -> Optional[version.Version]: - stables = [] - for item in items: - if item.channel is not None: - continue - parsed = version.try_parse_tag(f"v{item.short_version}") if item.short_version else None - if parsed is not None and not parsed.is_beta: - stables.append(parsed) - return max(stables, key=lambda candidate: candidate.build) if stables else None - - -def asset_problems(files: Dict[str, str], assets: List[dict]) -> List[str]: - found = {asset.get("name"): asset for asset in assets} - problems = [] - for name, digest in sorted(files.items()): - asset = found.get(name) - if asset is None: - problems.append(f"the release has no {name}") - elif asset.get("state") != "uploaded": - problems.append(f"{name} is {asset.get('state')}, not uploaded") - elif asset.get("digest") != f"sha256:{digest}": - found_digest = asset.get("digest") or "missing its digest" - problems.append(f"{name} is {found_digest} on GitHub but sha256:{digest} here") - return problems - - -def _sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -class GitHub: - def __init__(self, repository: str, run: Callable = subprocess.run): - self.repository = repository - self._run = run - - def _gh(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: - result = self._run(["gh", *args], capture_output=True, text=True) - if check and result.returncode != 0: - raise PublishError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") - return result - - def _release(self, *args: str) -> List[str]: - return ["release", *args, "--repo", self.repository] - - def is_published(self, tag: str) -> bool: - result = self._gh("api", f"repos/{self.repository}/releases/tags/{tag}", "--jq", ".draft", check=False) - if result.returncode == 0: - return result.stdout.strip() == "false" - if "HTTP 404" in result.stderr: - return False - raise PublishError(f"gh api releases/tags/{tag}: {result.stderr.strip()}") - - # gh races its published and draft lookups, so this runs only once no published release holds the tag. - def draft(self, tag: str) -> Optional[dict]: - result = self._gh(*self._release("view", tag, "--json", "isDraft,assets"), check=False) - if result.returncode != 0: - if "release not found" in result.stderr: - return None - raise PublishError(f"gh release view {tag}: {result.stderr.strip()}") - release = json.loads(result.stdout) - if not release.get("isDraft"): - raise PublishError(f"{tag} was published while this ran") - return release - - def create_draft(self, tag: str, notes: Path, files: List[Path], prerelease: bool) -> None: - self._gh(*self._release("create", tag, "--draft", "--verify-tag", "--title", tag, "--notes-file", str(notes), - f"--prerelease={'true' if prerelease else 'false'}", *[str(path) for path in files])) - - def update_draft(self, tag: str, notes: Path, files: List[Path], prerelease: bool) -> None: - self._gh(*self._release("edit", tag, "--title", tag, "--notes-file", str(notes), - f"--prerelease={'true' if prerelease else 'false'}")) - self._gh(*self._release("upload", tag, "--clobber", *[str(path) for path in files])) - - def edit(self, tag: str, flags: List[str]) -> None: - self._gh(*self._release("edit", tag, *flags)) - - def latest_tag(self) -> Optional[str]: - result = self._gh("api", f"repos/{self.repository}/releases/latest", "--jq", ".tag_name", check=False) - if result.returncode == 0: - return result.stdout.strip() or None - if "HTTP 404" in result.stderr: - return None - raise PublishError(f"gh api releases/latest: {result.stderr.strip()}") - - -def draft(github: GitHub, tag: str, notes: Path, files: List[Path], sleep: Callable[[float], None] = time.sleep, - log: Callable[[str], None] = print) -> str: - parsed = version.parse_tag(tag) - missing = [str(path) for path in [notes, *files] if not path.is_file()] - if missing: - raise PublishError(f"{', '.join(missing)} does not exist") - names = [path.name for path in files] - if len(set(names)) != len(names): - raise PublishError("two assets share a file name") - - published = github.is_published(tag) - existing = None if published else github.draft(tag) - action = draft_action(published, existing is not None) - if action == "published": - log(f"{tag} is already published") - return "published" - if action == "create": - github.create_draft(tag, notes, files, parsed.is_beta) - log(f"created the draft for {tag}") - else: - github.update_draft(tag, notes, files, parsed.is_beta) - log(f"reused the draft an earlier attempt left for {tag}") - - current = None - for _ in range(FIND_ATTEMPTS): - current = github.draft(tag) - if current is not None: - break - sleep(FIND_DELAY_SECONDS) - if current is None: - raise PublishError(f"the draft for {tag} was created but cannot be found") - problems = asset_problems({path.name: _sha256(path) for path in files}, current.get("assets", [])) - if problems: - raise PublishError(f"the draft for {tag} does not hold this build, so it stays unpublished:\n " - + "\n ".join(problems)) - log(f"the draft for {tag} holds all {len(files)} assets") - return "draft" - - -def release(github: GitHub, tag: str, log: Callable[[str], None] = print) -> None: - parsed = version.parse_tag(tag) - if github.is_published(tag): - log(f"{tag} is already published") - return - if github.draft(tag) is None: - raise PublishError(f"{tag} has no draft; run publish.py draft first") - flags = release_flags(parsed, github.latest_tag()) - github.edit(tag, flags) - log(f"published {tag} ({' '.join(flags[1:])})") - - -def latest(github: GitHub, items: List[feed.Item], log: Callable[[str], None] = print) -> str: - highest = highest_stable(items) - if highest is None: - raise PublishError("the feed has no stable item") - github.edit(highest.tag, ["--latest=true"]) - log(f"{highest.tag} is Latest") - return highest.tag - - -def _stderr(message: str) -> None: - print(message, file=sys.stderr) - - -def main(argv: Optional[List[str]] = None) -> int: - parser = argparse.ArgumentParser(description="Drafts, publishes and marks GitHub releases.") - commands = parser.add_subparsers(dest="command", required=True) - command = commands.add_parser("draft", help="put every asset on a draft and check its digests") - command.add_argument("tag") - command.add_argument("--notes", type=Path, required=True) - command.add_argument("files", type=Path, nargs="+") - commands.add_parser("release", help="publish the draft").add_argument("tag") - commands.add_parser("latest", help="mark the highest stable left in the feed as Latest") - args = parser.parse_args(argv) - - github = GitHub(os.environ.get("GITHUB_REPOSITORY", version.REPOSITORY)) - try: - if args.command == "draft": - print(f"state={draft(github, args.tag, args.notes, args.files, log=_stderr)}") - elif args.command == "release": - release(github, args.tag, _stderr) - else: - latest(github, feed.published_items(ROOT), _stderr) - except (PublishError, feed.FeedError, ValueError, OSError) as error: - print(f"error: {error}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/release/registry.py b/scripts/release/registry.py index 2bd77268ed..ee3193b896 100755 --- a/scripts/release/registry.py +++ b/scripts/release/registry.py @@ -8,19 +8,22 @@ import sys import zipfile from pathlib import Path -from typing import Callable, Dict, List, Optional, Set +from typing import Callable, Dict, List, Optional import version ROOT = Path(__file__).resolve().parent.parent.parent METADATA = ROOT / ".github/plugin-registry.json" -PROJECT = ROOT / "project.yml" +PLUGINKIT_CONFIG = "Configs/PluginKit.xcconfig" SCHEMA_VERSION = 3 CHANNELS = ("stable", "beta") RELEASES_PER_KEY = 2 PROVIDES = ("TableProProvidesDatabaseTypeIds", "TableProProvidesExportFormatIds", "TableProProvidesImportFormatIds") +LISTED = ( + ("TableProProvidesDatabaseTypeIds", "databaseTypeIds"), + ("TableProProvidesExportFormatIds", "exportFormatIds"), +) SYSTEM_VERSION = re.compile(r"[0-9]+(\.[0-9]+){0,2}") -SHA256 = re.compile(r"[0-9a-f]{64}") class RegistryError(Exception): @@ -32,29 +35,29 @@ def load_metadata(path: Path) -> dict: metadata = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError) as error: raise RegistryError(f"{path}: {error}") from error - shape = "{retainFromPluginKit, plugins: {: {name, iconName}}}" + shape = "{retainFromPluginKit, plugins: {: {name}}}" plugins = metadata.get("plugins") if isinstance(metadata, dict) else None if not isinstance(plugins, dict) or not _is_int(metadata.get("retainFromPluginKit")): raise RegistryError(f"{path} is not in the {shape} shape") for target, entry in plugins.items(): - if not isinstance(entry, dict) or not all(isinstance(entry.get(key), str) for key in ("name", "iconName")): - raise RegistryError(f"{path}: {target} needs a name and an iconName, as in {shape}") + if not isinstance(entry, dict) or not isinstance(entry.get("name"), str): + raise RegistryError(f"{path}: {target} needs a name, as in {shape}") return metadata -def bundled_targets(project: str) -> Set[str]: - block = re.search(r"^aggregateTargets:\n(?:^(?: .*)?\n)*?^ BundledPlugins:\n^ targets:\n((?:^ - .*\n)+)", - project, re.MULTILINE) - if block is None: - raise RegistryError("project.yml has no BundledPlugins aggregate with a targets list") - return {line.strip()[2:].strip() for line in block.group(1).splitlines()} +def pluginkit_version(root: Path) -> int: + try: + text = (root / PLUGINKIT_CONFIG).read_text(encoding="utf-8") + except OSError as error: + raise RegistryError(f"{PLUGINKIT_CONFIG}: {error}") from error + found = re.search(r"^TABLEPRO_PLUGINKIT_VERSION\s*=\s*([0-9]+)\s*$", text, re.MULTILINE) + if found is None: + raise RegistryError(f"{PLUGINKIT_CONFIG} sets no TABLEPRO_PLUGINKIT_VERSION") + return int(found.group(1)) -def publish_set(metadata: dict, bundled: Set[str]) -> List[str]: +def publish_set(metadata: dict) -> List[str]: targets = sorted(metadata["plugins"]) - inside = [target for target in targets if target in bundled] - if inside: - raise RegistryError(f"the app bundles {', '.join(inside)}, so the registry must not publish them") if not targets: raise RegistryError("the registry metadata publishes no plugin") return targets @@ -99,9 +102,12 @@ def describe_one(archive: Path, target: str, entry: dict, release: str, build: i system = info.get("LSMinimumSystemVersion") if not isinstance(system, str) or not SYSTEM_VERSION.fullmatch(system): problems.append(f"LSMinimumSystemVersion is {system!r}") - type_ids = info.get(PROVIDES[0], []) - if not isinstance(type_ids, list) or not all(isinstance(type_id, str) for type_id in type_ids): - problems.append(f"{PROVIDES[0]} is not a list of strings") + listed = {} + for key, field in LISTED: + ids = info.get(key, []) + if not isinstance(ids, list) or not all(isinstance(each, str) for each in ids): + problems.append(f"{key} is not a list of strings") + listed[field] = ids # The app finds the plugin that serves a database type or a file format through these keys. if not any(info.get(key) for key in PROVIDES): problems.append(f"it declares none of {', '.join(PROVIDES)}") @@ -110,8 +116,7 @@ def describe_one(archive: Path, target: str, entry: dict, release: str, build: i return { "id": identifier, "name": entry["name"], - "iconName": entry["iconName"], - "databaseTypeIds": type_ids, + **listed, "version": release, "build": build, "pluginKit": kit, @@ -121,9 +126,9 @@ def describe_one(archive: Path, target: str, entry: dict, release: str, build: i } -def describe(zips: Path, release: str, build: int, kit: int, metadata: dict, bundled: Set[str]) -> List[dict]: +def describe(zips: Path, release: str, build: int, kit: int, metadata: dict) -> List[dict]: entries = [] - for target in publish_set(metadata, bundled): + for target in publish_set(metadata): archive = zips / f"{target}-{release}.zip" if not archive.is_file(): raise RegistryError(f"{archive} does not exist") @@ -199,28 +204,8 @@ def _release(entry: dict, channel: str, url_base: str) -> dict: } -def _check_entry(entry: dict, channel: str) -> None: - missing = [key for key in ("id", "name", "iconName", "databaseTypeIds", "version", "build", "pluginKit", - "minimumSystemVersion", "file", "sha256") if key not in entry] - if missing: - raise RegistryError(f"an entry has no {', '.join(missing)}") - if not SHA256.fullmatch(str(entry["sha256"])): - raise RegistryError(f"{entry['id']}: sha256 {entry['sha256']!r} is not 64 lowercase hex digits") - parsed = version.parse_version(entry["version"]) - if parsed.build != entry["build"]: - raise RegistryError(f"{entry['id']}: build {entry['build']} is not {parsed.build}, the build of {parsed}") - if parsed.channel != channel: - raise RegistryError(f"{entry['id']}: {parsed} belongs to the {parsed.channel} channel, not {channel}") - - def add(path: Path, entries: List[dict], channel: str, url_base: str, retain_from: int, log: Callable[[str], None] = print) -> bool: - if channel not in CHANNELS: - raise RegistryError(f"channel {channel!r} is not one of {', '.join(CHANNELS)}") - if not url_base.startswith("https://"): - raise RegistryError(f"the URL base {url_base!r} is not https") - for entry in entries: - _check_entry(entry, channel) loaded = load_manifest(path) before = json.dumps(loaded, sort_keys=True) if loaded is not None else None manifest = loaded if loaded is not None else empty_manifest(retain_from) @@ -248,7 +233,7 @@ def add(path: Path, entries: List[dict], channel: str, url_base: str, retain_fro if not any(kept is release for kept in plugin["releases"]): log(f"{entry['id']} {release['version']}: pruned on arrival by retention, not added") elif plugin["releases"][0] is release: - plugin.update({key: entry[key] for key in ("name", "iconName", "databaseTypeIds")}) + plugin.update({key: entry[key] for key in ("name", "databaseTypeIds", "exportFormatIds")}) manifest = { "schemaVersion": SCHEMA_VERSION, @@ -262,7 +247,7 @@ def add(path: Path, entries: List[dict], channel: str, url_base: str, retain_fro def _ordered(plugin: dict) -> dict: - keys = ("id", "name", "iconName", "databaseTypeIds") + keys = ("id", "name", "databaseTypeIds", "exportFormatIds") return {**{key: plugin[key] for key in keys if key in plugin}, "releases": plugin["releases"]} @@ -290,43 +275,21 @@ def main(argv: Optional[List[str]] = None) -> int: commands = parser.add_subparsers(dest="command", required=True) command = commands.add_parser("targets", help="print the targets the registry publishes") command.add_argument("--metadata", type=Path, default=METADATA) - command.add_argument("--project", type=Path, default=PROJECT) - command = commands.add_parser("describe", help="read the built plugin zips into manifest entries") + command = commands.add_parser("describe", help="check the built plugin zips as the manifest will read them") command.add_argument("--zips", type=Path, required=True) command.add_argument("--version", required=True) command.add_argument("--build", type=int, required=True) - command.add_argument("--kit", type=int, required=True) - command.add_argument("--metadata", type=Path, required=True) - command.add_argument("--project", type=Path, default=PROJECT) - command.add_argument("--out", type=Path, required=True) - command = commands.add_parser("add", help="add described releases to the manifest") - command.add_argument("--manifest", type=Path, required=True) - command.add_argument("--entries", type=Path, required=True) - command.add_argument("--channel", choices=CHANNELS, required=True) - command.add_argument("--url-base", required=True) - command.add_argument("--retain-from", type=int, required=True) - command = commands.add_parser("withdraw", help="remove one version's releases from the manifest") - command.add_argument("--manifest", type=Path, required=True) - command.add_argument("--version", required=True) + command.add_argument("--kit", type=int, help=f"default: TABLEPRO_PLUGINKIT_VERSION in {PLUGINKIT_CONFIG}") + command.add_argument("--metadata", type=Path, default=METADATA) args = parser.parse_args(argv) try: if args.command == "targets": - bundled = bundled_targets(args.project.read_text(encoding="utf-8")) - print("\n".join(publish_set(load_metadata(args.metadata), bundled))) - elif args.command == "describe": - bundled = bundled_targets(args.project.read_text(encoding="utf-8")) - entries = describe(args.zips, args.version, args.build, args.kit, load_metadata(args.metadata), bundled) - args.out.write_text(json.dumps(entries, indent=2) + "\n", encoding="utf-8") - print(f"described {len(entries)} plugins in {args.out}") - elif args.command == "add": - entries = json.loads(args.entries.read_text(encoding="utf-8")) - if not isinstance(entries, list): - raise RegistryError(f"{args.entries} is not a list of entries") - changed = add(args.manifest, entries, args.channel, args.url_base, args.retain_from) - print(f"{args.manifest} {'updated' if changed else 'unchanged'}") - elif args.command == "withdraw": - withdraw(args.manifest, args.version) + print("\n".join(publish_set(load_metadata(args.metadata)))) + else: + kit = args.kit if args.kit is not None else pluginkit_version(ROOT) + entries = describe(args.zips, args.version, args.build, kit, load_metadata(args.metadata)) + print(f"described {len(entries)} plugins") except (RegistryError, ValueError, OSError) as error: print(f"error: {error}", file=sys.stderr) return 1 diff --git a/scripts/release/requirements.in b/scripts/release/requirements.in new file mode 100644 index 0000000000..1ac9bdff44 --- /dev/null +++ b/scripts/release/requirements.in @@ -0,0 +1 @@ +towncrier==26.9.0 diff --git a/scripts/release/requirements.txt b/scripts/release/requirements.txt index d823c47b0a..e46d15f4df 100644 --- a/scripts/release/requirements.txt +++ b/scripts/release/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# echo 'towncrier==26.9.0' | uv pip compile --universal --python-version 3.9 --generate-hashes -o scripts/release/requirements.txt - +# uv pip compile scripts/release/requirements.in --universal --python-version 3.9 --generate-hashes -o scripts/release/requirements.txt click==8.1.8 ; python_full_version < '3.10' \ --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \ --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a @@ -247,6 +247,7 @@ tomli==2.5.0 ; python_full_version < '3.11' \ towncrier==26.9.0 \ --hash=sha256:ace9031631c718cc0709107b8755e6e31350eb8acdc877af43df3177addb9a55 \ --hash=sha256:ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660 + # via -r scripts/release/requirements.in zipp==3.23.1 ; python_full_version < '3.10' \ --hash=sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc \ --hash=sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110 diff --git a/scripts/release/test_feed.py b/scripts/release/test_feed.py index d4eb227211..49165a49b7 100644 --- a/scripts/release/test_feed.py +++ b/scripts/release/test_feed.py @@ -4,31 +4,21 @@ import json import os import plistlib -import subprocess import sys import tarfile import tempfile import unittest import zipfile -from contextlib import redirect_stderr +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from unittest import mock import feed -import publish import registry import version KEY = "cHJvYmUta2V5LW5vdC1hLXJlYWwtb25l" REPOSITORY = "TableProApp/TablePro" -GIT_ENV = { - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_AUTHOR_NAME": "Fixture", - "GIT_AUTHOR_EMAIL": "fixture@example.com", - "GIT_COMMITTER_NAME": "Fixture", - "GIT_COMMITTER_EMAIL": "fixture@example.com", -} SIGNATURE = r''' import hashlib, re @@ -41,50 +31,51 @@ def signed(text, key): return body + "\n" ''' -FAKE_GH = SIGNATURE + r''' -import json, os, shutil, sys +FAKE_GH = r''' +import hashlib, json, os, shutil, sys from pathlib import Path args = sys.argv[1:] with open(os.environ["FAKE_LOG"], "a") as log: log.write(json.dumps(["gh", *args]) + "\n") releases = Path(os.environ["FAKE_RELEASES"]) -if args[:2] == ["release", "view"]: - folder = releases / args[2] - if not folder.is_dir(): - sys.exit("release not found") - assets = [] - for path in sorted(folder.iterdir()): - digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() - if path.name == os.environ.get("FAKE_BAD_DIGEST"): - digest = "sha256:" + "0" * 64 - assets.append({"name": path.name, "digest": digest, "state": "uploaded"}) - print(json.dumps({"isDraft": os.environ.get("FAKE_DRAFT") == "1", "assets": assets})) -elif args[:2] == ["release", "download"]: +def digest(path): + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() +if args[:2] == ["release", "download"]: out = Path(args[args.index("--dir") + 1]) - for index, arg in enumerate(args): - if arg == "--pattern": - shutil.copy(releases / args[2] / args[index + 1], out / args[index + 1]) -elif args[:2] != ["release", "edit"]: + names = [args[index + 1] for index, arg in enumerate(args) if arg == "--pattern"] + found = [name for name in names if (releases / args[2] / name).is_file()] + if not found: + sys.exit("no assets match the file pattern") + for name in found: + shutil.copy(releases / args[2] / name, out / name) + if name == os.environ.get("FAKE_BAD_DIGEST"): + with open(out / name, "ab") as handle: + handle.write(b"x") +elif args[:2] == ["release", "verify-asset"]: + tag, path = args[2], Path(args[3]) + if os.environ.get("FAKE_DRAFT"): + sys.exit("no attestations found for tag " + tag) + if digest(path) != digest(releases / tag / path.name): + sys.exit("attestation for " + tag + " does not contain subject " + digest(path)) + print("Verification succeeded! " + path.name + " is present in release " + tag) +else: sys.exit("fake gh cannot answer " + " ".join(args)) ''' FAKE_GENERATE_APPCAST = SIGNATURE + r''' -import json, os, subprocess, sys +import json, os, sys from pathlib import Path args = sys.argv[1:] key = sys.stdin.read() with open(os.environ["FAKE_LOG"], "a") as log: log.write(json.dumps(["generate_appcast", *args, "stdin=" + key.strip()]) + "\n") -race = os.environ.get("FAKE_RACE_REPO") -if race and not os.path.exists(race + ".done"): - open(race + ".done", "w").close() - subprocess.run(["git", "-C", race, "push", "--quiet", "origin", "HEAD:refs/heads/feeds"], check=True) def option(name): return args[args.index(name) + 1] if name in args else None stage = Path(args[-1]) appcast = stage / "appcast.xml" +text = content(appcast.read_text()) if os.environ.get("FAKE_PRUNE"): - appcast.write_text(content(appcast.read_text())) + appcast.write_text(signed(text, key)) print("Wrote 0 new updates") sys.exit(0) archive = next(stage.glob("*.tar.xz")) @@ -105,9 +96,9 @@ def option(name): lines.append('") lines.append("") -text = content(appcast.read_text()) index = text.index("") if "" in text else text.index("") -appcast.write_text(text[:index] + "\n".join(lines) + "\n" + text[index:]) +text = text[:index] + "\n".join(lines) + "\n" + text[index:] +appcast.write_text(text if os.environ.get("FAKE_UNSIGNED_FEED") else signed(text, key)) print("Wrote 1 new update") ''' @@ -121,7 +112,7 @@ def option(name): path = Path(args[-1]) text = path.read_text() if "--verify" in args: - if os.environ.get("FAKE_VERIFY_FAIL") or signed(text, key) != text: + if signed(text, key) != text: print("Error: failed to pass signing verification.") sys.exit(1) sys.exit(0) @@ -154,18 +145,13 @@ def option(name): METADATA = { "retainFromPluginKit": 39, "plugins": { - "OracleDriver": {"name": "Oracle Driver", "iconName": "oracle-icon"}, - "ParquetExport": {"name": "Parquet Export", "iconName": "parquet-icon"}, + "OracleDriver": {"name": "Oracle Driver"}, + "ParquetExport": {"name": "Parquet Export"}, }, } -PROJECT = """aggregateTargets: - BundledPlugins: - targets: - - MySQLDriver -""" -def item(build, short, channel=None, critical=None, signature="c2ln", length="10"): +def item(build, short, channel=None, critical=None, signature="c2ln", url=None): lines = [""] if channel is not None: lines.append(f"{channel}") @@ -175,7 +161,8 @@ def item(build, short, channel=None, critical=None, signature="c2ln", length="10 lines += [f"{build}", f"{short}"] signed = f' sparkle:edSignature="{signature}"' if signature else "" - lines += [f'', ""] + url = url or f"https://example.com/TablePro-{short}.tar.xz" + lines += [f'', ""] return "\n".join(lines) @@ -184,38 +171,34 @@ def appcast(*items): f'TablePro{"".join(items)}').encode() -class FeedRulesTests(unittest.TestCase): - def test_a_clean_feed_passes(self): - self.assertEqual(feed.feed_problems(appcast(item(820001, "0.82.0-beta.1", "beta"), item(140, "0.81.0"))), []) - - def test_each_rule(self): - cases = [ - (appcast(item(140, "0.81.0", signature="")), "has no sparkle:edSignature"), - (appcast(item(140, "0.81.0", length="")), "has no length"), - (appcast(item("1.4", "0.81.0")), "sparkle:version '1.4' is not an integer"), - (appcast(item(140, "0.81")), "shortVersionString '0.81' is not a release version"), - (appcast(item(820001, "0.82.0-beta.1", "alpha")), "the only channel is beta"), - (appcast(item(820001, "0.82.0-beta.1", "beta", critical="")), "a beta marked critical"), - (appcast(item(141, "0.81.0"), item(140, "0.82.0")), "0.81.0 has the higher build (141 over 140)"), - (appcast(item(140, "0.81.0"), item(140, "0.80.0")), "build 140 carries 0.80.0, 0.81.0"), - ] - for data, message in cases: - with self.subTest(message=message): - self.assertTrue(any(message in problem for problem in feed.feed_problems(data)), - feed.feed_problems(data)) +def fake_signed(text, key=KEY): + namespace = {} + exec(SIGNATURE, namespace) + return namespace["signed"](text, key) - def test_the_last_default_channel_item_cannot_go(self): - before = appcast(item(140, "0.81.0")) - after = appcast(item(820001, "0.82.0-beta.1", "beta")) - self.assertIn("the feed no longer has a default-channel item", feed.feed_problems(after, before)) - def test_the_new_item_must_be_present(self): +class FeedRulesTests(unittest.TestCase): + def test_the_new_item_must_be_there_once(self): release = version.parse_tag("v0.82.0-beta.1") url = "https://example.com/TablePro-0.82.0-beta.1.tar.xz" - self.assertEqual(feed.feed_problems(appcast(item(820001, "0.82.0-beta.1", "beta")), None, - feed.Expected(release, url)), []) - problems = feed.feed_problems(appcast(item(820001, "0.82.0-beta.1")), None, feed.Expected(release, url)) - self.assertEqual(problems, [f"no beta item for 0.82.0-beta.1 (820001) at {url}"]) + present = feed.parse_items(appcast(item(820001, "0.82.0-beta.1", "beta"), item(140, "0.81.0"))) + self.assertEqual(feed.new_item_problems(present, release, url), []) + cases = [ + appcast(item(820001, "0.82.0-beta.1")), + appcast(item(820001, "0.82.0-beta.1", "beta", url="https://example.com/other.tar.xz")), + appcast(item(820001, "0.82.0-beta.1", "beta"), item(820001, "0.82.0-beta.1", "beta")), + ] + for data, count in zip(cases, (0, 0, 2)): + with self.subTest(data=data): + self.assertEqual(feed.new_item_problems(feed.parse_items(data), release, url), + [f"appcast.xml has {count} beta items for 0.82.0-beta.1 (820001) at {url}, not one"]) + + def test_a_new_item_with_no_signature_names_the_key(self): + release = version.parse_tag("v0.82.0") + items = feed.parse_items(appcast(item(820099, "0.82.0", signature=""))) + self.assertEqual(feed.new_item_problems(items, release, "https://example.com/TablePro-0.82.0.tar.xz"), [ + "0.82.0's enclosure has no sparkle:edSignature; " + "SPARKLE_PRIVATE_KEY does not match the archive's SUPublicEDKey"]) def test_the_critical_build_carries_forward(self): items = feed.parse_items(appcast(item(820099, "0.82.0", critical="140"), item(140, "0.81.0", critical=""), @@ -224,6 +207,13 @@ def test_the_critical_build_carries_forward(self): self.assertEqual(feed.carried_critical(feed.parse_items(SEED.encode())), 140) self.assertIsNone(feed.carried_critical(feed.parse_items(appcast(item(139, "0.80.0"))))) + def test_the_highest_stable_in_the_feed(self): + items = feed.parse_items(appcast(item(830001, "0.83.0-beta.1", "beta"), item(820199, "0.82.1"), + item(820099, "0.82.0"), item(140, "0.81.0"))) + self.assertEqual(str(feed.highest_stable(items)), "0.82.1") + self.assertEqual(str(feed.highest_stable(feed.parse_items(SEED.encode()))), "0.81.0") + self.assertIsNone(feed.highest_stable(feed.parse_items(appcast(item(830001, "0.83.0-beta.1", "beta"))))) + def test_the_generate_appcast_arguments(self): stage = Path("/stage") prefix = ["--ed-key-file", "-", "--embed-release-notes", "--maximum-deltas", "0"] @@ -231,38 +221,27 @@ def test_the_generate_appcast_arguments(self): self.assertEqual(beta, prefix + [ "--versions", "820001", "--download-url-prefix", "https://github.com/TableProApp/TablePro/releases/download/v0.82.0-beta.1/", - "--full-release-notes-url", "https://github.com/TableProApp/TablePro/releases/tag/v0.82.0-beta.1", + "--full-release-notes-url", "https://docs.tablepro.app/changelog", "--channel", "beta", "/stage"]) stable = feed.appcast_arguments(stage, "v0.82.0", version.parse_tag("v0.82.0"), 140, REPOSITORY) + self.assertEqual(stable[stable.index("--full-release-notes-url") + 1], "https://docs.tablepro.app/changelog") self.assertEqual(stable[stable.index("--phased-rollout-interval"):], ["--phased-rollout-interval", "21600", "--critical-update-version", "140", "/stage"]) plain = feed.appcast_arguments(stage, "v0.82.1", version.parse_tag("v0.82.1"), None, REPOSITORY) self.assertNotIn("--critical-update-version", plain) self.assertNotIn("--channel", plain) - def test_removes_items_by_version_and_keeps_the_rest_of_the_text(self): - text = SEED - edited, removed = feed.remove_items(text, "0.80.0") + def test_removes_items_by_version_and_keeps_the_rest(self): + edited, removed = feed.remove_items(SEED.encode(), "0.80.0") self.assertEqual(removed, 1) - self.assertEqual([each.short_version for each in feed.parse_items(edited.encode())], ["0.81.0"]) - self.assertEqual(edited, text[:text.index(" \n 0.80.0")] - + text[text.index(" </channel>"):]) - - def test_refuses_to_edit_text_it_cannot_match(self): - text = SEED.replace("# 0.81.0", "# 0.81.0 </item> in the notes") - with self.assertRaises(feed.FeedError): - feed.remove_items(text, "0.81.0") + self.assertEqual(feed.parse_items(edited), feed.parse_items(SEED.encode())[:1]) + self.assertIn(b"<![CDATA[# 0.81.0]]>", edited) - -def git(path, *args): - return subprocess.run(["git", "-C", str(path), *args], env={**os.environ, **GIT_ENV}, check=True, - capture_output=True, text=True).stdout.strip() - - -def fake_signed(text, key=KEY): - namespace = {} - exec(SIGNATURE, namespace) - return namespace["signed"](text, key) + def test_an_item_end_tag_in_the_notes_does_not_block_a_withdraw(self): + text = SEED.replace("# 0.81.0", "# 0.81.0 </item> in the notes").encode() + edited, removed = feed.remove_items(text, "0.81.0") + self.assertEqual(removed, 1) + self.assertEqual([each.short_version for each in feed.parse_items(edited)], ["0.80.0"]) def app_archive(path, text, build): @@ -281,7 +260,7 @@ def plugin_zip(path, target, text, build, kit=39, **extra): archive.writestr(f"{target}.tableplugin/Contents/Info.plist", plistlib.dumps(info)) -class FeedBranchTests(unittest.TestCase): +class FeedSiteTests(unittest.TestCase): def setUp(self): directory = tempfile.TemporaryDirectory() self.addCleanup(directory.cleanup) @@ -296,7 +275,6 @@ def setUp(self): self.log_path = self.base / "calls.log" self.releases = self.base / "releases" environment = mock.patch.dict(os.environ, { - **GIT_ENV, "PATH": f"{self.tools}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log_path), "FAKE_RELEASES": str(self.releases), @@ -304,28 +282,17 @@ def setUp(self): environment.start() self.addCleanup(environment.stop) - self.origin = self.base / "origin.git" - git(self.base, "init", "--quiet", "--bare", "--initial-branch=main", str(self.origin)) - seed = self.base / "seed" - git(self.base, "init", "--quiet", "--initial-branch=feeds", str(seed)) - (seed / "appcast.xml").write_text(SEED) - (seed / "plugins").mkdir() - (seed / "plugins/v3.json").write_text(json.dumps(registry.empty_manifest(39))) - git(seed, "add", ".") - git(seed, "commit", "--quiet", "-m", "seed") - git(seed, "push", "--quiet", str(self.origin), "feeds") + self.site = self.base / "gh-pages" + (self.site / "plugins").mkdir(parents=True) + (self.site / "appcast.xml").write_text(fake_signed(SEED)) + registry.write_manifest(self.site / "plugins/v3.json", registry.empty_manifest(39)) self.root = self.base / "tag" - git(self.base, "init", "--quiet", "--initial-branch=main", str(self.root)) - (self.root / ".github").mkdir() + (self.root / ".github").mkdir(parents=True) (self.root / ".github/plugin-registry.json").write_text(json.dumps(METADATA)) (self.root / "Configs").mkdir() (self.root / "Configs/PluginKit.xcconfig").write_text( "TABLEPRO_PLUGINKIT_VERSION = 39\nTABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19\n") - (self.root / "project.yml").write_text(PROJECT) - git(self.root, "add", ".") - git(self.root, "commit", "--quiet", "-m", "tag") - git(self.root, "remote", "add", "origin", str(self.origin)) self.notes = self.base / "feed.md" self.notes.write_text("- A fix. (#1)\n") @@ -340,18 +307,16 @@ def publish(self, tag, kit=39, archive=None): TableProProvidesExportFormatIds=["parquet"]) def add(self, tag, critical=False): - release = version.parse_tag(tag) messages = [] - changed = feed.add(self.root, tag, release.build, release.channel, self.notes, critical, self.tools, KEY, + changed = feed.add(self.root, self.site, tag, self.notes, critical, self.tools, KEY, feed.Releases(REPOSITORY), REPOSITORY, log=messages.append) return changed, messages - def replace_appcast(self, text): - clone = self.base / "replace" - git(self.base, "clone", "--quiet", "--branch", "feeds", str(self.origin), str(clone)) - (clone / "appcast.xml").write_text(text) - git(clone, "commit", "--quiet", "-am", "replace the appcast") - git(clone, "push", "--quiet", "origin", "feeds") + def withdraw(self, text): + return feed.withdraw(self.site, text, self.tools, KEY, log=lambda message: None) + + def files(self): + return [(self.site / name).read_bytes() for name in ("appcast.xml", "plugins/v3.json")] def calls(self, tool): if not self.log_path.exists(): @@ -359,35 +324,34 @@ def calls(self, tool): return [call[1:] for call in map(json.loads, self.log_path.read_text().splitlines()) if call[0] == tool] def published(self, path): - return subprocess.run(["git", "-C", str(self.origin), "show", f"feeds:{path}"], check=True, - capture_output=True, text=True).stdout - - def history(self): - return git(self.origin, "log", "--format=%an <%ae> %s", "feeds").splitlines() + return (self.site / path).read_text() def test_adds_a_beta_and_its_plugins(self): self.publish("v0.82.0-beta.1") - changed, _ = self.add("v0.82.0-beta.1") + changed, messages = self.add("v0.82.0-beta.1") self.assertTrue(changed) - self.assertEqual(self.history(), [ - "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> release: v0.82.0-beta.1", - "Fixture <fixture@example.com> seed", - ]) (generate,) = self.calls("generate_appcast") self.assertEqual(generate[-1], f"stdin={KEY}") - signing = [call for call in self.calls("sign_update") if "--verify" not in call] - self.assertEqual([call[:2] + call[-1:] for call in signing], [["--ed-key-file", "-", f"stdin={KEY}"]]) + self.assertEqual([call for call in self.calls("sign_update") if "--verify" not in call], []) self.assertIn("--channel", generate) self.assertNotIn("--critical-update-version", generate) self.assertNotIn("--phased-rollout-interval", generate) - items = feed.parse_items(self.published("appcast.xml").encode()) + verified = [call[3] for call in self.calls("gh") if call[:2] == ["release", "verify-asset"]] + self.assertEqual([Path(path).name for path in verified], [ + "TablePro-0.82.0-beta.1.tar.xz", "OracleDriver-0.82.0-beta.1.zip", "ParquetExport-0.82.0-beta.1.zip"]) + self.assertIn("Verification succeeded! OracleDriver-0.82.0-beta.1.zip is present in release v0.82.0-beta.1", + messages) + published = self.published("appcast.xml") + self.assertEqual(fake_signed(published), published) + items = feed.parse_items(published.encode()) self.assertEqual([(each.short_version, each.channel) for each in items], [("0.82.0-beta.1", "beta"), ("0.81.0", None), ("0.80.0", None)]) manifest = json.loads(self.published("plugins/v3.json")) self.assertEqual([plugin["id"] for plugin in manifest["plugins"]], ["com.TablePro.OracleDriver", "com.TablePro.ParquetExport"]) - oracle = manifest["plugins"][0] - self.assertEqual(oracle["databaseTypeIds"], ["Oracle"]) + oracle, parquet = manifest["plugins"] + self.assertEqual((oracle["databaseTypeIds"], oracle["exportFormatIds"]), (["Oracle"], [])) + self.assertEqual((parquet["databaseTypeIds"], parquet["exportFormatIds"]), ([], ["parquet"])) self.assertEqual(oracle["releases"][0]["url"], "https://github.com/TableProApp/TablePro/releases/download/v0.82.0-beta.1/" "OracleDriver-0.82.0-beta.1.zip") @@ -410,45 +374,25 @@ def test_a_critical_beta_is_refused(self): with self.assertRaisesRegex(feed.FeedError, "a beta is never critical"): self.add("v0.82.0-beta.1", critical=True) - def test_a_build_or_channel_that_disagrees_with_the_tag_is_refused(self): - with self.assertRaisesRegex(feed.FeedError, "is build 820001 on beta"): - feed.add(self.root, "v0.82.0-beta.1", 820099, "stable", self.notes, False, self.tools, KEY, - feed.Releases(REPOSITORY), REPOSITORY, log=lambda message: None) - def test_a_rerun_changes_nothing(self): self.publish("v0.82.0-beta.1") self.add("v0.82.0-beta.1") - head = git(self.origin, "rev-parse", "feeds") + written = self.files() changed, messages = self.add("v0.82.0-beta.1") self.assertFalse(changed) - self.assertEqual(git(self.origin, "rev-parse", "feeds"), head) + self.assertEqual(self.files(), written) self.assertEqual(len(self.calls("generate_appcast")), 1) self.assertIn("appcast.xml already has 0.82.0-beta.1 (820001)", messages) - def test_starts_again_when_another_run_pushed_first(self): - self.publish("v0.82.0-beta.1") - race = self.base / "race" - git(self.base, "clone", "--quiet", "--branch", "feeds", str(self.origin), str(race)) - (race / "other.txt").write_text("another run\n") - git(race, "add", "other.txt") - git(race, "commit", "--quiet", "-m", "release: v0.81.1") - with mock.patch.dict(os.environ, {"FAKE_RACE_REPO": str(race)}): - changed, messages = self.add("v0.82.0-beta.1") - self.assertTrue(changed) - self.assertEqual([line.split("> ")[1] for line in self.history()], - ["release: v0.82.0-beta.1", "release: v0.81.1", "seed"]) - self.assertEqual(len(self.calls("generate_appcast")), 2) - self.assertTrue(any("attempt 1 of 3" in message for message in messages)) - self.assertEqual(self.published("other.txt"), "another run\n") - def test_an_archive_of_another_build_is_refused(self): self.publish("v0.82.0-beta.1", archive=("0.82.0-beta.1", 140)) + written = self.files() with self.assertRaisesRegex(feed.FeedError, r"holds 0.82.0-beta.1 \(140\), not 0.82.0-beta.1 \(820001\)"): self.add("v0.82.0-beta.1") self.assertEqual(self.calls("generate_appcast"), []) + self.assertEqual(self.files(), written) def test_an_item_sparkle_prunes_on_arrival_leaves_the_appcast_alone(self): - self.replace_appcast(fake_signed(SEED)) self.publish("v0.79.1") with mock.patch.dict(os.environ, {"FAKE_PRUNE": "1"}): changed, messages = self.add("v0.79.1") @@ -457,38 +401,58 @@ def test_an_item_sparkle_prunes_on_arrival_leaves_the_appcast_alone(self): self.assertIn("generate_appcast keeps newer items on 0.79.1's branch, so appcast.xml stays as it is", messages) self.assertEqual(len(json.loads(self.published("plugins/v3.json"))["plugins"]), 2) - def test_a_failed_check_pushes_nothing(self): + def test_a_new_item_with_no_signature_writes_nothing(self): self.publish("v0.82.0-beta.1") + written = self.files() with mock.patch.dict(os.environ, {"FAKE_UNSIGNED_ENCLOSURE": "1"}): with self.assertRaisesRegex(feed.FeedError, "0.82.0-beta.1's enclosure has no sparkle:edSignature"): self.add("v0.82.0-beta.1") - self.assertEqual(len(self.history()), 1) + self.assertEqual(self.files(), written) - def test_a_rejected_signature_pushes_nothing(self): + def test_an_unsigned_feed_from_generate_appcast_writes_nothing(self): self.publish("v0.82.0-beta.1") - with mock.patch.dict(os.environ, {"FAKE_VERIFY_FAIL": "1"}): - with self.assertRaisesRegex(feed.FeedError, "sign_update failed"): + written = self.files() + with mock.patch.dict(os.environ, {"FAKE_UNSIGNED_FEED": "1"}): + with self.assertRaisesRegex(feed.FeedError, "fails its checks, so nothing was written:\n sign_update failed"): self.add("v0.82.0-beta.1") - self.assertEqual(len(self.history()), 1) + self.assertEqual(self.files(), written) + + def test_a_feed_changed_outside_ci_is_refused(self): + self.publish("v0.82.0-beta.1") + (self.site / "appcast.xml").write_text(fake_signed(SEED).replace("TablePro-0.81.0.zip", "evil.zip")) + written = self.files() + for attempt in (lambda: self.add("v0.82.0-beta.1"), lambda: self.withdraw("0.80.0")): + with self.subTest(attempt=attempt): + with self.assertRaisesRegex(feed.FeedError, "signature does not verify; it changed outside CI"): + attempt() + self.assertEqual(self.files(), written) + self.assertEqual(self.calls("gh"), []) def test_a_wrong_digest_is_refused(self): self.publish("v0.82.0-beta.1") with mock.patch.dict(os.environ, {"FAKE_BAD_DIGEST": "TablePro-0.82.0-beta.1.tar.xz"}): - with self.assertRaisesRegex(feed.FeedError, "but v0.82.0-beta.1 lists sha256:0000"): + with self.assertRaisesRegex(feed.FeedError, "attestation for v0.82.0-beta.1 does not contain subject"): self.add("v0.82.0-beta.1") self.assertEqual(self.calls("generate_appcast"), []) def test_a_draft_is_refused(self): self.publish("v0.82.0-beta.1") with mock.patch.dict(os.environ, {"FAKE_DRAFT": "1"}): - with self.assertRaisesRegex(feed.FeedError, "is a draft"): + with self.assertRaisesRegex(feed.FeedError, "no attestations found for tag v0.82.0-beta.1"): self.add("v0.82.0-beta.1") + def test_a_missing_asset_is_refused(self): + self.publish("v0.82.0-beta.1") + (self.releases / "v0.82.0-beta.1/ParquetExport-0.82.0-beta.1.zip").unlink() + with self.assertRaisesRegex(feed.FeedError, "v0.82.0-beta.1 has no ParquetExport-0.82.0-beta.1.zip"): + self.add("v0.82.0-beta.1") + def test_a_plugin_built_against_another_kit_is_refused(self): self.publish("v0.82.0-beta.1", kit=38) + written = self.files() with self.assertRaisesRegex(registry.RegistryError, "TableProPluginKitVersion is 38"): self.add("v0.82.0-beta.1") - self.assertEqual(len(self.history()), 1) + self.assertEqual(self.files(), written) def test_withdraw_removes_the_version_and_re_signs(self): self.publish("v0.82.0-beta.1") @@ -496,53 +460,39 @@ def test_withdraw_removes_the_version_and_re_signs(self): self.add("v0.82.0-beta.1") self.add("v0.82.0") signed_before = sum("--verify" not in call for call in self.calls("sign_update")) - self.assertTrue(feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None)) - self.assertEqual(self.history()[0].split("> ")[1], "withdraw: v0.82.0") + self.assertTrue(self.withdraw("0.82.0")) published = self.published("appcast.xml") self.assertEqual([each.short_version for each in feed.parse_items(published.encode())], ["0.82.0-beta.1", "0.81.0", "0.80.0"]) self.assertEqual(fake_signed(published), published) self.assertEqual(sum("--verify" not in call for call in self.calls("sign_update")), signed_before + 1) self.assertEqual(json.loads(self.published("plugins/v3.json"))["plugins"], []) - self.assertFalse(feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None)) + self.assertFalse(self.withdraw("0.82.0")) def test_withdraw_keeps_a_default_channel_item(self): - self.replace_appcast(fake_signed(feed.remove_items(SEED, "0.80.0")[0])) + (self.site / "appcast.xml").write_text(fake_signed(feed.remove_items(SEED.encode(), "0.80.0")[0].decode())) with self.assertRaisesRegex(feed.FeedError, "no default-channel item"): - feed.withdraw(self.root, "0.81.0", self.tools, KEY, log=lambda message: None) + self.withdraw("0.81.0") - def test_latest_after_a_withdraw_reads_the_pushed_branch(self): + def test_latest_prints_the_highest_stable_left(self): self.publish("v0.82.0-beta.1") self.publish("v0.82.0") self.add("v0.82.0-beta.1") self.add("v0.82.0") - before = git(self.origin, "rev-parse", "feeds") - feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None) - git(self.root, "branch", "feeds", before) - git(self.root, "update-ref", feed.REMOTE_BRANCH, before) - self.assertEqual([each.short_version for each in feed.published_items(self.root)], - ["0.82.0-beta.1", "0.81.0", "0.80.0"]) - - git(self.root, "update-ref", feed.REMOTE_BRANCH, before) - with mock.patch.object(publish, "ROOT", self.root), \ - mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": REPOSITORY}), redirect_stderr(io.StringIO()): - self.assertEqual(publish.main(["latest"]), 0) - self.assertEqual(self.calls("gh")[-1], ["release", "edit", "v0.81.0", "--latest=true", "--repo", REPOSITORY]) - - def test_check_reads_the_published_branch(self): - self.assertTrue(any("sign_update failed" in problem for problem in feed.check(self.root, self.tools, KEY))) - self.publish("v0.82.0-beta.1") - self.add("v0.82.0-beta.1") - self.assertEqual(feed.check(self.root, self.tools, KEY), []) + self.withdraw("0.82.0") + with redirect_stdout(io.StringIO()) as output: + self.assertEqual(feed.main(["latest", "--site", str(self.site)]), 0) + self.assertEqual(output.getvalue(), "0.81.0\n") def test_main_needs_the_sparkle_tools_and_key(self): + command = ["withdraw", "0.82.0", "--site", str(self.site)] with mock.patch.dict(os.environ, {"SPARKLE_BIN": "", "SPARKLE_PRIVATE_KEY": KEY}), \ redirect_stderr(io.StringIO()) as errors: - self.assertEqual(feed.main(["check"]), 1) + self.assertEqual(feed.main(command), 1) self.assertIn("SPARKLE_BIN is not set", errors.getvalue()) with mock.patch.dict(os.environ, {"SPARKLE_BIN": str(self.tools), "SPARKLE_PRIVATE_KEY": ""}), \ redirect_stderr(io.StringIO()) as errors: - self.assertEqual(feed.main(["check"]), 1) + self.assertEqual(feed.main(command), 1) self.assertIn("SPARKLE_PRIVATE_KEY is not set", errors.getvalue()) diff --git a/scripts/release/test_publish.py b/scripts/release/test_publish.py deleted file mode 100644 index cdcb6cb161..0000000000 --- a/scripts/release/test_publish.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 - -import hashlib -import io -import json -import os -import sys -import tempfile -import unittest -from contextlib import redirect_stderr, redirect_stdout -from pathlib import Path -from unittest import mock - -import feed -import publish -import version - -REPOSITORY = "TableProApp/TablePro" - -FAKE_GH = r''' -import hashlib, json, os, sys -from pathlib import Path -args = sys.argv[1:] -state_path = Path(os.environ["FAKE_STATE"]) -state = json.loads(state_path.read_text()) -state["calls"].append(args) -releases = state["releases"] - -def save(): - state_path.write_text(json.dumps(state)) - -def fail(message): - save() - sys.exit(message) - -def option(name): - return args[args.index(name) + 1] if name in args else None - -def files(): - return [arg for arg in args[3:] if os.path.isfile(arg) and arg != option("--notes-file")] - -def upload(release, paths): - for path in paths: - digest = "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() - if Path(path).name == os.environ.get("FAKE_CORRUPT"): - digest = "sha256:" + "0" * 64 - release["assets"][Path(path).name] = digest - -if args[0] == "api": - path = args[1] - if path.endswith("/releases/latest"): - if state["latest"] is None: - fail("gh: Not Found (HTTP 404)") - print(state["latest"]) - else: - tag = path.rsplit("/", 1)[1] - if tag not in releases or releases[tag]["draft"]: - fail("gh: Not Found (HTTP 404)") - print("false") -elif args[:2] == ["release", "view"]: - release = releases.get(args[2]) - if release is None: - fail("release not found") - if state.get("hide_drafts", 0) > 0: - state["hide_drafts"] -= 1 - fail("release not found") - assets = [{"name": name, "digest": digest, "state": "uploaded"} for name, digest in release["assets"].items()] - print(json.dumps({"isDraft": release["draft"], "assets": assets})) -elif args[:2] == ["release", "create"]: - if args[2] not in state["tags"]: - fail("tag " + args[2] + " doesn't exist in the repo") - release = {"draft": "--draft" in args, "prerelease": "--prerelease=true" in args, "latest": None, "assets": {}} - releases[args[2]] = release - upload(release, files()) -elif args[:2] == ["release", "upload"]: - upload(releases[args[2]], files()) -elif args[:2] == ["release", "edit"]: - release = releases[args[2]] - for arg in args[3:]: - if arg == "--draft=false": - release["draft"] = False - elif arg in ("--prerelease", "--prerelease=true"): - release["prerelease"] = True - elif arg == "--prerelease=false": - release["prerelease"] = False - elif arg == "--latest=true": - state["latest"] = args[2] - elif arg == "--latest=false": - release["latest"] = False -else: - fail("fake gh cannot answer " + " ".join(args)) -save() -''' - - -def items(*pairs): - return [feed.Item(build=str(build), short_version=short, channel=channel, is_critical=False, - critical_version=None, url="https://example.com", signature="c2ln", length="1") - for build, short, channel in pairs] - - -class DecisionTests(unittest.TestCase): - def test_the_draft_action(self): - self.assertEqual(publish.draft_action(True, False), "published") - self.assertEqual(publish.draft_action(True, True), "published") - self.assertEqual(publish.draft_action(False, True), "reuse") - self.assertEqual(publish.draft_action(False, False), "create") - - def test_latest_goes_only_to_the_highest_stable(self): - cases = [ - ("v0.82.0", "v0.81.0", True), - ("v0.82.0", None, True), - ("v0.82.0", "plugin-oracle-v1.0.0", True), - ("v0.81.1", "v0.82.0", False), - ("v0.82.0", "v0.82.0", False), - ("v0.83.0-beta.1", "v0.82.0", False), - ] - for tag, latest, expected in cases: - with self.subTest(tag=tag, latest=latest): - self.assertEqual(publish.should_be_latest(version.parse_tag(tag), latest), expected) - - def test_the_release_flags(self): - self.assertEqual(publish.release_flags(version.parse_tag("v0.82.0-beta.1"), "v0.81.0"), - ["--draft=false", "--prerelease", "--latest=false"]) - self.assertEqual(publish.release_flags(version.parse_tag("v0.82.0"), "v0.81.0"), - ["--draft=false", "--latest=true"]) - self.assertEqual(publish.release_flags(version.parse_tag("v0.81.1"), "v0.82.0"), - ["--draft=false", "--latest=false"]) - - def test_the_highest_stable_in_the_feed(self): - feed_items = items((830001, "0.83.0-beta.1", "beta"), (820199, "0.82.1", None), (820099, "0.82.0", None), - (140, "0.81.0", None)) - self.assertEqual(str(publish.highest_stable(feed_items)), "0.82.1") - self.assertEqual(str(publish.highest_stable(items((140, "0.81.0", None), (139, "0.80.0", None)))), "0.81.0") - self.assertIsNone(publish.highest_stable(items((830001, "0.83.0-beta.1", "beta")))) - - def test_asset_problems(self): - assets = [{"name": "a.zip", "digest": "sha256:" + "1" * 64, "state": "uploaded"}, - {"name": "b.zip", "digest": "sha256:" + "2" * 64, "state": "starter"}] - self.assertEqual(publish.asset_problems({"a.zip": "1" * 64}, assets), []) - self.assertEqual(publish.asset_problems({"a.zip": "3" * 64, "b.zip": "2" * 64, "c.zip": "4" * 64}, assets), [ - f"a.zip is sha256:{'1' * 64} on GitHub but sha256:{'3' * 64} here", - "b.zip is starter, not uploaded", - "the release has no c.zip", - ]) - - -class GitHubTests(unittest.TestCase): - def setUp(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - self.base = Path(directory.name) - tools = self.base / "bin" - tools.mkdir() - (tools / "gh").write_text(f"#!{sys.executable}\n{FAKE_GH}") - (tools / "gh").chmod(0o755) - self.state_path = self.base / "state.json" - self.state({"releases": {"v0.81.0": {"draft": False, "prerelease": False, "assets": {}}}, - "latest": "v0.81.0", "tags": ["v0.81.0", "v0.82.0-beta.1", "v0.82.0", "v0.81.1"], "calls": []}) - environment = mock.patch.dict(os.environ, {"PATH": f"{tools}{os.pathsep}{os.environ['PATH']}", - "FAKE_STATE": str(self.state_path), - "GITHUB_REPOSITORY": REPOSITORY}) - environment.start() - self.addCleanup(environment.stop) - self.github = publish.GitHub(REPOSITORY) - self.notes = self.base / "body.md" - self.notes.write_text("Notes\n") - self.files = [] - for name in ("TablePro-0.82.0.dmg", "TablePro-0.82.0.tar.xz", "OracleDriver-0.82.0.zip"): - path = self.base / name - path.write_bytes(name.encode()) - self.files.append(path) - self.log = [] - - def state(self, value=None): - if value is not None: - self.state_path.write_text(json.dumps(value)) - return json.loads(self.state_path.read_text()) - - def verbs(self): - return [call[:2] if call[0] == "release" else call[:1] for call in self.state()["calls"]] - - def draft(self, tag="v0.82.0"): - return publish.draft(self.github, tag, self.notes, self.files, sleep=lambda seconds: None, - log=self.log.append) - - def digests(self): - return {path.name: "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() for path in self.files} - - def test_creates_a_draft_and_checks_its_digests(self): - self.assertEqual(self.draft(), "draft") - release = self.state()["releases"]["v0.82.0"] - self.assertEqual((release["draft"], release["prerelease"], release["assets"]), (True, False, self.digests())) - create = next(call for call in self.state()["calls"] if call[:2] == ["release", "create"]) - self.assertEqual(create[:9], ["release", "create", "v0.82.0", "--draft", "--verify-tag", "--title", "v0.82.0", - "--notes-file", str(self.notes)]) - - def test_a_beta_draft_is_a_prerelease(self): - self.files = [self.base / "TablePro-0.82.0.dmg"] - self.draft("v0.82.0-beta.1") - self.assertTrue(self.state()["releases"]["v0.82.0-beta.1"]["prerelease"]) - - def test_a_published_release_is_left_alone(self): - state = self.state() - state["releases"]["v0.82.0"] = {"draft": False, "prerelease": False, "assets": {}} - self.state(state) - self.assertEqual(self.draft(), "published") - self.assertEqual(self.verbs(), [["api"]]) - - def test_an_earlier_draft_is_reused(self): - self.draft() - state = self.state() - state["calls"] = [] - self.state(state) - self.files[0].write_bytes(b"rebuilt") - self.assertEqual(self.draft(), "draft") - self.assertNotIn(["release", "create"], self.verbs()) - self.assertIn(["release", "upload"], self.verbs()) - self.assertEqual(self.state()["releases"]["v0.82.0"]["assets"], self.digests()) - - def test_waits_for_a_new_draft_to_appear(self): - state = self.state() - state["hide_drafts"] = 2 - self.state(state) - self.assertEqual(self.draft(), "draft") - self.assertEqual(self.verbs().count(["release", "view"]), 4) - - def test_a_draft_that_never_appears_is_an_error(self): - state = self.state() - state["hide_drafts"] = publish.FIND_ATTEMPTS - self.state(state) - with self.assertRaisesRegex(publish.PublishError, "cannot be found"): - self.draft() - - def test_a_wrong_digest_keeps_the_draft_unpublished(self): - with mock.patch.dict(os.environ, {"FAKE_CORRUPT": "OracleDriver-0.82.0.zip"}): - with self.assertRaisesRegex(publish.PublishError, "OracleDriver-0.82.0.zip is sha256:0000"): - self.draft() - self.assertTrue(self.state()["releases"]["v0.82.0"]["draft"]) - - def test_a_missing_file_is_refused_before_any_call(self): - self.files.append(self.base / "missing.zip") - with self.assertRaisesRegex(publish.PublishError, "missing.zip does not exist"): - self.draft() - self.assertEqual(self.state()["calls"], []) - - def test_publishes_a_beta_as_a_prerelease_that_is_never_latest(self): - self.files = [self.base / "TablePro-0.82.0.dmg"] - self.draft("v0.82.0-beta.1") - publish.release(self.github, "v0.82.0-beta.1", self.log.append) - state = self.state() - self.assertEqual(state["releases"]["v0.82.0-beta.1"]["draft"], False) - self.assertEqual(state["latest"], "v0.81.0") - self.assertEqual(state["calls"][-1], ["release", "edit", "v0.82.0-beta.1", "--draft=false", "--prerelease", - "--latest=false", "--repo", REPOSITORY]) - - def test_a_stable_above_latest_becomes_latest(self): - self.draft() - publish.release(self.github, "v0.82.0", self.log.append) - self.assertEqual(self.state()["latest"], "v0.82.0") - - def test_a_patch_on_an_older_line_is_not_latest(self): - state = self.state() - state["latest"] = "v0.82.0" - self.state(state) - self.draft("v0.81.1") - publish.release(self.github, "v0.81.1", self.log.append) - self.assertEqual(self.state()["latest"], "v0.82.0") - self.assertIn("--latest=false", self.state()["calls"][-1]) - - def test_releasing_twice_edits_once(self): - self.draft() - publish.release(self.github, "v0.82.0", self.log.append) - publish.release(self.github, "v0.82.0", self.log.append) - self.assertEqual(sum(call[:2] == ["release", "edit"] for call in self.state()["calls"]), 1) - - def test_releasing_without_a_draft_is_refused(self): - with self.assertRaisesRegex(publish.PublishError, "has no draft"): - publish.release(self.github, "v0.82.0", self.log.append) - - def test_latest_marks_the_highest_stable_left_in_the_feed(self): - feed_items = items((830001, "0.83.0-beta.1", "beta"), (140, "0.81.0", None)) - self.assertEqual(publish.latest(self.github, feed_items, self.log.append), "v0.81.0") - self.assertEqual(self.state()["calls"][-1], ["release", "edit", "v0.81.0", "--latest=true", - "--repo", REPOSITORY]) - - def test_draft_prints_only_the_state_on_stdout(self): - arguments = ["draft", "v0.82.0", "--notes", str(self.notes), *[str(path) for path in self.files]] - with redirect_stdout(io.StringIO()) as output, redirect_stderr(io.StringIO()): - self.assertEqual(publish.main(arguments), 0) - self.assertEqual(output.getvalue(), "state=draft\n") - with redirect_stdout(io.StringIO()) as output, redirect_stderr(io.StringIO()): - self.assertEqual(publish.main(["release", "v0.82.0"]), 0) - self.assertEqual(publish.main(arguments), 0) - self.assertEqual(output.getvalue(), "state=published\n") - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/release/test_registry.py b/scripts/release/test_registry.py index 53eec9d538..b6dcf1af61 100644 --- a/scripts/release/test_registry.py +++ b/scripts/release/test_registry.py @@ -6,42 +6,25 @@ import tempfile import unittest import zipfile -from contextlib import redirect_stderr, redirect_stdout +from contextlib import redirect_stdout from pathlib import Path import registry import version URL_BASE = "https://github.com/TableProApp/TablePro/releases/download/v0.82.0/" -PROJECT = """\ -targets: - OracleDriver: - templates: [DriverPlugin] - -aggregateTargets: - BundledPlugins: - targets: - - MySQLDriver - - SQLiteDriver - - RegistryPlugins: - targets: - - OracleDriver - - ParquetExport - scheme: {} -""" METADATA = { "retainFromPluginKit": 39, "plugins": { - "OracleDriver": {"name": "Oracle Driver", "iconName": "oracle-icon"}, - "ParquetExport": {"name": "Parquet Export", "iconName": "parquet-icon"}, + "OracleDriver": {"name": "Oracle Driver"}, + "ParquetExport": {"name": "Parquet Export"}, }, } def entry(text="0.82.0", kit=39, system="13.0", sha="a", plugin="com.TablePro.OracleDriver", name="Oracle Driver"): return { - "id": plugin, "name": name, "iconName": "oracle-icon", "databaseTypeIds": ["Oracle"], + "id": plugin, "name": name, "databaseTypeIds": ["Oracle"], "exportFormatIds": [], "version": text, "build": version.parse_version(text).build, "pluginKit": kit, "minimumSystemVersion": system, "file": f"OracleDriver-{text}.zip", "sha256": sha * 64, } @@ -82,8 +65,8 @@ def test_creates_the_manifest_and_an_entry(self): "schemaVersion": 3, "retainFromPluginKit": 39, "plugins": [{ - "id": "com.TablePro.OracleDriver", "name": "Oracle Driver", "iconName": "oracle-icon", - "databaseTypeIds": ["Oracle"], + "id": "com.TablePro.OracleDriver", "name": "Oracle Driver", "databaseTypeIds": ["Oracle"], + "exportFormatIds": [], "releases": [{ "version": "0.82.0", "build": 820099, "channel": "stable", "pluginKit": 39, "minimumSystemVersion": "13.0", "url": URL_BASE + "OracleDriver-0.82.0.zip", "sha256": "a" * 64, @@ -173,16 +156,6 @@ def test_another_schema_is_refused(self): with self.assertRaisesRegex(registry.RegistryError, "schemaVersion 4"): self.add([entry()]) - def test_a_build_that_is_not_the_versions_build_is_refused(self): - wrong = entry() - wrong["build"] = 140 - with self.assertRaisesRegex(registry.RegistryError, "build 140 is not 820099"): - self.add([wrong]) - - def test_a_version_on_the_wrong_channel_is_refused(self): - with self.assertRaisesRegex(registry.RegistryError, "belongs to the beta channel"): - self.add([entry("0.82.0-beta.1")], channel="stable") - class WithdrawTests(ManifestCase): def test_a_withdraw_brings_the_previous_build_back_into_selection(self): @@ -205,29 +178,14 @@ def test_a_plugin_left_with_no_release_leaves_the_manifest(self): class TargetsTests(unittest.TestCase): - def test_reads_the_bundled_aggregate(self): - self.assertEqual(registry.bundled_targets(PROJECT), {"MySQLDriver", "SQLiteDriver"}) - - def test_reads_the_bundled_aggregate_of_this_repository(self): - bundled = registry.bundled_targets(registry.PROJECT.read_text(encoding="utf-8")) - self.assertIn("MySQLDriver", bundled) - self.assertNotIn("OracleDriver", bundled) - def test_prints_the_publish_set(self): with tempfile.TemporaryDirectory() as directory: - metadata, project = Path(directory) / "registry.json", Path(directory) / "project.yml" + metadata = Path(directory) / "registry.json" metadata.write_text(json.dumps(METADATA)) - project.write_text(PROJECT) with redirect_stdout(io.StringIO()) as output: - self.assertEqual(registry.main(["targets", "--metadata", str(metadata), "--project", str(project)]), 0) + self.assertEqual(registry.main(["targets", "--metadata", str(metadata)]), 0) self.assertEqual(output.getvalue(), "OracleDriver\nParquetExport\n") - def test_refuses_a_bundled_target(self): - metadata = {"retainFromPluginKit": 39, "plugins": {**METADATA["plugins"], - "MySQLDriver": {"name": "MySQL", "iconName": "mysql"}}} - with self.assertRaisesRegex(registry.RegistryError, "the app bundles MySQLDriver"): - registry.publish_set(metadata, registry.bundled_targets(PROJECT)) - def test_refuses_metadata_in_another_shape(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "registry.json" @@ -235,6 +193,17 @@ def test_refuses_metadata_in_another_shape(self): with self.assertRaisesRegex(registry.RegistryError, "shape"): registry.load_metadata(path) + def test_reads_the_pluginkit_version(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + config = root / registry.PLUGINKIT_CONFIG + config.parent.mkdir() + config.write_text("TABLEPRO_PLUGINKIT_VERSION = 39\nTABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19\n") + self.assertEqual(registry.pluginkit_version(root), 39) + config.write_text("TABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19\n") + with self.assertRaisesRegex(registry.RegistryError, "sets no TABLEPRO_PLUGINKIT_VERSION"): + registry.pluginkit_version(root) + class DescribeTests(unittest.TestCase): def setUp(self): @@ -257,8 +226,7 @@ def plugin(self, target, **overrides): return path def describe(self, metadata=None): - return registry.describe(self.zips, "0.82.0", 820099, 39, metadata or METADATA, - registry.bundled_targets(PROJECT)) + return registry.describe(self.zips, "0.82.0", 820099, 39, metadata or METADATA) def test_reads_each_zip(self): self.plugin("OracleDriver") @@ -266,13 +234,13 @@ def test_reads_each_zip(self): TableProProvidesDatabaseTypeIds=None, TableProProvidesExportFormatIds=["parquet"]) oracle, parquet = self.describe() self.assertEqual(oracle["id"], "com.TablePro.OracleDriver") - self.assertEqual(oracle["databaseTypeIds"], ["Oracle"]) self.assertEqual((oracle["version"], oracle["build"], oracle["pluginKit"], oracle["minimumSystemVersion"]), ("0.82.0", 820099, 39, "13.0")) self.assertEqual(oracle["file"], "OracleDriver-0.82.0.zip") self.assertRegex(oracle["sha256"], "^[0-9a-f]{64}$") - self.assertEqual((parquet["id"], parquet["databaseTypeIds"], parquet["name"]), - ("com.TablePro.ParquetExportPlugin", [], "Parquet Export")) + self.assertEqual((oracle["databaseTypeIds"], oracle["exportFormatIds"]), (["Oracle"], [])) + self.assertEqual((parquet["id"], parquet["name"]), ("com.TablePro.ParquetExportPlugin", "Parquet Export")) + self.assertEqual((parquet["databaseTypeIds"], parquet["exportFormatIds"]), ([], ["parquet"])) def test_rejects_a_wrong_version_build_or_kit(self): cases = [ @@ -288,6 +256,14 @@ def test_rejects_a_wrong_version_build_or_kit(self): with self.assertRaisesRegex(registry.RegistryError, message): self.describe() + def test_rejects_ids_that_are_not_a_list_of_strings(self): + self.plugin("OracleDriver") + for key in ("TableProProvidesDatabaseTypeIds", "TableProProvidesExportFormatIds"): + with self.subTest(key=key): + self.plugin("ParquetExport", **{key: "parquet"}) + with self.assertRaisesRegex(registry.RegistryError, f"{key} is not a list of strings"): + self.describe() + def test_rejects_a_plugin_that_provides_nothing(self): self.plugin("OracleDriver", TableProProvidesDatabaseTypeIds=None) self.plugin("ParquetExport", TableProProvidesExportFormatIds=["parquet"]) @@ -305,22 +281,16 @@ def test_rejects_two_plugins_with_one_bundle_id(self): with self.assertRaisesRegex(registry.RegistryError, "share the bundle id"): self.describe() - def test_the_cli_writes_entries_that_add_reads(self): - self.plugin("OracleDriver") - self.plugin("ParquetExport", TableProProvidesExportFormatIds=["parquet"]) - metadata, project = self.zips / "registry.json", self.zips / "project.yml" + def test_the_cli_reads_the_kit_from_this_repository(self): + kit = registry.pluginkit_version(registry.ROOT) + self.plugin("OracleDriver", TableProPluginKitVersion=kit) + self.plugin("ParquetExport", TableProPluginKitVersion=kit, TableProProvidesExportFormatIds=["parquet"]) + metadata = self.zips / "registry.json" metadata.write_text(json.dumps(METADATA)) - project.write_text(PROJECT) - entries, manifest = self.zips / "entries.json", self.zips / "v3.json" - with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): - self.assertEqual(registry.main([ - "describe", "--zips", str(self.zips), "--version", "0.82.0", "--build", "820099", "--kit", "39", - "--metadata", str(metadata), "--project", str(project), "--out", str(entries)]), 0) - self.assertEqual(registry.main([ - "add", "--manifest", str(manifest), "--entries", str(entries), "--channel", "stable", - "--url-base", URL_BASE, "--retain-from", "39"]), 0) - self.assertEqual(registry.main(["withdraw", "--manifest", str(manifest), "--version", "0.82.0"]), 0) - self.assertEqual(json.loads(manifest.read_text())["plugins"], []) + with redirect_stdout(io.StringIO()) as output: + self.assertEqual(registry.main(["describe", "--zips", str(self.zips), "--version", "0.82.0", + "--build", "820099", "--metadata", str(metadata)]), 0) + self.assertEqual(output.getvalue(), "described 2 plugins\n") if __name__ == "__main__": diff --git a/scripts/release/test_version.py b/scripts/release/test_version.py index 354e1a45ee..9c7cad66f2 100644 --- a/scripts/release/test_version.py +++ b/scripts/release/test_version.py @@ -108,10 +108,15 @@ def git(self, *args: str) -> str: return subprocess.run(["git", "-C", str(self.path), *args], env=GIT_ENV, check=True, capture_output=True, text=True).stdout.strip() - def commit(self, message: str, changelog: str = None) -> str: + def commit(self, message: str, changelog: str = None, fragment: str = None) -> str: if changelog is not None: (self.path / "CHANGELOG.md").write_text(changelog, encoding="utf-8") self.git("add", "CHANGELOG.md") + if fragment is not None: + path = self.path / "changelog.d" / fragment + path.parent.mkdir(exist_ok=True) + path.write_text("A fix.\n", encoding="utf-8") + self.git("add", str(path)) self.git("commit", "--quiet", "--allow-empty", "-m", message) return self.git("rev-parse", "HEAD") @@ -292,6 +297,26 @@ def test_a_stable_needs_its_heading(self): errors = self.check("v0.82.1", FakeGitHub(releases=[("v0.81.0", False)])) self.assertEqual(errors, ["CHANGELOG.md at v0.82.1 has no ## [0.82.1]"]) + def test_a_stable_refuses_fragments_left_at_its_tag(self): + changelog = MAIN_CHANGELOG.replace("## [0.81.0]", "## [0.81.1] - 2026-10-25\n\n## [0.81.0]") + self.repo.commit("chore: keep the fragment folder", fragment=".gitkeep") + self.repo.commit("chore(release): v0.81.1", changelog) + pick = self.repo.commit("fix: a pick after the release PR was cut", fragment="3610.fixed.md") + self.repo.line("0.81", pick) + self.repo.git("tag", "v0.81.1", pick) + runs, jobs = green_on(pick) + github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) + self.assertEqual(self.check("v0.81.1", github), + ["v0.81.1 still has 3610.fixed.md in changelog.d/; assemble again on the line head"]) + + def test_a_beta_ships_with_its_fragments(self): + tagged = self.repo.commit("fix: a fix", fragment="3610.fixed.md") + self.repo.line("0.82", tagged) + self.repo.git("tag", "v0.82.0-beta.1", tagged) + runs, jobs = green_on(tagged) + github = FakeGitHub(releases=[("v0.81.0", False)], runs=runs, jobs=jobs) + self.assertEqual(self.check("v0.82.0-beta.1", github), []) + def test_a_beta_refuses_a_line_whose_stable_is_assembled(self): assembled = self.repo.commit("chore(release): v0.82.0", MAIN_CHANGELOG.replace( "## [0.81.0]", "## [0.82.0] - 2026-10-20\n\n## [0.81.0]")) diff --git a/scripts/release/version.py b/scripts/release/version.py index bcdbdcd2d0..3a8b2687b8 100755 --- a/scripts/release/version.py +++ b/scripts/release/version.py @@ -25,6 +25,7 @@ ("release-build.yml", "Release Build", "release"), ) DETECTOR = "scripts/ci/detect-changed-paths.sh" +FRAGMENTS = "changelog.d/" WALK_DEPTH = 50 POLL_SECONDS = 60 PENDING_STATUSES = {"queued", "in_progress", "waiting", "requested", "pending"} @@ -127,6 +128,9 @@ def show(self, ref: str, path: str) -> Optional[str]: result = self._git("show", f"{ref}:{path}", check=False) return result.stdout if result.returncode == 0 else None + def list_tree(self, ref: str, path: str) -> List[str]: + return self._git("ls-tree", "--name-only", ref, "--", path).stdout.splitlines() + def first_parents(self, ref: str, limit: int) -> List[str]: return self._git("rev-list", "--first-parent", f"--max-count={limit}", f"{ref}^{{commit}}").stdout.split() @@ -238,6 +242,15 @@ def changelog_errors(changelog: Optional[str], version: Version, stables: List[V return errors +def fragment_errors(paths: List[str], version: Version) -> List[str]: + if version.is_beta: + return [] + fragments = [Path(path).name for path in paths if Path(path).name != ".gitkeep"] + if not fragments: + return [] + return [f"{version.tag} still has {', '.join(fragments)} in {FRAGMENTS}; assemble again on the line head"] + + def is_green(github: GitHub, workflow: str, job: str, sha: str, sleep: Callable[[float], None], log: Callable[[str], None]) -> bool: while True: @@ -296,6 +309,7 @@ def check(tag: str, git: Git, github: GitHub, changed: Optional[Callable[[str, s errors = branch_errors(git, version) errors += beta_errors(git, version, releases) errors += changelog_errors(git.show(tag, "CHANGELOG.md"), version, published_stables(releases)) + errors += fragment_errors(git.list_tree(tag, FRAGMENTS), version) if errors: return errors return ci_errors(git, github, version, changed or detector(git.root), sleep, log)