diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a47629b..290c9f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,9 @@ name: Release # tag-guard — resolve the input tag to its immutable PEELED commit SHA, then validate the # tag shape (vX.Y.Z). Fails closed before anything else runs. Exports version+sha. # test — full bun lane + packaging contract lane (test-before-publish). -# build-deb — build the .deb set (both arches), contract + smoke, per-release manifest. +# build-deb — DIFFERENTIAL .deb set (both arches): resolve the previous release once, detect +# which upstream sources actually changed, carry the rest forward byte-identically, +# rebuild only the changed ones, then contract + smoke + per-release manifest. # publish-npm — integrity-idempotent OIDC trusted publish of @ceralive/modem-control@X.Y.Z. # create-release — stage sanitized assets, immutably reconcile the GitHub release. # @@ -27,6 +29,11 @@ on: description: "Unified release tag, exactly vX.Y.Z (no pre-release, no build metadata)" required: true type: string + force_rebuild: + description: "Rebuild EVERY upstream source, ignoring per-source change detection (defense-in-depth escape hatch; a shared-input change force-alls on its own)" + required: false + type: boolean + default: false permissions: contents: read @@ -154,10 +161,16 @@ jobs: # host docker daemon, not to be a container itself. runs-on: ubuntu-latest steps: + # fetch-depth: 0 is LOAD-BEARING here and nowhere else. detect-changed-sources.sh takes + # `git diff --name-only ..HEAD` and reads `:packaging/upstream-pins.yaml`; + # under the shallow default neither the history nor the previous tag is present, so the + # detector's `previous-ref-unresolvable` rule would force-all on EVERY release — safe, but it + # would silently disable the differential pipeline this job exists to run. - name: Checkout the resolved commit uses: actions/checkout@v7 with: ref: ${{ needs.tag-guard.outputs.sha }} + fetch-depth: 0 - name: Assert checkout is pinned to the resolved SHA env: @@ -174,13 +187,105 @@ jobs: - name: Set up QEMU (arm64 emulation) uses: docker/setup-qemu-action@v4 - # Real rebuilds. RELEASE_VERSION injects -~ceraliveX.Y.Z into a COPY of - # each source's debian/changelog inside the container (the committed tree is never - # mutated), builds in bootstrap order on native amd64 + QEMU arm64, and asserts the - # per-source package sets. --force-bad-version is applied by inject-deb-version. + # The previous release is resolved and its manifest downloaded EXACTLY ONCE, here, and the + # resulting path is handed to all three consumers below (detection, carry-forward staging, + # per-source counter derivation). `gh release list` — never `git describe`: the previous + # release is the latest PUBLISHED release, which is a different question from "nearest tag". + # The resolution flags mirror detect-changed-sources.sh's own so the two cannot disagree. + # + # NO previous release, or a release carrying no manifest asset, leaves BOTH outputs empty. + # That is the bootstrap case, not an error: an empty PREV_TAG/PREV_MANIFEST_FILE reads as + # unset downstream, and the detector's own force-all rules take over. + - name: Resolve previous release + fetch its manifest (once) + id: prev-release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + MANIFEST_DEST: ${{ runner.temp }}/previous-release-manifest.txt + run: | + set -euo pipefail + prev_tag="$(gh release list --repo "$GH_REPO" --limit 1 \ + --exclude-drafts --exclude-pre-releases --json tagName --jq '.[0].tagName' 2>/dev/null || true)" + if [ -z "$prev_tag" ] || [ "$prev_tag" = "null" ]; then + echo "no published previous release — the differential pipeline bootstraps by force-alling" + { + echo "prev_tag=" + echo "prev_manifest=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "previous published release: ${prev_tag}" + dl="$(mktemp -d)" + if ! gh release download "$prev_tag" --repo "$GH_REPO" \ + --pattern 'release-manifest*.txt' --dir "$dl" >/dev/null 2>&1; then + echo "::warning::release ${prev_tag} carries no downloadable release-manifest asset — every source will rebuild" + { + echo "prev_tag=${prev_tag}" + echo "prev_manifest=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + found="$(find "$dl" -maxdepth 1 -name 'release-manifest*.txt' -type f | head -n1)" + if [ -z "$found" ]; then + echo "::warning::no release-manifest*.txt downloaded from ${prev_tag} — every source will rebuild" + { + echo "prev_tag=${prev_tag}" + echo "prev_manifest=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + cp "$found" "$MANIFEST_DEST" + echo "previous release manifest: ${MANIFEST_DEST} ($(wc -l < "$MANIFEST_DEST") lines)" + { + echo "prev_tag=${prev_tag}" + echo "prev_manifest=${MANIFEST_DEST}" + } >> "$GITHUB_OUTPUT" + + # Per-source verdicts (`=changed|unchanged` + `mode=`) into verdicts.txt. Fail-SAFE + # here means REBUILD EVERYTHING: absent previous release, absent/v1-shaped manifest, a + # shared `packaging/ci/**` input change, or force_rebuild all yield mode=force-all. + - name: Detect changed sources (per-source verdicts) + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PREV_TAG: ${{ steps.prev-release.outputs.prev_tag }} + PREV_MANIFEST_FILE: ${{ steps.prev-release.outputs.prev_manifest }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild == 'true' && 'all' || '' }} + run: | + bash packaging/ci/detect-changed-sources.sh --out verdicts.txt + echo "--- verdicts.txt ---" + cat verdicts.txt + + # STRICTLY BEFORE any build-bookworm.sh invocation. Carried debs are not merely release + # assets, they are a build INPUT: build-bookworm.sh seeds its Pin-Priority-1001 local apt + # repo from packaging/build//, so a changed source resolves its build-deps and gir + # typelibs against the carried -dev/gir1.2-* packages instead of stock bookworm. Staging + # after the build would silently reintroduce stock dependencies. + - name: Stage carry-forward debs (unchanged sources, sha256-verified) + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PREV_TAG: ${{ steps.prev-release.outputs.prev_tag }} + PREV_MANIFEST_FILE: ${{ steps.prev-release.outputs.prev_manifest }} + run: bash packaging/ci/stage-carryforward-debs.sh --verdicts verdicts.txt --build-root packaging/build + + # Differential rebuilds. VERDICTS_FILE supplies the build set EXPLICITLY — the script's + # build-all default when neither VERDICTS_FILE nor BUILD_SOURCES is set is a local-dev + # convenience and must never be what CI relies on. PREV_MANIFEST_FILE is the file resolved + # above, reused rather than re-fetched, and is what each rebuilt source's next ~ceralive.N + # counter is derived from. RELEASE_VERSION injects into a COPY of each changelog inside the + # container (the committed tree is never mutated). A zero-build run starts no container at + # all and still asserts the merged runtime closure over the carried set. + # + # RESIDUAL RISK, ACCEPTED BY THE PLAN: the zero-build path reaching manifest generation with + # staged-only debs is proven by contract fixtures (packaging/ci/test-build-bookworm-differential.sh, + # test-suffix-coherence-manifest.sh) and by static wiring proof (test-release-workflow-wiring.sh). + # Full end-to-end proof lands at the FIRST REAL RELEASE RUN; it is deliberately not simulated. - name: Build the MM 1.24 stack (.deb) — amd64 + arm64 env: RELEASE_VERSION: ${{ github.event.inputs.tag }} + VERDICTS_FILE: verdicts.txt + PREV_MANIFEST_FILE: ${{ steps.prev-release.outputs.prev_manifest }} run: | packaging/ci/build-bookworm.sh amd64 packaging/ci/build-bookworm.sh arm64 diff --git a/.github/workflows/upstream-watch.yml b/.github/workflows/upstream-watch.yml new file mode 100644 index 0000000..06b10e9 --- /dev/null +++ b/.github/workflows/upstream-watch.yml @@ -0,0 +1,111 @@ +name: Upstream freshness watch + +# Weekly check: has a newer STABLE release of ModemManager / libmbim / libqmi / libqrtr-glib +# (or of its Debian salsa packaging tag) appeared since we pinned? +# +# ISSUE-ONLY, BY DESIGN. This workflow opens or updates ONE labelled issue and does nothing +# else. It never edits packaging/upstream-pins.yaml and it never dispatches a build — bumping +# a pin is a separate, human-reviewed change that must re-run +# packaging/ci/verify-upstream-pins.sh (the four-link provenance chain). That is why the job +# escalates only `issues: write` and holds no dispatch token. +# +# THE DEV-SERIES FILTER IS THE POINT. All four sources publish their unstable train on the same +# tag namespace as their releases (ModemManager 1.25.95 -> Debian *experimental*), so a naive +# "newest tag wins" watch would file a bump request for a development snapshot roughly every +# other week and train everyone to ignore it. packaging/ci/check-upstream-freshness.sh applies +# an explicit stable-only filter; packaging/ci/test-check-upstream-freshness.sh pins it offline. +# +# GITHUB DISABLES SCHEDULED WORKFLOWS AFTER 60 DAYS OF REPOSITORY INACTIVITY. If this watch goes +# quiet, that is the first thing to check: re-enable it from the Actions tab (or push a commit), +# then confirm with a manual `workflow_dispatch` run. A silent watch reads exactly like an +# up-to-date one, which is the failure mode worth knowing about. +on: + schedule: + # Mondays, 06:17 UTC. Off the hour on purpose — GitHub's scheduler is heavily contended at + # :00 and delays a run that has no deadline anyway. + - cron: "17 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +# Schedule + manual dispatch, never a PR gate: a run in flight is doing the real check and must +# not be cancelled by the next trigger. +concurrency: + group: upstream-watch + cancel-in-progress: false + +jobs: + watch: + name: Compare the pins against upstream + Debian salsa stable tags + runs-on: ubuntu-latest + permissions: + contents: read + # The ONLY escalation in this repository's workflows. Needed to create the + # `upstream-freshness` label and to open/edit the single tracking issue. + issues: write + env: + ISSUE_LABEL: upstream-freshness + ISSUE_TITLE: "Upstream freshness: a newer stable ModemManager-stack release is available" + steps: + - uses: actions/checkout@v7 + + # `runner.temp` is unavailable in a job-level `env:` block, so the body path is declared + # per step. It stays out of the checkout so the workspace is never dirtied. + - name: Enumerate upstream + salsa tags and compare against the pins + id: check + env: + ISSUE_BODY_FILE: ${{ runner.temp }}/upstream-freshness-issue.md + run: | + set -uo pipefail + rc=0 + bash packaging/ci/check-upstream-freshness.sh --issue-body "$ISSUE_BODY_FILE" || rc=$? + case "$rc" in + 0) echo "Every pinned source is current (or upstream is ahead with no Debian packaging yet)." + echo "behind=false" >> "$GITHUB_OUTPUT" ;; + 10) echo "At least one pinned source is behind a newer stable release." + echo "behind=true" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::check-upstream-freshness.sh could not complete (exit $rc)." + exit "$rc" ;; + esac + + - name: Open or update the single tracking issue + if: steps.check.outputs.behind == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ISSUE_BODY_FILE: ${{ runner.temp }}/upstream-freshness-issue.md + run: | + set -euo pipefail + # The label is what makes re-runs UPDATE instead of duplicate, so create it on demand + # rather than assuming an operator pre-created it. + gh label create "$ISSUE_LABEL" \ + --description "Scheduled watch: a newer stable upstream/Debian pin is available" \ + --color "0E8A16" >/dev/null 2>&1 || true + + existing="$(gh issue list --label "$ISSUE_LABEL" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + + if [ -n "$existing" ]; then + gh issue edit "$existing" --title "$ISSUE_TITLE" --body-file "$ISSUE_BODY_FILE" + echo "Updated existing issue #$existing." + else + gh issue create --title "$ISSUE_TITLE" --label "$ISSUE_LABEL" --body-file "$ISSUE_BODY_FILE" + fi + + - name: Close the tracking issue once every pin is current again + if: steps.check.outputs.behind == 'false' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + existing="$(gh issue list --label "$ISSUE_LABEL" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + if [ -n "$existing" ]; then + gh issue close "$existing" \ + --comment "Every pinned source is current again — closed by the scheduled upstream-freshness watch." + echo "Closed issue #$existing." + else + echo "Nothing to close." + fi diff --git a/AGENTS.md b/AGENTS.md index 84ead34..1c728c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,9 +77,32 @@ do not reach up the tree. SemVer, **not** CalVer — this repo is the documented exception (alongside `srtla-send-rs`). ONE unified tag `vX.Y.Z` releases **both** artifacts: `@ceralive/modem-control@X.Y.Z` on -npm and the `.deb` set. `.deb` versions encode the tag as `-~ceralive` -(upstream-ordered, apt-safe; injected with `dch --force-bad-version`). Non-tag CI builds use -`~ceralive0.0.0~dev`. Full contract: `docs/VERSIONING.md`. +npm and the `.deb` set. Non-tag CI builds use `~ceralive0.0.0~dev`. Full contract: +`docs/VERSIONING.md`. + +**`.deb` versions no longer encode the tag.** Releases are DIFFERENTIAL, so each upstream +source carries its own rebuild counter: `-~ceralive.N` (upstream-ordered, +apt-safe; injected with `dch --force-bad-version`). A REBUILT source takes its previous +counter + 1, derived from the previous release manifest's rows for that source; an +UNTOUCHED source is carried forward byte-identically and keeps the counter it already had, +never re-stamped with the new tag. Two sources at different counters in one release is the +normal shape, not drift — coherence is a PER-SOURCE property, and a source disagreeing with +ITSELF fails closed naming that source. Derivation reads every row of a source (both arches, +runtime and aux) and refuses on a counter disagreement, a counter/legacy mixture, or a +malformed suffix; entirely-legacy rows and an absent previous manifest bootstrap at `.1`. + +The pre-`v1.0.0` releases WERE built with one uniform `~ceralive` suffix shared by all +four sources; those published artifacts are unchanged. No release mixes the two schemes — the +first differential release force-rebuilds every source at `.1`, because this effort's own +`packaging/ci/**` changes are a shared build input and force-all on their own. The +migration-continuity chain `~ceralive0.2.0 < ~ceralive1.0.0 < ~ceralive1.1.0 < ~ceralive.1 < +~ceralive.2 < ~ceralive.10 < -` is proven with real `dpkg --compare-versions` +by the ONE sourced library `packaging/ci/suffix-contract.sh`, from both +`test-suffix-coherence-manifest.sh` (host) and `test-package-contract.sh` CHECK 5/6 +(container). The release manifest states `suffix_scheme: per-source-counter` and carries **no +`deb_version_suffix:`** — under per-source counters no single suffix value is truthful. The +companion `ceralive-modem-support` stays outside this entirely: bare SemVer tag version, no +`~ceralive` suffix, and always rebuilt. ## FROZEN V1.1 DOMAIN CONTRACTS @@ -249,6 +272,17 @@ arch-dependent stanzas + enumerated `-dbgsym`) for exact per-source set **equali `packaging/ci/check-package-sets.sh` (add/remove/rename fails closed). Full detail: `packaging/README.md`. +**The pins are watched, never auto-bumped.** `.github/workflows/upstream-watch.yml` runs +weekly (plus `workflow_dispatch`) and calls `packaging/ci/check-upstream-freshness.sh`, which +enumerates each source's upstream release tags and salsa `debian/*` packaging tags via +`git ls-remote --tags`, filters the development series out, and compares the survivors to the +four pins above. On `behind` it opens **or updates** ONE issue labelled `upstream-freshness`, +and closes it when everything is current again. It is **issue-only**: it never edits +`upstream-pins.yaml` and never dispatches a build, which is why it is the only workflow here +holding `issues: write` and no dispatch token. A newer upstream release with no matching +Debian packaging tag reports the distinct `upstream-ahead-no-packaging` — there is no +`-` pair to pin, so there is no bump to recommend. + ## MUTATION ADMISSION + EXCLUSIVE OWNERSHIP `control/src/ports/mutation-admission.ts` defines `MutationAdmissionPort`. It is an injected @@ -1350,13 +1384,44 @@ major action versions, per-manager caches, weekly grouped Dependabot, test-befor before any other job). Exports `version` + `sha`. 2. **test** (needs tag-guard) — full bun lane + packaging contract lane (test-before-publish). - 3. **build-deb** (needs [tag-guard, test]) — injects `-~ceralive` - (non-tag runs `~ceralive0.0.0~dev`) via `packaging/ci/inject-deb-version.sh`, builds both - arches, runs the package contract suite + daemon smoke, builds the `Architecture: all` - companion ONCE (`packaging/ci/build-companion.sh`) and runs its clean-chroot contract - (`packaging/ci/test-companion-chroot.sh`), generates the manifest-complete - release manifest (`packaging/ci/generate-release-manifest.sh`), and uploads the `.deb` - artifacts + manifest. + 3. **build-deb** (needs [tag-guard, test]) — the **DIFFERENTIAL** `.deb` job. Steps, in file + order: **Checkout the resolved commit** (`fetch-depth: 0` — load-bearing here and nowhere + else, since the detector diffs `..HEAD`; a shallow checkout would silently + force-all forever) → **Assert checkout is pinned to the resolved SHA** → **Set up QEMU + (arm64 emulation)** → **Resolve previous release + fetch its manifest (once)** + (`id: prev-release`; `gh release list`, never `git describe` — the previous release is the + latest PUBLISHED one; no release or no manifest asset leaves both outputs empty, which is + the bootstrap case, not an error) → **Detect changed sources (per-source verdicts)** + (`packaging/ci/detect-changed-sources.sh --out verdicts.txt`) → **Stage carry-forward debs + (unchanged sources, sha256-verified)** (`packaging/ci/stage-carryforward-debs.sh`) → + **Build the MM 1.24 stack (.deb) — amd64 + arm64** (`build-bookworm.sh amd64` /`arm64`, + with `VERDICTS_FILE` + the already-resolved `PREV_MANIFEST_FILE`) → **Package contract + suite** (amd64 full, arm64 metadata) → **Daemon smoke (amd64)** → **Build the first-party + companion .deb (Architecture: all)** (UNCONDITIONAL — the companion is never detected and + never carried) → **Companion package contract (clean Debian chroot)** → **Generate release + manifest** (`packaging/ci/generate-release-manifest.sh`) → **Upload .deb artifacts + + release manifest**. + + Three things about that order are load-bearing. The previous release is resolved and its + manifest downloaded **exactly once**, and that one path feeds all three consumers + (detection, carry-forward staging, per-source counter derivation). Carry-forward staging + runs **strictly before any `build-bookworm.sh` call**, because carried debs are a build + INPUT — `build-bookworm.sh` seeds its Pin-Priority-1001 local apt repo from + `packaging/build//`, so a changed source resolves its build-deps and gir typelibs + against the carried `-dev`/`gir1.2-*` packages rather than stock bookworm; staging late + still goes green and silently reintroduces stock dependencies, which is why + `packaging/ci/test-release-workflow-wiring.sh` pins the ordering statically. And a + zero-build run starts no container at all yet still asserts the merged runtime closure + over the carried set. + + Detection is **fail-SAFE toward rebuilding**: an absent previous release, a manifest with + no `closure_version:` header (an absent header IS closure version 1), a shared-input + change under `packaging/ci/**` or `packaging/BOOKWORM-ADAPTATIONS.md`, or the operator's + escape hatch all yield `mode=force-all`. That escape hatch is the `force_rebuild` + `workflow_dispatch` boolean input (default `false`), mapped to the script's + `FORCE_REBUILD=all` env via + `${{ github.event.inputs.force_rebuild == 'true' && 'all' || '' }}` — defense in depth, since + a shared-input change force-alls on its own. 4. **publish-npm** (needs [tag-guard, build-deb]) — OIDC trusted publishing (`id-token: write`), verifies `control/package.json` version === tag, then an **integrity-idempotent** publish: `npm pack` → classify registry state (404 → publish; @@ -1379,6 +1444,22 @@ major action versions, per-manager caches, weekly grouped Dependabot, test-befor the release exists. An operator's own `gh api` call would test the operator's CLI token and prove nothing about the repository secret — which is exactly why this lives here. `cancel-in-progress: false` (never cancel a release/publish mid-run). +- **`.github/workflows/upstream-watch.yml`** — the weekly **upstream freshness watch** + (schedule + `workflow_dispatch`, `cancel-in-progress: false`). Runs + `packaging/ci/check-upstream-freshness.sh`, which enumerates each source's upstream release + tags and salsa `debian/*` packaging tags via `git ls-remote --tags`, filters out the + development series, and compares the survivors to the pins. On `behind` it opens **or + updates** ONE issue labelled `upstream-freshness`; when everything is current again it closes + it. **Issue-only** — it never edits `packaging/upstream-pins.yaml` and never dispatches a + build, which is why it is the ONLY workflow here that escalates `issues: write` and why it + holds no dispatch token. The stable filter is the substance: all four projects publish their + unstable train on the same tag namespace (ModemManager `1.25.95` → Debian *experimental*), so + `-rc`/`-dev`, non-`X.Y.Z`, **odd-minor** and `.9x`-micro tags are rejected, as are `~`-bearing + Debian revisions. A newer upstream release with no Debian packaging tag reports the distinct + `upstream-ahead-no-packaging` — NOT `behind`, because with no `-` pair there is + no bump to recommend. `packaging/ci/test-check-upstream-freshness.sh` pins all of it offline + through a fixture seam. NOTE: GitHub disables scheduled workflows after 60 days of repository + inactivity — a silent watch reads exactly like an up-to-date one. Action pins track the latest stable **major** (resolved via the `gh api` releases/latest endpoint); Dependabot keeps them current. JS/TS CI runs on **Node 26** — the CeraLive CI diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md index 8a2fb51..1c4cf2e 100644 --- a/docs/VERSIONING.md +++ b/docs/VERSIONING.md @@ -31,53 +31,131 @@ Do **not** apply the CalVer scheme here. The rationale, and the parallel excepti convention. Everything in this repo — the git tag, the npm version, and the `~ceralive` suffix below — is SemVer. -## `.deb` version encoding +## `.deb` version encoding — per-source rebuild counters The `.deb` internal `Version:` field must stay **upstream-ordered** so `apt` compares -releases correctly, while still encoding which repo tag produced the rebuild. The -encoding is: +releases correctly, while still recording that this is a CeraLive rebuild and how many +times that particular source has been rebuilt. The encoding is: ``` --~ceralive +-~ceralive.N ``` -For the current pins at repo tag `v0.2.0` (the latest release — the ModemManager revision -is `-2`, the other three `-1`): - -| Source | Upstream | Encoded `.deb` version | +`N` is that **source's own rebuild counter**, not the release tag. Releases are +differential: only the sources whose inputs actually moved are rebuilt, and only a rebuilt +source's counter advances. Release provenance lives in the release manifest; the package +version records the rebuild count. + +- A **rebuilt** source takes its previous counter **+ 1**. The previous counter is derived + from the previous release manifest's rows for that source, by + `packaging/ci/inject-deb-version.sh`. +- An **untouched (carried-forward)** source keeps whatever counter it already had. Its + `.deb`s are the byte-identical artifacts from the release that last built them; they are + **not** re-stamped with the new release's tag. Builds are not reproducible, so reusing + recorded bytes is the only honest way to keep a release self-contained. +- Different sources therefore legitimately carry **different** counters within one release. + That is the normal shape of a differential release, not drift. + +Counter derivation is **coherence-checked and fail-closed**. Every previous-manifest row +for the source is read — both arches, `role=runtime` and `role=aux` alike — and the counter +is accepted only if all of them agree. Three conditions refuse the release outright, each +naming the offending source: + +| Condition | Why it refuses | +|-----------|----------------| +| rows disagree on the counter (`.2` vs `.3`) | picking either risks publishing a downgrade | +| rows mix counter and legacy `~ceraliveX.Y.Z` suffixes | the source's history is ambiguous | +| a row's suffix is neither `~ceralive.[1-9][0-9]*` nor `~ceraliveX.Y.Z` | malformed input | + +Two cases **bootstrap at `.1`** instead: a source whose previous-manifest rows are entirely +legacy, and a rebuild with no previous manifest at all (the force-all bootstrap). + +The current pins are ModemManager 1.24.2 (revision `-2`), libmbim 1.34.0, libqmi 1.38.0 and +libqrtr-glib 1.4.0 (all `-1`), so a release that rebuilt only libqmi while the other three +carried forward from `.1` would produce: + +| Source | Rebuilt? | Encoded `.deb` version | |--------|----------|------------------------| -| ModemManager | 1.24.2 | `1.24.2-2~ceralive0.2.0` | -| libmbim | 1.34.0 | `1.34.0-1~ceralive0.2.0` | -| libqmi | 1.38.0 | `1.38.0-1~ceralive0.2.0` | -| libqrtr-glib | 1.4.0 | `1.4.0-1~ceralive0.2.0` | +| ModemManager | carried | `1.24.2-2~ceralive.1` | +| libmbim | carried | `1.34.0-1~ceralive.1` | +| libqmi | rebuilt | `1.38.0-1~ceralive.2` | +| libqrtr-glib | carried | `1.4.0-1~ceralive.1` | + +> The authoritative pin manifest is `packaging/upstream-pins.yaml`; the release workflow +> derives `-` from each source's `debian/changelog`, never from a value +> hardcoded in the version script. + +### The legacy suffix, and the migration into counters + +Every release through `v0.2.0` used a different scheme: one `~ceralive` suffix, +identical across all four sources, taken from the release tag. Those artifacts are +published and unchanged — `1.24.2-2~ceralive0.2.0` is still exactly what a fleet device has +installed. What changed is what the pipeline produces from here on. -> The upstream versions above are the current provenance-verified pins. The authoritative -> manifest is `packaging/upstream-pins.yaml`; the release workflow derives `-` -> from each source's `debian/changelog`, never from a value hardcoded in the version script. +There is no release that mixes the two schemes. The first release built from the +differential pipeline **force-rebuilds every source at `.1`**, because this effort's own +changes under `packaging/ci/**` are a shared build input, and a shared-input change in the +diff against the last published release force-alls on its own. So the transition happens in +one release, for all four sources at once, and no manifest ever carries a legacy suffix +beside a counter suffix. -### Why the tilde (`~`) +### Why the tilde (`~`), and why the chain still orders -`dpkg` orders a `~` suffix **lower** than the un-suffixed version: +`dpkg` orders a `~` suffix **lower** than the un-suffixed version, and orders the counter +suffix above every legacy one. The full **migration-continuity chain**, exactly as +`packaging/ci/suffix-contract.sh` proves it: ``` -1.24.2-2~ceralive0.1.0 < 1.24.2-2~ceralive0.2.0 < 1.24.2-2 +~ceralive0.2.0 < ~ceralive1.0.0 < ~ceralive1.1.0 + < ~ceralive.1 < ~ceralive.2 < ~ceralive.10 < ``` -So every CeraLive rebuild sorts **below** a hypothetical stock Debian `1.24.2-2`, and a -newer repo tag (`0.2.0`) sorts **above** an older one (`0.1.0`) — exactly the ordering -`apt` needs. This is why the release workflow injects the version with -`dch --force-bad-version`: the tilde-encoded version is numerically **lower** than the -pinned `-` changelog top, and plain `dch --newversion` refuses a -lower-than-current version (per `dch(1)`). `--force-bad-version` is **required**, not -optional. +(`` is the source's `-`, e.g. `1.24.2-2`.) Its legacy members are every +version that exists as a published artifact today, so the chain is the proof that a fleet +device upgrades cleanly from any shipped release into the counter scheme, and that every +CeraLive rebuild still sorts **below** a hypothetical stock Debian `1.24.2-2`. -The exact injection command, run once per source, is: +Nothing here is asserted on paper. `prove_chain_ordered` runs the whole chain through real +`dpkg --compare-versions`, and it is exercised from both lanes: the host-runnable +`packaging/ci/test-suffix-coherence-manifest.sh` (which also carries a non-vacuity control +showing a lexical compare inverts `.2` against `.10`) and the container suite +`packaging/ci/test-package-contract.sh` CHECK 6. CHECK 5 is the coherence half — sources at +differing counters pass, a source disagreeing with **itself** fails closed naming that +source. + +Tilde ordering is also why the workflow injects with `dch --force-bad-version`: the +tilde-encoded version is numerically **lower** than the pinned `-` changelog +top, and plain `dch --newversion` refuses a lower-than-current version (per `dch(1)`). +`--force-bad-version` is **required**, not optional. + +The exact injection command, run once per **rebuilt** source, is: ```sh -dch --force-bad-version --newversion "-~ceralive" "CeraLive rebuild" +dch --force-bad-version --newversion "-~ceralive.N" "CeraLive rebuild" ``` -All four sources take the **same** `~ceralive` suffix for a given release. +### The manifest header states the scheme, not a value + +Under per-source counters no single suffix value is truthful for a release, so the release +manifest carries: + +``` +suffix_scheme: per-source-counter +``` + +and **no `deb_version_suffix:` header** — that field no longer exists in any manifest this +repo produces. Every row keeps carrying its own version, which it always did (rows are +parsed from real filenames), so a carried-forward deb at an old counter and a freshly built +one at a new counter both emit correctly. `version:` is unrelated and stays: it is the +release's own SemVer, not a per-deb suffix. + +### The companion is outside this scheme entirely + +`ceralive-modem-support` is a first-party native package with no upstream version to order +against. It takes the repo's SemVer tag **verbatim** (`v1.1.0` → `1.1.0`) with no +`~ceralive` suffix at all, and it is **always rebuilt** — it is never detected, never +verdicted, and never carried forward. That is unrelated to the counter scheme, and neither +rule affects the other. ## Tag guard (fail-closed) @@ -92,7 +170,7 @@ Anything that does not match **fails closed before any other job runs**. In part | Input | Result | Why | |-------|--------|-----| | `v1.0.0` | ✅ accepted | canonical `vX.Y.Z` | -| `v1.0.0-rc.1` | ❌ rejected | a pre-release tag inverts dpkg ordering — the rc's tilde-encoded version would outrank the final release (`~ceralive1.0.0-rc.1` vs `~ceralive1.0.0`), which is wrong | +| `v1.0.0-rc.1` | ❌ rejected | a pre-release tag inverts dpkg ordering — the companion takes the tag verbatim, and `dpkg` reads the `-rc.1` as a Debian revision, so the rc would sort **above** the final `1.0.0`, which is wrong | | `v1.0.0+build5` | ❌ rejected | build metadata has no meaning in a `.deb` version and is not part of the contract | | `1.0.0` | ❌ rejected | missing the `v` prefix | diff --git a/packaging/README.md b/packaging/README.md index 9950715..848bc32 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -28,7 +28,8 @@ re-verified end-to-end by [`ci/verify-upstream-pins.sh`](ci/verify-upstream-pins > package **contract suite** (`ci/test-package-contract.sh`) + **daemon smoke** > (`ci/daemon-smoke.sh`) all landed. Both arches build clean with the exact 9-package runtime > closure; the contract suite is green on amd64 (full) and arm64 (metadata). The -> deb-artifact + per-release-manifest job runs inside `release.yml`'s `build-deb`. +> deb-artifact + per-release-manifest job runs inside `release.yml`'s `build-deb`, which is +> now **differential** — see [The differential build flow](#the-differential-build-flow). ## Recipes & build @@ -43,15 +44,60 @@ adaptations (debhelper relax, `systemd-dev → udev`, and systemd/udev install-d documented, with rationale, stock-bookworm citation, and diff shape, in [`BOOKWORM-ADAPTATIONS.md`](BOOKWORM-ADAPTATIONS.md). -[`ci/build-bookworm.sh`](ci/build-bookworm.sh) `` rebuilds all four in a -`debian:bookworm` container in the mandatory bootstrap order +[`ci/build-bookworm.sh`](ci/build-bookworm.sh) `` rebuilds the **selected** +sources in a `debian:bookworm` container in the mandatory bootstrap order `libqrtr-glib → libmbim → libqmi → modemmanager`. Each source's freshly built `.deb`s feed a temporary **local apt repo** so the next source resolves its build-deps against them (not the older bookworm-main versions). Arches are **native** — amd64 directly, arm64 via full-system -QEMU (`--platform linux/arm64`) — never cross-built. The script injects +QEMU (`--platform linux/arm64`) — never cross-built. On a dev run the script injects `~ceralive0.0.0~dev` (via [`ci/inject-deb-version.sh`](ci/inject-deb-version.sh)), runs real -`dpkg-buildpackage`, and asserts the **9-package runtime closure** from the `.changes` -(drift ⇒ non-zero exit). `.deb`s land in the gitignored `build//`. +`dpkg-buildpackage`, and asserts the **9-package runtime closure** (drift ⇒ non-zero exit). +`.deb`s land in the gitignored `build//`. Run with neither `VERDICTS_FILE` nor +`BUILD_SOURCES` set, it builds all four — the local-development default, and deliberately not +what CI relies on. + +### The differential build flow + +A release rebuilds **only the sources whose inputs actually moved**. Everything else is +carried forward byte-identically from the release that last built it. Builds are not +reproducible, so an unchanged source can never be re-created at its old version; reusing the +exact recorded artifacts is the only honest way to keep a release self-contained. The flow, +wired in `release.yml`'s `build-deb`: + +1. **Resolve the previous release and fetch its manifest — ONCE.** The one resolved path + feeds all three consumers below, so they cannot disagree. `gh release list`, never + `git describe`: the previous release is the latest PUBLISHED release, which is a different + question from "nearest tag". +2. **Detect** — [`ci/detect-changed-sources.sh`](ci/detect-changed-sources.sh) emits a + `=changed|unchanged` line per source plus `mode=differential|force-all`. A source + is `changed` when the diff touched its `packaging//**` recipe or its own + block-scoped entry in `upstream-pins.yaml`. It is **fail-SAFE toward rebuilding**: a shared + input change (`packaging/ci/**`, `BOOKWORM-ADAPTATIONS.md`), an absent previous release or + manifest, a v1-shaped manifest, or `FORCE_REBUILD=all` each force-all. A wrong `unchanged` + would ship stale bytes; a wrong `changed` only costs build time. +3. **Stage the carry-forward** — [`ci/stage-carryforward-debs.sh`](ci/stage-carryforward-debs.sh) + downloads every row of each `unchanged` source (runtime **and** aux — `-dbgsym`, `-dev`, + `gir1.2-*`) from the previous release, reverses the uploader's `~`→`.` asset-name + sanitization, sha256-verifies each file against its manifest row, and lands it under the + canonical `~` name in `build//`. Missing, ambiguous, or mismatching ⇒ fail closed + naming the row. This step runs **strictly before any build**: the carried debs are a build + INPUT, because the builder seeds its Pin-Priority-1001 local apt repo from `build//`, + so a changed source resolves its build-deps and gir typelibs against the carried + `-dev`/`gir1.2-*` packages rather than stock bookworm. Staging late still goes green and + silently reintroduces stock dependencies, which is why the ordering is pinned by + [`ci/test-release-workflow-wiring.sh`](ci/test-release-workflow-wiring.sh). +4. **Build the selected set** — `VERDICTS_FILE` supplies the build set explicitly. Skipped + sources' already-staged debs are seeded into the local repo before the first build; only + stale `*.changes`/`*.buildinfo` and artifacts of the selected sources are removed. Each + selected source gets one `inject-deb-version.sh` call which derives that source's next + `~ceralive.N` counter from the already-resolved previous manifest. +5. **Assert the MERGED closure** — the 9-package runtime closure is asserted at the end of + every run over built **plus** carried debs. A **zero-build run** starts no container at all + and still performs that assertion. + +The companion `ceralive-modem-support` sits outside all of it: never detected, never carried, +always rebuilt — see [First-party companion](#first-party-companion-ceralive-modem-support) +below. ## Provenance pins @@ -166,9 +212,16 @@ A clean chroot has no sysfs devices and no running daemons, so the contract is s ### Versioning The companion is a **native** package versioned with the repo's SemVer tag verbatim -(`v1.1.0` → `1.1.0`). It does NOT use the upstream rebuilds' `-~ceraliveX.Y.Z` -form: there is no upstream version to order against, and a bare SemVer sorts correctly on -its own. Non-tag builds are `0.0.0~dev`. +(`v1.1.0` → `1.1.0`). It does NOT use the upstream rebuilds' `-~ceralive.N` +form: there is no upstream version to order against, no rebuild counter to carry, and a bare +SemVer sorts correctly on its own. Non-tag builds are `0.0.0~dev`. + +It is **always rebuilt**, on every release, and that rationale survives the differential +pipeline intact — apt-worker's closure health check greps `^Version: $`, which only the +companion's bare tag version can satisfy, so a carried-forward companion at last release's +version would fail that check. It is therefore never enumerated by change detection, never +verdicted, and never staged as a carry-forward; a companion-only change correctly leaves all +four upstream sources `unchanged`. It is built **exactly once** into `build/all/`. A per-arch build would produce two byte-different files under one package/version key, which the APT publisher's immutable-key @@ -176,8 +229,18 @@ rule refuses. ## Versioning -`.deb` internal versions encode the repo's SemVer tag as `-~ceralive` -(upstream-ordered, apt-safe). Full contract: `docs/VERSIONING.md`. +`.deb` internal versions carry a **per-source rebuild counter**, +`-~ceralive.N` (upstream-ordered, apt-safe) — not the release tag. A rebuilt +source takes its previous counter + 1, derived from the previous release manifest's rows for +that source and accepted only if those rows are coherent; a carried-forward source keeps the +counter it already had. Two sources at different counters within one release is what a +differential release produces, and is accepted; a source disagreeing with **itself** fails +closed naming that source. The migration-continuity chain +`~ceralive0.2.0 < ~ceralive1.0.0 < ~ceralive1.1.0 < ~ceralive.1 < ~ceralive.2 < ~ceralive.10 +< -` is proven with real `dpkg --compare-versions` by the one sourced library +[`ci/suffix-contract.sh`](ci/suffix-contract.sh). The release manifest states +`suffix_scheme: per-source-counter` and carries no `deb_version_suffix:`. Full contract: +`docs/VERSIONING.md`. ## `ci/` scripts @@ -185,19 +248,29 @@ rule refuses. |--------|------| | [`ci/tag-guard.sh`](ci/tag-guard.sh) | The release-tag contract: accepts only `vX.Y.Z`, fails closed on pre-release / build-metadata / missing-`v`. Sourced by `release.yml` (job 1) and by the version-injection + test scripts. | | [`ci/test-tag-guard.sh`](ci/test-tag-guard.sh) | Executable proof of the tag-guard negatives (`v1.0.0-rc.1`, `v1.0.0+build5`, `1.0.0`, …). Run in CI and locally. | -| [`ci/read-pin.sh`](ci/read-pin.sh) | The **shared pin reader**. `read-pin.sh ` prints any scalar from `upstream-pins.yaml` (e.g. `read-pin.sh modemmanager upstream_tag` → `1.24.2`); `read-pin.sh --base-version` prints the full Debian base `-` (e.g. `1.24.2-2`) from that source's `debian/changelog` top entry, **cross-checked** to equal the pin's `salsa_tag` suffix (mismatch fails closed). bash+awk only — its YAML reader is byte-identical to `verify-upstream-pins.sh`'s. Sourced by `daemon-smoke.sh`, `test-package-contract.sh`, and `contract.sh` so every version assertion tracks the pins (no hardcoded literals). | -| [`ci/inject-deb-version.sh`](ci/inject-deb-version.sh) | Writes `-~ceralive` (or `~ceralive0.0.0~dev` for non-tag builds) into each source's `debian/changelog` top entry via `dch --force-bad-version`. Reads upstream versions from each source's changelog — never hardcoded here. | +| [`ci/read-pin.sh`](ci/read-pin.sh) | The **shared pin reader**. `read-pin.sh ` prints any scalar from `upstream-pins.yaml` (e.g. `read-pin.sh modemmanager upstream_tag` → `1.24.2`); `read-pin.sh --base-version` prints the full Debian base `-` (e.g. `1.24.2-2`) from that source's `debian/changelog` top entry, **cross-checked** to equal the pin's `salsa_tag` suffix (mismatch fails closed); `read-pin.sh --list-sources` prints the pinned source NAMES one per line, exposing the reader's own `yaml_sources` so an iterating caller (`check-upstream-freshness.sh`) needs no second copy of the YAML parser. bash+awk only — its YAML reader is byte-identical to `verify-upstream-pins.sh`'s. Sourced by `daemon-smoke.sh`, `test-package-contract.sh`, and `contract.sh` so every version assertion tracks the pins (no hardcoded literals). | +| [`ci/detect-changed-sources.sh`](ci/detect-changed-sources.sh) | **Step 1 of the differential pipeline.** Prints `=changed\|unchanged` for all four sources in bootstrap order plus `mode=differential\|force-all`; `--out ` also writes them for a later job step. A source is `changed` when the diff `..HEAD` touched `packaging//**` or its own **block-scoped** entry in `upstream-pins.yaml` (so a comment edit or a neighbour's pin bump does not implicate it). Force-all — every source `changed`, reason logged — on a shared-input change (`packaging/ci/**`, `BOOKWORM-ADAPTATIONS.md`), an absent previous release or manifest, a **v1-shaped** manifest (no `closure_version:` header — an absent header IS version 1), or `FORCE_REBUILD=all`. Fail-SAFE means REBUILD EVERYTHING: a wrong `unchanged` ships stale bytes, a wrong `changed` only costs build time. The previous release is resolved with `gh release list`, never `git describe` (latest PUBLISHED release ≠ nearest tag). Offline seams: `PREV_TAG`, `PREV_MANIFEST_FILE`, `HEAD_REF`, `GH_REPO`. The companion is never enumerated — it always rebuilds. | +| [`ci/test-detect-changed-sources.sh`](ci/test-detect-changed-sources.sh) | The detector's contract — builds its own throwaway git repo and stubs `gh` on `PATH`, so no docker, no network and no built `.deb`. Runs in the lightweight PR lane. | +| [`ci/stage-carryforward-debs.sh`](ci/stage-carryforward-debs.sh) | **Step 2 of the differential pipeline.** Stages EVERY row of each `unchanged` source — runtime **and** aux (`-dbgsym`, `-dev`, `gir1.2-*`); the `role` column is deliberately not filtered on, or a release would silently drop ~36 debs — from the previous release into `build//`, sha256-verified against its manifest row. Reads the verdict stream on stdin or `--verdicts ` rather than re-running detection, which must happen exactly once per run. It reverses the uploader's `~`→`.` asset-name sanitization by turning each `~` into a single-char `?` glob (a blanket `.`→`~` would hit legitimate dots) and requires EXACTLY ONE match; the staged file always lands under the canonical `~` name. Fails closed naming the offending row on a missing/ambiguous asset, a sha256 mismatch, an unknown source, an `unchanged` source with zero previous rows, a v1-shaped manifest, or a destination file with different bytes. The companion row is skipped explicitly. `mode=force-all` stages nothing and exits 0 without reading a manifest at all. | +| [`ci/test-stage-carryforward-debs.sh`](ci/test-stage-carryforward-debs.sh) | The stager's contract — builds its own fixture manifest + GitHub-mangled asset dir and drives the script through the `PREV_MANIFEST_FILE` / `CARRYFORWARD_ASSET_DIR` seams. No docker, no network, no built `.deb`. | +| [`ci/inject-deb-version.sh`](ci/inject-deb-version.sh) | Writes ONE selected source's rebuild version into its `debian/changelog` top entry via `dch --force-bad-version`. `--source ` is required — injection may only touch a source actually being built. Release builds derive `-~ceralive.N` from `PREV_MANIFEST_FILE`, the previous manifest the CALLER already resolved (this script never fetches a release): every row for that source is read across both arches and both roles, and the counter is accepted only if they agree. A disagreement, a counter/legacy mixture, or a malformed suffix **fails closed naming the source**; entirely-legacy rows, or no previous manifest during the force-all bootstrap, initialize at `.1`. `--dev` keeps the fixed `~ceralive0.0.0~dev`. Upstream versions come from the changelog — never hardcoded here. | | [`ci/verify-upstream-pins.sh`](ci/verify-upstream-pins.sh) | Re-verifies every field of `upstream-pins.yaml` in an isolated `GNUPGHOME`: git-tag lineage (`git ls-remote`), `.dsc` GPG signature vs pinned signer, `.dsc` checksums vs manifest, the downloaded `.orig.tar` sha256, and — the 4th link — the `.debian.tar.xz` sha256 plus a canonical `debian/`-tree manifest compared against the pinned salsa tag (exec-bit + symlink-target aware). Exit 0 on success; non-zero with a NAMED failing field on any drift. | | [`ci/test-verify-upstream-pins.sh`](ci/test-verify-upstream-pins.sh) | Offline fail-closed proof: runs the four [`ci/fixtures/`](ci/fixtures) tampers (wrong-signer / altered-`.dsc` / altered-`.orig.tar` / altered-salsa-tree) and asserts each is rejected on the correct named field. Run standalone; the packaging-wave container lane can adopt it. | -| [`ci/build-bookworm.sh`](ci/build-bookworm.sh) | Rebuilds all four sources in a `debian:bookworm` container in bootstrap order via a temporary local apt repo. `build-bookworm.sh ` — native amd64 or full-system-QEMU arm64, never cross-built. Fetches + sha256-verifies each pinned `.orig.tar`, overlays the checked-in `debian/`, injects the version (`RELEASE_VERSION=vX.Y.Z` → `~ceraliveX.Y.Z`; unset → `~ceralive0.0.0~dev`) into a **copy** of each changelog, installs the freshly-built `gir1.2-*-1.0` typelibs into the build env before each dependent source (so bookworm's GI-1.74 `dh_girepository` can resolve cross-namespace typelib deps — Qmi→Qrtr, MM→Qmi/Mbim/Qrtr), runs real `dpkg-buildpackage`, and asserts BOTH the 9-package runtime closure AND per-source package-set **equality** (via `ci/check-package-sets.sh`) from the `.changes` (drift ⇒ non-zero). Output to gitignored `build//`. | +| [`ci/check-upstream-freshness.sh`](ci/check-upstream-freshness.sh) | The **upstream freshness watch** behind [`.github/workflows/upstream-watch.yml`](../.github/workflows/upstream-watch.yml). For every source it enumerates the upstream release tags and the salsa `debian/*` packaging tags with `git ls-remote --tags`, keeps only STABLE members, and compares them to the pins (read through `read-pin.sh` — it parses no YAML itself). Three verdicts: `current`, `behind ()`, and the deliberately distinct `upstream-ahead-no-packaging ()` — upstream released but Debian has not packaged it, so there is no `-` pair to pin and no bump to recommend. The stable filter rejects `-rc`/`-dev`/`-alpha`/`-beta`/`-pre`, anything that is not a plain `X.Y.Z`, **odd-minor development series** (the GNOME/freedesktop convention all four projects follow — ModemManager `1.25.95` is the unstable train toward 1.26.0 and went to Debian *experimental*), `.9x` snapshot micros, and `~`-bearing Debian revisions. Exit `0` = no bump, `10` = at least one source behind. `--dry-run` prints the would-be issue body and makes no API call; **the script is issue-only and can neither edit a pin nor dispatch a build** (locked by a source-scan fence in its test). | +| [`ci/test-check-upstream-freshness.sh`](ci/test-check-upstream-freshness.sh) | The watch's **offline** contract — no network, no container. Drives the script through its documented `UPSTREAM_FRESHNESS_FIXTURE_DIR` seam, which swaps `git ls-remote` for verbatim fixture output so the tag parse under test is the real one. Cases: all-current ⇒ no issue body; upstream `1.26.0` + `debian/1.26.0-1` ⇒ `behind` with both versions named in the body; **only `1.25.95` ⇒ still `current`** (the real dev-series trap, proven on both the upstream tag and the `debian/1.25.95-1` experimental tag); newer upstream with no packaging tag ⇒ `upstream-ahead-no-packaging`, never `behind`; `-rc`/`-dev`/`.90`/`v`-prefixed noise ⇒ each rejected on its own named reason; packaging-only revision bump ⇒ `behind` naming the packaging tag; plus the issue-only source fence with a two-way non-vacuity control. | +| [`ci/build-bookworm.sh`](ci/build-bookworm.sh) | **Step 3 of the differential pipeline.** Rebuilds the SELECTED sources in a `debian:bookworm` container in bootstrap order via a temporary local apt repo. `build-bookworm.sh ` — native amd64 or full-system-QEMU arm64, never cross-built. The build set comes from `VERDICTS_FILE` (the detector's output) or an explicit `BUILD_SOURCES` list; supplying both fails closed, supplying neither builds all four (the local-dev default, never what CI relies on). Before the first build it seeds the Pin-Priority-1001 local repo with every already-staged deb of each SKIPPED source, so a selected source resolves stack build-deps and gir typelibs against carried CeraLive packages rather than stock bookworm; carried debs are preserved and only stale `*.changes`/`*.buildinfo` plus the selected sources' own artifacts are removed. Release builds call `inject-deb-version.sh` once per selected source, passing the caller's already-resolved `PREV_MANIFEST_FILE`. The runtime closure is asserted from the **MERGED** staged set (built + carried), and a zero-source run starts no container yet still performs that assertion. Fetches + sha256-verifies each pinned `.orig.tar`, overlays the checked-in `debian/`, injects the version (`RELEASE_VERSION=vX.Y.Z` → that source's derived `~ceralive.N`; unset → `~ceralive0.0.0~dev`) into a **copy** of each changelog, installs the freshly-built `gir1.2-*-1.0` typelibs into the build env before each dependent source (so bookworm's GI-1.74 `dh_girepository` can resolve cross-namespace typelib deps — Qmi→Qrtr, MM→Qmi/Mbim/Qrtr), runs real `dpkg-buildpackage`, and asserts per-source package-set **equality** (via `ci/check-package-sets.sh`) from each freshly built source's `.changes` plus the 9-package runtime closure over the merged staged set (drift ⇒ non-zero). Output to gitignored `build//`. | +| [`ci/test-build-bookworm-differential.sh`](ci/test-build-bookworm-differential.sh) | The differential builder's contract. Its `BUILD_BOOKWORM_STUB_DIR` seam replaces only the expensive source-build body with source-keyed fixture artifacts; build-set parsing, carry seeding, bootstrap dispatch, counter derivation, package-set checking and the merged-closure assertion all run through their production paths. No docker, no network. | | [`ci/check-package-sets.sh`](ci/check-package-sets.sh) | Exact per-source package-set **equality** enforcement. `check-package-sets.sh [expected-packages.txt]` asserts every `*.changes` binary set EQUALS its `[ all-artifact]` set in [`ci/expected-packages.txt`](ci/expected-packages.txt) (the finalized two-set model: declared arch-dependent stanzas + enumerated `-dbgsym`). Equality — not `≥`/count — so an add/remove/rename fails closed naming the offending package. Invoked by `build-bookworm.sh` in-container after the closure check, and standalone per-arch. | -| [`ci/contract.sh`](ci/contract.sh) | The packaging **PR lane** (bookworm container) entry point. Lightweight, needs no built `.deb`: asserts the scaffold, the tag-guard contract, that `dch` version-injection runs on a **copy** (the committed changelogs stay pristine), and the real `dpkg --compare-versions` tilde ordering. The deb-consuming contract lives in the two scripts below. | -| [`ci/test-package-contract.sh`](ci/test-package-contract.sh) | The **package contract suite** over the A5.1 build output. `test-package-contract.sh ` launches a `debian:bookworm` container and runs: metadata/arch over the 9-package closure (revision-exact — every deb's base must equal its `read-pin.sh` `-`); clean-bookworm `apt-get install ./*.deb`; upgrade (stock 1.20.4 → ceralive set) with a **direction-aware** `--allow-downgrades` (computed per-package from real `dpkg --compare-versions` vs `madison` stock — post-bump every source sorts ABOVE stock, so the flag is dropped); rollback (`madison`-derived stock versions + `--allow-downgrades`); coherence (identical `~ceralive` suffix + mismatched-libqmi negative); real ordering proofs; tag-guard negative; piuparts-style install→purge leftover-scan. All version literals are `read-pin.sh`-derived. amd64 = full; arm64 defaults to `metadata` mode (`CONTRACT_MODE=full` forces the apt scenarios under QEMU). | +| [`ci/contract.sh`](ci/contract.sh) | The packaging **PR lane** (bookworm container) entry point. Lightweight, needs no built `.deb`: asserts the scaffold, the tag-guard contract, that `dch` version-injection runs on a **copy** (the committed changelogs stay pristine), and the real `dpkg --compare-versions` tilde ordering. It also runs the **six** registered offline test suites — `test-tag-guard.sh`, `test-detect-changed-sources.sh`, `test-stage-carryforward-debs.sh`, `test-build-bookworm-differential.sh`, `test-suffix-coherence-manifest.sh`, `test-release-workflow-wiring.sh` — every one of which needs no docker, no network and no built artifact, which is exactly why they belong in this lane. The deb-consuming contract lives in `test-package-contract.sh` + `daemon-smoke.sh`. | +| [`ci/test-package-contract.sh`](ci/test-package-contract.sh) | The **package contract suite** over the A5.1 build output. `test-package-contract.sh ` launches a `debian:bookworm` container and runs: metadata/arch over the 9-package closure (revision-exact — every deb's base must equal its `read-pin.sh` `-`); clean-bookworm `apt-get install ./*.deb`; upgrade (stock 1.20.4 → ceralive set) with a **direction-aware** `--allow-downgrades` (computed per-package from real `dpkg --compare-versions` vs `madison` stock — post-bump every source sorts ABOVE stock, so the flag is dropped); rollback (`madison`-derived stock versions + `--allow-downgrades`); **per-source** coherence (one `~ceralive` suffix WITHIN each upstream source — two sources at different rebuild counters is what a differential release produces and is accepted; the retained negative is now a source disagreeing with ITSELF, and it fails closed naming that source); real ordering proofs including the migration-continuity chain; tag-guard negative; piuparts-style install→purge leftover-scan. All version literals are `read-pin.sh`-derived. amd64 = full; arm64 defaults to `metadata` mode (`CONTRACT_MODE=full` forces the apt scenarios under QEMU). | | [`ci/daemon-smoke.sh`](ci/daemon-smoke.sh) | The **daemon smoke**. `daemon-smoke.sh ` installs system D-Bus + polkit + NetworkManager (bookworm 1.42.4) and the built MM debs, starts a system `dbus-daemon` + `ModemManager`, then asserts: `busctl introspect` shows the root `ObjectManager`; `mmcli --version` matches the **pinned** ModemManager upstream version (via `ci/read-pin.sh`, never hardcoded); the udev-rules + FCC-unlock dispatcher dirs exist; and — **functional GI validation**, not presence-only (it installs `python3-gi valac build-essential pkg-config`) — the `gir1.2-modemmanager-1.0` typelib **loads** through PyGObject (`gi.require_version('ModemManager','1.0')` + a real `ModemManager.ModemCapability.LTE` enum read) and the `libmm-glib` `.vapi` **compiles+links** via `valac -C` → `cc $(pkg-config --cflags --libs mm-glib)` against a Vala program that genuinely calls a libmm-glib symbol (a broken/absent GI-1.74 adaptation fails closed here). amd64 by default. | | [`ci/build-companion.sh`](ci/build-companion.sh) | Builds the first-party `ceralive-modem-support` companion `.deb` — ONCE, `Architecture: all`, into `build/all/`. Container by default (`debian:bookworm`), `--native` for the local QA loop. `RELEASE_VERSION=vX.Y.Z` → Version `X.Y.Z` (unset → `0.0.0~dev`), injected into a COPY of the changelog. Fails closed if the produced deb's `Architecture` is not `all` or its `Version` is not the requested one. | | [`ci/test-fcc-reconcile.sh`](ci/test-fcc-reconcile.sh) | The FCC reconciler's **behaviour** contract, runnable on any host with no container and no root: every path is redirected into a scratch tree via `CERALIVE_FCC_{POLICY_FILE,AVAILABLE_DIR,ACTIVE_DIR}`. Covers absent/malformed/opt-out/idempotence, the one-model policy that must not parse as empty, the refused vendor-only key, an enabled model MM ships no script for, and both foreign-entry cases (a real file and a symlink pointing outside the available tier). Complements — never replaces — the chroot contract, which proves the same logic from the PACKAGED location after a real `dpkg` install. | | [`ci/test-companion-chroot.sh`](ci/test-companion-chroot.sh) | The companion's **CHROOT-stage** contract in a clean `debian:trixie` container: install / declared-inventory equality / chroot guard (+ non-vacuity) / single-owner `dpkg -S` / `/etc` override precedence / both override-removal branches / FCC absent-enabled-malformed matrix / upgrade with no conffile prompt / downgrade / purge with zero leftovers. Builds 0.9.0 and 1.1.0 so the upgrade and downgrade legs exercise real dpkg. The consumer stage is bench-gated. | | [`ci/companion-inventory.txt`](ci/companion-inventory.txt) | The companion's frozen declared file inventory, compared for EQUALITY by the chroot QA — an added, dropped or relocated asset fails the gate naming itself. | -| [`ci/generate-release-manifest.sh`](ci/generate-release-manifest.sh) | Emits the **manifest-complete per-release manifest** (`generate-release-manifest.sh ` → `dist/release-manifest.txt`): a checksum row for **every** built deb (both arches), the 9-package runtime closure MARKED (`role=runtime`, the rest `role=aux`) — the `build_arch package source version role filename sha256` matrix Phase-B apt publication AND `create-release` asset reconciliation consume. Emits **`closure_version: 2`** — the versioned contract apt-worker validates against — which adds the `Architecture: all` companion as ONE row with `build_arch` `all`, alongside the unchanged 9 × 2 arch-dependent rows. Build architecture and index membership are separate: `all` enters EVERY index arch, anything else its own. Per-source equality is scoped by `[arch-all sources]` so `build/all` is checked against the companion only and `build/` against the four upstream sources only. Fails closed on any set drift. dpkg-free (filename parse + `sha256sum`), so it runs anywhere. | +| [`ci/generate-release-manifest.sh`](ci/generate-release-manifest.sh) | Emits the **manifest-complete per-release manifest** (`generate-release-manifest.sh ` → `dist/release-manifest.txt`): a checksum row for **every** built deb (both arches), the 9-package runtime closure MARKED (`role=runtime`, the rest `role=aux`) — the `build_arch package source version role filename sha256` matrix Phase-B apt publication AND `create-release` asset reconciliation consume. Emits **`closure_version: 2`** — the versioned contract apt-worker validates against — which adds the `Architecture: all` companion as ONE row with `build_arch` `all`, alongside the unchanged 9 × 2 arch-dependent rows. Build architecture and index membership are separate: `all` enters EVERY index arch, anything else its own. Per-source equality is scoped by `[arch-all sources]` so `build/all` is checked against the companion only and `build/` against the four upstream sources only. Fails closed on any set drift. dpkg-free (filename parse + `sha256sum`), so it runs anywhere. Emits **`suffix_scheme: per-source-counter`** and NO `deb_version_suffix:` — under per-source rebuild counters no single suffix value is truthful, so the header states the scheme and each row keeps carrying its own version (apt-worker's validator reads neither field). | +| [`ci/suffix-contract.sh`](ci/suffix-contract.sh) | SOURCED library — the `~ceralive` suffix contract in ONE place. `assert_group_coherence =…` groups packages by owning source (derived from `expected-packages.txt`, never a second frozen list) and asserts each source's OWN internal coherence, so differing counters ACROSS sources pass while a source disagreeing with itself fails closed naming it. `prove_chain_ordered ` runs the **migration-continuity chain** — `~ceralive0.2.0 < ~ceralive1.0.0 < ~ceralive1.1.0 < ~ceralive.1 < ~ceralive.2 < ~ceralive.10 < -` — through real `dpkg --compare-versions`; its legacy members are every version that exists as a published artifact today, so the chain is the proof that every fleet device upgrades cleanly into the counter scheme. Sourced by `test-package-contract.sh` (CHECK 5/6) and `test-suffix-coherence-manifest.sh` so the container lane and the host lane cannot prove different rules. | +| [`ci/test-suffix-coherence-manifest.sh`](ci/test-suffix-coherence-manifest.sh) | The per-source-suffix + mixed-version-manifest contract — host-runnable, offline, **no docker**. Proves: sources at differing counters are accepted while an internally-mixed source fails closed naming itself; the full migration chain under real `dpkg` (with a non-vacuity control showing a lexical compare inverts `.2` vs `.10`); `generate-release-manifest.sh` over a MIXED staged set emits the new header, no `deb_version_suffix:`, and a row per staged deb at each source's own version; and a ZERO-UPSTREAM-BUILD set (only the companion fresh) keeps every upstream row at its carried counter with only the companion at the bare tag version. Every expected count is COUNTED from the fixture that produced it — no total is written down. | +| [`ci/test-release-workflow-wiring.sh`](ci/test-release-workflow-wiring.sh) | The **static** proof that `release.yml`'s `build-deb` is wired the way the differential pipeline needs. It reads the workflow text and never dispatches a run: step presence, the ordering chain (previous-release resolution → detection → carry-forward staging → **every** executable `build-bookworm.sh` mention, comment lines excluded), `fetch-depth: 0` inside the `build-deb` slice specifically, and an awk detector for any `${{ }}` interpolated into a `run:` body. It exists because the invariant it guards fails SILENTLY — staging after the build still produces a green release, built against stock bookworm dependencies. `RELEASE_WORKFLOW_FILE` points it at a scratch copy for failure demonstrations, so the tracked workflow is never mutated. | | [`ci/resolve-tag.sh`](ci/resolve-tag.sh) | The **shared tag → peeled-commit-SHA resolver** used by `release.yml`. `resolve-tag.sh ` asks the remote (`git ls-remote`, no clone) for both `refs/tags/` and `refs/tags/^{}`, prefers the **peeled** commit SHA (an annotated tag otherwise resolves to its tag object), prints it, and fails closed if the tag is absent or ambiguous. ONE script, called from tag-guard (pin every checkout), publish-npm (last-instant pre-publish TOCTOU re-check), and create-release (pre-create re-check) — no divergent copies. A caller detects a moved tag by comparing the output against the pinned SHA. | | [`ci/reconcile-release-assets.sh`](ci/reconcile-release-assets.sh) | The **immutable, manifest-complete release-asset reconciler** used by `release.yml`'s `create-release`. `reconcile-release-assets.sh ` takes a flat dir of the raw built debs + the manifest and: verifies the deb set equals the manifest sha256-exactly (missing/extra/corrupt ⇒ fail closed); stages each asset under its **own sanitized basename** (`~` → `.`, never relying on GitHub's upload mapping) and rejects any name **collision**; creates the release if absent; then for each staged asset uploads it if MISSING or integrity-compares (download + sha256) if it already EXISTS — matching ⇒ skip (idempotent), differing ⇒ fail closed (published assets are never overwritten); and finally verifies the live asset set equals the staged set. `RECONCILE_RELEASE_DIR=` selects a local mock backend for standalone testing. | diff --git a/packaging/ci/build-bookworm.sh b/packaging/ci/build-bookworm.sh index 73abde9..fb8c50c 100755 --- a/packaging/ci/build-bookworm.sh +++ b/packaging/ci/build-bookworm.sh @@ -1,150 +1,417 @@ #!/usr/bin/env bash -# build-bookworm.sh — rebuild the ModemManager stack for bookworm. +# build-bookworm.sh — differential ModemManager-stack rebuild for bookworm. # -# WHAT IT DOES -# Builds the four provenance-pinned sources (upstream-pins.yaml) in the mandatory -# bootstrap order libqrtr-glib -> libmbim -> libqmi -> modemmanager inside a -# `debian:bookworm` container. Each source's freshly built .debs are dropped into a -# temporary LOCAL apt repo (dpkg-scanpackages + `deb [trusted=yes] file:` line) so the -# NEXT source's build-deps resolve against the just-built package, not the older -# bookworm-main version. Real `dpkg-buildpackage` — no faking. +# BUILD SET +# BUILD_SOURCES may be an explicitly set space/comma-separated list of upstream-pins.yaml +# pin keys: libqrtr-glib libmbim libqmi modemmanager. An explicitly empty value selects zero +# sources. Alternatively, VERDICTS_FILE may point directly at detect-changed-sources.sh's +# five-line `=changed|unchanged` + `mode=` output. Supplying neither preserves the +# local-development default and builds all four sources. Supplying both fails closed. # -# ARCHES (native — never cross) -# amd64 -> `--platform linux/amd64`; arm64 -> `--platform linux/arm64` (full-system QEMU -# via the host's binfmt_misc `qemu-aarch64` handler — a genuine aarch64 userland, not a -# cross-compile). +# DIFFERENTIAL INVARIANT +# Selected sources build in the mandatory bootstrap order. Before the first build, every deb +# belonging to each skipped source is copied from the already-staged output directory into the +# temporary Pin-Priority 1001 local apt repo. A selected source therefore resolves stack +# build-deps and gir typelibs against carried CeraLive packages, never stock bookworm merely +# because its dependency source was skipped. # -# INPUTS (all under packaging/, this script's parent) -# /debian/ the pinned salsa debian/ dir + the bookworm adaptations (ModemManager -# only: debhelper relax, systemd-dev->udev, and the systemd/udev -# install-dir rules pins — see packaging/BOOKWORM-ADAPTATIONS.md). -# upstream-pins.yaml orig_tar_url / orig_tar_name / orig_tar_sha256 per source. -# ci/inject-deb-version.sh writes ~ceralive0.0.0~dev (dev build) into each changelog. +# OUTPUT + ASSERTIONS +# Carried .debs already staged by stage-carryforward-debs.sh are preserved. Only stale +# *.changes/*.buildinfo and artifacts belonging to sources selected for this run are removed. +# Freshly built sources retain the existing exact check-package-sets.sh verification from their +# .changes. The full runtime closure is asserted from the MERGED staged deb set (built + carried) +# at the end. A zero-source run starts no container and still performs that merged assertion. # -# OUTPUT -# Binary .debs + the four *.changes into $OUT (default: packaging/build/, gitignored). -# The runtime closure is asserted from the *.changes: EXACTLY the 9 runtime packages -# (modemmanager libmm-glib0 libmbim-{glib4,proxy,utils} libqmi-{glib5,proxy,utils} -# libqrtr-glib0). Any drift => exit 3 (STOP-and-surface). +# VERSIONING +# Release builds invoke inject-deb-version.sh once per selected source. It consumes the caller's +# already-resolved PREV_MANIFEST_FILE and derives that source's next ~ceralive.N counter; this +# script never fetches the previous manifest. Dev builds retain ~ceralive0.0.0~dev. # -# USAGE -# packaging/ci/build-bookworm.sh amd64 -# packaging/ci/build-bookworm.sh arm64 -# OUT=/some/dir packaging/ci/build-bookworm.sh amd64 # override output dir +# TEST SEAM +# BUILD_BOOKWORM_STUB_DIR is used only by test-build-bookworm-differential.sh. It replaces the +# expensive build body with source-keyed fixture artifacts while exercising production build-set +# parsing, seeding, dispatch, check-package-sets.sh, and merged-closure logic. # # EXIT -# 0 success (all 4 built, closure == the 9). 2 usage/env. 3 closure drift. non-zero build fail. +# 0 success. 2 usage/environment. 3 fail-closed package/carry/closure drift. Other non-zero is a +# real tooling or source-build failure. set -euo pipefail -# ------------------------------------------------------------------------------------------ -# Bootstrap order + the 9-package runtime closure are contract constants. BUILD_ORDER=(libqrtr-glib libmbim libqmi ModemManager) EXPECTED_RUNTIME=(libmbim-glib4 libmbim-proxy libmbim-utils libmm-glib0 libqmi-glib5 \ libqmi-proxy libqmi-utils libqrtr-glib0 modemmanager) -# Map a packaging dir name -> its upstream-pins.yaml source key (only ModemManager differs). +LOG_PREFIX="build-bookworm" +build_log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# Map a packaging directory name to its upstream-pins.yaml source key. Only ModemManager differs. pin_key() { case "$1" in ModemManager) echo modemmanager ;; *) echo "$1" ;; esac; } +is_known_key() { + local wanted="$1" dir + for dir in "${BUILD_ORDER[@]}"; do + [ "$(pin_key "$dir")" = "$wanted" ] && return 0 + done + return 1 +} + +expected_source_set() { # + awk -v want="[$1 all-artifact]" ' + /^\[/ { h=$0; sub(/[ \t]*#.*$/, "", h); insec=(h==want)?1:0; next } + insec { l=$0; sub(/#.*$/, "", l); gsub(/[ \t]+/, "", l); if (l!="") print l } + ' "$2" | LC_ALL=C sort -u +} + +declare -A SELECTED=() +BUILD_DIRS=() +BUILD_SOURCE_KEYS=() + +select_key() { + local key="$1" + is_known_key "$key" || die "unknown build source '$key' (expected pin-key names: libqrtr-glib libmbim libqmi modemmanager)" + [ -z "${SELECTED[$key]:-}" ] || die "build source '$key' was selected more than once" + SELECTED["$key"]=1 +} + +resolve_verdicts_file() { # + local verdicts_file="$1" line key value mode="" dir + local -A verdict=() + + [ -r "$verdicts_file" ] || die "VERDICTS_FILE='$verdicts_file' is set but not readable" + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + \#*) continue ;; + mode=*) + [ -z "$mode" ] || die "VERDICTS_FILE='$verdicts_file' contains more than one mode= line" + mode="${line#mode=}" + ;; + *=changed | *=unchanged) + key="${line%%=*}" + value="${line#*=}" + is_known_key "$key" || die "VERDICTS_FILE='$verdicts_file' names unknown source '$key'" + [ -z "${verdict[$key]:-}" ] || die "VERDICTS_FILE='$verdicts_file' verdicts source '$key' more than once" + verdict["$key"]="$value" + ;; + *) die "VERDICTS_FILE='$verdicts_file' has unparseable line '$line'" ;; + esac + done <"$verdicts_file" + + case "$mode" in + differential | force-all) ;; + "") die "VERDICTS_FILE='$verdicts_file' contains no mode= line" ;; + *) die "VERDICTS_FILE='$verdicts_file' has unrecognized mode '$mode'" ;; + esac + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + [ -n "${verdict[$key]:-}" ] || die "VERDICTS_FILE='$verdicts_file' contains no verdict for source '$key'" + if [ "$mode" = force-all ] && [ "${verdict[$key]}" != changed ]; then + die "VERDICTS_FILE='$verdicts_file' says mode=force-all but source '$key' is '${verdict[$key]}'" + fi + if [ "${verdict[$key]}" = changed ]; then select_key "$key"; fi + done + build_log "build set read from VERDICTS_FILE='$verdicts_file' (mode=$mode)" +} + +resolve_build_set() { + local normalized token dir key + local requested=() + + if [ -n "${VERDICTS_FILE:-}" ] && [ "${BUILD_SOURCES+x}" = x ]; then + die "BUILD_SOURCES and VERDICTS_FILE are mutually exclusive; pass one build-set contract" + fi + + if [ -n "${VERDICTS_FILE:-}" ]; then + resolve_verdicts_file "$VERDICTS_FILE" + elif [ "${BUILD_SOURCES+x}" = x ]; then + normalized="${BUILD_SOURCES//,/ }" + if [[ "$normalized" =~ [^[:space:]] ]]; then + read -r -a requested <<<"$normalized" + fi + for token in "${requested[@]}"; do select_key "$token"; done + build_log "build set read from BUILD_SOURCES='${BUILD_SOURCES}'" + else + for dir in "${BUILD_ORDER[@]}"; do select_key "$(pin_key "$dir")"; done + build_log "no build-set input supplied; defaulting to all sources" + fi + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + if [ -n "${SELECTED[$key]:-}" ]; then + BUILD_DIRS+=("$dir") + BUILD_SOURCE_KEYS+=("$key") + fi + done +} + +is_selected() { [ -n "${SELECTED[$1]:-}" ]; } + +assert_merged_runtime_closure() { # + local staged_dir="$1" deb filename package expected got + local debs=() runtime=() runtime_sorted=() + local -A seen=() + + shopt -s nullglob + debs=("$staged_dir"/*.deb) + shopt -u nullglob + for deb in "${debs[@]}"; do + filename="$(basename "$deb")" + case "$filename" in + *_*.deb) package="${filename%%_*}" ;; + *) refuse "merged staged file '$filename' is not a canonical '__.deb' name" ;; + esac + case "$package" in + *-dev | *-doc | *-dbgsym | gir1.2-*) continue ;; + esac + if [ -n "${seen[$package]:-}" ]; then + refuse "merged runtime set contains more than one staged deb for package '$package' ('$filename' and '${seen[$package]}')" + fi + seen["$package"]="$filename" + runtime+=("$package") + done + + if [ "${#runtime[@]}" -gt 0 ]; then + mapfile -t runtime_sorted < <(printf '%s\n' "${runtime[@]}" | LC_ALL=C sort -u) + fi + expected="$(printf '%s\n' "${EXPECTED_RUNTIME[@]}" | LC_ALL=C sort -u)" + got="$(printf '%s\n' "${runtime_sorted[@]:-}" | sed '/^$/d')" + + if [ "$expected" != "$got" ]; then + printf '%s: FAIL CLOSED — merged runtime closure drift in %s\n' "$LOG_PREFIX" "$staged_dir" >&2 + comm -23 <(printf '%s\n' "$expected") <(printf '%s\n' "$got") \ + | sed 's/^/ MISSING (expected, not staged): /' >&2 || true + comm -13 <(printf '%s\n' "$expected") <(printf '%s\n' "$got") \ + | sed 's/^/ UNEXPECTED (staged, not expected): /' >&2 || true + exit 3 + fi + build_log "MERGED RUNTIME CLOSURE OK: ${#EXPECTED_RUNTIME[@]} expected packages across built + carried debs in '$staged_dir'" +} + +clean_run_outputs() { # + local out_dir="$1" expected_file="$2" key package source_set + rm -f "$out_dir"/*.changes "$out_dir"/*.buildinfo 2>/dev/null || true + for key in "${BUILD_SOURCE_KEYS[@]}"; do + source_set="$(expected_source_set "$key" "$expected_file")" + [ -n "$source_set" ] || die "source '$key' has no [$key all-artifact] block in '$expected_file'" + while IFS= read -r package; do + [ -n "$package" ] || continue + rm -f "$out_dir/${package}_"*.deb + done <<<"$source_set" + build_log "cleared stale artifacts for selected source '$key'; carried debs for skipped sources were preserved" + done +} + +resolve_build_set +CANONICAL_BUILD_SOURCES="${BUILD_SOURCE_KEYS[*]}" + # ========================================================================================== -# HOST ROLE — arg parse, launch the container, then post-process the results it wrote. +# HOST ROLE — preserve staged carries, launch selected builds, or prove a zero-build closure. # ========================================================================================== -if [ "${BUILD_IN_CONTAINER:-0}" != "1" ]; then +if [ "${BUILD_IN_CONTAINER:-0}" != 1 ]; then ARCH="${1:-}" case "$ARCH" in - amd64) PLATFORM="linux/amd64" ;; - arm64) PLATFORM="linux/arm64" ;; - *) echo "usage: build-bookworm.sh " >&2; exit 2 ;; + amd64) PLATFORM="linux/amd64" ;; + arm64) PLATFORM="linux/arm64" ;; + *) echo "usage: build-bookworm.sh " >&2; exit 2 ;; esac - command -v docker >/dev/null 2>&1 || { echo "build-bookworm: docker not found" >&2; exit 2; } - - HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - PKG_ROOT="$(cd "$HERE/.." && pwd)" + PKG_ROOT="$SCRIPT_PKG_ROOT" OUT="${OUT:-$PKG_ROOT/build/$ARCH}" mkdir -p "$OUT" - # Clean any prior artifacts so the closure check sees only this run's output. - rm -f "$OUT"/*.deb "$OUT"/*.changes "$OUT"/*.buildinfo 2>/dev/null || true - - echo "build-bookworm: arch=$ARCH platform=$PLATFORM" - echo "build-bookworm: packaging root=$PKG_ROOT" - echo "build-bookworm: output=$OUT" - - # Mount packaging/ read-only at /pkg; output read-write at /out. Re-invoke self in-container. - # RELEASE_VERSION (optional): a vX.Y.Z tag → build ~ceraliveX.Y.Z debs. Unset → dev build - # (~ceralive0.0.0~dev). release.yml passes the release tag here; local/CI dev builds omit it. - docker run --rm --platform "$PLATFORM" \ - -e BUILD_IN_CONTAINER=1 \ - -e ARCH="$ARCH" \ - -e RELEASE_VERSION="${RELEASE_VERSION:-}" \ - -v "$PKG_ROOT":/pkg:ro \ - -v "$OUT":/out \ - debian:bookworm \ + clean_run_outputs "$OUT" "$PKG_ROOT/ci/expected-packages.txt" + + build_log "arch=$ARCH platform=$PLATFORM" + build_log "packaging root=$PKG_ROOT" + build_log "output=$OUT" + build_log "selected sources (${#BUILD_SOURCE_KEYS[@]}): ${CANONICAL_BUILD_SOURCES:-none}" + + if [ "${#BUILD_SOURCE_KEYS[@]}" -eq 0 ]; then + build_log "zero-build path: no source selected; no container will be started" + assert_merged_runtime_closure "$OUT" + build_log "PASS [$ARCH]: zero sources built; carried-only merged runtime closure is complete" + exit 0 + fi + + command -v docker >/dev/null 2>&1 || die "docker not found" + if [ -n "${PREV_MANIFEST_FILE:-}" ] && [ ! -r "$PREV_MANIFEST_FILE" ]; then + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable" + fi + if [ -n "${RELEASE_VERSION:-}" ] && [ -z "${PREV_MANIFEST_FILE:-}" ] && + [ "${#BUILD_SOURCE_KEYS[@]}" -ne "${#BUILD_ORDER[@]}" ]; then + die "release differential build selects ${#BUILD_SOURCE_KEYS[@]} source(s) but PREV_MANIFEST_FILE is absent; only a force-all bootstrap may initialize counters without a previous manifest" + fi + + docker_args=( + run --rm --platform "$PLATFORM" + -e BUILD_IN_CONTAINER=1 + -e ARCH="$ARCH" + -e BUILD_SOURCES="$CANONICAL_BUILD_SOURCES" + -e RELEASE_VERSION="${RELEASE_VERSION:-}" + ) + if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + docker_args+=( + -e PREV_MANIFEST_FILE=/previous-release-manifest.txt + -v "$PREV_MANIFEST_FILE":/previous-release-manifest.txt:ro + ) + fi + docker_args+=( + -v "$PKG_ROOT":/pkg:ro + -v "$OUT":/out + debian:bookworm bash /pkg/ci/build-bookworm.sh "$ARCH" + ) + docker "${docker_args[@]}" - echo "build-bookworm: container finished; artifacts in $OUT" + build_log "container finished; merged artifacts are in '$OUT'" exit 0 fi # ========================================================================================== -# CONTAINER ROLE — the real build, inside debian:bookworm. +# CONTAINER ROLE — seed skipped sources, then build only selected sources in bootstrap order. # ========================================================================================== -ARCH="${ARCH:-$(dpkg --print-architecture)}" -echo "== in-container build (arch=$(dpkg --print-architecture), target=$ARCH, $(uname -m)) ==" +PKG_MOUNT="${BUILD_BOOKWORM_PKG_ROOT:-/pkg}" +OUT_DIR="${BUILD_BOOKWORM_OUT_DIR:-/out}" +STUB_BUILD_DIR="${BUILD_BOOKWORM_STUB_DIR:-}" +[ -d "$PKG_MOUNT" ] || die "container packaging root '$PKG_MOUNT' is not a directory" +[ -d "$OUT_DIR" ] || die "container output '$OUT_DIR' is not a directory" +if [ -n "$STUB_BUILD_DIR" ] && [ ! -d "$STUB_BUILD_DIR" ]; then + die "BUILD_BOOKWORM_STUB_DIR='$STUB_BUILD_DIR' is set but not a directory" +fi -export DEBIAN_FRONTEND=noninteractive -# nocheck: skip the upstream test phase (needs a live session bus; that is A5.2 daemon-smoke, -# not a build-time concern). nodoc: skip gtk-doc (arch:all -doc pkgs are not in the -# runtime closure). Both are standard for a binary rebuild. -NPROC="$(nproc)" -export DEB_BUILD_OPTIONS="nocheck nodoc parallel=$NPROC" -# dch (version injection) needs a maintainer identity; the container has none by default. -export DEBEMAIL="ci@ceralive.tv" -export DEBFULLNAME="CeraLive CI" +ARCH="${ARCH:-}" +if [ -z "$ARCH" ]; then + [ -z "$STUB_BUILD_DIR" ] || die "ARCH is required with BUILD_BOOKWORM_STUB_DIR" + ARCH="$(dpkg --print-architecture)" +fi log() { echo " [build] $*"; } step() { echo; echo "==== $* ===="; } -# apt drops to the unprivileged `_apt` user for acquire, which cannot read the local -# file: repo under a 0700 mktemp dir — turn the sandbox off (standard container fix). -echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01-no-sandbox +if [ "${#BUILD_SOURCE_KEYS[@]}" -eq 0 ]; then + build_log "zero-build container role: no source selected; skipping all tooling and build setup" + assert_merged_runtime_closure "$OUT_DIR" + exit 0 +fi + +if [ -n "$STUB_BUILD_DIR" ]; then + echo "== stubbed in-container build (target=$ARCH) ==" + log "BUILD_BOOKWORM_STUB_DIR active: real apt and dpkg-buildpackage commands are disabled" +else + echo "== in-container build (arch=$(dpkg --print-architecture), target=$ARCH, $(uname -m)) ==" +fi + +export DEBIAN_FRONTEND=noninteractive +if [ -n "$STUB_BUILD_DIR" ]; then + NPROC=1 +else + NPROC="$(nproc)" +fi +export DEB_BUILD_OPTIONS="nocheck nodoc parallel=$NPROC" +export DEBEMAIL="ci@ceralive.tv" +export DEBFULLNAME="CeraLive CI" -step "install build tooling" -apt-get update -qq -apt-get install -y -qq --no-install-recommends \ - build-essential dpkg-dev devscripts equivs \ - meson ninja-build pkgconf ca-certificates curl xz-utils bzip2 >/dev/null -DPKGBP_VER="$(dpkg-buildpackage --version 2>/dev/null | sed -n '1p' || true)" -log "toolchain ready: $DPKGBP_VER" +if [ -z "$STUB_BUILD_DIR" ]; then + # apt drops to `_apt`, which cannot read a file: repo under mktemp's 0700 directory. + echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01-no-sandbox + step "install build tooling" + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + build-essential dpkg-dev devscripts equivs \ + meson ninja-build pkgconf ca-certificates curl xz-utils bzip2 >/dev/null + DPKGBP_VER="$(dpkg-buildpackage --version 2>/dev/null | sed -n '1p' || true)" + log "toolchain ready: $DPKGBP_VER" +else + step "install build tooling (stubbed)" + log "toolchain stub ready" +fi -# ---- writable copy of the packaging tree (source of truth is the ro mount) --------------- -WORK="$(mktemp -d /tmp/mmbuild.XXXXXX)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/mmbuild.XXXXXX")" +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT PKGW="$WORK/pkg" -cp -a /pkg "$PKGW" -REPO="$WORK/repo" # the temporary LOCAL apt repo +cp -a "$PKG_MOUNT" "$PKGW" +REPO="$WORK/repo" mkdir -p "$REPO" -: > "$REPO/Packages" # empty index so `apt-get update` is happy before build 1 -echo "deb [trusted=yes] file:$REPO ./" > /etc/apt/sources.list.d/local-mm.list -# Prefer the freshly built local packages over bookworm-main for the stack's own libs. -cat > /etc/apt/preferences.d/local-mm.pref <<'EOF' +: >"$REPO/Packages" + +if [ -z "$STUB_BUILD_DIR" ]; then + echo "deb [trusted=yes] file:$REPO ./" > /etc/apt/sources.list.d/local-mm.list + cat > /etc/apt/preferences.d/local-mm.pref <<'EOF' Package: * Pin: origin "" Pin-Priority: 1001 EOF -apt-get update -qq + apt-get update -qq +fi refresh_repo() { - ( cd "$REPO" && dpkg-scanpackages -m . /dev/null > Packages 2>/dev/null ) - apt-get update -qq + if [ -n "$STUB_BUILD_DIR" ]; then + find "$REPO" -maxdepth 1 -type f -name '*.deb' -printf '%f\n' | LC_ALL=C sort >"$REPO/Packages" + else + (cd "$REPO" && dpkg-scanpackages -m . /dev/null >Packages 2>/dev/null) + apt-get update -qq + fi } -# ---- version injection: A1.1's script, run against the writable packaging copy ----------- -# RELEASE_VERSION set (a vX.Y.Z tag) → inject ~ceraliveX.Y.Z; unset → the dev suffix. Either -# way this runs against the WRITABLE copy ($PKGW), never the committed source-of-truth tree. -INJECT_ARG="${RELEASE_VERSION:-}"; [ -n "$INJECT_ARG" ] || INJECT_ARG="--dev" -step "inject version ($INJECT_ARG) via ci/inject-deb-version.sh" -( cd "$PKGW" && bash ci/inject-deb-version.sh "$INJECT_ARG" ) +seed_carried_source() { # + local source_name="$1" package source_set count=0 + local matches=() + source_set="$(expected_source_set "$source_name" "$PKGW/ci/expected-packages.txt")" + [ -n "$source_set" ] || die "source '$source_name' has no [$source_name all-artifact] block in '$PKGW/ci/expected-packages.txt'" + + while IFS= read -r package; do + [ -n "$package" ] || continue + shopt -s nullglob + matches=("$OUT_DIR/${package}_"*.deb) + shopt -u nullglob + if [ "${#matches[@]}" -eq 0 ]; then + refuse "skipped source '$source_name' carried package '$package' is missing from '$OUT_DIR'; build-deps must never fall back to stock bookworm" + fi + if [ "${#matches[@]}" -gt 1 ]; then + refuse "skipped source '$source_name' carried package '$package' has ${#matches[@]} staged debs in '$OUT_DIR'; the local repo seed requires exactly one version" + fi + cp "${matches[0]}" "$REPO/" + count=$((count + 1)) + done <<<"$source_set" + log "seed $source_name: copied $count carried deb(s) into the local apt repo" +} + +step "seed local apt repo with carried debs for skipped sources" +seeded_sources=0 +for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + if ! is_selected "$key"; then + seed_carried_source "$key" + seeded_sources=$((seeded_sources + 1)) + fi +done +if [ "$seeded_sources" -gt 0 ]; then + refresh_repo + log "carried-deb seed complete before first BUILD; Pin-Priority 1001 local repo contains every skipped source" +else + log "no skipped source; local apt repo starts empty and will be populated in bootstrap order" +fi + +INJECT_ARG="${RELEASE_VERSION:-}" +[ -n "$INJECT_ARG" ] || INJECT_ARG="--dev" +for dir in "${BUILD_DIRS[@]}"; do + key="$(pin_key "$dir")" + step "inject version for $key ($INJECT_ARG)" + (cd "$PKGW" && bash ci/inject-deb-version.sh --source "$key" "$INJECT_ARG") +done -# ---- tiny pin reader (same awk shape as verify-upstream-pins.sh) ------------------------- pin_scalar() { awk -v src="$1" -v key="$2" ' $0 ~ "^ " src ":[ \t]*$" { inblk=1; next } @@ -156,120 +423,94 @@ pin_scalar() { ' "$PKGW/upstream-pins.yaml" } -# ---- build one source ------------------------------------------------------------------- build_one() { - local dir="$1" key; key="$(pin_key "$dir")" + local dir="$1" key + key="$(pin_key "$dir")" step "BUILD $dir (pin key: $key)" - # Resolve the injected source + upstream version from the (now version-injected) changelog. + if [ -n "$STUB_BUILD_DIR" ]; then + local fixture="$STUB_BUILD_DIR/$key" file + local fixture_debs=() fixture_changes=() fixture_buildinfo=() + [ -d "$fixture" ] || die "stub build for source '$key' has no fixture directory '$fixture'" + shopt -s nullglob + fixture_debs=("$fixture"/*.deb) + fixture_changes=("$fixture"/*.changes) + fixture_buildinfo=("$fixture"/*.buildinfo) + shopt -u nullglob + [ "${#fixture_debs[@]}" -gt 0 ] || die "stub build for source '$key' contains no .deb" + [ "${#fixture_changes[@]}" -gt 0 ] || die "stub build for source '$key' contains no .changes" + for file in "${fixture_debs[@]}"; do cp "$file" "$OUT_DIR/"; cp "$file" "$REPO/"; done + for file in "${fixture_changes[@]}"; do cp "$file" "$OUT_DIR/"; done + for file in "${fixture_buildinfo[@]}"; do cp "$file" "$OUT_DIR/"; done + refresh_repo + log "$dir stub-built; fixture artifacts entered output + local repo" + return 0 + fi + local src ver upstream src="$(dpkg-parsechangelog -l "$PKGW/$dir/debian/changelog" -S Source)" ver="$(dpkg-parsechangelog -l "$PKGW/$dir/debian/changelog" -S Version)" - upstream="${ver%%-*}" # strip -~ceralive... + upstream="${ver%%-*}" log "source=$src version=$ver upstream=$upstream" - # Fetch + verify the provenance-pinned orig tarball. local url name sha url="$(pin_scalar "$key" orig_tar_url)" name="$(pin_scalar "$key" orig_tar_name)" sha="$(pin_scalar "$key" orig_tar_sha256)" - [ -n "$url" ] && [ -n "$name" ] && [ -n "$sha" ] || { echo "missing pin for $key" >&2; exit 2; } + if [ -z "$url" ] || [ -z "$name" ] || [ -z "$sha" ]; then + die "missing pin for source '$key'" + fi log "orig: $name" curl -fsSL --retry 3 --retry-delay 2 -o "$WORK/$name" "$url" - local got; got="$(sha256sum "$WORK/$name" | awk '{print $1}')" - [ "$got" = "$sha" ] || { echo "STOP: orig sha256 drift for $name (pin $sha, got $got)" >&2; exit 3; } + local got + got="$(sha256sum "$WORK/$name" | awk '{print $1}')" + [ "$got" = "$sha" ] || refuse "orig sha256 drift for '$name' (source '$key': pin $sha, got $got)" log "orig sha256 OK ($sha)" - # Assemble the build tree: -/ with debian/ overlaid; orig in the parent. local bdir="$WORK/build" mkdir -p "$bdir" local tree="$bdir/${src}-${upstream}" - rm -rf "$tree"; mkdir -p "$tree" + rm -rf "$tree" + mkdir -p "$tree" tar -xf "$WORK/$name" -C "$tree" --strip-components=1 cp -a "$PKGW/$dir/debian" "$tree/debian" - # Non-native 3.0 (quilt): dpkg-source wants ../_.orig.tar.. cp "$WORK/$name" "$bdir/${src}_${upstream}.orig.${name#*.orig.}" - # Resolve build-deps against bookworm-main + the local repo (freshly built deps). - log "apt-get build-dep (resolves against local repo for stack deps)" + log "apt-get build-dep (Pin-Priority 1001 local repo supplies built + carried stack deps)" apt-get build-dep -y --no-install-recommends "$tree" >/dev/null - # dh_girepository (bookworm GI 1.74) resolves cross-namespace typelib deps (Qmi-1.0 imports - # Qrtr-1.0; ModemManager-1.0 imports Qmi/Mbim/Qrtr) via the *installed* dependency .typelib, - # which ships only in the stack's own gir1.2-*-1.0 packages. On bookworm the rebuilt -dev - # build-deps do NOT pull them (GI 1.74 regenerates an empty ${gir:Depends} for -dev), so - # libqmi dies with "Could not find Qrtr-1.0.typelib dependency". Install every gir typelib - # already in the local repo first — mirrors an archive build (all gir1.2-* co-installable); - # affects only the build env, never a produced package's contents or the emitted set. local gir_names gir_names="$(find "$REPO" -maxdepth 1 -name 'gir1.2-*.deb' -printf '%f\n' 2>/dev/null \ | sed 's/_.*//' | sort -u | tr '\n' ' ')" if [ -n "${gir_names// /}" ]; then - log "install freshly-built gir typelibs so dh_girepository resolves them: $gir_names" + log "install local gir typelibs so dh_girepository resolves them: $gir_names" + # shellcheck disable=SC2086 # package names are intentionally split for apt-get argv apt-get install -y --no-install-recommends $gir_names >/dev/null fi - # Real binary build, arch-only (-B): all 9 runtime pkgs are arch-specific; -B skips the - # arch:all -doc pkgs and the -indep DEP-8 patch target. log "dpkg-buildpackage -B (DEB_BUILD_OPTIONS='$DEB_BUILD_OPTIONS')" - ( cd "$tree" && dpkg-buildpackage -B -us -uc ) + (cd "$tree" && dpkg-buildpackage -B -us -uc) - # dpkg-buildpackage writes artifacts to $bdir (the source tree's PARENT); prior sources' - # debs were already moved to $REPO, so $bdir holds only this source's fresh output. - find "$bdir" -maxdepth 1 -name '*.changes' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.buildinfo' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.deb' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.deb' -exec mv -t "$REPO" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.changes' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.buildinfo' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.deb' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.deb' -exec mv -t "$REPO" {} + 2>/dev/null || true refresh_repo - log "$dir built; local repo now has $(ls "$REPO"/*.deb 2>/dev/null | wc -l) .deb(s)" + local repo_debs=() + shopt -s nullglob + repo_debs=("$REPO"/*.deb) + shopt -u nullglob + log "$dir built; local repo now has ${#repo_debs[@]} .deb(s)" } -for d in "${BUILD_ORDER[@]}"; do build_one "$d"; done - -# ---- runtime-closure verification from the four *.changes -------------------------------- -step "runtime closure verification (from *.changes)" -# All binary package names across the 4 binary .changes (RFC822 Binary: field, fold-safe). -mapfile -t ALL_BINS < <( - for ch in /out/*.changes; do - awk ' - /^[A-Za-z][A-Za-z0-9-]*:/ { inb=0 } - /^Binary:/ { inb=1; l=$0; sub(/^Binary:[ \t]*/, "", l); print l; next } - inb && /^[ \t]/ { l=$0; sub(/^[ \t]+/, "", l); print l } - ' "$ch" - done | tr ' ' '\n' | sed '/^$/d' | sort -u -) -# Runtime = not -dev, not -doc, not an auto-generated -dbgsym, not a gir typelib. -RUNTIME=() -for b in "${ALL_BINS[@]}"; do - case "$b" in - *-dev|*-doc|*-dbgsym|gir1.2-*) : ;; - *) RUNTIME+=("$b") ;; - esac -done -mapfile -t RUNTIME_SORTED < <(printf '%s\n' "${RUNTIME[@]}" | sort -u) - -echo "all binary packages produced:"; printf ' %s\n' "${ALL_BINS[@]}" -echo "runtime closure (dev/doc/gir excluded):"; printf ' %s\n' "${RUNTIME_SORTED[@]}" +for dir in "${BUILD_DIRS[@]}"; do build_one "$dir"; done -expected="$(printf '%s\n' "${EXPECTED_RUNTIME[@]}" | sort -u)" -got="$(printf '%s\n' "${RUNTIME_SORTED[@]}")" -if [ "$expected" = "$got" ]; then - echo "CLOSURE OK: exactly the 9 expected runtime packages." -else - echo "STOP: runtime closure drift." >&2 - echo "--- expected ---" >&2; printf '%s\n' "$expected" >&2 - echo "--- got ---" >&2; printf '%s\n' "$got" >&2 - echo "--- diff (want<->got) ---" >&2; diff <(printf '%s\n' "$expected") <(printf '%s\n' "$got") >&2 || true - exit 3 -fi +step "fresh-source package-set equality (selected sources only, from *.changes)" +bash "$PKGW/ci/check-package-sets.sh" "$OUT_DIR" "$PKGW/ci/expected-packages.txt" -# ---- exact per-source package-set EQUALITY (finalized two-set model) ---------------------- -# Stronger than the runtime closure above: asserts EVERY source's *.changes binary set equals -# its frozen [ all-artifact] set in expected-packages.txt, exactly (add/remove/rename -# all fail). check-package-sets.sh fails closed and names the discrepancy. -step "package-set equality verification (per source+arch, from *.changes)" -bash "$PKGW/ci/check-package-sets.sh" /out "$PKGW/ci/expected-packages.txt" +step "merged runtime closure verification (freshly built + carried debs)" +assert_merged_runtime_closure "$OUT_DIR" echo -echo "PASS [$ARCH]: 4 sources built in bootstrap order; runtime closure == the 9; per-source .changes sets EQUAL the finalized all-artifact expectation." +echo "PASS [$ARCH]: ${#BUILD_SOURCE_KEYS[@]} selected source(s) built in bootstrap order; skipped-source debs seeded before BUILD; merged runtime closure complete." diff --git a/packaging/ci/check-upstream-freshness.sh b/packaging/ci/check-upstream-freshness.sh new file mode 100755 index 0000000..d773d06 --- /dev/null +++ b/packaging/ci/check-upstream-freshness.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# check-upstream-freshness.sh — is any pinned ModemManager-stack source behind a newer STABLE +# release? +# +# ISSUE-ONLY BY CONTRACT. This script reports. It never writes packaging/upstream-pins.yaml, it +# never invokes dch/inject-deb-version.sh, and it never dispatches a build. Bumping a pin is a +# separate, human-reviewed change that must re-run packaging/ci/verify-upstream-pins.sh (the +# four-link provenance chain). Nothing here is a substitute for that. +# +# WHAT IT COMPARES +# For each source in upstream-pins.yaml it enumerates two remotes with `git ls-remote --tags`: +# upstream — the freedesktop release tags (`1.24.2`) +# salsa — the Debian packaging tags (`debian/1.24.2-2`) +# and compares the newest STABLE member of each against the pinned `upstream_tag` / `salsa_tag`. +# +# THE STABLE FILTER IS THE WHOLE POINT — the four sources ship a DEVELOPMENT series on the same +# tag namespace as their releases, so a naive "newest tag wins" is wrong roughly half the time. +# ModemManager `1.25.95` is the live example: it is the unstable train toward 1.26.0, was +# uploaded to Debian EXPERIMENTAL, and must never produce a "behind" verdict. Four explicit +# rejection rules, each with a named reason (see `stable_reason`): +# prerelease-suffix -rc / -dev / -alpha / -beta / -pre (`1.26.0-rc1`) +# not-a-plain-triple anything but strict X.Y.Z with no prefix or suffix (`v1.26.0`) +# odd-minor-development-series odd MINOR — the GNOME/freedesktop convention every one of these +# four projects follows: even minor = release, odd = development +# (`1.25.95`, `1.37.1`). All four current pins are even-minor. +# snapshot-micro-series MICRO >= 90 — the pre-release snapshot train (`1.24.90`) +# `1.25.95` is caught independently by BOTH the odd-minor and the snapshot-micro rule, so the +# trap stays closed if either is ever relaxed. +# +# A Debian packaging tag is stable when its upstream part passes the SAME filter and its revision +# is a plain Debian revision (no `~`, which is how experimental/backports uploads spell +# themselves: `1.24.2-1~exp1`, `1~bpo12+1`). +# +# THREE VERDICTS, and the third is deliberately NOT the second: +# current nothing newer is both released upstream AND packaged in Debian. +# behind () a newer stable upstream release EXISTS AND has a stable Debian +# packaging tag — a bump is actionable. Also raised for a +# packaging-only bump (same upstream, newer `debian/` revision), +# because the rebuild consumes that revision too. +# upstream-ahead-no-packaging upstream released, Debian has not packaged it yet. This is NOT +# `behind`: these rebuilds are `-` pairs, so with +# no packaging tag there is no revision to pin and no bump to +# recommend. Reporting it as `behind` would file an issue nobody +# can act on. +# +# OFFLINE SEAM (how the test suite runs with no network) +# UPSTREAM_FRESHNESS_FIXTURE_DIR= (or --fixture-dir ) replaces every `git ls-remote` +# with a read of `/..tags`. The fixture holds verbatim +# `git ls-remote --tags --refs` output (`\trefs/tags/`), so the parse under test is +# the same parse the network path uses — only the transport is stubbed. A missing fixture file +# FAILS CLOSED rather than silently reaching the network. +# +# Usage: +# check-upstream-freshness.sh [--dry-run] [--issue-body PATH] [--fixture-dir DIR] [--source NAME] +# --dry-run also print the would-be GitHub issue body to stdout. Makes NO GitHub +# API call — this script never makes one in any mode. +# --issue-body PATH write the issue body to PATH when at least one source is behind +# (the workflow feeds it to `gh issue create/edit --body-file`). +# Nothing is written when no source is behind. +# --fixture-dir DIR the offline seam above. +# --source NAME check only this source (default: every source in the manifest). +# +# Exit status: +# 0 no bump is recommended (every source `current` and/or `upstream-ahead-no-packaging`). +# 10 at least one source is `behind` — the workflow opens/updates its issue on this code. +# 1 a check could not be completed (unreachable remote, missing fixture, unreadable pin). +# 2 usage error. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +READ_PIN="$HERE/read-pin.sh" + +DRY_RUN=0 +ISSUE_BODY_PATH="" +FIXTURE_DIR="${UPSTREAM_FRESHNESS_FIXTURE_DIR:-}" +ONLY_SOURCE="" + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --issue-body) ISSUE_BODY_PATH="${2-}"; shift 2 ;; + --fixture-dir) FIXTURE_DIR="${2-}"; shift 2 ;; + --source) ONLY_SOURCE="${2-}"; shift 2 ;; + -h|--help) sed -n '2,60p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "check-upstream-freshness: unknown argument '$1'" >&2; exit 2 ;; + esac +done + +[ -x "$READ_PIN" ] || [ -r "$READ_PIN" ] || { + echo "check-upstream-freshness: shared pin reader not found at $READ_PIN" >&2; exit 2 +} +for tool in awk sort git; do + command -v "$tool" >/dev/null 2>&1 \ + || { echo "check-upstream-freshness: missing required tool '$tool'" >&2; exit 2; } +done + +die() { echo "check-upstream-freshness: $*" >&2; exit 1; } +note() { printf '%s\n' "$*" >&2; } + +# Every pin VALUE comes from the shared reader — this file parses no YAML of its own. +pin() { bash "$READ_PIN" "$@"; } + +# ---- the stable filter ---------------------------------------------------------------------- +# Prints `stable`, or the NAMED reason the version was rejected. Total: every input gets a word. +stable_reason() { + local v="$1" minor micro + case "${v,,}" in + *-rc*|*-dev*|*-alpha*|*-beta*|*-pre*) echo prerelease-suffix; return 0 ;; + esac + [[ "$v" =~ ^[0-9]+\.([0-9]+)\.([0-9]+)$ ]] || { echo not-a-plain-triple; return 0; } + minor="${BASH_REMATCH[1]}" + micro="${BASH_REMATCH[2]}" + if [ $(( 10#$minor % 2 )) -ne 0 ]; then echo odd-minor-development-series; return 0; fi + if [ $(( 10#$micro )) -ge 90 ]; then echo snapshot-micro-series; return 0; fi + echo stable +} +is_stable() { [ "$(stable_reason "$1")" = stable ]; } + +# `a` strictly newer than `b`, by version sort (handles `1.24.2-10` > `1.24.2-2`). +ver_gt() { + [ "$1" != "$2" ] || return 1 + [ "$(printf '%s\n%s\n' "$1" "$2" | LC_ALL=C sort -V | tail -n1)" = "$1" ] +} +newest() { LC_ALL=C sort -V | tail -n1; } + +# ---- tag enumeration (network, or the offline fixture seam) ---------------------------------- +ls_remote_tags() { + local src="$1" kind="$2" url="$3" raw + if [ -n "$FIXTURE_DIR" ]; then + local fixture="$FIXTURE_DIR/$src.$kind.tags" + [ -r "$fixture" ] || die "[$src] missing $kind fixture '$fixture'" + raw="$(cat "$fixture")" + else + [ -n "$url" ] || die "[$src] no $kind repo URL in the pin manifest" + raw="$(GIT_TERMINAL_PROMPT=0 git ls-remote --tags --refs "$url" 2>/dev/null || true)" + [ -n "$raw" ] || die "[$src] git ls-remote returned no tags for $url" + fi + printf '%s\n' "$raw" | awk ' + $2 ~ /^refs\/tags\// { t=$2; sub(/^refs\/tags\//, "", t); sub(/\^\{\}$/, "", t); print t } + ' | LC_ALL=C sort -u +} + +# ---- per-source result table ----------------------------------------------------------------- +declare -a RESULT_SOURCES=() +declare -A R_STATE=() R_PIN_UP=() R_PIN_SALSA=() R_NEW_UP=() R_NEW_SALSA=() +BEHIND_COUNT=0 +AHEAD_COUNT=0 + +check_source() { + local src="$1" + local pin_up pin_salsa up_repo salsa_repo + pin_up="$(pin "$src" upstream_tag)" || die "[$src] cannot read upstream_tag" + pin_salsa="$(pin "$src" salsa_tag)" || die "[$src] cannot read salsa_tag" + up_repo="$(pin "$src" upstream_repo)" || die "[$src] cannot read upstream_repo" + salsa_repo="$(pin "$src" salsa_repo)" || die "[$src] cannot read salsa_repo" + + local pin_base="${pin_salsa#debian/}" + + # Newest stable upstream release tag. + local stable_up="" t reason + while IFS= read -r t; do + [ -n "$t" ] || continue + reason="$(stable_reason "$t")" + if [ "$reason" = stable ]; then + stable_up+="$t"$'\n' + else + note " skip [$src] upstream tag '$t' — $reason" + fi + done < <(ls_remote_tags "$src" upstream "$up_repo") + local newest_up + newest_up="$(printf '%s' "$stable_up" | awk 'NF' | newest)" + [ -n "$newest_up" ] || die "[$src] no STABLE upstream tag found — refusing to guess" + + # Newest stable Debian packaging tag, and the newest one for the newest upstream release. + local stable_salsa="" salsa_for_new="" rest ver rev + while IFS= read -r t; do + [ -n "$t" ] || continue + case "$t" in debian/*) : ;; *) continue ;; esac + rest="${t#debian/}" + case "$rest" in *-*) : ;; *) note " skip [$src] salsa tag '$t' — no-debian-revision"; continue ;; esac + ver="${rest%-*}" + rev="${rest##*-}" + reason="$(stable_reason "$ver")" + if [ "$reason" != stable ]; then + note " skip [$src] salsa tag '$t' — $reason" + continue + fi + if ! [[ "$rev" =~ ^[0-9][A-Za-z0-9.+]*$ ]]; then + note " skip [$src] salsa tag '$t' — non-stable-debian-revision" + continue + fi + stable_salsa+="$rest"$'\n' + [ "$ver" = "$newest_up" ] && salsa_for_new+="$rest"$'\n' + done < <(ls_remote_tags "$src" salsa "$salsa_repo") + + local newest_salsa_for_new newest_salsa_for_pin + newest_salsa_for_new="$(printf '%s' "$salsa_for_new" | awk 'NF' | newest)" + newest_salsa_for_pin="$(printf '%s' "$stable_salsa" | awk -v v="$pin_up" 'index($0, v "-")==1' | newest)" + + local state new_up_display new_salsa_display bump_display + new_up_display="$newest_up" + new_salsa_display="${newest_salsa_for_new:+debian/$newest_salsa_for_new}" + bump_display="$newest_up" + + if ver_gt "$newest_up" "$pin_up"; then + if [ -n "$newest_salsa_for_new" ]; then + state="behind" + else + state="upstream-ahead-no-packaging" + new_salsa_display="none" + fi + elif [ -n "$newest_salsa_for_pin" ] && ver_gt "$newest_salsa_for_pin" "$pin_base"; then + # Same upstream release, newer Debian revision. The rebuild consumes `-`, + # so a packaging-only revision bump is a real, actionable bump — and naming the upstream + # version alone would report a bump to the version already pinned. + state="behind" + new_up_display="$pin_up" + new_salsa_display="debian/$newest_salsa_for_pin" + bump_display="debian/$newest_salsa_for_pin" + else + state="current" + [ -n "$new_salsa_display" ] || new_salsa_display="$pin_salsa" + fi + + RESULT_SOURCES+=("$src") + R_STATE["$src"]="$state" + R_PIN_UP["$src"]="$pin_up" + R_PIN_SALSA["$src"]="$pin_salsa" + R_NEW_UP["$src"]="$new_up_display" + R_NEW_SALSA["$src"]="$new_salsa_display" + + case "$state" in + behind) BEHIND_COUNT=$((BEHIND_COUNT + 1)); printf '%s: behind (%s)\n' "$src" "$bump_display" ;; + upstream-ahead-no-packaging) AHEAD_COUNT=$((AHEAD_COUNT + 1)); printf '%s: upstream-ahead-no-packaging (%s)\n' "$src" "$bump_display" ;; + *) printf '%s: current\n' "$src" ;; + esac +} + +render_issue_body() { + local src + cat <<-'EOF' + The scheduled upstream-freshness watch found a newer **stable** release for at least one + pinned ModemManager-stack source. + + | Source | Pinned upstream | Pinned packaging | Newest stable upstream | Newest stable packaging | State | + |---|---|---|---|---|---| + EOF + for src in "${RESULT_SOURCES[@]}"; do + printf '| %s | %s | %s | %s | %s | %s |\n' \ + "$src" "${R_PIN_UP[$src]}" "${R_PIN_SALSA[$src]}" \ + "${R_NEW_UP[$src]}" "${R_NEW_SALSA[$src]}" "${R_STATE[$src]}" + done + + printf '\n## Bumps available\n\n' + for src in "${RESULT_SOURCES[@]}"; do + [ "${R_STATE[$src]}" = behind ] || continue + printf -- '- **%s**: `%s` (`%s`) -> `%s` (`%s`)\n' \ + "$src" "${R_PIN_UP[$src]}" "${R_PIN_SALSA[$src]}" \ + "${R_NEW_UP[$src]}" "${R_NEW_SALSA[$src]}" + done + + if [ "$AHEAD_COUNT" -gt 0 ]; then + printf '\n## Upstream ahead, Debian packaging not ready\n\n' + for src in "${RESULT_SOURCES[@]}"; do + [ "${R_STATE[$src]}" = upstream-ahead-no-packaging ] || continue + printf -- '- **%s**: upstream released `%s`, but no stable `debian/` packaging tag exists for it yet.\n' \ + "$src" "${R_NEW_UP[$src]}" + done + printf -- '\nNo bump is recommended for these. The rebuilds are `-` pairs, so\n' + printf -- 'without a Debian packaging tag there is no revision to pin.\n' + fi + + cat <<-'EOF' + + ## What was deliberately ignored + + Pre-release and development-series tags are filtered out: `-rc`/`-dev`/`-alpha`/`-beta`/`-pre` + suffixes, anything that is not a plain `X.Y.Z`, **odd-minor development series** (the + GNOME/freedesktop convention these four projects follow — ModemManager `1.25.95` is the + unstable train toward 1.26.0 and was uploaded to Debian *experimental*), and `.9x` snapshot + micro versions. Debian packaging tags whose revision carries a `~` (experimental/backports + uploads) are ignored for the same reason. + + ## What this issue is NOT + + This watch is **issue-only**. It never edits `packaging/upstream-pins.yaml` and never + dispatches a build. Bumping a pin is a separate, human-reviewed change that must re-run + `packaging/ci/verify-upstream-pins.sh` — the four-link provenance chain (lineage, `.dsc` + authority, `.orig.tar` artifact, `debian/` packaging tree) — and refresh the checked-in + `debian/` recipes plus `packaging/BOOKWORM-ADAPTATIONS.md`. + + + EOF +} + +# ---- main ------------------------------------------------------------------------------------ +note "check-upstream-freshness: pins=$( [ -n "$FIXTURE_DIR" ] && echo "fixtures:$FIXTURE_DIR" || echo "live git ls-remote" )" + +sources="$(pin --list-sources)" || die "cannot enumerate sources from the pin manifest" +[ -n "$sources" ] || die "the pin manifest lists no sources" + +checked=0 +for src in $sources; do + [ -z "$ONLY_SOURCE" ] || [ "$src" = "$ONLY_SOURCE" ] || continue + check_source "$src" + checked=$((checked + 1)) +done +[ "$checked" -gt 0 ] || die "no matching source '$ONLY_SOURCE'" + +note "check-upstream-freshness: $checked source(s) checked — $BEHIND_COUNT behind, $AHEAD_COUNT upstream-ahead-no-packaging" + +if [ "$BEHIND_COUNT" -eq 0 ]; then + if [ "$DRY_RUN" -eq 1 ]; then + echo "--- no issue would be opened (no source is behind) ---" + fi + exit 0 +fi + +if [ -n "$ISSUE_BODY_PATH" ]; then + render_issue_body > "$ISSUE_BODY_PATH" + note "check-upstream-freshness: issue body written to $ISSUE_BODY_PATH" +fi +if [ "$DRY_RUN" -eq 1 ]; then + echo "--- issue body (dry-run, not sent) ---" + render_issue_body + echo "--- end issue body ---" +fi +exit 10 diff --git a/packaging/ci/contract.sh b/packaging/ci/contract.sh index db91d12..b58f4d3 100755 --- a/packaging/ci/contract.sh +++ b/packaging/ci/contract.sh @@ -33,12 +33,22 @@ require "README.md" require "BOOKWORM-ADAPTATIONS.md" require "ci/tag-guard.sh" require "ci/test-tag-guard.sh" +require "ci/detect-changed-sources.sh" +require "ci/test-detect-changed-sources.sh" +require "ci/stage-carryforward-debs.sh" +require "ci/test-stage-carryforward-debs.sh" require "ci/read-pin.sh" require "ci/inject-deb-version.sh" require "ci/build-bookworm.sh" +require "ci/test-build-bookworm-differential.sh" require "ci/test-package-contract.sh" require "ci/daemon-smoke.sh" require "ci/generate-release-manifest.sh" +require "ci/suffix-contract.sh" +require "ci/test-suffix-coherence-manifest.sh" +require "ci/test-release-workflow-wiring.sh" +require "ci/check-upstream-freshness.sh" +require "ci/test-check-upstream-freshness.sh" require "ci/build-companion.sh" require "ci/test-companion-chroot.sh" require "ci/companion-inventory.txt" @@ -76,6 +86,43 @@ echo " ok: companion ships no image-owned udev basename" echo " running tag-guard contract..." bash "$HERE/test-tag-guard.sh" >/dev/null +# The differential-release change detector must hold. It builds its own throwaway git repo and +# stubs `gh` on PATH, so it needs no docker, no network and no built .deb — which is exactly why +# it belongs in this lightweight lane rather than the deb-consuming suite. +echo " running change-detection contract..." +bash "$HERE/test-detect-changed-sources.sh" >/dev/null + +# The carry-forward stager must hold. It builds its own fixture manifest + GitHub-mangled asset +# dir and drives the script through the PREV_MANIFEST_FILE / CARRYFORWARD_ASSET_DIR seams, so it +# needs no docker, no network and no built .deb — the same reason the detector's test lives here. +echo " running carry-forward staging contract..." +bash "$HERE/test-stage-carryforward-debs.sh" >/dev/null + +# The differential builder contract stubs only the expensive source-build body. Build-set parsing, +# carry seeding, bootstrap dispatch, counter derivation, package-set checking and merged closure all +# run through their production paths, with no docker or network. +echo " running differential build + rebuild-counter contract..." +bash "$HERE/test-build-bookworm-differential.sh" >/dev/null + +# The per-source suffix + mixed-version manifest contract. It stages placeholder debs and runs the +# real generate-release-manifest.sh over them, and proves the migration-continuity ordering with +# real `dpkg --compare-versions` — no docker, no network, no built .deb. The same suffix-contract.sh +# library backs test-package-contract.sh's CHECK 5/6, so the heavy lane cannot drift from this one. +echo " running per-source suffix coherence + mixed-version manifest contract..." +bash "$HERE/test-suffix-coherence-manifest.sh" >/dev/null + +# The release.yml wiring proof is STATIC — it reads the workflow text and never dispatches a run. +# It belongs in this lane because the invariant it guards (carry-forward staged before every +# build-bookworm.sh call) fails SILENTLY: a late stage still produces a green release, built +# against stock bookworm dependencies instead of the carried ones. +echo " running release.yml differential wiring contract..." +bash "$HERE/test-release-workflow-wiring.sh" >/dev/null + +# The upstream freshness proof is offline and fixture-driven, so it needs no docker, no network +# and no built .deb — exactly the kind of invariant this lightweight lane should exercise. +echo " running upstream freshness contract..." +bash "$HERE/test-check-upstream-freshness.sh" >/dev/null + # Reading the base here also runs read-pin.sh's changelog-top vs salsa_tag cross-check on every # PR-lane run, so a `-1`-vs-`-2` revision drift fails closed before any ordering proof. MM_BASE="$(bash "$HERE/read-pin.sh" modemmanager --base-version)" @@ -94,7 +141,9 @@ for item in "$PKG_ROOT"/*; do cp -a "$item" "$TMP/" done if command -v dch >/dev/null 2>&1; then - ( cd "$TMP" && bash ci/inject-deb-version.sh --dev >/dev/null ) + for source_key in libqrtr-glib libmbim libqmi modemmanager; do + ( cd "$TMP" && bash ci/inject-deb-version.sh --source "$source_key" --dev >/dev/null ) + done # The copy's changelogs must now carry the dev suffix; the SOURCE tree must not. for src in ModemManager libmbim libqmi libqrtr-glib; do cl="$PKG_ROOT/$src/debian/changelog" diff --git a/packaging/ci/detect-changed-sources.sh b/packaging/ci/detect-changed-sources.sh new file mode 100755 index 0000000..89284eb --- /dev/null +++ b/packaging/ci/detect-changed-sources.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# detect-changed-sources.sh — which upstream packaging sources actually changed since the +# previous release, and therefore need rebuilding. +# +# WHAT THIS IS +# The FIRST script of the differential-release pipeline. A `vX.Y.Z` release should rebuild +# only the sources whose inputs moved; every other source's .debs are carried forward +# byte-identically from the release that last built them (builds are NOT reproducible, so an +# unchanged source must reuse recorded bytes and can never be rebuilt at its old version). +# This script makes ONLY the verdict; staging, building and manifest generation are elsewhere. +# +# OUTPUT (stdout — the machine contract; consumers grep these exact lines) +# libqrtr-glib=changed|unchanged +# libmbim=changed|unchanged +# libqmi=changed|unchanged +# modemmanager=changed|unchanged +# mode=differential|force-all +# +# Source names are the upstream-pins.yaml PIN KEYS (lowercase `modemmanager`), not the +# packaging directory names (`ModemManager`) — the mapping is NON-IDENTITY and is the same +# one build-bookworm.sh's `pin_key()` and read-pin.sh's `recipe_dir()` carry. Lines are +# emitted in BOOTSTRAP ORDER, so a consumer can build the selected set by reading top to +# bottom. Every human-readable reason goes to STDERR, so stdout stays parseable. +# +# THE VERDICT +# A source is `changed` when the diff `..HEAD` touched either +# * `packaging//**` — its checked-in debian/ recipe, or +# * that source's own block in `packaging/upstream-pins.yaml` — compared BLOCK-SCOPED, so a +# comment edit or a neighbouring source's pin bump does not implicate it. +# +# FORCE-ALL (every source `changed`, `mode=force-all`, reason logged) +# 1. A SHARED INPUT changed — `packaging/ci/**` (which subsumes `ci/expected-packages.txt`) +# or `packaging/BOOKWORM-ADAPTATIONS.md`. These feed every source's build, so a change to +# one of them invalidates every carried artifact. +# 2. The previous release is ABSENT, or carries no manifest asset. With no manifest there is +# nothing to carry forward from, so rebuilding everything is the only honest answer. +# 3. The previous manifest is V1-SHAPED — it carries no `closure_version:` header. An ABSENT +# header IS closure version 1 (that default is apt-worker's backward-compatibility +# mechanism, `modem_manifest_closure_version`), and a v1 manifest predates the companion +# row this pipeline depends on. +# 4. `FORCE_REBUILD=all` — the operator's escape hatch. +# Fail-SAFE here means REBUILD EVERYTHING, never "assume unchanged": a wrong `unchanged` +# ships stale bytes, a wrong `changed` only costs build time. +# +# THE COMPANION IS NOT PART OF DETECTION +# `ceralive-modem-support` is never enumerated and never verdicted — it ALWAYS rebuilds +# (apt-worker's health check greps `^Version: $`, which only the companion's bare +# version can satisfy). A companion-only change therefore leaves all four sources +# `unchanged`; that is correct, not a miss. +# +# PREVIOUS-RELEASE RESOLUTION — `gh release list`, NEVER `git describe` +# `git describe` answers "nearest tag in this history", which is not the same question: a tag +# can exist with no release, a release can be a draft or a pre-release, and a release can +# carry no manifest asset. The previous release is the latest PUBLISHED release, so it is +# resolved through `gh`. `PREV_TAG` overrides the resolution outright. +# +# SEAMS (all optional; they exist so the contract test runs offline, with no gh and no network) +# PREV_TAG Use this tag as the previous release; skips `gh release list`. +# PREV_MANIFEST_FILE Read the previous release manifest from this local path; skips +# `gh release download`. Set-but-unreadable is a hard error (exit 2), not +# a force-all — a named seam pointing at nothing is operator error, and +# silently force-alling would hide it. +# FORCE_REBUILD=all Force-all (rule 4). +# HEAD_REF The current side of the diff (default `HEAD`). +# GH_REPO Passed to `gh --repo` when resolving/downloading (default: gh's own +# repo inference from the checkout). +# +# USAGE +# detect-changed-sources.sh [--out ] +# --out additionally write the stdout contract lines to (for a later job +# step to source); the file is written atomically-enough for CI (truncate+write). +# +# EXIT +# 0 verdicts printed (differential OR force-all — force-all is a normal outcome, not an error). +# 2 usage / environment error: not a git repo, unreadable pin manifest, unresolvable HEAD ref, +# a set-but-unreadable PREV_MANIFEST_FILE, or a pinned-source set that no longer matches the +# declared bootstrap order. Every one names the offending field. +set -euo pipefail + +LOG_PREFIX="detect-changed-sources" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# ---- arguments ----------------------------------------------------------------------------- +OUT_FILE="" +while [ $# -gt 0 ]; do + case "$1" in + --out) + shift + [ $# -gt 0 ] || die "--out requires a file path" + OUT_FILE="$1" + ;; + --out=*) OUT_FILE="${1#--out=}" ;; + -h | --help) + echo "usage: detect-changed-sources.sh [--out ]" + exit 0 + ;; + *) die "unknown argument '$1' (usage: detect-changed-sources.sh [--out ])" ;; + esac + shift +done + +# ---- repository + packaging paths ---------------------------------------------------------- +PINS="$PKG_ROOT/upstream-pins.yaml" +[ -r "$PINS" ] || die "cannot read pin manifest '$PINS'" + +repo_top="$(git -C "$PKG_ROOT" rev-parse --show-toplevel 2>/dev/null)" || + die "packaging root '$PKG_ROOT' is not inside a git repository (a diff needs one)" +REPO_ROOT="$(cd "$repo_top" && pwd)" +cd "$REPO_ROOT" + +# Repo-RELATIVE packaging prefix, derived rather than hardcoded, so the same script works from +# any checkout layout (and from the contract test's throwaway fixture repo). +PKG_REL="${PKG_ROOT#"$REPO_ROOT"/}" +[ "$PKG_REL" != "$PKG_ROOT" ] || die "packaging root '$PKG_ROOT' is not under repository root '$REPO_ROOT'" +PINS_REL="$PKG_REL/upstream-pins.yaml" + +# ---- contract constants (mirrored from build-bookworm.sh:41,46) ----------------------------- +# Bootstrap order — the order a differential build must walk the selected set in. +BUILD_ORDER=(libqrtr-glib libmbim libqmi ModemManager) +# packaging dir -> upstream-pins.yaml source key. NON-IDENTITY: only ModemManager differs. +pin_key() { case "$1" in ModemManager) echo modemmanager ;; *) echo "$1" ;; esac; } + +# The list of source names under `sources:` — byte-identical to read-pin.sh's / +# verify-upstream-pins.sh's reader, deliberately the SAME parser rather than a second one. +yaml_sources() { + awk ' + /^sources:[ \t]*$/ { ins=1; next } + ins && /^[^ ]/ { ins=0 } + ins && /^ [^ ]+:[ \t]*$/ { s=$0; sub(/^ /, "", s); sub(/:[ \t]*$/, "", s); print s } + ' "$PINS" +} + +# One source's block of the pin manifest, read from stdin. Block-scoped so a comment edit or a +# neighbouring source's bump cannot implicate this source. +pins_block() { # (manifest text on stdin) + awk -v src="$1" ' + $0 ~ "^ " src ":[ \t]*$" { inblk=1; print; next } + inblk && /^ [^ ]/ { inblk=0 } + inblk && /^[^ ]/ { inblk=0 } + inblk { print } + ' +} + +# The pinned set and the declared bootstrap set must agree. A fifth pinned source added without +# updating this detector would otherwise be silently un-detected — i.e. carried forward forever. +pinned_sorted="$(yaml_sources | LC_ALL=C sort)" +declared_sorted="$(for d in "${BUILD_ORDER[@]}"; do pin_key "$d"; done | LC_ALL=C sort)" +if [ "$pinned_sorted" != "$declared_sorted" ]; then + die "pinned source set in '$PINS_REL' != declared BUILD_ORDER set — pinned: [$(echo "$pinned_sorted" | tr '\n' ' ')] declared: [$(echo "$declared_sorted" | tr '\n' ' ')]" +fi + +# ---- shared-input classifier --------------------------------------------------------------- +# `packaging/ci/**` subsumes ci/expected-packages.txt; both are named in the header on purpose. +is_shared_input() { + case "$1" in + "$PKG_REL"/ci/*) return 0 ;; + "$PKG_REL"/BOOKWORM-ADAPTATIONS.md) return 0 ;; + esac + return 1 +} + +# ---- resolution ---------------------------------------------------------------------------- +FORCED=0 +FORCE_REASON="" +PREV_TAG_RESOLVED="" +HEAD_REF_RESOLVED="${HEAD_REF:-HEAD}" +CHANGED_FILES="" +MANIFEST_TMP="" + +cleanup() { [ -n "$MANIFEST_TMP" ] && rm -rf "$MANIFEST_TMP"; return 0; } +trap cleanup EXIT + +force() { + FORCED=1 + FORCE_REASON="$*" +} + +gh_args() { + if [ -n "${GH_REPO:-}" ]; then printf '%s\n%s\n' "--repo" "$GH_REPO"; fi +} + +resolve() { + # (4) Operator override, checked first so it needs no network and no previous release. + if [ "${FORCE_REBUILD:-}" = "all" ]; then + force "FORCE_REBUILD=all — operator override" + return + fi + if [ -n "${FORCE_REBUILD:-}" ]; then + die "FORCE_REBUILD='$FORCE_REBUILD' is not a recognized value (the only accepted value is 'all')" + fi + + # --- previous release tag --------------------------------------------------------------- + if [ -n "${PREV_TAG:-}" ]; then + PREV_TAG_RESOLVED="$PREV_TAG" + log "previous release tag: $PREV_TAG_RESOLVED (PREV_TAG seam)" + else + if ! command -v gh >/dev/null 2>&1; then + force "previous-release-unresolved — PREV_TAG is unset and 'gh' is not installed (the previous release is the latest PUBLISHED release, resolved via 'gh release list'; 'git describe' is never used)" + return + fi + local ghargs=() tag="" + mapfile -t ghargs < <(gh_args) + if ! tag="$(gh release list "${ghargs[@]}" --limit 1 --exclude-drafts --exclude-pre-releases --json tagName --jq '.[0].tagName' 2>/dev/null)"; then + force "previous-release-unresolved — 'gh release list' failed" + return + fi + tag="$(printf '%s' "$tag" | tr -d '[:space:]')" + if [ -z "$tag" ] || [ "$tag" = "null" ]; then + force "previous-release-absent — 'gh release list' reports no published release to diff against" + return + fi + PREV_TAG_RESOLVED="$tag" + log "previous release tag: $PREV_TAG_RESOLVED (gh release list)" + fi + + # --- previous release manifest ------------------------------------------------------------ + local manifest="" + if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + [ -r "$PREV_MANIFEST_FILE" ] || + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable (a named seam pointing at nothing is operator error, not a force-all)" + manifest="$PREV_MANIFEST_FILE" + log "previous release manifest: $manifest (PREV_MANIFEST_FILE seam)" + else + if ! command -v gh >/dev/null 2>&1; then + force "previous-manifest-absent — 'gh' is not installed, so release '$PREV_TAG_RESOLVED' manifest cannot be fetched" + return + fi + local ghargs=() + mapfile -t ghargs < <(gh_args) + MANIFEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/detect-changed-sources.XXXXXX")" + if ! gh release download "$PREV_TAG_RESOLVED" "${ghargs[@]}" --pattern 'release-manifest*.txt' --dir "$MANIFEST_TMP" >/dev/null 2>&1; then + force "previous-manifest-absent — release '$PREV_TAG_RESOLVED' has no downloadable release-manifest asset" + return + fi + manifest="$(find "$MANIFEST_TMP" -maxdepth 1 -type f -name 'release-manifest*.txt' | LC_ALL=C sort | head -n1)" + if [ -z "$manifest" ]; then + force "previous-manifest-absent — 'gh release download' produced no release-manifest*.txt for '$PREV_TAG_RESOLVED'" + return + fi + log "previous release manifest: $manifest (gh release download from $PREV_TAG_RESOLVED)" + fi + + # (3) An ABSENT `closure_version:` header IS closure version 1 — apt-worker's + # `modem_manifest_closure_version` treats it exactly that way, and that default is the + # backward-compatibility mechanism for every pre-header release. + if ! grep -q '^closure_version:' "$manifest"; then + force "previous-manifest-v1-shaped — '$manifest' carries no 'closure_version:' header (an absent header IS closure version 1), so nothing may be carried forward from it" + return + fi + + # --- the diff ----------------------------------------------------------------------------- + git rev-parse --verify --quiet "$HEAD_REF_RESOLVED^{commit}" >/dev/null || + die "HEAD ref '$HEAD_REF_RESOLVED' does not resolve to a commit in '$REPO_ROOT'" + if ! git rev-parse --verify --quiet "$PREV_TAG_RESOLVED^{commit}" >/dev/null; then + force "previous-ref-unresolvable — '$PREV_TAG_RESOLVED' does not resolve to a commit in this checkout (fetch the tag, or the diff cannot be taken)" + return + fi + if ! CHANGED_FILES="$(git diff --name-only "$PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED")"; then + force "diff-unavailable — 'git diff --name-only $PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED' failed" + return + fi + log "diff $PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED touched $(printf '%s' "$CHANGED_FILES" | grep -c . || true) file(s)" + + # (1) Shared inputs feed every source's build; one of them moving invalidates every + # carried artifact, so it is a force-all rather than a per-source verdict. + local f + while IFS= read -r f; do + [ -n "$f" ] || continue + if is_shared_input "$f"; then + force "shared-input-changed — '$f' is a SHARED build input (packaging/ci/** or packaging/BOOKWORM-ADAPTATIONS.md); it feeds every source, so no source may be carried forward" + return + fi + done <<<"$CHANGED_FILES" +} + +resolve + +# ---- per-source verdicts -------------------------------------------------------------------- +declare -A VERDICT=() + +pins_block_changed() { # -> 0 when this source's pin block differs prev..HEAD + local key="$1" prev="" cur="" + prev="$(git show "$PREV_TAG_RESOLVED:$PINS_REL" 2>/dev/null | pins_block "$key")" || prev="" + cur="$(git show "$HEAD_REF_RESOLVED:$PINS_REL" 2>/dev/null | pins_block "$key")" || cur="" + [ "$prev" != "$cur" ] +} + +if [ "$FORCED" -eq 1 ]; then + for dir in "${BUILD_ORDER[@]}"; do VERDICT["$(pin_key "$dir")"]=changed; done + log "mode=force-all — $FORCE_REASON" + log "every source rebuilds; nothing is carried forward" +else + pins_touched=0 + while IFS= read -r f; do + if [ "$f" = "$PINS_REL" ]; then pins_touched=1; fi + done <<<"$CHANGED_FILES" + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + verdict=unchanged + why="" + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + "$PKG_REL/$dir"/*) + verdict=changed + why="recipe '$f'" + break + ;; + esac + done <<<"$CHANGED_FILES" + + if [ "$verdict" = unchanged ] && [ "$pins_touched" -eq 1 ] && pins_block_changed "$key"; then + verdict=changed + why="pin block [$key] in '$PINS_REL'" + fi + + VERDICT["$key"]="$verdict" + if [ "$verdict" = changed ]; then + log " $key=changed ($why)" + else + log " $key=unchanged (no recipe change under $PKG_REL/$dir/, no [$key] pin-block change)" + fi + done + + changed_n=0 + for dir in "${BUILD_ORDER[@]}"; do + if [ "${VERDICT["$(pin_key "$dir")"]}" = changed ]; then changed_n=$((changed_n + 1)); fi + done + log "mode=differential — $changed_n of ${#BUILD_ORDER[@]} source(s) changed since $PREV_TAG_RESOLVED; the rest carry forward" + log "the companion ceralive-modem-support is outside detection and always rebuilds" +fi + +MODE=differential +if [ "$FORCED" -eq 1 ]; then MODE=force-all; fi + +emit() { + local dir + for dir in "${BUILD_ORDER[@]}"; do + printf '%s=%s\n' "$(pin_key "$dir")" "${VERDICT["$(pin_key "$dir")"]}" + done + printf 'mode=%s\n' "$MODE" +} + +if [ -n "$OUT_FILE" ]; then + mkdir -p "$(dirname "$OUT_FILE")" + emit | tee "$OUT_FILE" + log "wrote verdicts to '$OUT_FILE'" +else + emit +fi diff --git a/packaging/ci/generate-release-manifest.sh b/packaging/ci/generate-release-manifest.sh index 3ce0a2c..f97c970 100755 --- a/packaging/ci/generate-release-manifest.sh +++ b/packaging/ci/generate-release-manifest.sh @@ -26,6 +26,18 @@ # infer the shape: `runtime_closure_size` (per-arch, arch-dependent), `arch_all_closure_size`, # and `index_arches`. # +# NO `deb_version_suffix:` HEADER — `suffix_scheme: per-source-counter` REPLACES IT. +# Releases are differential: each upstream source carries its own rebuild counter +# `-~ceralive.N`, and a source that was not rebuilt keeps the counter it +# already had. There is therefore NO single truthful suffix value a header could state, so +# the generator declares the SCHEME instead of a value and every row keeps carrying its own +# version (which it always did — rows are parsed from real filenames, so a carried-forward +# deb at an old counter and a freshly built one at a new counter both emit correctly). +# Dropping the old header is safe on both sides: apt-worker's publisher/validator reads +# NEITHER field (it extracts only tag / closure_version / sizes / rows), and legacy manifests +# are untouched because validation never read the old header either. `version:` is unrelated +# and stays — it is the RELEASE's own SemVer, not a per-deb suffix. +# # It also FAILS CLOSED if the produced set is not exactly the frozen all-artifact set: per arch, # per source, the enumerated packages must EQUAL `[ all-artifact]` in expected-packages.txt # (the two-set model finalized in todo 1.4). An added, dropped, renamed, or unmapped deb is a @@ -50,11 +62,12 @@ OUT="${3:-$PKG_ROOT/../dist/release-manifest.txt}" EXPECTED="${EXPECTED_PACKAGES:-$HERE/expected-packages.txt}" [ -r "$EXPECTED" ] || { echo "generate-release-manifest: cannot read expected-packages '$EXPECTED'" >&2; exit 2; } -# Strip a leading v for the encoded suffix (tag guard already vetted the shape upstream). +# Strip a leading v for the release's own version string (tag guard vetted the shape upstream). +# NOTE: this is NOT a deb suffix — per-deb versions come from each file's own name (see (d)). VERSION="${TAG#v}" -SUFFIX="~ceralive${VERSION}" CLOSURE_VERSION=2 +SUFFIX_SCHEME="per-source-counter" # The 9 arch-dependent runtime packages, and the arch-all runtime companion. RUNTIME_PKGS=(modemmanager libmm-glib0 libmbim-glib4 libmbim-proxy libmbim-utils \ @@ -110,7 +123,7 @@ mkdir -p "$(dirname "$OUT")" echo "# MANIFEST-COMPLETE: one row per built deb, both arches; runtime closure marked." echo "tag: ${TAG}" echo "version: ${VERSION}" - echo "deb_version_suffix: ${SUFFIX}" + echo "suffix_scheme: ${SUFFIX_SCHEME}" echo "sources: [${SOURCES[*]}]" echo "closure_version: ${CLOSURE_VERSION}" echo "runtime_closure_size: ${#RUNTIME_PKGS[@]}" diff --git a/packaging/ci/inject-deb-version.sh b/packaging/ci/inject-deb-version.sh index 0c83f76..fa55a59 100755 --- a/packaging/ci/inject-deb-version.sh +++ b/packaging/ci/inject-deb-version.sh @@ -1,69 +1,157 @@ #!/usr/bin/env bash -# inject-deb-version.sh — write the CeraLive-encoded version into each source's -# debian/changelog top entry. +# inject-deb-version.sh — inject one selected source's CeraLive rebuild version. # -# Encoded version: -~ceralive (see docs/VERSIONING.md). -# The tilde makes it sort BELOW the pinned - changelog top, so plain -# `dch --newversion` REFUSES it (dch(1) rejects a lower-than-current version). Hence -# `--force-bad-version` is REQUIRED, not optional. All four sources take the same suffix. +# Release suffixes are per-source counters: -~ceralive.N. N is derived from +# EVERY previous-manifest row for that source (both arches, runtime and aux), and is accepted +# only when those rows are coherent. Entirely legacy rows (~ceraliveX.Y.Z), or an absent previous +# manifest during the force-all bootstrap, initialize at .1. Non-tag builds keep the fixed +# ~ceralive0.0.0~dev suffix. # # Usage: -# inject-deb-version.sh vX.Y.Z # release: suffix ~ceraliveX.Y.Z -# inject-deb-version.sh X.Y.Z # same (leading v optional) -# inject-deb-version.sh --dev # non-tag CI: fixed suffix ~ceralive0.0.0~dev +# PREV_MANIFEST_FILE=release-manifest.txt \ +# inject-deb-version.sh --source libqmi vX.Y.Z +# inject-deb-version.sh --source modemmanager --dev # -# The 4 repackaged sources are ModemManager, libmbim, libqmi, libqrtr-glib. Their upstream -# versions are NOT hardcoded here — each is read from that source's own debian/changelog -# top entry (added in the packaging wave). Until those recipes exist, this script documents -# the exact `dch` invocation per source and exits 0. +# Source names are upstream-pins.yaml pin keys. The ModemManager recipe-directory mapping is +# deliberately non-identity. The previous manifest is an input seam supplied by the caller; this +# script never resolves or downloads a release independently. set -euo pipefail +LOG_PREFIX="inject-deb-version" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PKG_ROOT="$(cd "$HERE/.." && pwd)" -# Names only — versions/pins are provenance-verified in a later task, never hardcoded here. -SOURCES=(ModemManager libmbim libqmi libqrtr-glib) +SOURCE_KEYS=(libqrtr-glib libmbim libqmi modemmanager) +recipe_dir() { case "$1" in modemmanager) echo ModemManager ;; *) echo "$1" ;; esac; } -arg="${1-}" -if [ -z "$arg" ]; then - echo "usage: inject-deb-version.sh " >&2 - exit 2 -fi +usage() { + echo "usage: inject-deb-version.sh --source " +} + +SOURCE="" +VERSION_ARG="" +while [ $# -gt 0 ]; do + case "$1" in + --source) + shift + [ $# -gt 0 ] || die "--source requires a pin-key name" + [ -z "$SOURCE" ] || die "--source was supplied more than once" + SOURCE="$1" + ;; + --source=*) + [ -z "$SOURCE" ] || die "--source was supplied more than once" + SOURCE="${1#--source=}" + ;; + -h | --help) + usage + exit 0 + ;; + --dev | v* | [0-9]*) + [ -z "$VERSION_ARG" ] || die "version argument was supplied more than once ('$VERSION_ARG' then '$1')" + VERSION_ARG="$1" + ;; + *) die "unknown argument '$1'" ;; + esac + shift +done + +[ -n "$SOURCE" ] || die "--source is required (version injection is per-source and may only touch a source being built)" +[ -n "$VERSION_ARG" ] || die "a release version or --dev is required" + +known=0 +for key in "${SOURCE_KEYS[@]}"; do + if [ "$key" = "$SOURCE" ]; then known=1; break; fi +done +[ "$known" -eq 1 ] || die "unknown source '$SOURCE' (expected one of: ${SOURCE_KEYS[*]})" + +derive_release_suffix() { # + local source_name="$1" manifest version kind="" counter="" next + local versions=() -if [ "$arg" = "--dev" ]; then + if [ -z "${PREV_MANIFEST_FILE:-}" ]; then + log "source '$source_name': no previous manifest supplied (force-all bootstrap); initializing counter at .1" + printf '%s\n' '~ceralive.1' + return 0 + fi + + manifest="$PREV_MANIFEST_FILE" + [ -r "$manifest" ] || + die "source '$source_name': PREV_MANIFEST_FILE='$manifest' is set but not readable" + + mapfile -t versions < <( + awk -v source_name="$source_name" \ + '$0 !~ /^#/ && NF==7 && $1 !~ /:$/ && $3==source_name { print $4 }' "$manifest" + ) + if [ "${#versions[@]}" -eq 0 ]; then + refuse "source '$source_name' is being rebuilt but previous manifest '$manifest' contains ZERO rows for it; no counter can be derived" + fi + + for version in "${versions[@]}"; do + if [[ "$version" =~ ~ceralive\.([1-9][0-9]*)$ ]]; then + if [ "$kind" = legacy ]; then + refuse "source '$source_name' mixes counter and legacy suffixes across its previous-manifest rows (encountered '$version')" + fi + kind=counter + if [ -z "$counter" ]; then + counter="${BASH_REMATCH[1]}" + elif [ "$counter" != "${BASH_REMATCH[1]}" ]; then + refuse "source '$source_name' has differing counters in its previous-manifest rows (.$counter vs .${BASH_REMATCH[1]}); refusing to risk a downgrade" + fi + elif [[ "$version" =~ ~ceralive[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [ "$kind" = counter ]; then + refuse "source '$source_name' mixes counter and legacy suffixes across its previous-manifest rows (encountered '$version')" + fi + kind=legacy + else + refuse "source '$source_name' has malformed previous-manifest version '$version'; expected terminal '~ceralive.[1-9][0-9]*' or legacy '~ceraliveX.Y.Z'" + fi + done + + if [ "$kind" = legacy ]; then + log "source '$source_name': all ${#versions[@]} previous-manifest row(s) use the legacy suffix; initializing counter at .1" + printf '%s\n' '~ceralive.1' + return 0 + fi + + next=$((10#$counter + 1)) + log "source '$source_name': all ${#versions[@]} previous-manifest row(s) agree on counter .$counter; next counter is .$next" + printf '~ceralive.%s\n' "$next" +} + +if [ "$VERSION_ARG" = --dev ]; then SUFFIX="~ceralive0.0.0~dev" else - # Reuse the tag guard so a bad version is rejected identically everywhere. + # Reuse the tag guard so release invocation syntax is rejected identically everywhere. The + # tag no longer supplies the suffix: release provenance lives in the manifest, while the + # package version records this source's rebuild count. # shellcheck source=./tag-guard.sh source "$HERE/tag-guard.sh" - case "$arg" in - v*) XYZ="$(validate_tag "$arg")" ;; - *) XYZ="$(validate_tag "v$arg")" ;; + case "$VERSION_ARG" in + v*) validate_tag "$VERSION_ARG" >/dev/null ;; + *) validate_tag "v$VERSION_ARG" >/dev/null ;; esac - SUFFIX="~ceralive${XYZ}" + SUFFIX="$(derive_release_suffix "$SOURCE")" fi -echo "inject-deb-version: suffix ${SUFFIX}" - -injected=0 -pending=0 -for src in "${SOURCES[@]}"; do - changelog="$PKG_ROOT/$src/debian/changelog" - if [ -f "$changelog" ]; then - # Derive - from the top entry, strip any prior ~ceralive suffix, - # then append ours. - top="$(dpkg-parsechangelog -l "$changelog" -S Version)" - base="${top%%~ceralive*}" - version="${base}${SUFFIX}" - echo " ${src}: ${top} -> ${version}" - (cd "$PKG_ROOT/$src" && dch --force-bad-version --newversion "$version" "CeraLive rebuild") - injected=$((injected + 1)) - else - # No recipe yet (packaging wave). Document the exact invocation this WILL run. - echo " ${src}: no debian/changelog yet — will run:" - echo " dch --force-bad-version --newversion \"-${SUFFIX}\" \"CeraLive rebuild\"" - pending=$((pending + 1)) - fi -done +DIR="$(recipe_dir "$SOURCE")" +CHANGELOG="$PKG_ROOT/$DIR/debian/changelog" +[ -r "$CHANGELOG" ] || die "source '$SOURCE' changelog '$CHANGELOG' is not readable" +command -v dpkg-parsechangelog >/dev/null 2>&1 || die "source '$SOURCE': dpkg-parsechangelog is not installed" +command -v dch >/dev/null 2>&1 || die "source '$SOURCE': dch is not installed" -echo "inject-deb-version: ${injected} injected, ${pending} pending recipes" +TOP="$(dpkg-parsechangelog -l "$CHANGELOG" -S Version)" +BASE="${TOP%%~ceralive*}" +VERSION="${BASE}${SUFFIX}" +log "source '$SOURCE': $TOP -> $VERSION" +(cd "$PKG_ROOT/$DIR" && dch --force-bad-version --newversion "$VERSION" "CeraLive rebuild") +log "source '$SOURCE': injected suffix '$SUFFIX'" diff --git a/packaging/ci/read-pin.sh b/packaging/ci/read-pin.sh index 55bdb5d..881e391 100755 --- a/packaging/ci/read-pin.sh +++ b/packaging/ci/read-pin.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash # read-pin.sh — dependency-free reader for packaging/upstream-pins.yaml + the Debian base. # -# TWO MODES +# THREE MODES # read-pin.sh Print a scalar field of a source from upstream-pins.yaml # (e.g. `read-pin.sh modemmanager upstream_tag` -> 1.24.2). # read-pin.sh --base-version Print the FULL Debian base `-` (e.g. # 1.24.2-2) taken from that source's debian/changelog TOP # entry, cross-checked to equal the pin's salsa_tag suffix # (`debian/1.24.2-2` -> `1.24.2-2`). Mismatch FAILS CLOSED. +# read-pin.sh --list-sources Print the pinned source NAMES, one per line, in manifest +# order. Exposes the reader's own `yaml_sources` so a +# caller that must iterate every source (e.g. +# check-upstream-freshness.sh) does not grow a second +# copy of the YAML parser to enumerate the keys. # # WHY A SHARED READER # The packaging CI assertion scripts (daemon-smoke.sh, test-package-contract.sh, contract.sh) @@ -38,9 +43,15 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PKG_ROOT="$(cd "$HERE/.." && pwd)" MANIFEST="$PKG_ROOT/upstream-pins.yaml" -[ $# -ge 2 ] || { echo "read-pin: usage: read-pin.sh " >&2; exit 2; } -SRC="$1" -FIELD="$2" +if [ "${1-}" = "--list-sources" ]; then + MODE="list-sources"; SRC=""; FIELD="" +else + [ $# -ge 2 ] || { + echo "read-pin: usage: read-pin.sh | read-pin.sh --list-sources" >&2 + exit 2 + } + MODE="scalar"; SRC="$1"; FIELD="$2" +fi [ -r "$MANIFEST" ] || { echo "read-pin: cannot read manifest '$MANIFEST'" >&2; exit 2; } # ---- tiny dependency-free YAML readers (byte-identical to verify-upstream-pins.sh) --------- @@ -64,6 +75,13 @@ yaml_sources() { ' "$MANIFEST" } +if [ "$MODE" = "list-sources" ]; then + names="$(yaml_sources)" + [ -n "$names" ] || { echo "read-pin: no sources found in $MANIFEST" >&2; exit 1; } + printf '%s\n' "$names" + exit 0 +fi + # The pinned source must exist (fail-closed on a typo'd / wrong source name). src_known=0 while IFS= read -r s; do diff --git a/packaging/ci/stage-carryforward-debs.sh b/packaging/ci/stage-carryforward-debs.sh new file mode 100755 index 0000000..b980f4f --- /dev/null +++ b/packaging/ci/stage-carryforward-debs.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash +# stage-carryforward-debs.sh — stage every UNCHANGED source's .debs from the PREVIOUS release +# into packaging/build//, byte-verified, so the differential build never rebuilds them. +# +# WHAT THIS IS +# The SECOND script of the differential-release pipeline. `detect-changed-sources.sh` decides +# WHICH sources moved; this one carries the bytes of the ones that did not. Builds are NOT +# reproducible, so an unchanged source can never be rebuilt at its old version — the only +# honest way to keep a release self-contained is to reuse the exact artifacts the release that +# last built them recorded. Those bytes are a build INPUT too: `build-bookworm.sh` seeds its +# temporary local apt repo from `build//`, so a changed source resolves its build-deps +# against the carried `-dev`/`gir1.2-*` packages rather than stock bookworm. +# +# INPUT — the verdict lines from detect-changed-sources.sh, on STDIN by default +# The caller pipes that script's stdout straight in (or points `--verdicts ` at the file +# it wrote with `--out`). Reading a stream rather than re-running the detector is deliberate: +# detection resolves the previous release over the network and must happen exactly ONCE per +# run, and release.yml already holds that result. +# +# libqrtr-glib=unchanged +# libmbim=unchanged +# libqmi=changed +# modemmanager=unchanged +# mode=differential +# +# Source names are upstream-pins.yaml PIN KEYS (`modemmanager`, not `ModemManager`). The +# `mode=` line is REQUIRED — a verdict stream without it is malformed, not a default. +# +# WHAT IS CARRIED +# EVERY row of an `unchanged` source — runtime AND aux (`-dbgsym`, `-dev`, `gir1.2-*`). The +# `role` column is deliberately NOT filtered on: the release manifest is manifest-complete and +# a release that re-attached only the 9-package runtime closure would silently drop ~36 debs. +# Rows of a `changed` source are skipped (the build produces them fresh). +# +# THE COMPANION IS NEVER CARRIED +# `ceralive-modem-support` ALWAYS rebuilds — apt-worker's health check greps `^Version: $`, +# which only the companion's bare tag version can satisfy. Its row is skipped explicitly, and +# that skip is checked BEFORE the unknown-source rule below, so a previous manifest carrying it +# (every closure_version-2 manifest does) is normal input rather than an error. +# +# GITHUB ASSET-NAME RECONCILIATION (the inverse of reconcile-release-assets.sh) +# The uploader stages every asset under a sanitized basename — `target="${src//\~/.}"` — so the +# canonical `libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb` is STORED on the release as +# `libqmi-glib5_1.38.0-1.ceralive1.1.0_amd64.deb`. Going back the other way cannot simply +# substitute `.`→`~` (the name is full of legitimate dots), so each `~` becomes a single-char +# glob wildcard `?` and every other character stays anchored — apt-worker's `modem_asset_glob` +# discipline. EXACTLY ONE asset must match: zero means the release does not carry it, and two +# (both the `~` and the `.` spelling present) means the name is ambiguous. Either is fail-closed. +# The staged file always lands under the CANONICAL `~` name, which is what dpkg orders on and +# what the next manifest must report. +# +# FAIL CLOSED, ALWAYS NAMING THE OFFENDING ROW/FILE +# * a manifest row's asset is missing, or its name is ambiguous; +# * a staged asset's sha256 differs from the manifest row; +# * a row names a source that is neither the companion nor any verdicted source (defense in +# depth: a source the caller never adjudicated must never be carried on this script's guess); +# * an `unchanged` source has ZERO rows in the previous manifest (nothing would be carried and +# nothing would be built, so the merged set would be quietly incomplete); +# * the previous manifest is V1-SHAPED — no `closure_version:` header. An ABSENT header IS +# closure version 1 (apt-worker's backward-compatibility default), which predates the +# companion row this pipeline depends on. In practice detection force-alls on a v1 manifest, +# so nothing is ever carried from one; if this script is nevertheless ASKED to, it refuses +# rather than guessing which shape it is reading. +# * a destination file already exists with DIFFERENT bytes (a carried deb never overwrites a +# freshly built one — reconcile-release-assets.sh's integrity-compare stance). +# +# NOTHING TO CARRY IS A NORMAL OUTCOME +# `mode=force-all` (or simply no `unchanged` source) stages nothing and exits 0 WITHOUT reading +# a manifest at all — under force-all there is nothing to carry from by definition. +# +# STAGING LAYOUT +# `//` — the same gitignored tree +# `build-bookworm.sh` writes freshly built debs into and `generate-release-manifest.sh` reads. +# `build_arch` is taken verbatim from manifest column 1, so an `all` row would land in +# `build/all/`. Only the companion is ever `Architecture: all`, and the companion is never +# carried — an upstream source's rows are always arch-dependent — so `build/all/` should never +# be created here. It is handled rather than assumed, and loudly logged if it ever happens. +# +# SEAMS (all optional; they exist so the contract test runs offline, with no gh and no network) +# PREV_MANIFEST_FILE Read the previous release manifest from this local path; skips +# `gh release download`. Set-but-unreadable is exit 2, not a silent skip. +# CARRYFORWARD_ASSET_DIR Read the release assets from this local directory instead of +# downloading them. Names in it may be either the GitHub-mangled `.` +# form or the canonical `~` form — the glob resolves both. +# PREV_TAG The release to download the manifest/assets from (required when +# neither seam above is set). +# GH_REPO Passed to `gh --repo`. +# BUILD_ROOT Staging root (default: packaging/build). `--build-root` overrides it. +# +# OUTPUT (stdout — the machine contract) +# One `/` line per staged deb, in manifest order. Every human-readable +# line goes to stderr, so stdout stays parseable. +# +# USAGE +# detect-changed-sources.sh | stage-carryforward-debs.sh +# stage-carryforward-debs.sh --verdicts verdicts.txt --build-root packaging/build +# +# EXIT +# 0 staged (or nothing to carry). +# 2 usage / environment error: bad argument, malformed verdict stream, a set-but-unreadable +# seam, or no way to reach the previous manifest. +# 3 fail closed: the carry cannot be performed honestly (see the list above). Every message +# names the row, the file, or the source at fault. +set -euo pipefail + +LOG_PREFIX="stage-carryforward-debs" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# The one source that is never carried. It always rebuilds (apt-worker's health check greps +# `^Version: $`, which only its bare tag version satisfies). +COMPANION_SOURCE="ceralive-modem-support" + +# ---- arguments ----------------------------------------------------------------------------- +VERDICTS_FILE="" +BUILD_ROOT="${BUILD_ROOT:-$PKG_ROOT/build}" +while [ $# -gt 0 ]; do + case "$1" in + --verdicts) + shift + [ $# -gt 0 ] || die "--verdicts requires a file path" + VERDICTS_FILE="$1" + ;; + --verdicts=*) VERDICTS_FILE="${1#--verdicts=}" ;; + --build-root) + shift + [ $# -gt 0 ] || die "--build-root requires a directory path" + BUILD_ROOT="$1" + ;; + --build-root=*) BUILD_ROOT="${1#--build-root=}" ;; + -h | --help) + echo "usage: stage-carryforward-debs.sh [--verdicts ] [--build-root ]" + echo " (verdict lines are read from stdin when --verdicts is absent)" + exit 0 + ;; + *) die "unknown argument '$1' (usage: stage-carryforward-debs.sh [--verdicts ] [--build-root ])" ;; + esac + shift +done + +if [ -n "$VERDICTS_FILE" ]; then + [ -r "$VERDICTS_FILE" ] || + die "--verdicts '$VERDICTS_FILE' is not readable (a named input pointing at nothing is operator error, not an empty verdict set)" + VERDICTS_SRC="$VERDICTS_FILE" +else + VERDICTS_SRC="/dev/stdin" +fi + +# ---- the verdict stream --------------------------------------------------------------------- +declare -A VERDICT=() +VERDICT_ORDER=() +MODE="" +while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + \#*) continue ;; + mode=*) MODE="${line#mode=}" ;; + *=changed | *=unchanged) + key="${line%%=*}" + val="${line#*=}" + [ -n "$key" ] || die "verdict line '$line' has an empty source name" + if [ -n "${VERDICT[$key]:-}" ] && [ "${VERDICT[$key]}" != "$val" ]; then + die "source '$key' is verdicted twice and inconsistently ('${VERDICT[$key]}' then '$val')" + fi + if [ -z "${VERDICT[$key]:-}" ]; then VERDICT_ORDER+=("$key"); fi + VERDICT["$key"]="$val" + ;; + *) die "unparseable verdict line '$line' (expected '=changed|unchanged' or 'mode=differential|force-all')" ;; + esac +done <"$VERDICTS_SRC" + +[ "${#VERDICT_ORDER[@]}" -gt 0 ] || + die "the verdict stream carried no '=changed|unchanged' line (is detect-changed-sources.sh's stdout actually piped in?)" +case "$MODE" in +differential | force-all) ;; +"") die "the verdict stream carried no 'mode=' line — a stream without it is malformed, not a default" ;; +*) die "unrecognized mode '$MODE' (the only accepted values are 'differential' and 'force-all')" ;; +esac + +UNCHANGED=() +for key in "${VERDICT_ORDER[@]}"; do + if [ "${VERDICT[$key]}" = unchanged ]; then UNCHANGED+=("$key"); fi +done + +# A force-all run rebuilds EVERYTHING by definition, so an `unchanged` verdict under it is a +# contract violation in the caller's wiring rather than an instruction to carry something. +if [ "$MODE" = force-all ] && [ "${#UNCHANGED[@]}" -gt 0 ]; then + die "mode=force-all but source(s) [${UNCHANGED[*]}] are verdicted 'unchanged' — under force-all every source rebuilds; refusing to carry anything from a self-contradicting verdict stream" +fi + +log "verdicts: mode=$MODE, ${#UNCHANGED[@]} of ${#VERDICT_ORDER[@]} source(s) unchanged" + +if [ "${#UNCHANGED[@]}" -eq 0 ]; then + log "nothing to carry — every source rebuilds; no previous manifest is read and no deb is staged" + exit 0 +fi +log "carrying: ${UNCHANGED[*]}" + +is_unchanged() { # -> 0 when the caller verdicted it `unchanged` + local s + for s in "${UNCHANGED[@]}"; do [ "$s" = "$1" ] && return 0; done + return 1 +} +is_verdicted() { [ -n "${VERDICT[$1]:-}" ]; } + +# ---- the previous release manifest ------------------------------------------------------------ +TMP_DIR="" +cleanup() { + [ -n "$TMP_DIR" ] && rm -rf "$TMP_DIR" + return 0 +} +trap cleanup EXIT + +gh_args() { + if [ -n "${GH_REPO:-}" ]; then printf '%s\n%s\n' "--repo" "$GH_REPO"; fi +} +need_tmp() { + if [ -z "$TMP_DIR" ]; then TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/stage-carryforward.XXXXXX")"; fi +} + +MANIFEST="" +if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + [ -r "$PREV_MANIFEST_FILE" ] || + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable (a named seam pointing at nothing is operator error)" + MANIFEST="$PREV_MANIFEST_FILE" + log "previous release manifest: $MANIFEST (PREV_MANIFEST_FILE seam)" +else + [ -n "${PREV_TAG:-}" ] || + die "no previous manifest: set PREV_MANIFEST_FILE, or PREV_TAG so the manifest can be downloaded from that release" + command -v gh >/dev/null 2>&1 || + die "no previous manifest: 'gh' is not installed and PREV_MANIFEST_FILE is unset, so release '$PREV_TAG' cannot be read" + need_tmp + mkdir -p "$TMP_DIR/manifest" + ghargs=() + mapfile -t ghargs < <(gh_args) + gh release download "$PREV_TAG" "${ghargs[@]}" --pattern 'release-manifest*.txt' --dir "$TMP_DIR/manifest" >/dev/null 2>&1 || + refuse "release '$PREV_TAG' has no downloadable release-manifest asset, so there is nothing to carry forward from" + MANIFEST="$(find "$TMP_DIR/manifest" -maxdepth 1 -type f -name 'release-manifest*.txt' | LC_ALL=C sort | head -n1)" + [ -n "$MANIFEST" ] || + refuse "'gh release download' produced no release-manifest*.txt for '$PREV_TAG'" + log "previous release manifest: $MANIFEST (gh release download from $PREV_TAG)" +fi + +# An ABSENT `closure_version:` header IS closure version 1 — apt-worker's +# `modem_manifest_closure_version` treats it exactly that way. A v1 manifest predates the +# companion row this pipeline depends on, so carrying from one would be a guess. +grep -q '^closure_version:' "$MANIFEST" || + refuse "previous manifest '$MANIFEST' carries no 'closure_version:' header (an absent header IS closure version 1); refusing to carry from a v1-shaped manifest rather than guessing its shape" + +# Data rows are the 7-column `build_arch package source version role filename sha256` lines. +# Header lines are `key: value` and are excluded by the trailing-colon test on column 1, so a +# multi-token header (`sources: [...]`) can never be mistaken for a row. +ROWS="$(awk '$0 !~ /^#/ && NF==7 && $1 !~ /:$/ { print }' "$MANIFEST")" +[ -n "$ROWS" ] || + refuse "previous manifest '$MANIFEST' contains no deb rows" + +# ---- release assets --------------------------------------------------------------------------- +ASSET_NAMES=() +ASSET_DIR="" +if [ -n "${CARRYFORWARD_ASSET_DIR:-}" ]; then + [ -d "$CARRYFORWARD_ASSET_DIR" ] || + die "CARRYFORWARD_ASSET_DIR='$CARRYFORWARD_ASSET_DIR' is set but is not a directory" + ASSET_DIR="$CARRYFORWARD_ASSET_DIR" + mapfile -t ASSET_NAMES < <(find "$ASSET_DIR" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort) + log "release assets: ${#ASSET_NAMES[@]} local file(s) in $ASSET_DIR (CARRYFORWARD_ASSET_DIR seam)" +else + [ -n "${PREV_TAG:-}" ] || + die "no release assets: set CARRYFORWARD_ASSET_DIR, or PREV_TAG so the assets can be downloaded" + command -v gh >/dev/null 2>&1 || + die "no release assets: 'gh' is not installed and CARRYFORWARD_ASSET_DIR is unset" + log "release assets: downloaded per row from release '$PREV_TAG'" +fi + +# Canonical `~ceralive…` filename -> the glob that matches whatever name the release stores it +# under. The uploader sanitizes `~`→`.`, so each `~` becomes a single-char wildcard while every +# other character stays anchored. INVERSE of reconcile-release-assets.sh's `${src//\~/.}`. +asset_glob() { printf '%s\n' "${1//\~/?}"; } + +# Put the row's asset at , resolving the stored (possibly mangled) name into the global +# RESOLVED_ASSET so callers can report which asset a failure came from. Deliberately NOT a +# command-substitution helper: every failure here is a `refuse`, and an `exit 3` inside `$( )` +# would leave the main shell to infer the outcome from a status. +RESOLVED_ASSET="" +fetch_asset() { # + local canonical="$1" dest="$2" glob name matches=() dl ghargs=() + glob="$(asset_glob "$canonical")" + RESOLVED_ASSET="" + + if [ -n "$ASSET_DIR" ]; then + for name in "${ASSET_NAMES[@]}"; do + # shellcheck disable=SC2053 # $glob is a glob PATTERN here, quoting would break it + if [[ "$name" == $glob ]]; then matches+=("$name"); fi + done + if [ "${#matches[@]}" -eq 0 ]; then + refuse "asset for '$canonical' is missing — no file in '$ASSET_DIR' matches '$glob' (the previous release does not carry it, so it cannot be carried forward)" + fi + if [ "${#matches[@]}" -gt 1 ]; then + refuse "asset for '$canonical' is ambiguous — ${#matches[@]} files match '$glob': ${matches[*]}" + fi + cp "$ASSET_DIR/${matches[0]}" "$dest" + RESOLVED_ASSET="${matches[0]}" + return 0 + fi + + need_tmp + dl="$TMP_DIR/dl" + rm -rf "$dl" + mkdir -p "$dl" + mapfile -t ghargs < <(gh_args) + gh release download "$PREV_TAG" "${ghargs[@]}" --pattern "$glob" --dir "$dl" >/dev/null 2>&1 || + refuse "asset for '$canonical' could not be downloaded from release '$PREV_TAG' (pattern '$glob')" + mapfile -t matches < <(find "$dl" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort) + if [ "${#matches[@]}" -ne 1 ]; then + refuse "asset for '$canonical' resolved to ${#matches[@]} release asset(s) via '$glob' (expected exactly 1): ${matches[*]:-none}" + fi + mv "$dl/${matches[0]}" "$dest" + RESOLVED_ASSET="${matches[0]}" +} + +sha_of() { sha256sum "$1" | awk '{print $1}'; } + +# ---- the carry -------------------------------------------------------------------------------- +declare -A STAGED_PER_SOURCE=() +for key in "${UNCHANGED[@]}"; do STAGED_PER_SOURCE["$key"]=0; done + +need_tmp +STAGE_TMP="$TMP_DIR/stage" +mkdir -p "$STAGE_TMP" + +staged_total=0 +skipped_changed=0 +skipped_companion=0 +staged_lines=() + +while IFS= read -r row; do + [ -n "$row" ] || continue + # shellcheck disable=SC2086 # deliberate word-splitting of a fixed 7-column row + set -- $row + build_arch="$1" + package="$2" + source_name="$3" + version="$4" + role="$5" + filename="$6" + want_sha="$7" + + # The companion is checked FIRST: it is never verdicted, so the unknown-source rule below + # would otherwise reject a perfectly ordinary closure_version-2 manifest. + if [ "$source_name" = "$COMPANION_SOURCE" ]; then + skipped_companion=$((skipped_companion + 1)) + log " skip $filename — the companion '$COMPANION_SOURCE' always rebuilds and is never carried" + continue + fi + + if ! is_verdicted "$source_name"; then + refuse "manifest row '$filename' names source '$source_name', which the caller never verdicted (verdicted: ${VERDICT_ORDER[*]}) — a source the caller did not adjudicate is never carried" + fi + + if ! is_unchanged "$source_name"; then + skipped_changed=$((skipped_changed + 1)) + continue + fi + + case "$filename" in + */* | "") refuse "manifest row for package '$package' has an unusable filename '$filename'" ;; + esac + if [ "$build_arch" = all ]; then + log " NOTE: upstream source '$source_name' has a build_arch 'all' row ($filename) — only the companion is ever Architecture: all; staging it into '$BUILD_ROOT/all/' verbatim" + fi + + tmp_asset="$STAGE_TMP/$filename" + fetch_asset "$filename" "$tmp_asset" + stored="$RESOLVED_ASSET" + + got_sha="$(sha_of "$tmp_asset")" + if [ "$got_sha" != "$want_sha" ]; then + refuse "carried deb '$filename' (release asset '$stored') sha256 $got_sha != previous manifest $want_sha — the recorded bytes are the only thing that may be carried" + fi + + dest_dir="$BUILD_ROOT/$build_arch" + mkdir -p "$dest_dir" + dest="$dest_dir/$filename" + if [ -e "$dest" ]; then + dest_sha="$(sha_of "$dest")" + if [ "$dest_sha" != "$want_sha" ]; then + refuse "'$dest' already exists with different bytes (staged $dest_sha != manifest $want_sha); a carried deb never overwrites what is already in the build tree" + fi + log " keep $build_arch/$filename ($package $version, $role) — already staged, integrity matches" + else + cp "$tmp_asset" "$dest" + # Re-hash the DESTINATION: a short write must never look like a successful carry. + dest_sha="$(sha_of "$dest")" + if [ "$dest_sha" != "$want_sha" ]; then + refuse "staged copy '$dest' hashes $dest_sha, not the manifest's $want_sha" + fi + log " stage $build_arch/$filename ($package $version, $role) <- release asset '$stored'" + fi + + rm -f "$tmp_asset" + per_source="${STAGED_PER_SOURCE[$source_name]}" + STAGED_PER_SOURCE["$source_name"]=$((per_source + 1)) + staged_total=$((staged_total + 1)) + staged_lines+=("$build_arch/$filename") +done <<<"$ROWS" + +# An unchanged source with no rows in the previous manifest would be neither carried nor built — +# a silently incomplete merged set, which is exactly what this pipeline exists to prevent. +for key in "${UNCHANGED[@]}"; do + if [ "${STAGED_PER_SOURCE[$key]}" -eq 0 ]; then + refuse "source '$key' is verdicted 'unchanged' but the previous manifest '$MANIFEST' carries ZERO rows for it — it would be neither carried nor rebuilt" + fi +done + +for key in "${UNCHANGED[@]}"; do + log " carried $key: ${STAGED_PER_SOURCE[$key]} deb(s)" +done +log "staged $staged_total deb(s) into '$BUILD_ROOT' (skipped $skipped_changed row(s) of changed sources, $skipped_companion companion row(s))" + +if [ "${#staged_lines[@]}" -gt 0 ]; then printf '%s\n' "${staged_lines[@]}"; fi diff --git a/packaging/ci/suffix-contract.sh b/packaging/ci/suffix-contract.sh new file mode 100644 index 0000000..2a940ef --- /dev/null +++ b/packaging/ci/suffix-contract.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# suffix-contract.sh — SOURCED library: the `~ceralive` version-suffix contract. +# +# Releases are DIFFERENTIAL, so there is no longer one suffix across the whole set: each +# upstream source carries its own rebuild counter `-~ceralive.N`, and a source +# that was not rebuilt keeps the counter it already had. Coherence is therefore a PER-SOURCE +# property — every deb of ONE source shares one suffix, while two sources legitimately differ. +# The companion `ceralive-modem-support` is outside this contract entirely (bare SemVer). +# +# It lives here rather than inside test-package-contract.sh because that suite only runs +# inside a docker container against a built .deb set, and both halves of the contract — the +# per-source grouping AND the migration-continuity ordering — must also be provable on a bare +# host. A second copy in the host test would only ever test itself; one sourced library means +# the container suite and the host suite exercise the same code. +# +# USAGE . "$(dirname "$0")/suffix-contract.sh" +# ENV EXPECTED_PACKAGES — override the package→source map (default: alongside this file). + +SUFFIX_CONTRACT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUFFIX_CONTRACT_EXPECTED="${EXPECTED_PACKAGES:-$SUFFIX_CONTRACT_DIR/expected-packages.txt}" + +# ---- package -> owning source ------------------------------------------------------------ +# Derived from expected-packages.txt's `[ all-artifact]` blocks, never a second frozen +# list: the grouping a coherence check needs is exactly the ownership the manifest generator +# and check-package-sets.sh already read from that file, and two copies would drift. +suffix_source_of() { # -> its owning source, or non-zero + local pkg="$1" src + src="$(awk -v want="$pkg" ' + /^\[/ { h=$0; sub(/[ \t]*#.*$/, "", h) + insec = (h ~ /^\[[^]]+ all-artifact\]$/) ? 1 : 0 + if (insec) { split(h, a, /[][ ]+/); src=a[2] } + next } + insec { l=$0; sub(/#.*$/, "", l); gsub(/[ \t]+/, "", l) + if (l == want) { print src; exit } } + ' "$SUFFIX_CONTRACT_EXPECTED")" + [ -n "$src" ] || { echo "suffix-contract: package '$pkg' belongs to no [ all-artifact] set" >&2; return 1; } + printf '%s\n' "$src" +} + +# ---- coherence ---------------------------------------------------------------------------- +# assert_coherent — prints the shared ~ceralive suffix; non-zero iff the versions +# do not all share one. UNCHANGED core logic: "does this set of versions share one suffix" is +# the same question per-source as it used to be globally; only the grouping of the inputs moved. +assert_coherent() { + local v suf first="" + for v in "$@"; do + suf="~ceralive${v##*~ceralive}" + [ "$suf" != "~ceralive$v" ] || { echo " no ~ceralive suffix in '$v'"; return 1; } + if [ -z "$first" ]; then first="$suf" + elif [ "$suf" != "$first" ]; then + echo " incoherent: '$suf' != '$first'"; return 1 + fi + done + echo "$first" +} + +# assert_source_coherent — the per-source wrapper. On failure it NAMES +# the source on stderr, because "something is incoherent" is unactionable in a differential +# release where a cross-source difference is normal and only an intra-source one is a bug. +assert_source_coherent() { + local source_name="$1"; shift + local detail + if ! detail="$(assert_coherent "$@")"; then + echo "suffix-contract: INCOHERENT source '$source_name' — ${detail# } (versions: $*)" >&2 + return 1 + fi + printf '%s\n' "$detail" +} + +# assert_group_coherence =... — the whole per-source rule in one call: group +# the given packages by owning source, then assert each group's OWN internal coherence. Prints +# ` ` per source (sorted); non-zero iff some source is internally +# incoherent, with that source named on stderr. Cross-source differences are accepted by +# construction — they are what a differential release produces. +assert_group_coherence() { + local spec pkg ver src suffix fail=0 vers=() + declare -A COHERENCE_GROUP=() + for spec in "$@"; do + pkg="${spec%%=*}"; ver="${spec#*=}" + src="$(suffix_source_of "$pkg")" || return 1 + COHERENCE_GROUP["$src"]+="$ver " + done + while IFS= read -r src; do + read -r -a vers <<<"${COHERENCE_GROUP[$src]}" + if suffix="$(assert_source_coherent "$src" "${vers[@]}")"; then + printf '%s %d %s\n' "$src" "${#vers[@]}" "$suffix" + else + fail=1 + fi + done < <(printf '%s\n' "${!COHERENCE_GROUP[@]}" | LC_ALL=C sort) + return "$fail" +} + +# ---- migration continuity ----------------------------------------------------------------- +# Every legacy suffix below EXISTS as a published artifact today (v0.2.0's closure and v1.0.0's +# repair are live on apt; v1.1.0 released 2026-08-21), so this chain is the proof that every +# fleet device upgrades cleanly into the per-source counter scheme — and that counters keep +# ordering among themselves once there. It is ONE definition so the container suite and the +# host suite cannot prove different chains. +migration_continuity_chain() { # -> the ordered chain, one version per line + local base="$1" + printf '%s\n' \ + "${base}~ceralive0.2.0" \ + "${base}~ceralive1.0.0" \ + "${base}~ceralive1.1.0" \ + "${base}~ceralive.1" \ + "${base}~ceralive.2" \ + "${base}~ceralive.10" \ + "${base}" +} + +# prove_chain_ordered — REAL `dpkg --compare-versions` over every consecutive pair. +# Never a string compare: a lexical sort puts `~ceralive.10` BELOW `~ceralive.2`, which is the +# exact inversion this proof exists to rule out. +prove_chain_ordered() { + local base="$1" prev="" v fail=0 + command -v dpkg >/dev/null 2>&1 || { echo "suffix-contract: dpkg not found — cannot prove ordering" >&2; return 2; } + while IFS= read -r v; do + if [ -n "$prev" ]; then + if dpkg --compare-versions "$prev" lt "$v"; then + echo " ok: '$prev' lt '$v'" + else + echo " FAIL: '$prev' NOT lt '$v'"; fail=1 + fi + fi + prev="$v" + done < <(migration_continuity_chain "$base") + return "$fail" +} diff --git a/packaging/ci/test-build-bookworm-differential.sh b/packaging/ci/test-build-bookworm-differential.sh new file mode 100755 index 0000000..a1f7500 --- /dev/null +++ b/packaging/ci/test-build-bookworm-differential.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# test-build-bookworm-differential.sh — differential builder + rebuild-counter contract. +# +# HOST-RUNNABLE, OFFLINE, NO DOCKER. The builder's BUILD_BOOKWORM_STUB_DIR seam replaces only +# the expensive source-build body with fixture artifacts; build-set parsing, carried-deb repo +# seeding, bootstrap-order dispatch, the unchanged check-package-sets.sh call, and the merged +# runtime-closure assertion are the production paths. A poisoned docker stub proves the +# zero-build host path never starts a container. +# +# Counter fixtures drive the real inject-deb-version.sh with stubbed dpkg-parsechangelog/dch. +# Every previous-manifest row is parsed by production code; the dch transcript proves the suffix +# that would be injected without touching a checked-in changelog. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_SCRIPT="$HERE/build-bookworm.sh" +INJECT_SCRIPT="$HERE/inject-deb-version.sh" +EXPECTED="$HERE/expected-packages.txt" + +for required in "$BUILD_SCRIPT" "$INJECT_SCRIPT" "$EXPECTED" "$HERE/check-package-sets.sh" "$HERE/tag-guard.sh"; do + [ -r "$required" ] || { echo "missing: $required" >&2; exit 1; } +done + +pass=0; fail=0 +ok() { printf ' ok %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' FAIL %s\n' "$1"; fail=$((fail + 1)); } + +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ceralive-build-differential.XXXXXX")" +trap 'rm -rf "$ROOT"' EXIT + +FIXTURE_PKG="$ROOT/pkg" +STUB_BIN="$ROOT/bin" +STUB_BUILDS="$ROOT/stub-builds" +OUT="$ROOT/stdout.txt" +ERR="$ROOT/stderr.txt" +TRACE="$ROOT/build-trace.txt" +DCH_LOG="$ROOT/dch.log" +DOCKER_LOG="$ROOT/docker.log" +mkdir -p "$FIXTURE_PKG/ci" "$STUB_BIN" "$STUB_BUILDS" + +# ---- a minimal standalone packaging tree ------------------------------------------------------ +cp "$INJECT_SCRIPT" "$FIXTURE_PKG/ci/inject-deb-version.sh" +cp "$HERE/tag-guard.sh" "$FIXTURE_PKG/ci/tag-guard.sh" +cp "$HERE/check-package-sets.sh" "$FIXTURE_PKG/ci/check-package-sets.sh" +cp "$EXPECTED" "$FIXTURE_PKG/ci/expected-packages.txt" + +base_version() { + case "$1" in + libqrtr-glib) printf '1.4.0-1\n' ;; + libmbim) printf '1.34.0-1\n' ;; + libqmi) printf '1.38.0-1\n' ;; + modemmanager) printf '1.24.2-2\n' ;; + *) echo "fixture: unknown source '$1'" >&2; return 1 ;; + esac +} + +recipe_dir() { case "$1" in modemmanager) echo ModemManager ;; *) echo "$1" ;; esac; } + +for source_name in libqrtr-glib libmbim libqmi modemmanager; do + dir="$(recipe_dir "$source_name")" + mkdir -p "$FIXTURE_PKG/$dir/debian" + printf '%s (%s) unstable; urgency=medium\n\n * fixture\n\n -- CeraLive CI Fri, 21 Aug 2026 00:00:00 +0000\n' \ + "$source_name" "$(base_version "$source_name")" >"$FIXTURE_PKG/$dir/debian/changelog" +done + +# These stubs remove the host's devscripts/dpkg dependency without moving counter parsing out of +# inject-deb-version.sh. dpkg-parsechangelog reads only the fixture changelog's top version; dch +# records the exact --newversion and working source directory it was asked to mutate. +cat >"$STUB_BIN/dpkg-parsechangelog" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +changelog="" +while [ $# -gt 0 ]; do + case "$1" in + -l) shift; changelog="${1:-}" ;; + -S) shift; [ "${1:-}" = Version ] || { echo "stub: only -S Version is supported" >&2; exit 97; } ;; + esac + shift +done +[ -r "$changelog" ] || { echo "stub: unreadable changelog '$changelog'" >&2; exit 97; } +IFS= read -r first <"$changelog" +version="${first#*(}" +version="${version%%)*}" +printf '%s\n' "$version" +STUB +cat >"$STUB_BIN/dch" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s|%s\n' "$PWD" "$*" >>"$DCH_LOG" +STUB +cat >"$STUB_BIN/docker" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$DOCKER_LOG" +echo "docker(stub): zero-build path must not invoke docker" >&2 +exit 97 +STUB +chmod +x "$STUB_BIN/dpkg-parsechangelog" "$STUB_BIN/dch" "$STUB_BIN/docker" + +# ---- expected-set fixture helpers -------------------------------------------------------------- +expected_set() { # + awk -v want="[$1 all-artifact]" ' + /^\[/ { h=$0; sub(/[ \t]*#.*$/, "", h); insec=(h==want)?1:0; next } + insec { l=$0; sub(/#.*$/, "", l); gsub(/[ \t]+/, "", l); if (l!="") print l } + ' "$EXPECTED" | LC_ALL=C sort -u +} + +write_source_debs() { # + local source_name="$1" dest="$2" suffix="$3" package version + version="$(base_version "$source_name")${suffix}" + mkdir -p "$dest" + while IFS= read -r package; do + [ -n "$package" ] || continue + printf 'fixture deb: %s %s\n' "$package" "$version" >"$dest/${package}_${version}_amd64.deb" + done < <(expected_set "$source_name") +} + +write_changes() { # + local source_name="$1" dest="$2" binaries + binaries="$(expected_set "$source_name" | tr '\n' ' ')" + cat >"$dest/${source_name}_amd64.changes" < + local dest="$1" suffix="$2" source_name + for source_name in libqrtr-glib libmbim libqmi modemmanager; do + write_source_debs "$source_name" "$dest" "$suffix" + done +} + +assert_rc() { if [ "$1" -eq "$2" ]; then ok "$3"; else bad "$3 — exit $1, expected $2"; fi; } +assert_nz() { if [ "$1" -ne 0 ]; then ok "$2"; else bad "$2 — expected a non-zero exit, got 0"; fi; } +assert_err() { if grep -qF -- "$1" "$ERR"; then ok "$2"; else bad "$2 — stderr does not name '$1'"; fi; } +assert_log() { if grep -qF -- "$1" "$2"; then ok "$3"; else bad "$3 — '$1' absent from $2"; fi; } + +# ---- previous-manifest fixtures for one source ------------------------------------------------- +write_qmi_manifest() { # + local path="$1" v_ar="$2" v_aa="$3" v_rr="$4" v_ra="$5" + cat >"$path" <"$M_ZERO" <<'EOF' +# CeraLive modem-stack release manifest (source intentionally absent) +tag: v1.1.0 +version: 1.1.0 +closure_version: 2 +amd64 libmbim-glib4 libmbim 1.34.0-1~ceralive.2 runtime libmbim-glib4_1.34.0-1~ceralive.2_amd64.deb 0000 +EOF + +if [ "$(awk '$3=="libqmi" && $1=="amd64" {a=1} $3=="libqmi" && $1=="arm64" {b=1} $3=="libqmi" && $5=="runtime" {r=1} $3=="libqmi" && $5=="aux" {x=1} END {print a+b+r+x}' "$M_UNIFORM")" -eq 4 ]; then + ok "counter fixture covers both arches and runtime+aux rows" +else + bad "counter fixture does not cover both arches and runtime+aux rows" +fi + +run_inject() { # + : >"$DCH_LOG" + if [ "$1" = ABSENT ]; then + env -u PREV_MANIFEST_FILE PATH="$STUB_BIN:$PATH" DCH_LOG="$DCH_LOG" \ + bash "$FIXTURE_PKG/ci/inject-deb-version.sh" --source libqmi v1.2.0 >"$OUT" 2>"$ERR" + else + PREV_MANIFEST_FILE="$1" PATH="$STUB_BIN:$PATH" DCH_LOG="$DCH_LOG" \ + bash "$FIXTURE_PKG/ci/inject-deb-version.sh" --source libqmi v1.2.0 >"$OUT" 2>"$ERR" + fi +} + +assert_injected_suffix() { #