feat(update): signed offline packages, a signed release channel, and a support-access bridge

.github/workflows/channel-signature.yml

@@ -0,0 +1,73 @@
+name: Channel signature
+
+# An appliance holding our public key REFUSES a channel manifest that is not
+# signed by it, and refuses an update.sh whose hash does not match the
+# `updater_sha256` inside that signed manifest. Both are fail-closed, which is the
+# right behaviour — and it means a forgotten `tools/sign-channel.sh` does not break
+# the fleet, it STOPS it: every box quietly keeps its current version and nobody
+# notices for weeks.
+#
+# So the two things that must never diverge are checked here, on every change:
+#   1. channel.json.sig actually verifies against channel.json
+#   2. channel.json's updater_sha256 matches the update.sh in this tree
+#
+# Needs only the PUBLIC key, so it lives in a repo variable rather than a secret:
+#   Settings > Secrets and variables > Actions > Variables > PACKAGE_PUBLIC_KEY_PEM
+on:
+  push:
+    paths: ['channel.json', 'channel.json.sig', 'update.sh']
+  pull_request:
+    paths: ['channel.json', 'channel.json.sig', 'update.sh']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: updater_sha256 matches update.sh
+        run: |
+          set -euo pipefail
+          bash -n update.sh
+          want="$(sed -n 's/.*"updater_sha256"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' channel.json | head -1)"
+          got="$(sha256sum update.sh | awk '{print $1}')"
+          if [[ -z "$want" ]]; then
+            echo "::error::channel.json has an empty updater_sha256 — run tools/sign-channel.sh"
+            exit 1
+          fi
+          if [[ "$want" != "$got" ]]; then
+            echo "::error::updater_sha256 is stale. channel.json pins $want but update.sh hashes to $got."
+            echo "::error::Run: PACKAGE_PRIVATE_KEY=… tools/sign-channel.sh   and commit both files."
+            exit 1
+          fi
+          echo "updater_sha256 matches ($got)"
+
+      - name: channel.json.sig verifies
+        env:
+          PACKAGE_PUBLIC_KEY_PEM: ${{ vars.PACKAGE_PUBLIC_KEY_PEM }}
+        run: |
+          set -euo pipefail
+          if [[ -z "${PACKAGE_PUBLIC_KEY_PEM:-}" ]]; then
+            echo "::warning::PACKAGE_PUBLIC_KEY_PEM variable not set — signature not verified."
+            echo "::warning::Set it so a stale signature cannot reach the fleet."
+            exit 0
+          fi
+          if [[ ! -f channel.json.sig ]]; then
+            echo "::error::channel.json.sig is missing. Appliances holding the key will refuse this channel."
+            exit 1
+          fi
+          printf '%s\n' "$PACKAGE_PUBLIC_KEY_PEM" > /tmp/package.pub
+          if ! openssl pkeyutl -verify -rawin -pubin -inkey /tmp/package.pub \
+                 -sigfile channel.json.sig -in channel.json >/dev/null 2>&1; then
+            echo "::error::channel.json.sig does NOT verify against channel.json."
+            echo "::error::Every appliance with the key would refuse this channel. Re-sign before merging."
+            exit 1
+          fi
+          echo "signature verifies"
+
+      - name: verifier self-test
+        run: tools/test-package-verify.sh

.github/workflows/offline-package.yml

@@ -0,0 +1,100 @@
+name: Offline update package
+
+# Builds the signed USB update package for air-gapped appliances. Manual by
+# design: a package is cut for a specific customer visit, not on every push.
+#
+# Runs on a NATIVE arm64 runner — the appliance is a GB10 (aarch64), and pulling
+# arm64 images through qemu on x86 is slow and needlessly fragile when GitHub
+# hands out arm64 runners for public repositories.
+#
+# ⚠️ Disk: a hosted runner has ~14 GB usable. The app + sandbox + infra images
+# fit; the vLLM image (~10 GB on its own) does not, so `include_vllm` defaults
+# to false. When a release actually changes the vLLM image, build the package on
+# a machine with room — typically the same box that will write the USB drive:
+#   PACKAGE_PRIVATE_KEY=… tools/build-offline-package.sh --arch arm64
+on:
+  workflow_dispatch:
+    inputs:
+      min_from:
+        description: 'Refuse to apply on appliances older than this app version (blank = no floor)'
+        required: false
+        default: ''
+      include_vllm:
+        description: 'Bundle the vLLM image (~10 GB — usually exceeds the runner disk)'
+        type: boolean
+        required: false
+        default: false
+      arch:
+        description: 'Image platform'
+        type: choice
+        options: [arm64, amd64]
+        default: arm64
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Reclaim runner disk
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                      /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true
+          df -h /
+
+      - name: Read target versions from channel.json
+        id: channel
+        run: |
+          app="$(sed -n 's/.*"app_version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' channel.json | head -1)"
+          echo "app_version=$app" >> "$GITHUB_OUTPUT"
+          echo "Building package for app $app"
+
+      - name: Install the signing key
+        env:
+          # Ed25519 private key (PEM), generated by tools/gen-package-key.sh.
+          # Its public half lives on every appliance as package-release.pub.
+          PACKAGE_PRIVATE_KEY_PEM: ${{ secrets.PACKAGE_PRIVATE_KEY_PEM }}
+        run: |
+          [[ -n "$PACKAGE_PRIVATE_KEY_PEM" ]] || {
+            echo "::error::secret PACKAGE_PRIVATE_KEY_PEM is not set"; exit 1; }
+          install -m 0700 -d "$RUNNER_TEMP/keys"
+          printf '%s\n' "$PACKAGE_PRIVATE_KEY_PEM" > "$RUNNER_TEMP/keys/package.key"
+          chmod 0600 "$RUNNER_TEMP/keys/package.key"
+          # Derive the public half NOW, while the private key is on disk: the
+          # verification step below must not need the secret again (interpolating
+          # it into a later `run:` block would print it under any shell trace).
+          openssl pkey -in "$RUNNER_TEMP/keys/package.key" -pubout \
+            -out "$RUNNER_TEMP/package.pub"
+
+      - name: Build + sign the package
+        run: |
+          args=( --arch '${{ inputs.arch }}' --out "$RUNNER_TEMP/dist" )
+          [[ -n '${{ inputs.min_from }}' ]] && args+=( --min-from '${{ inputs.min_from }}' )
+          [[ '${{ inputs.include_vllm }}' == 'true' ]] || args+=( --no-vllm )
+          PACKAGE_PRIVATE_KEY="$RUNNER_TEMP/keys/package.key" \
+            tools/build-offline-package.sh "${args[@]}"
+
+      - name: Drop the signing key
+        if: always()
+        run: shred -u "$RUNNER_TEMP/keys/package.key" 2>/dev/null || true
+
+      - name: Verify the package as an appliance would
+        run: |
+          pkg="$RUNNER_TEMP/dist/suite366-update-${{ steps.channel.outputs.app_version }}"
+          openssl pkeyutl -verify -rawin -pubin -inkey "$RUNNER_TEMP/package.pub" \
+            -sigfile "$pkg/SHA256SUMS.sig" -in "$pkg/SHA256SUMS"
+          ( cd "$pkg" && sha256sum -c --strict --quiet SHA256SUMS )
+          echo "package verifies; size: $(du -sh "$pkg" | cut -f1)"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: suite366-update-${{ steps.channel.outputs.app_version }}-${{ inputs.arch }}
+          path: ${{ runner.temp }}/dist/
+          retention-days: 30
+          # Already-compressed image layers: recompressing costs minutes and
+          # saves nothing.
+          compression-level: 0

.gitignore

@@ -3,6 +3,15 @@
 .env.*
 !.env.example
 
+# Package signing keys. tools/gen-package-key.sh writes <basename>.key/.pub and
+# is easy to run in the checkout by accident; this repo is PUBLIC and the private
+# half is the root of trust for every appliance, so neither belongs here. The
+# private key lives in the vault and in CI as a secret, the public half is a repo
+# VARIABLE and is deployed to appliances — never a tracked file.
+*.key
+*.pem
+package-release*.pub
+
 # Install directory created by install.sh when run in-place
 /suite366/
 

README.md

@@ -241,9 +241,14 @@ lib/suite.sh                          Suite 366 drive Helm chart + CoreDNS patch
 lib/mdns.sh                           Avahi/mDNS publishing of *.DOMAIN
 lib/updater.sh                        install update.sh + daily notify-only timer
 lib/summary.sh                        final post-install summary
-update.sh                             update checker/applier (check | apply | install-units); run by the daily timer + app triggers
+update.sh                             update checker/applier (check | apply | scan-usb | install-units); run by the daily timer + app triggers
+tools/build-offline-package.sh        build a SIGNED offline update package for an air-gapped appliance
+tools/sign-channel.sh                 pin updater_sha256 + sign channel.json (run on every channel bump)
+tools/gen-package-key.sh              generate the Ed25519 keypair that signs packages AND channels
+tools/test-package-verify.sh          self-test: real signatures, real tampering, no hardware
 uninstall.sh                          clean uninstaller — reverses install.sh (systemd units, vLLM stack, k3s, DATA_DIR, …)
-channel.json                          fleet release manifest (chart_version / app_version / vllm_image) polled by update.sh
+channel.json                          fleet release manifest (chart_version / app_version / vllm_image / updater_sha256) polled by update.sh
+channel.json.sig                      Ed25519 signature over channel.json — required by any appliance holding the public key
 values.yaml                           Helm values (@DOMAIN@/@HOST_IP@/etc. tokens substituted at run-time)
 llm/docker-compose.yml                vllm-llm + vllm-embed + vllm-proxy (host Docker)
 llm/tool_chat_template_gemma4.jinja   chat template required by --tool-call-parser=gemma4
@@ -287,15 +292,91 @@ picked up by systemd `.path` units (`suite366-update-check.path`,
 `suite366-update-apply.path`, installed by `update.sh install-units`). The
 apply reuses the box's install-time parameters (`values.yaml`, `llm/.env`,
 `update.env`) — nothing is re-asked. After each apply, `update.sh` refreshes
-itself from the repo and re-installs the trigger units, so the update
-mechanism itself rolls forward with regular updates (disable with
-`SELF_UPDATE=0` in `update.env`).
+itself from the repo and re-installs the trigger units, so the update mechanism
+itself rolls forward with regular updates. That refresh is signature-verified on
+any appliance holding the package public key (see *Signed channels* below);
+disable it entirely with `SELF_UPDATE=0` in `update.env`.
 
 **App version pinning**: the appliance pins the app + sandbox image tags in
 `values.yaml` (offline safety), so a bare `helm upgrade` never moves the app.
 `channel.json`'s `app_version` is what rolls the app forward: on apply,
 `update.sh` rewrites the pins to the new tag before upgrading.
 
+### Offline updates from a USB drive
+
+A site with no outbound access updates from a **signed package** instead. The
+online check is unchanged and still primary — USB is an *additional* source, and
+the two coexist: `check` tries the network and never fails fatally when it is
+unreachable, so a verified package still produces an "update available" prompt,
+and a reachable network never invalidates a staged one. `state.json` carries both
+sources plus the resolved best target (highest app version wins; online wins a tie
+since it needs no image import).
+
+Build one (needs docker + helm + the signing key):
+
+```bash
+tools/gen-package-key.sh ~/.secrets/package-release      # once, ever
+PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key \
+  tools/build-offline-package.sh --arch arm64 --min-from 1.8.0
+```
+
+Copy the resulting `suite366-update-<version>/` directory to the **root** of a USB
+drive, then on the appliance:
+
+```bash
+sudo /opt/suite366/update.sh scan-usb /media/usb   # verify + stage; applies nothing
+```
+
+The admin then confirms in the app exactly as if the box were online. Deploy the
+**public** half of the key to each appliance as
+`/opt/suite366/package-release.pub` (`PACKAGE_PUBLIC_KEY`); with no key installed
+every package is refused, which is the right default.
+
+Verification is **all-or-nothing**: one Ed25519 signature over a `SHA256SUMS` that
+covers every file in the package, `manifest.json` included. One bad byte anywhere,
+a foreign signature, a downgrade, or an unmet `min_from_version` and the whole
+package is refused — and the refusal is shown in the admin UI, not just written to
+the journal. A verified package is copied off the drive before use, so the key can
+be unplugged and a mid-copy removal cannot truncate an image tar.
+
+```bash
+tools/test-package-verify.sh   # 18 assertions against real signatures + tampering
+```
+
+### Signed channels
+
+TLS proves you reached the right host. It says nothing about who wrote the file —
+and `channel.json` decides which chart version and which vLLM image every
+appliance is told to run, while `update.sh` is fetched over the same channel and
+then runs **as root** on the next apply.
+
+So the channel is signed, and one signature covers both: `channel.json` carries
+`updater_sha256`, which the signature protects, so verifying the manifest
+transitively verifies the updater.
+
+```bash
+PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key tools/sign-channel.sh
+# -> recomputes updater_sha256 from update.sh, signs channel.json,
+#    and verifies its own output the way an appliance will
+git add channel.json channel.json.sig && git commit
+```
+
+Behaviour on the appliance is **graduated**, so the public one-command install is
+unchanged:
+
+| `package-release.pub` on the box | Channel manifest | `update.sh` refresh |
+|---|---|---|
+| present (fleet) | must be signed by our key, else **refused** | must match the signed `updater_sha256`, else **refused** |
+| absent (default) | TLS-only, as before | TLS-only, as before |
+
+Both strict paths **fail closed**: a bad signature makes the manifest unusable
+rather than merely suspicious, and a verified USB package can still carry the box
+forward. The practical consequence is that forgetting to re-sign does not break the
+fleet, it *stops* it — every box keeps its current version silently. The
+`channel-signature` workflow exists to catch that before it ships, and
+`tools/test-package-verify.sh` covers the refusal paths (23 assertions: foreign
+key, tampering after signing, missing signature, stale hash).
+
 By default each box polls the `channel.json` shipped in this repo, so it tracks
 the releases published here. Point a box at a manifest you control with
 `MANIFEST_URL=…`, or get a push notification by setting `UPDATE_WEBHOOK=…`

channel.json

@@ -3,5 +3,6 @@
   "chart_version": "0.8.0",
   "app_version": "1.8.22",
   "vllm_image": "vllm/vllm-openai:cu130-nightly",
-  "notes": "app 1.8.22: roll the stable channel to the latest published Suite 366 release (app + sandbox-api + sandbox-runner image pins bumped 1.8.10 -> 1.8.22; chart unchanged at 0.8.0). app_version drives the app/sandbox image pins in values.yaml (the appliance pins them for offline safety, update.sh rewrites the pins on apply). Bump chart_version / app_version / vllm_image here to roll out to the fleet; appliances poll this file daily and notify (no auto-apply)."
+  "updater_sha256": "f844c7210141193689e605209ea37569cb8e1f37e5e50a1df9a3de180ab69847",
+  "notes": "app 1.8.22: roll the stable channel to the latest published Suite 366 release (app + sandbox-api + sandbox-runner image pins bumped 1.8.10 -> 1.8.22; chart unchanged at 0.8.0). app_version drives the app/sandbox image pins in values.yaml (the appliance pins them for offline safety, update.sh rewrites the pins on apply). Bump chart_version / app_version / vllm_image here to roll out to the fleet; appliances poll this file daily and notify (no auto-apply). updater_sha256 is filled in by tools/sign-channel.sh — never by hand; it is what lets an appliance trust the update.sh it fetches."
 }

channel.json.sig

@@ -0,0 +1 @@
+£«°½«b½öñÈKê6 V
Öù'Ío'³AOƒ¢1S•ßñ8[êûwÚº”ĪùßWgªå3¢:ñ

lib/config.sh

@@ -79,6 +79,14 @@ EMBED_MAX_MODEL_LEN="${EMBED_MAX_MODEL_LEN:-8192}"
 
 DATA_DIR="${DATA_DIR:-/opt/suite366}"
 MODELS_DIR="${MODELS_DIR:-$DATA_DIR/models}"
+# Ed25519 PUBLIC key that signs OFFLINE update packages (built by
+# tools/build-offline-package.sh). Path to a PEM file — when the file is
+# ABSENT, `update.sh scan-usb` refuses every package, which is the correct
+# default for a box with no offline-update entitlement. Deployments that want
+# USB updates drop the key there (suite366-fleet does it at install time).
+# Separate keypair from the LICENSE key: different lifecycle, different blast
+# radius, and a license key must never acquire code-execution meaning.
+PACKAGE_PUBLIC_KEY="${PACKAGE_PUBLIC_KEY:-$DATA_DIR/package-release.pub}"
 ASSUME_YES="${ASSUME_YES:-0}"
 CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.2}"
 

lib/suite.sh

@@ -49,13 +49,20 @@ deploy_suite() {
         > "$vals" )
   chmod 0600 "$vals"
 
-  # App <-> host update bridge dir, hostPath-mounted into drive-app (see the
+  # App <-> host bridge dirs, hostPath-mounted into drive-app (see the
   # extraVolumes block in values.yaml). Created BEFORE helm so kubelet's
-  # DirectoryOrCreate doesn't make it root:root 0755 (the pod, uid/gid 1001,
+  # DirectoryOrCreate doesn't make them root:root 0755 (the pod, uid/gid 1001,
   # must be able to drop trigger files — k8s does not fsGroup-chown hostPath).
-  mkdir -p "$DATA_DIR/updates"
-  chown root:1001 "$DATA_DIR/updates"
-  chmod 0770 "$DATA_DIR/updates"
+  #
+  # `support` stays EMPTY here: the remote-support toggle is a fleet feature
+  # (suite366-fleet drops state.json in it). With no state.json the app hides
+  # the feature, so a customer-run appliance is unaffected by the mount.
+  local d
+  for d in updates support; do
+    mkdir -p "$DATA_DIR/$d"
+    chown root:1001 "$DATA_DIR/$d"
+    chmod 0770 "$DATA_DIR/$d"
+  done
 
   patch_coredns_for_local_domain
   # CA locale auto-générée par cert-manager : la passer au chart pour qu'il

lib/updater.sh

@@ -32,6 +32,7 @@ RELEASE=$RELEASE
 DATA_DIR=$DATA_DIR
 KUBECONFIG_PATH=$KUBECONFIG_PATH
 UPDATE_WEBHOOK=$UPDATE_WEBHOOK
+PACKAGE_PUBLIC_KEY=$PACKAGE_PUBLIC_KEY
 EOF
   )