diff --git a/.github/workflows/channel-signature.yml b/.github/workflows/channel-signature.yml new file mode 100644 index 0000000..4cf4bff --- /dev/null +++ b/.github/workflows/channel-signature.yml @@ -0,0 +1,73 @@ +name: Channel signature + +# An appliance holding our public key REFUSES a channel manifest that is not +# signed by it, and refuses an update.sh whose hash does not match the +# `updater_sha256` inside that signed manifest. Both are fail-closed, which is the +# right behaviour — and it means a forgotten `tools/sign-channel.sh` does not break +# the fleet, it STOPS it: every box quietly keeps its current version and nobody +# notices for weeks. +# +# So the two things that must never diverge are checked here, on every change: +# 1. channel.json.sig actually verifies against channel.json +# 2. channel.json's updater_sha256 matches the update.sh in this tree +# +# Needs only the PUBLIC key, so it lives in a repo variable rather than a secret: +# Settings > Secrets and variables > Actions > Variables > PACKAGE_PUBLIC_KEY_PEM +on: + push: + paths: ['channel.json', 'channel.json.sig', 'update.sh'] + pull_request: + paths: ['channel.json', 'channel.json.sig', 'update.sh'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: updater_sha256 matches update.sh + run: | + set -euo pipefail + bash -n update.sh + want="$(sed -n 's/.*"updater_sha256"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' channel.json | head -1)" + got="$(sha256sum update.sh | awk '{print $1}')" + if [[ -z "$want" ]]; then + echo "::error::channel.json has an empty updater_sha256 — run tools/sign-channel.sh" + exit 1 + fi + if [[ "$want" != "$got" ]]; then + echo "::error::updater_sha256 is stale. channel.json pins $want but update.sh hashes to $got." + echo "::error::Run: PACKAGE_PRIVATE_KEY=… tools/sign-channel.sh and commit both files." + exit 1 + fi + echo "updater_sha256 matches ($got)" + + - name: channel.json.sig verifies + env: + PACKAGE_PUBLIC_KEY_PEM: ${{ vars.PACKAGE_PUBLIC_KEY_PEM }} + run: | + set -euo pipefail + if [[ -z "${PACKAGE_PUBLIC_KEY_PEM:-}" ]]; then + echo "::warning::PACKAGE_PUBLIC_KEY_PEM variable not set — signature not verified." + echo "::warning::Set it so a stale signature cannot reach the fleet." + exit 0 + fi + if [[ ! -f channel.json.sig ]]; then + echo "::error::channel.json.sig is missing. Appliances holding the key will refuse this channel." + exit 1 + fi + printf '%s\n' "$PACKAGE_PUBLIC_KEY_PEM" > /tmp/package.pub + if ! openssl pkeyutl -verify -rawin -pubin -inkey /tmp/package.pub \ + -sigfile channel.json.sig -in channel.json >/dev/null 2>&1; then + echo "::error::channel.json.sig does NOT verify against channel.json." + echo "::error::Every appliance with the key would refuse this channel. Re-sign before merging." + exit 1 + fi + echo "signature verifies" + + - name: verifier self-test + run: tools/test-package-verify.sh diff --git a/.github/workflows/offline-package.yml b/.github/workflows/offline-package.yml new file mode 100644 index 0000000..2f68c4d --- /dev/null +++ b/.github/workflows/offline-package.yml @@ -0,0 +1,100 @@ +name: Offline update package + +# Builds the signed USB update package for air-gapped appliances. Manual by +# design: a package is cut for a specific customer visit, not on every push. +# +# Runs on a NATIVE arm64 runner — the appliance is a GB10 (aarch64), and pulling +# arm64 images through qemu on x86 is slow and needlessly fragile when GitHub +# hands out arm64 runners for public repositories. +# +# ⚠️ Disk: a hosted runner has ~14 GB usable. The app + sandbox + infra images +# fit; the vLLM image (~10 GB on its own) does not, so `include_vllm` defaults +# to false. When a release actually changes the vLLM image, build the package on +# a machine with room — typically the same box that will write the USB drive: +# PACKAGE_PRIVATE_KEY=… tools/build-offline-package.sh --arch arm64 +on: + workflow_dispatch: + inputs: + min_from: + description: 'Refuse to apply on appliances older than this app version (blank = no floor)' + required: false + default: '' + include_vllm: + description: 'Bundle the vLLM image (~10 GB — usually exceeds the runner disk)' + type: boolean + required: false + default: false + arch: + description: 'Image platform' + type: choice + options: [arm64, amd64] + default: arm64 + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Reclaim runner disk + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" || true + df -h / + + - name: Read target versions from channel.json + id: channel + run: | + app="$(sed -n 's/.*"app_version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' channel.json | head -1)" + echo "app_version=$app" >> "$GITHUB_OUTPUT" + echo "Building package for app $app" + + - name: Install the signing key + env: + # Ed25519 private key (PEM), generated by tools/gen-package-key.sh. + # Its public half lives on every appliance as package-release.pub. + PACKAGE_PRIVATE_KEY_PEM: ${{ secrets.PACKAGE_PRIVATE_KEY_PEM }} + run: | + [[ -n "$PACKAGE_PRIVATE_KEY_PEM" ]] || { + echo "::error::secret PACKAGE_PRIVATE_KEY_PEM is not set"; exit 1; } + install -m 0700 -d "$RUNNER_TEMP/keys" + printf '%s\n' "$PACKAGE_PRIVATE_KEY_PEM" > "$RUNNER_TEMP/keys/package.key" + chmod 0600 "$RUNNER_TEMP/keys/package.key" + # Derive the public half NOW, while the private key is on disk: the + # verification step below must not need the secret again (interpolating + # it into a later `run:` block would print it under any shell trace). + openssl pkey -in "$RUNNER_TEMP/keys/package.key" -pubout \ + -out "$RUNNER_TEMP/package.pub" + + - name: Build + sign the package + run: | + args=( --arch '${{ inputs.arch }}' --out "$RUNNER_TEMP/dist" ) + [[ -n '${{ inputs.min_from }}' ]] && args+=( --min-from '${{ inputs.min_from }}' ) + [[ '${{ inputs.include_vllm }}' == 'true' ]] || args+=( --no-vllm ) + PACKAGE_PRIVATE_KEY="$RUNNER_TEMP/keys/package.key" \ + tools/build-offline-package.sh "${args[@]}" + + - name: Drop the signing key + if: always() + run: shred -u "$RUNNER_TEMP/keys/package.key" 2>/dev/null || true + + - name: Verify the package as an appliance would + run: | + pkg="$RUNNER_TEMP/dist/suite366-update-${{ steps.channel.outputs.app_version }}" + openssl pkeyutl -verify -rawin -pubin -inkey "$RUNNER_TEMP/package.pub" \ + -sigfile "$pkg/SHA256SUMS.sig" -in "$pkg/SHA256SUMS" + ( cd "$pkg" && sha256sum -c --strict --quiet SHA256SUMS ) + echo "package verifies; size: $(du -sh "$pkg" | cut -f1)" + + - uses: actions/upload-artifact@v4 + with: + name: suite366-update-${{ steps.channel.outputs.app_version }}-${{ inputs.arch }} + path: ${{ runner.temp }}/dist/ + retention-days: 30 + # Already-compressed image layers: recompressing costs minutes and + # saves nothing. + compression-level: 0 diff --git a/.gitignore b/.gitignore index cb4adda..a91b348 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,15 @@ .env.* !.env.example +# Package signing keys. tools/gen-package-key.sh writes .key/.pub and +# is easy to run in the checkout by accident; this repo is PUBLIC and the private +# half is the root of trust for every appliance, so neither belongs here. The +# private key lives in the vault and in CI as a secret, the public half is a repo +# VARIABLE and is deployed to appliances — never a tracked file. +*.key +*.pem +package-release*.pub + # Install directory created by install.sh when run in-place /suite366/ diff --git a/README.md b/README.md index 4514e27..73d7050 100644 --- a/README.md +++ b/README.md @@ -241,9 +241,14 @@ lib/suite.sh Suite 366 drive Helm chart + CoreDNS patch lib/mdns.sh Avahi/mDNS publishing of *.DOMAIN lib/updater.sh install update.sh + daily notify-only timer lib/summary.sh final post-install summary -update.sh update checker/applier (check | apply | install-units); run by the daily timer + app triggers +update.sh update checker/applier (check | apply | scan-usb | install-units); run by the daily timer + app triggers +tools/build-offline-package.sh build a SIGNED offline update package for an air-gapped appliance +tools/sign-channel.sh pin updater_sha256 + sign channel.json (run on every channel bump) +tools/gen-package-key.sh generate the Ed25519 keypair that signs packages AND channels +tools/test-package-verify.sh self-test: real signatures, real tampering, no hardware uninstall.sh clean uninstaller — reverses install.sh (systemd units, vLLM stack, k3s, DATA_DIR, …) -channel.json fleet release manifest (chart_version / app_version / vllm_image) polled by update.sh +channel.json fleet release manifest (chart_version / app_version / vllm_image / updater_sha256) polled by update.sh +channel.json.sig Ed25519 signature over channel.json — required by any appliance holding the public key values.yaml Helm values (@DOMAIN@/@HOST_IP@/etc. tokens substituted at run-time) llm/docker-compose.yml vllm-llm + vllm-embed + vllm-proxy (host Docker) llm/tool_chat_template_gemma4.jinja chat template required by --tool-call-parser=gemma4 @@ -287,15 +292,91 @@ picked up by systemd `.path` units (`suite366-update-check.path`, `suite366-update-apply.path`, installed by `update.sh install-units`). The apply reuses the box's install-time parameters (`values.yaml`, `llm/.env`, `update.env`) — nothing is re-asked. After each apply, `update.sh` refreshes -itself from the repo and re-installs the trigger units, so the update -mechanism itself rolls forward with regular updates (disable with -`SELF_UPDATE=0` in `update.env`). +itself from the repo and re-installs the trigger units, so the update mechanism +itself rolls forward with regular updates. That refresh is signature-verified on +any appliance holding the package public key (see *Signed channels* below); +disable it entirely with `SELF_UPDATE=0` in `update.env`. **App version pinning**: the appliance pins the app + sandbox image tags in `values.yaml` (offline safety), so a bare `helm upgrade` never moves the app. `channel.json`'s `app_version` is what rolls the app forward: on apply, `update.sh` rewrites the pins to the new tag before upgrading. +### Offline updates from a USB drive + +A site with no outbound access updates from a **signed package** instead. The +online check is unchanged and still primary — USB is an *additional* source, and +the two coexist: `check` tries the network and never fails fatally when it is +unreachable, so a verified package still produces an "update available" prompt, +and a reachable network never invalidates a staged one. `state.json` carries both +sources plus the resolved best target (highest app version wins; online wins a tie +since it needs no image import). + +Build one (needs docker + helm + the signing key): + +```bash +tools/gen-package-key.sh ~/.secrets/package-release # once, ever +PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key \ + tools/build-offline-package.sh --arch arm64 --min-from 1.8.0 +``` + +Copy the resulting `suite366-update-/` directory to the **root** of a USB +drive, then on the appliance: + +```bash +sudo /opt/suite366/update.sh scan-usb /media/usb # verify + stage; applies nothing +``` + +The admin then confirms in the app exactly as if the box were online. Deploy the +**public** half of the key to each appliance as +`/opt/suite366/package-release.pub` (`PACKAGE_PUBLIC_KEY`); with no key installed +every package is refused, which is the right default. + +Verification is **all-or-nothing**: one Ed25519 signature over a `SHA256SUMS` that +covers every file in the package, `manifest.json` included. One bad byte anywhere, +a foreign signature, a downgrade, or an unmet `min_from_version` and the whole +package is refused — and the refusal is shown in the admin UI, not just written to +the journal. A verified package is copied off the drive before use, so the key can +be unplugged and a mid-copy removal cannot truncate an image tar. + +```bash +tools/test-package-verify.sh # 18 assertions against real signatures + tampering +``` + +### Signed channels + +TLS proves you reached the right host. It says nothing about who wrote the file — +and `channel.json` decides which chart version and which vLLM image every +appliance is told to run, while `update.sh` is fetched over the same channel and +then runs **as root** on the next apply. + +So the channel is signed, and one signature covers both: `channel.json` carries +`updater_sha256`, which the signature protects, so verifying the manifest +transitively verifies the updater. + +```bash +PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key tools/sign-channel.sh +# -> recomputes updater_sha256 from update.sh, signs channel.json, +# and verifies its own output the way an appliance will +git add channel.json channel.json.sig && git commit +``` + +Behaviour on the appliance is **graduated**, so the public one-command install is +unchanged: + +| `package-release.pub` on the box | Channel manifest | `update.sh` refresh | +|---|---|---| +| present (fleet) | must be signed by our key, else **refused** | must match the signed `updater_sha256`, else **refused** | +| absent (default) | TLS-only, as before | TLS-only, as before | + +Both strict paths **fail closed**: a bad signature makes the manifest unusable +rather than merely suspicious, and a verified USB package can still carry the box +forward. The practical consequence is that forgetting to re-sign does not break the +fleet, it *stops* it — every box keeps its current version silently. The +`channel-signature` workflow exists to catch that before it ships, and +`tools/test-package-verify.sh` covers the refusal paths (23 assertions: foreign +key, tampering after signing, missing signature, stale hash). + By default each box polls the `channel.json` shipped in this repo, so it tracks the releases published here. Point a box at a manifest you control with `MANIFEST_URL=…`, or get a push notification by setting `UPDATE_WEBHOOK=…` diff --git a/channel.json b/channel.json index a77d71c..70563a6 100644 --- a/channel.json +++ b/channel.json @@ -3,5 +3,6 @@ "chart_version": "0.8.0", "app_version": "1.8.22", "vllm_image": "vllm/vllm-openai:cu130-nightly", - "notes": "app 1.8.22: roll the stable channel to the latest published Suite 366 release (app + sandbox-api + sandbox-runner image pins bumped 1.8.10 -> 1.8.22; chart unchanged at 0.8.0). app_version drives the app/sandbox image pins in values.yaml (the appliance pins them for offline safety, update.sh rewrites the pins on apply). Bump chart_version / app_version / vllm_image here to roll out to the fleet; appliances poll this file daily and notify (no auto-apply)." + "updater_sha256": "f844c7210141193689e605209ea37569cb8e1f37e5e50a1df9a3de180ab69847", + "notes": "app 1.8.22: roll the stable channel to the latest published Suite 366 release (app + sandbox-api + sandbox-runner image pins bumped 1.8.10 -> 1.8.22; chart unchanged at 0.8.0). app_version drives the app/sandbox image pins in values.yaml (the appliance pins them for offline safety, update.sh rewrites the pins on apply). Bump chart_version / app_version / vllm_image here to roll out to the fleet; appliances poll this file daily and notify (no auto-apply). updater_sha256 is filled in by tools/sign-channel.sh — never by hand; it is what lets an appliance trust the update.sh it fetches." } diff --git a/channel.json.sig b/channel.json.sig new file mode 100644 index 0000000..ea85a8f --- /dev/null +++ b/channel.json.sig @@ -0,0 +1 @@ +bK6V'o'AO1S8[wںĪ Wg3: \ No newline at end of file diff --git a/lib/config.sh b/lib/config.sh index f954879..213e2c2 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -79,6 +79,14 @@ EMBED_MAX_MODEL_LEN="${EMBED_MAX_MODEL_LEN:-8192}" DATA_DIR="${DATA_DIR:-/opt/suite366}" MODELS_DIR="${MODELS_DIR:-$DATA_DIR/models}" +# Ed25519 PUBLIC key that signs OFFLINE update packages (built by +# tools/build-offline-package.sh). Path to a PEM file — when the file is +# ABSENT, `update.sh scan-usb` refuses every package, which is the correct +# default for a box with no offline-update entitlement. Deployments that want +# USB updates drop the key there (suite366-fleet does it at install time). +# Separate keypair from the LICENSE key: different lifecycle, different blast +# radius, and a license key must never acquire code-execution meaning. +PACKAGE_PUBLIC_KEY="${PACKAGE_PUBLIC_KEY:-$DATA_DIR/package-release.pub}" ASSUME_YES="${ASSUME_YES:-0}" CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.2}" diff --git a/lib/suite.sh b/lib/suite.sh index 73e4a97..c5c2bfc 100644 --- a/lib/suite.sh +++ b/lib/suite.sh @@ -49,13 +49,20 @@ deploy_suite() { > "$vals" ) chmod 0600 "$vals" - # App <-> host update bridge dir, hostPath-mounted into drive-app (see the + # App <-> host bridge dirs, hostPath-mounted into drive-app (see the # extraVolumes block in values.yaml). Created BEFORE helm so kubelet's - # DirectoryOrCreate doesn't make it root:root 0755 (the pod, uid/gid 1001, + # DirectoryOrCreate doesn't make them root:root 0755 (the pod, uid/gid 1001, # must be able to drop trigger files — k8s does not fsGroup-chown hostPath). - mkdir -p "$DATA_DIR/updates" - chown root:1001 "$DATA_DIR/updates" - chmod 0770 "$DATA_DIR/updates" + # + # `support` stays EMPTY here: the remote-support toggle is a fleet feature + # (suite366-fleet drops state.json in it). With no state.json the app hides + # the feature, so a customer-run appliance is unaffected by the mount. + local d + for d in updates support; do + mkdir -p "$DATA_DIR/$d" + chown root:1001 "$DATA_DIR/$d" + chmod 0770 "$DATA_DIR/$d" + done patch_coredns_for_local_domain # CA locale auto-générée par cert-manager : la passer au chart pour qu'il diff --git a/lib/updater.sh b/lib/updater.sh index 6062d61..763c13d 100644 --- a/lib/updater.sh +++ b/lib/updater.sh @@ -32,6 +32,7 @@ RELEASE=$RELEASE DATA_DIR=$DATA_DIR KUBECONFIG_PATH=$KUBECONFIG_PATH UPDATE_WEBHOOK=$UPDATE_WEBHOOK +PACKAGE_PUBLIC_KEY=$PACKAGE_PUBLIC_KEY EOF ) diff --git a/tools/build-offline-package.sh b/tools/build-offline-package.sh new file mode 100755 index 0000000..52c977c --- /dev/null +++ b/tools/build-offline-package.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +# ============================================================================= +# Build a SIGNED, self-sufficient offline update package for an appliance with +# no outbound access. The result is copied to a USB drive; the appliance's +# `update.sh scan-usb` verifies it and surfaces the very same "update +# available" prompt in the admin UI as an online check would. +# +# Source of truth is channel.json — the same file the online updater polls — so +# an offline package can never describe a release the channel does not. +# +# Usage: +# PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key tools/build-offline-package.sh +# … --arch amd64 --out /tmp/pkgs --min-from 1.8.0 --no-vllm +# +# Options: +# --key FILE Ed25519 private key (PEM). Default: $PACKAGE_PRIVATE_KEY +# --arch ARCH image platform: arm64 (default, DGX/GB10) | amd64 +# --out DIR output directory (default: ./dist) +# --min-from VER refuse to apply on appliances older than VER +# --no-vllm skip the multi-GB vLLM image (box already runs the right one) +# --channel FILE channel manifest to build from (default: ./channel.json) +# +# Layout produced (see update.sh `pkg_verify` for the verifier): +# suite366-update-/ +# ├── manifest.json flat JSON: channel.json keys + min_from_version +# ├── SHA256SUMS covers EVERY other file, manifest.json included +# ├── SHA256SUMS.sig raw Ed25519 signature over SHA256SUMS <- the only sig +# ├── chart/drive-.tgz +# ├── images/*.tar imported into containerd's k8s.io namespace +# ├── docker-images/*.tar loaded into the Docker daemon (vLLM/compose stack) +# └── scripts/update.sh the updater this package expects +# +# ONE signature, over SHA256SUMS. Every other file earns trust from a checksum +# line inside that signed list, so there is no ambiguity about which signature +# is authoritative and a file the builder forgot to list is simply not trusted. +# ============================================================================= +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +KEY="${PACKAGE_PRIVATE_KEY:-}" +ARCH="arm64" +OUT="$REPO_ROOT/dist" +MIN_FROM="" +WITH_VLLM=1 +CHANNEL_FILE="$REPO_ROOT/channel.json" + +c_b="\033[1m"; c_g="\033[32m"; c_y="\033[33m"; c_r="\033[31m"; c_0="\033[0m" +log() { printf "${c_g}==>${c_0} ${c_b}%s${c_0}\n" "$*"; } +info() { printf " %s\n" "$*"; } +warn() { printf "${c_y}!! %s${c_0}\n" "$*"; } +die() { printf "${c_r}xx %s${c_0}\n" "$*" >&2; exit 1; } +have() { command -v "$1" >/dev/null 2>&1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --key) KEY="$2"; shift 2 ;; + --arch) ARCH="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --min-from) MIN_FROM="$2"; shift 2 ;; + --channel) CHANNEL_FILE="$2"; shift 2 ;; + --no-vllm) WITH_VLLM=0; shift ;; + -h|--help) sed -n '2,40p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown option: $1" ;; + esac +done + +for t in helm docker openssl sha256sum; do have "$t" || die "$t required."; done +[[ -n "$KEY" ]] || die "no signing key: pass --key or set PACKAGE_PRIVATE_KEY." +[[ -s "$KEY" ]] || die "signing key not readable: $KEY" +[[ -s "$CHANNEL_FILE" ]] || die "channel manifest not found: $CHANNEL_FILE" +case "$ARCH" in arm64|amd64) ;; *) die "--arch must be arm64 or amd64." ;; esac + +# Same minimal flat-JSON reader as update.sh, for the same reason (no jq +# dependency) and so both sides agree on what a manifest key means. +json_get() { sed -n 's/.*"'"$1"'"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1; } +json_esc() { + local s=${1//\\/\\\\}; s=${s//\"/\\\"}; s=${s//$'\n'/\\n}; printf '%s' "$s" +} + +CHANNEL="$(json_get channel < "$CHANNEL_FILE")" +CHART_VERSION="$(json_get chart_version < "$CHANNEL_FILE")" +APP_VERSION="$(json_get app_version < "$CHANNEL_FILE")" +VLLM_IMAGE="$(json_get vllm_image < "$CHANNEL_FILE")" +NOTES="$(json_get notes < "$CHANNEL_FILE")" +CHART_REF="${CHART_REF:-oci://ghcr.io/scriptor-group/chart/drive}" +[[ -n "$CHART_VERSION" && -n "$APP_VERSION" ]] \ + || die "channel.json must carry chart_version and app_version." + +PKG_NAME="suite366-update-$APP_VERSION" +PKG="$OUT/$PKG_NAME" + +log "Building $PKG_NAME" +info "channel : ${CHANNEL:-?}" +info "chart : $CHART_VERSION" +info "app : $APP_VERSION" +info "vLLM image : ${VLLM_IMAGE:-none}$([[ "$WITH_VLLM" == 0 ]] && echo ' (skipped)')" +info "platform : linux/$ARCH" +[[ -n "$MIN_FROM" ]] && info "min_from : $MIN_FROM" + +# A stale directory would leave orphaned files that SHA256SUMS still covers. +rm -rf "$PKG" +mkdir -p "$PKG"/{chart,images,docker-images,scripts} + +# --- Chart ------------------------------------------------------------------- +log "Pulling chart $CHART_VERSION" +helm pull "$CHART_REF" --version "$CHART_VERSION" -d "$PKG/chart" \ + || die "helm pull failed for $CHART_REF --version $CHART_VERSION" + +# --- Images ------------------------------------------------------------------ +# Enumerate from the chart itself (rendered with the appliance's own values) so +# a new sidecar added upstream lands in the package without editing this script. +# `helm template` needs no cluster. +log "Enumerating images from the rendered chart" +rendered="$(mktemp)" +trap 'rm -f "$rendered"' EXIT +# The repo's values.yaml carries @TOKEN@ placeholders that are not valid YAML +# values for every field; substitute the few that matter for image resolution +# and let the rest render as literals (we only read `image:` lines back out). +sed -e "s|@DOMAIN@|suite366.local|g" \ + -e "s|@HOST_IP@|127.0.0.1|g" -e "s|@SUITE_IP@|10.99.0.1|g" \ + -e "s|@PROXY_PORT@|8000|g" -e "s|@LLM_MODEL@|m|g" -e "s|@EMBED_MODEL@|m|g" \ + -e "s|@VLLM_API_KEY@|x|g" -e "s|@VLLM_EMBEDDING_DIMENSIONS@|4096|g" \ + -e "s|@VLLM_MAX_CONTEXT_WINDOW@|200000|g" -e "s|@LICENSE_PUBLIC_KEY@|x|g" \ + -e "s|@SANDBOX_NAMESPACE@|sandbox|g" -e "s|@DATA_DIR@|/opt/suite366|g" \ + "$REPO_ROOT/values.yaml" > "$rendered" + +mapfile -t images < <( + helm template pkg "$PKG/chart"/*.tgz -f "$rendered" 2>/dev/null \ + | sed -n 's/^[[:space:]]*image:[[:space:]]*"\?\([^"[:space:]]*\)"\?.*/\1/p' \ + | sort -u +) +[[ ${#images[@]} -gt 0 ]] || die "no images resolved from the chart — check values rendering." + +# Images Helm never schedules but the box needs offline: the livekit +# initContainer and the sandbox runner (spawned on demand by sandbox-api). +# Kept in sync with prepull_images() in lib/suite.sh. +images+=( "busybox:1.37" "ghcr.io/scriptor-group/suite-366-sandbox-runner:$APP_VERSION" ) +mapfile -t images < <(printf '%s\n' "${images[@]}" | sort -u) + +info "${#images[@]} image(s) to export" +for img in "${images[@]}"; do + # One tar per image: a single multi-image archive would force a full re-export + # on any change, and `ctr images import` is happy to take them one by one. + safe="$(printf '%s' "$img" | tr '/:' '__')" + log "docker pull $img (linux/$ARCH)" + docker pull --platform "linux/$ARCH" "$img" >/dev/null \ + || die "docker pull failed: $img" + docker save "$img" -o "$PKG/images/$safe.tar" || die "docker save failed: $img" + info " -> images/$safe.tar ($(du -h "$PKG/images/$safe.tar" | cut -f1))" +done + +if [[ "$WITH_VLLM" == 1 && -n "$VLLM_IMAGE" ]]; then + safe="$(printf '%s' "$VLLM_IMAGE" | tr '/:' '__')" + log "docker pull $VLLM_IMAGE (linux/$ARCH) — several GB" + docker pull --platform "linux/$ARCH" "$VLLM_IMAGE" >/dev/null \ + || die "docker pull failed: $VLLM_IMAGE" + docker save "$VLLM_IMAGE" -o "$PKG/docker-images/$safe.tar" \ + || die "docker save failed: $VLLM_IMAGE" + info " -> docker-images/$safe.tar ($(du -h "$PKG/docker-images/$safe.tar" | cut -f1))" +else + rmdir "$PKG/docker-images" +fi + +# --- Updater ----------------------------------------------------------------- +# The appliance installs THIS update.sh after applying the package: it is the +# only signed path to move the updater forward on an air-gapped box. +install -m 0644 "$REPO_ROOT/update.sh" "$PKG/scripts/update.sh" +bash -n "$PKG/scripts/update.sh" || die "bundled update.sh does not parse." + +# --- Manifest ---------------------------------------------------------------- +cat > "$PKG/manifest.json" < SHA256SUMS ) +grep -qE '[[:space:]]\*?\./?manifest\.json$' "$PKG/SHA256SUMS" \ + || die "internal error: manifest.json missing from SHA256SUMS." + +openssl pkeyutl -sign -rawin -inkey "$KEY" \ + -in "$PKG/SHA256SUMS" -out "$PKG/SHA256SUMS.sig" \ + || die "signing failed — is $KEY an Ed25519 private key?" + +# Fail here rather than on the appliance: verify with the public half now. +pub="$(mktemp)" +openssl pkey -in "$KEY" -pubout -out "$pub" 2>/dev/null || die "cannot derive the public key." +openssl pkeyutl -verify -rawin -pubin -inkey "$pub" \ + -sigfile "$PKG/SHA256SUMS.sig" -in "$PKG/SHA256SUMS" >/dev/null 2>&1 \ + || { rm -f "$pub"; die "self-verification failed — the package would be refused."; } +rm -f "$pub" +( cd "$PKG" && sha256sum -c --strict --quiet SHA256SUMS ) \ + || die "self-verification failed — checksums do not match their own files." + +log "Package ready: $PKG" +info "size: $(du -sh "$PKG" | cut -f1)" +info "Copy the DIRECTORY to the root of a USB drive (FAT32/exFAT/ext4), then plug" +info "it into the appliance — or run: sudo /opt/suite366/update.sh scan-usb " diff --git a/tools/gen-package-key.sh b/tools/gen-package-key.sh new file mode 100755 index 0000000..32ef711 --- /dev/null +++ b/tools/gen-package-key.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# ============================================================================= +# Generate the Ed25519 keypair that signs offline update packages. +# +# tools/gen-package-key.sh ~/.secrets/package-release +# -> ~/.secrets/package-release.key PRIVATE — never leaves the vault / CI +# -> ~/.secrets/package-release.pub ships with every appliance +# +# The PUBLIC half goes onto each appliance as $DATA_DIR/package-release.pub +# (see PACKAGE_PUBLIC_KEY in lib/config.sh). It can only verify, never sign. +# +# This is deliberately NOT the license keypair: a license key must never gain +# the power to authorise code execution, and the two rotate on different +# schedules for different reasons. +# ============================================================================= +set -euo pipefail + +BASE="${1:-}" +[[ -n "$BASE" ]] || { echo "usage: $0 " >&2; exit 1; } +[[ -e "$BASE.key" ]] && { echo "refusing to overwrite $BASE.key" >&2; exit 1; } + +mkdir -p "$(dirname "$BASE")" +umask 077 +openssl genpkey -algorithm ed25519 -out "$BASE.key" +chmod 0600 "$BASE.key" +openssl pkey -in "$BASE.key" -pubout -out "$BASE.pub" +chmod 0644 "$BASE.pub" + +echo "private : $BASE.key (0600 — store in the vault, load into CI as a secret)" +echo "public : $BASE.pub (deploy to appliances as \$DATA_DIR/package-release.pub)" diff --git a/tools/sign-channel.sh b/tools/sign-channel.sh new file mode 100755 index 0000000..f1de6bc --- /dev/null +++ b/tools/sign-channel.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# ============================================================================= +# Sign channel.json, so an appliance can trust what it is told to run. +# +# PACKAGE_PRIVATE_KEY=~/.secrets/package-release.key tools/sign-channel.sh +# +# Two holes, one signature: +# +# 1. THE MANIFEST ITSELF. Whoever controls MANIFEST_URL decides the chart +# version and the vLLM image every appliance is told to run. TLS proves we +# reached the right host; it says nothing about who wrote the file. +# 2. THE UPDATER. `update.sh` is fetched over HTTPS and then runs as root on the +# next apply. Rather than a second detached signature, the manifest carries +# `updater_sha256` — covered by the manifest's own signature — so verifying +# the manifest transitively verifies the updater. +# +# This script therefore, in order: +# • recomputes `updater_sha256` from the update.sh in this repo, +# • signs the resulting channel.json, +# • verifies its own output, then re-checks it the way an appliance would. +# +# Both channel.json and channel.json.sig must be published together. An appliance +# holding our public key REFUSES an unsigned or stale-signed channel (fail +# closed), so publishing one without the other stops the fleet rather than +# breaking it silently. +# ============================================================================= +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KEY="${PACKAGE_PRIVATE_KEY:-}" +CHANNEL="$REPO_ROOT/channel.json" +UPDATER="$REPO_ROOT/update.sh" + +c_b="\033[1m"; c_g="\033[32m"; c_y="\033[33m"; c_r="\033[31m"; c_0="\033[0m" +log() { printf "${c_g}==>${c_0} ${c_b}%s${c_0}\n" "$*"; } +info() { printf " %s\n" "$*"; } +warn() { printf "${c_y}!! %s${c_0}\n" "$*"; } +die() { printf "${c_r}xx %s${c_0}\n" "$*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --key) KEY="$2"; shift 2 ;; + --channel) CHANNEL="$2"; shift 2 ;; + --updater) UPDATER="$2"; shift 2 ;; + -h|--help) sed -n '2,28p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown option: $1" ;; + esac +done + +command -v openssl >/dev/null || die "openssl required." +[[ -n "$KEY" ]] || die "no signing key: pass --key or set PACKAGE_PRIVATE_KEY." +[[ -s "$KEY" ]] || die "signing key not readable: $KEY" +[[ -s "$CHANNEL" ]] || die "channel manifest not found: $CHANNEL" +[[ -s "$UPDATER" ]] || die "updater not found: $UPDATER" + +# --- 1. Pin the updater ------------------------------------------------------ +bash -n "$UPDATER" || die "$UPDATER does not parse — refusing to publish it." +sha="$(sha256sum "$UPDATER" | awk '{print $1}')" +log "Pinning updater_sha256" +info "$(basename "$UPDATER") -> $sha" + +if grep -q '"updater_sha256"' "$CHANNEL"; then + # In place, preserving the rest of the file byte for byte — the signature is + # over exact bytes, so a reformat here is a needless churn in every diff. + sed -i -E "s|(\"updater_sha256\"[[:space:]]*:[[:space:]]*\")[^\"]*(\")|\1$sha\2|" "$CHANNEL" +else + die "$CHANNEL has no updater_sha256 field — add \"updater_sha256\": \"\" first." +fi + +got="$(sed -n 's/.*"updater_sha256"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CHANNEL" | head -1)" +[[ "$got" == "$sha" ]] || die "failed to write updater_sha256 into $CHANNEL (got '$got')." + +# --- 2. Sign ------------------------------------------------------------------ +log "Signing $(basename "$CHANNEL")" +openssl pkeyutl -sign -rawin -inkey "$KEY" \ + -in "$CHANNEL" -out "$CHANNEL.sig" \ + || die "signing failed — is $KEY an Ed25519 private key?" + +# --- 3. Verify our own output, as the appliance will -------------------------- +pub="$(mktemp)"; trap 'rm -f "$pub"' EXIT +openssl pkey -in "$KEY" -pubout -out "$pub" 2>/dev/null || die "cannot derive the public key." +openssl pkeyutl -verify -rawin -pubin -inkey "$pub" \ + -sigfile "$CHANNEL.sig" -in "$CHANNEL" >/dev/null 2>&1 \ + || die "self-verification failed — appliances would refuse this channel." + +log "Signed and verified" +info "$CHANNEL" +info "$CHANNEL.sig" +printf '\n' +warn "Publish BOTH files together, and in this order matters little — but never one alone:" +warn " an appliance holding the public key refuses an unsigned or mismatched channel," +warn " which stops the fleet from updating rather than letting it update wrongly." +info "Sanity check as an appliance sees it:" +info " curl -fsSL -o /tmp/c.json" +info " curl -fsSL .sig -o /tmp/c.sig" +info " openssl pkeyutl -verify -rawin -pubin -inkey package-release.pub \\" +info " -sigfile /tmp/c.sig -in /tmp/c.json" diff --git a/tools/test-package-verify.sh b/tools/test-package-verify.sh new file mode 100755 index 0000000..0780f4c --- /dev/null +++ b/tools/test-package-verify.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# ============================================================================= +# Self-test for update.sh's offline-package verifier. Builds real signed +# packages, then tampers with them one way at a time and asserts each is +# refused for the right reason. No cluster, no appliance, no GPU: +# +# tools/test-package-verify.sh # from the repo root +# +# pkg_verify + ver_gt are lifted VERBATIM out of update.sh (sed range extract) +# so this exercises the shipping code, not a copy that can drift from it. +# ============================================================================= +set -uo pipefail + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +UPDATE_SH="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/update.sh}" +[[ -s "$UPDATE_SH" ]] || { echo "update.sh not found: $UPDATE_SH" >&2; exit 1; } + +pass=0; fail=0 +ok() { printf ' \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); } +ko() { printf ' \033[31mFAIL\033[0m %s — %s\n' "$1" "${2:-}"; fail=$((fail+1)); } + +# --- keys --------------------------------------------------------------------- +openssl genpkey -algorithm ed25519 -out "$WORK/good.key" 2>/dev/null +openssl pkey -in "$WORK/good.key" -pubout -out "$WORK/good.pub" 2>/dev/null +openssl genpkey -algorithm ed25519 -out "$WORK/evil.key" 2>/dev/null + +# --- build a minimal but structurally valid package --------------------------- +mkpkg() { # mkpkg DIR SIGNING_KEY APP_VERSION [MIN_FROM] + local d="$1" key="$2" app="$3" minfrom="${4:-}" + rm -rf "$d"; mkdir -p "$d/chart" "$d/images" "$d/scripts" + printf 'fake chart archive\n' > "$d/chart/drive-9.9.9.tgz" + printf 'fake image tar\n' > "$d/images/app.tar" + printf '#!/bin/bash\ntrue\n' > "$d/scripts/update.sh" + cat > "$d/manifest.json" < SHA256SUMS ) + openssl pkeyutl -sign -rawin -inkey "$key" -in "$d/SHA256SUMS" -out "$d/SHA256SUMS.sig" +} + +# --- harness: run pkg_verify from update.sh in a subshell ---------------------- +# update.sh refuses to run as non-root and dispatches on $1, so we source it +# with a guard: extract just the functions we need by stubbing the environment. +verify() { # verify PKGDIR PUBKEY CUR_APP -> prints pkg_error, exit status + bash -c ' + set -uo pipefail + PACKAGE_PUBLIC_KEY="$2"; cur_app="$3" + json_get() { sed -n "s/.*\"$1\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\".*/\1/p" | head -1; } + ver_gt() { + [[ -n "$1" ]] || return 1 + [[ -n "$2" ]] || return 0 + [[ "$1" != "$2" ]] || return 1 + [[ "$(printf "%s\n%s\n" "$1" "$2" | sort -V | tail -1)" == "$1" ]] + } + # Pull pkg_verify out of the real update.sh, verbatim. + eval "$(sed -n "/^pkg_verify() {/,/^}/p" "$4")" + if pkg_verify "$1"; then echo "OK"; exit 0; else echo "$pkg_error"; exit 1; fi + ' _ "$1" "$2" "$3" "$UPDATE_SH" +} + +echo "== offline package verifier ==" + +mkpkg "$WORK/pkg" "$WORK/good.key" 1.8.23 +out="$(verify "$WORK/pkg" "$WORK/good.pub" 1.8.22)" \ + && [[ "$out" == OK ]] && ok "valid package accepted" || ko "valid package accepted" "$out" + +# signed by the wrong key +mkpkg "$WORK/evil" "$WORK/evil.key" 1.8.23 +out="$(verify "$WORK/evil" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"signature check failed"* ]] \ + && ok "foreign signature refused" || ko "foreign signature refused" "$out" + +# one byte flipped in an image tar (checksum, not signature, catches this) +mkpkg "$WORK/bitrot" "$WORK/good.key" 1.8.23 +printf 'X' >> "$WORK/bitrot/images/app.tar" +out="$(verify "$WORK/bitrot" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"checksum mismatch"* ]] \ + && ok "corrupt image tar refused" || ko "corrupt image tar refused" "$out" + +# manifest swapped after signing +mkpkg "$WORK/swap" "$WORK/good.key" 1.8.23 +sed -i 's/"app_version": "1.8.23"/"app_version": "9.9.9"/' "$WORK/swap/manifest.json" +out="$(verify "$WORK/swap" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"checksum mismatch"* ]] \ + && ok "tampered manifest refused" || ko "tampered manifest refused" "$out" + +# manifest present but NOT covered by the signed list +mkpkg "$WORK/uncovered" "$WORK/good.key" 1.8.23 +grep -v 'manifest.json' "$WORK/uncovered/SHA256SUMS" > "$WORK/uncovered/S.tmp" +mv "$WORK/uncovered/S.tmp" "$WORK/uncovered/SHA256SUMS" +openssl pkeyutl -sign -rawin -inkey "$WORK/good.key" \ + -in "$WORK/uncovered/SHA256SUMS" -out "$WORK/uncovered/SHA256SUMS.sig" +out="$(verify "$WORK/uncovered" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"not covered"* ]] \ + && ok "uncovered manifest refused" || ko "uncovered manifest refused" "$out" + +# missing signature entirely +mkpkg "$WORK/nosig" "$WORK/good.key" 1.8.23 +rm -f "$WORK/nosig/SHA256SUMS.sig" +out="$(verify "$WORK/nosig" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"SHA256SUMS.sig missing"* ]] \ + && ok "unsigned package refused" || ko "unsigned package refused" "$out" + +# no public key on the appliance => refuse everything +mkpkg "$WORK/pkg2" "$WORK/good.key" 1.8.23 +out="$(verify "$WORK/pkg2" "$WORK/absent.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"no package signing key"* ]] \ + && ok "no appliance key => refused" || ko "no appliance key => refused" "$out" + +# downgrade +mkpkg "$WORK/down" "$WORK/good.key" 1.8.20 +out="$(verify "$WORK/down" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"downgrade refused"* ]] \ + && ok "downgrade refused" || ko "downgrade refused" "$out" + +# min_from_version not satisfied +mkpkg "$WORK/floor" "$WORK/good.key" 1.9.0 1.8.30 +out="$(verify "$WORK/floor" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"requires app >= 1.8.30"* ]] \ + && ok "min_from_version enforced" || ko "min_from_version enforced" "$out" + +# min_from_version satisfied +mkpkg "$WORK/floor2" "$WORK/good.key" 1.9.0 1.8.20 +out="$(verify "$WORK/floor2" "$WORK/good.pub" 1.8.22)" \ + && [[ "$out" == OK ]] && ok "min_from_version satisfied" || ko "min_from_version satisfied" "$out" + +# same version = legitimate repair re-apply, not a downgrade +mkpkg "$WORK/same" "$WORK/good.key" 1.8.22 +out="$(verify "$WORK/same" "$WORK/good.pub" 1.8.22)" \ + && [[ "$out" == OK ]] && ok "same-version re-apply allowed" || ko "same-version re-apply allowed" "$out" + +# two chart archives = ambiguous helm upgrade +mkpkg "$WORK/twocharts" "$WORK/good.key" 1.8.23 +printf 'second\n' > "$WORK/twocharts/chart/drive-9.9.8.tgz" +( cd "$WORK/twocharts" && find . -type f ! -name SHA256SUMS ! -name SHA256SUMS.sig -print0 \ + | sort -z | xargs -0 sha256sum > SHA256SUMS ) +openssl pkeyutl -sign -rawin -inkey "$WORK/good.key" \ + -in "$WORK/twocharts/SHA256SUMS" -out "$WORK/twocharts/SHA256SUMS.sig" +out="$(verify "$WORK/twocharts" "$WORK/good.pub" 1.8.22)" +[[ $? -ne 0 && "$out" == *"expected 1"* ]] \ + && ok "ambiguous chart set refused" || ko "ambiguous chart set refused" "$out" + +# --- ver_gt ordering ---------------------------------------------------------- +echo "== version ordering ==" +vg() { bash -c ' + ver_gt() { + [[ -n "$1" ]] || return 1 + [[ -n "$2" ]] || return 0 + [[ "$1" != "$2" ]] || return 1 + [[ "$(printf "%s\n%s\n" "$1" "$2" | sort -V | tail -1)" == "$1" ]] + } + ver_gt "$1" "$2"' _ "$1" "$2"; } +vg 1.8.10 1.8.9 && ok "1.8.10 > 1.8.9 (numeric, not lexical)" || ko "1.8.10 > 1.8.9" +vg 1.8.9 1.8.10 && ko "1.8.9 > 1.8.10 must be false" || ok "1.8.9 !> 1.8.10" +vg 1.8.22 1.8.22 && ko "equal must be false" || ok "equal is not greater" +vg 2.0.0 1.99.99 && ok "2.0.0 > 1.99.99" || ko "2.0.0 > 1.99.99" +vg "" 1.0.0 && ko "empty must not be greater" || ok "empty !> anything" +vg 1.0.0 "" && ok "anything > empty" || ko "anything > empty" + +# --- manifest signature (channel.json) ---------------------------------------- +# Whoever controls MANIFEST_URL decides what every appliance is told to run, so +# the manifest is signed too. Graduated: strict when the appliance holds our +# public key, TLS-only when it does not (the public product's posture). +echo +echo "== channel manifest signature ==" + +mkchannel() { # mkchannel FILE KEY [CHART_VERSION] + local f="$1" key="$2" cv="${3:-0.8.0}" + cat > "$f" </dev/null 2>&1; } + # Stubbed transport. The real call is `curl -fsSL -m 20 URL -o DEST`, so the + # URL is NOT $1 — read it from MANIFEST_SIG_URL and only parse out -o. + curl() { + local dest="" i + local -a a=("$@") + for ((i=0; i<${#a[@]}; i++)); do [[ "${a[i]}" == "-o" ]] && dest="${a[i+1]}"; done + local src="${MANIFEST_SIG_URL#sig://}" + [[ -n "$dest" && -s "$src" ]] || return 22 + cp "$src" "$dest" + } + eval "$(sed -n "/^verify_manifest_signature() {/,/^}/p" "$4")" + if verify_manifest_signature "$1"; then echo "OK signed=$online_signed"; else echo "$online_error"; exit 1; fi + ' _ "$1" "$2" "${3:-1}" "$UPDATE_SH" +} + +mkchannel "$WORK/ch.json" "$WORK/good.key" +out="$(verify_manifest "$WORK/ch.json" "$WORK/good.pub")" \ + && [[ "$out" == "OK signed=1" ]] && ok "signed manifest accepted, marked verified" \ + || ko "signed manifest accepted" "$out" + +# No key on the appliance -> permissive, and NOT marked verified (so self_update +# stays in TLS-only mode rather than believing it has a guarantee). +out="$(verify_manifest "$WORK/ch.json" "$WORK/absent.pub")" \ + && [[ "$out" == "OK signed=0" ]] && ok "no appliance key => permissive, not marked verified" \ + || ko "no appliance key => permissive" "$out" + +# Signed by the wrong key. +mkchannel "$WORK/evilch.json" "$WORK/evil.key" +out="$(verify_manifest "$WORK/evilch.json" "$WORK/good.pub")" +[[ $? -ne 0 && "$out" == *"signature INVALID"* ]] \ + && ok "foreign-signed manifest refused" || ko "foreign-signed manifest refused" "$out" + +# Tampered after signing — the chart_version an appliance would act on. +mkchannel "$WORK/tamper.json" "$WORK/good.key" +sed -i 's/"chart_version": "0.8.0"/"chart_version": "9.9.9"/' "$WORK/tamper.json" +out="$(verify_manifest "$WORK/tamper.json" "$WORK/good.pub")" +[[ $? -ne 0 && "$out" == *"signature INVALID"* ]] \ + && ok "manifest tampered after signing refused" || ko "manifest tampered after signing refused" "$out" + +# Signature missing while the appliance requires one -> fail closed. +mkchannel "$WORK/nosig.json" "" +out="$(verify_manifest "$WORK/nosig.json" "$WORK/good.pub")" +[[ $? -ne 0 && "$out" == *"signature unavailable"* ]] \ + && ok "missing signature refused (fail closed)" || ko "missing signature refused" "$out" + +printf '\n%d passed, %d failed\n' "$pass" "$fail" +[[ "$fail" -eq 0 ]] diff --git a/update.sh b/update.sh index 52e35a9..0efd725 100755 --- a/update.sh +++ b/update.sh @@ -15,11 +15,27 @@ # the fleet should run": you push channel.json -> the fleet rolls out. No # per-box edits, no in-the-blind tracking of `latest`. # +# TWO SOURCES, never exclusive: +# • ONLINE — the channel manifest over HTTPS. The normal path. +# • USB — a signed offline package (built by tools/build-offline-package.sh) +# found on a removable drive. For boxes whose owner cut outbound access. +# `check` tries the network and NEVER dies when it is unreachable: a verified +# USB package still produces an "update available" state, and conversely a +# reachable network never invalidates a staged package. state.json carries both +# sources plus the resolved best target (highest app version wins; online wins a +# tie since it needs no image import). +# # Modes: -# check (default) fetch manifest, compare, write marker + state.json. -# No changes. +# check (default) fetch manifest, load any staged offline package, +# compare, write marker + state.json. No changes. # apply helm upgrade (+ vLLM image / app image pin updates), then a -# health check. Idempotent (no-op if already up to date). +# health check. Idempotent (no-op if already up to date). Uses +# the resolved source: OCI chart pull, or the staged package's +# local chart + image tars when that is the better target. +# scan-usb DIR verify a signed offline package under DIR, stage it into +# $DATA_DIR/offline/pkg, then refresh state.json so the app +# shows the same "update available" prompt as when online. +# Applies NOTHING (an admin still confirms in the UI). # install-units (re)install the systemd .path units that let the app UI # trigger check/apply, and prepare $DATA_DIR/updates. Called by # install.sh and after every apply (fleet convergence). @@ -40,7 +56,11 @@ # KUBECONFIG_PATH /etc/rancher/k3s/k3s.yaml # UPDATE_WEBHOOK optional URL — POSTed {"text":"…"} on update-available # SELF_URL where to refresh update.sh from after an apply (default: -# sibling of MANIFEST_URL). Set SELF_UPDATE=0 to disable. +# sibling of MANIFEST_URL). Set SELF_UPDATE=0 to disable +# (fleet boxes do: their updater only moves with a SIGNED +# package, never with an unauthenticated HTTPS fetch). +# PACKAGE_PUBLIC_KEY PEM Ed25519 public key verifying offline packages. +# Absent file => every offline package is refused. # ============================================================================= set -euo pipefail @@ -52,6 +72,9 @@ DATA_DIR="${DATA_DIR:-/opt/suite366}" [[ -f "$DATA_DIR/update.env" ]] && . "$DATA_DIR/update.env" MANIFEST_URL="${MANIFEST_URL:-https://raw.githubusercontent.com/Scriptor-Group/suite366-deploy/main/channel.json}" +# Detached Ed25519 signature over channel.json, published beside it. Required on +# any appliance that holds PACKAGE_PUBLIC_KEY; ignored on one that does not. +MANIFEST_SIG_URL="${MANIFEST_SIG_URL:-$MANIFEST_URL.sig}" CHART_REF="${CHART_REF:-oci://ghcr.io/scriptor-group/chart/drive}" NAMESPACE="${NAMESPACE:-suite366}" RELEASE="${RELEASE:-drive}" @@ -61,6 +84,17 @@ SELF_UPDATE="${SELF_UPDATE:-1}" SELF_URL="${SELF_URL:-${MANIFEST_URL%/*}/update.sh}" MARKER="$DATA_DIR/update-available" +# --- Offline (USB) package source --------------------------------------------- +# A verified package is COPIED off the removable drive into $OFFLINE_PKG so the +# key can be pulled out before an admin confirms the update in the UI, and so a +# mid-apply unplug cannot truncate an image tar. $OFFLINE_SRC records the +# outcome (including rejections, which have no staged content to speak for +# them) and is the only thing `check` needs to read. +PACKAGE_PUBLIC_KEY="${PACKAGE_PUBLIC_KEY:-$DATA_DIR/package-release.pub}" +OFFLINE_DIR="$DATA_DIR/offline" +OFFLINE_PKG="$OFFLINE_DIR/pkg" +OFFLINE_SRC="$OFFLINE_DIR/source.json" + # Shared dir with the drive-app pod (hostPath). uid/gid 1001 = runAsUser of # the drive-app container in the chart. UPDATES_DIR="$DATA_DIR/updates" @@ -118,15 +152,20 @@ consume_trigger() { # consume_trigger FILENAME write_state_json() { # write_state_json AVAILABLE(0|1) ensure_updates_dir local avail=false; [[ "$1" == "1" ]] && avail=true + local reachable=false; [[ "${online_reachable:-0}" == "1" ]] && reachable=true local tmp="$STATE_JSON.tmp" + # schema 2 adds `source` + `sources` (online / usb). Readers tolerating only + # schema 1 keep working: every field they know is unchanged and still carries + # the RESOLVED best-of-both target, not just the online one. cat > "$tmp" <-" (e.g. drive-0.7.0); # grab the trailing version (first char a digit) regardless of the chart name. + # `|| true` is load-bearing, not defensive noise: under `set -euo pipefail` an + # assignment takes the pipeline's status, so a `helm list` that cannot reach the + # cluster killed this script HERE — silently, exit 1, no output — and the warn + # below plus the values.yaml fallback further down were unreachable code. That + # is the degraded box this whole offline path exists for: k3s down, a verified + # USB package waiting, and `check` dying before it can report it. cur_chart="$(helm list -n "$NAMESPACE" --filter "^${RELEASE}$" -o json 2>/dev/null \ - | sed -n 's/.*"chart":"[^"]*-\([0-9][^"]*\)".*/\1/p' | head -1)" + | sed -n 's/.*"chart":"[^"]*-\([0-9][^"]*\)".*/\1/p' | head -1 || true)" [[ -n "$cur_chart" ]] || warn "Could not read current chart version (release '$RELEASE' in ns '$NAMESPACE')." cur_vllm="" @@ -179,32 +242,191 @@ read_current_state() { # the cluster is unreadable. cur_app="$(kc -n "$NAMESPACE" get deploy \ -o jsonpath='{range .items[*].spec.template.spec.containers[*]}{.image}{"\n"}{end}' 2>/dev/null \ - | sed -n 's|.*/suite-366:||p' | head -1)" + | sed -n 's|.*/suite-366:||p' | head -1 || true)" if [[ -z "$cur_app" && -f "$DATA_DIR/values.yaml" ]]; then cur_app="$(sed -n 's/^ tag: "\(.*\)"/\1/p' "$DATA_DIR/values.yaml" | head -1)" [[ -n "$cur_app" ]] && warn "App version read from values.yaml pin (cluster unreadable) — may be ahead of the running pod." fi } -# --- Target state from the manifest ------------------------------------------- -fetch_manifest() { +# --- Version comparison -------------------------------------------------------- +# True when A is strictly newer than B (dotted numeric versions; `sort -V` +# handles 1.8.9 < 1.8.10 correctly, which a string compare does not). +ver_gt() { # ver_gt A B + [[ -n "$1" ]] || return 1 + [[ -n "$2" ]] || return 0 + [[ "$1" != "$2" ]] || return 1 + [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -1)" == "$1" ]] +} + +# --- Target state: source 1, the online channel manifest ------------------------ +# Soft failure by design: a box whose owner cut outbound access must still be +# able to update from a signed USB package, so an unreachable manifest is +# RECORDED, not fatal. +fetch_manifest_online() { + online_reachable=0; online_error=""; online_signed=0 + online_chart=""; online_app=""; online_vllm=""; online_channel=""; online_notes="" + online_updater_sha="" log "Fetching channel manifest" info "$MANIFEST_URL" - manifest="$(curl -fsSL -m 20 "$MANIFEST_URL")" || die "Manifest unreachable: $MANIFEST_URL" - channel="$(json_get channel <<<"$manifest")" - want_chart="$(json_get chart_version <<<"$manifest")" - want_vllm="$(json_get vllm_image <<<"$manifest")" - want_app="$(json_get app_version <<<"$manifest")" - notes="$(json_get notes <<<"$manifest")" - [[ -n "$want_chart" ]] || die "Manifest has no chart_version: $MANIFEST_URL" + # To a file, not a variable: a signature is over exact bytes, and command + # substitution strips trailing newlines. + local tmp; tmp="$(mktemp)" + if ! curl -fsSL -m 20 "$MANIFEST_URL" -o "$tmp" 2>/dev/null || [[ ! -s "$tmp" ]]; then + rm -f "$tmp" + online_error="manifest unreachable ($MANIFEST_URL)" + warn "$online_error — offline package (if any) still applies." + return 1 + fi + + if ! verify_manifest_signature "$tmp"; then + rm -f "$tmp" + return 1 + fi + + local manifest; manifest="$(cat "$tmp")" + rm -f "$tmp" + online_channel="$(json_get channel <<<"$manifest")" + online_chart="$(json_get chart_version <<<"$manifest")" + online_vllm="$(json_get vllm_image <<<"$manifest")" + online_app="$(json_get app_version <<<"$manifest")" + online_notes="$(json_get notes <<<"$manifest")" + online_updater_sha="$(json_get updater_sha256 <<<"$manifest")" + if [[ -z "$online_chart" ]]; then + online_error="manifest has no chart_version" + warn "$online_error ($MANIFEST_URL)" + return 1 + fi + online_reachable=1 + return 0 +} + +# Whoever controls MANIFEST_URL decides which chart version and which vLLM image +# every appliance is told to run. TLS proves we reached the right HOST; it says +# nothing about whether the file is ours. So when the appliance holds our public +# key, the manifest must be signed by it — and a bad signature makes the manifest +# UNUSABLE rather than merely suspicious (fail closed; a verified USB package can +# still carry the box forward). +# +# Graduated on purpose: an appliance with no key installed keeps the old +# TLS-only behaviour, so the public one-command install is unchanged. Fleet boxes +# always have the key, so they are always strict. +verify_manifest_signature() { # verify_manifest_signature FILE + local f="$1" + if [[ ! -s "$PACKAGE_PUBLIC_KEY" ]]; then + # Said once, not per-run-per-line: this is the documented posture of the + # public product, not a misconfiguration. + info "manifest : unsigned (no $PACKAGE_PUBLIC_KEY — TLS-only trust)" + return 0 + fi + have openssl || { online_error="openssl missing, cannot verify the manifest signature"; warn "$online_error"; return 1; } + + local sig; sig="$(mktemp)" + if ! curl -fsSL -m 20 "$MANIFEST_SIG_URL" -o "$sig" 2>/dev/null || [[ ! -s "$sig" ]]; then + rm -f "$sig" + online_error="manifest signature unavailable ($MANIFEST_SIG_URL)" + warn "$online_error — refusing the manifest (this appliance requires signed channels)." + return 1 + fi + if ! openssl pkeyutl -verify -rawin -pubin -inkey "$PACKAGE_PUBLIC_KEY" \ + -sigfile "$sig" -in "$f" >/dev/null 2>&1; then + rm -f "$sig" + online_error="manifest signature INVALID — not signed by this appliance's key" + warn "$online_error" + warn " Refusing it. Either the channel was tampered with, or it was published without signing." + return 1 + fi + rm -f "$sig" + online_signed=1 + info "manifest : signature verified" + return 0 +} + +# --- Target state: source 2, a staged offline package -------------------------- +load_offline_source() { + usb_status=none; usb_error=""; usb_label="" + usb_chart=""; usb_app=""; usb_vllm=""; usb_notes=""; usb_channel=""; usb_verified_at="" + [[ -f "$OFFLINE_SRC" ]] || return 0 + local src; src="$(cat "$OFFLINE_SRC" 2>/dev/null)" || return 0 + usb_status="$(json_get status <<<"$src")" + usb_error="$(json_get error <<<"$src")" + usb_label="$(json_get label <<<"$src")" + usb_channel="$(json_get channel <<<"$src")" + usb_chart="$(json_get chart_version <<<"$src")" + usb_app="$(json_get app_version <<<"$src")" + usb_vllm="$(json_get vllm_image <<<"$src")" + usb_notes="$(json_get notes <<<"$src")" + usb_verified_at="$(json_get verified_at <<<"$src")" + usb_status="${usb_status:-none}" + # A staged package whose content vanished (manual cleanup, disk wipe) must not + # keep advertising itself. + if [[ "$usb_status" == "ready" && ! -f "$OFFLINE_PKG/manifest.json" ]]; then + usb_status=none; usb_error="staged package missing" + fi +} + +write_offline_source() { # write_offline_source STATUS ERROR + mkdir -p "$OFFLINE_DIR"; chmod 0700 "$OFFLINE_DIR" + local tmp="$OFFLINE_SRC.tmp" + cat > "$tmp" < $want_chart") [[ "$app_diff" == 1 ]] && parts+=("app ${cur_app:-?} -> $want_app") [[ "$vllm_diff" == 1 ]] && parts+=("vLLM image -> $want_vllm") + [[ "$UPDATE_SOURCE" == "usb" && ${#parts[@]} -gt 0 ]] && parts+=("from USB package") summary_line="$(IFS='; '; echo "${parts[*]}")" } +# Full pipeline shared by check / apply / scan-usb. +survey() { + read_current_state + fetch_manifest_online || true + load_offline_source + resolve_target + compute_diffs +} + up_to_date() { [[ "$chart_diff" == 0 && "$vllm_diff" == 0 && "$app_diff" == 0 ]]; } +# --- Offline package: verification + staging ----------------------------------- +# Layout produced by tools/build-offline-package.sh: +# manifest.json flat JSON, same keys as channel.json + min_from_version +# SHA256SUMS covers EVERY other file, manifest.json included +# SHA256SUMS.sig raw Ed25519 signature over SHA256SUMS +# chart/-.tgz +# images/*.tar containerd/docker image exports +# scripts/update.sh the updater this package expects (see self_update) +# +# ONE signature, over SHA256SUMS. Everything else derives its authenticity from +# a checksum line in that signed file — so there is never a question of which +# signature is authoritative, and a file the builder forgot to list simply is +# not trusted. Verification is all-or-nothing: one bad byte anywhere and the +# whole package is refused. +pkg_error="" + +# Accept either a package root or a drive whose top level holds exactly one +# (the "plug the key in and we find it" case: `./` on the drive). +pkg_root() { # pkg_root MOUNT -> prints the package root + local m="$1" d + [[ -f "$m/manifest.json" ]] && { printf '%s' "$m"; return 0; } + for d in "$m"/suite366-update-*/; do + [[ -f "$d/manifest.json" ]] && { printf '%s' "${d%/}"; return 0; } + done + return 1 +} + +pkg_verify() { # pkg_verify ROOT — sets pkg_* on success, pkg_error on failure + local root="$1" f + pkg_error="" + pkg_chart=""; pkg_app=""; pkg_vllm=""; pkg_channel=""; pkg_notes=""; pkg_min_from="" + + if [[ ! -s "$PACKAGE_PUBLIC_KEY" ]]; then + pkg_error="no package signing key on this appliance ($PACKAGE_PUBLIC_KEY)"; return 1 + fi + for f in manifest.json SHA256SUMS SHA256SUMS.sig; do + [[ -s "$root/$f" ]] || { pkg_error="incomplete package: $f missing"; return 1; } + done + + # 1. Is the checksum list itself authentic? + if ! openssl pkeyutl -verify -rawin -pubin -inkey "$PACKAGE_PUBLIC_KEY" \ + -sigfile "$root/SHA256SUMS.sig" -in "$root/SHA256SUMS" >/dev/null 2>&1; then + pkg_error="signature check failed — package not signed by this appliance's key"; return 1 + fi + # 2. Is the manifest actually covered by it? (a manifest outside SHA256SUMS + # would be attacker-controlled while everything else verified fine) + if ! grep -qE '[[:space:]]\*?\./?manifest\.json$' "$root/SHA256SUMS"; then + pkg_error="manifest.json is not covered by the signed SHA256SUMS"; return 1 + fi + # 3. Does every listed file match? + if ! ( cd "$root" && sha256sum -c --strict --quiet SHA256SUMS ) >/dev/null 2>&1; then + pkg_error="checksum mismatch — package corrupt or truncated"; return 1 + fi + + local mf; mf="$(cat "$root/manifest.json")" + pkg_channel="$(json_get channel <<<"$mf")" + pkg_chart="$(json_get chart_version <<<"$mf")" + pkg_app="$(json_get app_version <<<"$mf")" + pkg_vllm="$(json_get vllm_image <<<"$mf")" + pkg_notes="$(json_get notes <<<"$mf")" + pkg_min_from="$(json_get min_from_version <<<"$mf")" + [[ -n "$pkg_chart" && -n "$pkg_app" ]] \ + || { pkg_error="manifest lacks chart_version / app_version"; return 1; } + + # Exactly one chart archive, or `helm upgrade` would be ambiguous. + local charts=( "$root"/chart/*.tgz ) + [[ -f "${charts[0]:-}" ]] || { pkg_error="no chart archive under chart/"; return 1; } + [[ ${#charts[@]} -eq 1 ]] || { pkg_error="${#charts[@]} chart archives found, expected 1"; return 1; } + + # 4. Version policy. A strict downgrade is refused outright: rolling the app + # backwards past a Prisma migration is not recoverable from the UI. + if [[ -n "$cur_app" ]] && ver_gt "$cur_app" "$pkg_app"; then + pkg_error="package targets app $pkg_app but $cur_app is installed (downgrade refused)"; return 1 + fi + if [[ -n "$pkg_min_from" && -n "$cur_app" ]] && ver_gt "$pkg_min_from" "$cur_app"; then + pkg_error="package requires app >= $pkg_min_from first (installed: $cur_app)"; return 1 + fi + return 0 +} + +# Copy a verified package off the removable drive, then re-verify the COPY: a +# key pulled mid-copy, or a drive that lies about writes, both show up here. +pkg_stage() { # pkg_stage ROOT + local root="$1" need avail + # `|| true` guards: a failing pipeline inside an assignment would abort the + # whole script under `set -e` (see the same note in lib/preflight.sh). + need="$(du -sk "$root" 2>/dev/null | cut -f1 || true)"; need="${need:-0}" + avail="$(df -Pk "$DATA_DIR" 2>/dev/null | awk 'NR==2{print $4}' || true)"; avail="${avail:-0}" + if (( avail < need * 12 / 10 )); then + pkg_error="not enough free space in $DATA_DIR ($((need/1024)) MiB needed, $((avail/1024)) MiB free)" + return 1 + fi + mkdir -p "$OFFLINE_DIR"; chmod 0700 "$OFFLINE_DIR" + rm -rf "$OFFLINE_PKG.new" + log "Staging package into $OFFLINE_PKG ($((need/1024)) MiB)" + cp -a "$root/." "$OFFLINE_PKG.new/" || { pkg_error="copy from the drive failed"; return 1; } + if ! ( cd "$OFFLINE_PKG.new" && sha256sum -c --strict --quiet SHA256SUMS ) >/dev/null 2>&1; then + rm -rf "$OFFLINE_PKG.new" + pkg_error="staged copy failed verification — drive removed mid-copy?"; return 1 + fi + rm -rf "$OFFLINE_PKG" + mv "$OFFLINE_PKG.new" "$OFFLINE_PKG" + return 0 +} + +do_scan_usb() { # do_scan_usb MOUNT + local mount="${1:-}" + [[ -n "$mount" ]] || die "scan-usb needs a directory (usage: update.sh scan-usb /mnt/key)" + [[ -d "$mount" ]] || die "not a directory: $mount" + + read_current_state + load_offline_source + + local root + if ! root="$(pkg_root "$mount")"; then + info "no offline package found under $mount — nothing to do." + return 0 + fi + log "Offline package found: $root" + usb_label="$(basename "$root")" + + if pkg_verify "$root"; then + usb_channel="$pkg_channel"; usb_chart="$pkg_chart"; usb_app="$pkg_app" + usb_vllm="$pkg_vllm"; usb_notes="$pkg_notes" + if pkg_stage "$root"; then + usb_verified_at="$(now_utc)" + write_offline_source ready "" + log "Package verified and staged: app $pkg_app, chart $pkg_chart" + logger -t suite366-update "offline package staged: $usb_label (app $pkg_app)" 2>/dev/null || true + else + write_offline_source rejected "$pkg_error" + warn "Package REJECTED: $pkg_error" + logger -t suite366-update "offline package rejected: $usb_label ($pkg_error)" 2>/dev/null || true + fi + else + # Keep the versions we could not trust out of state.json. + usb_channel=""; usb_chart=""; usb_app=""; usb_vllm=""; usb_notes=""; usb_verified_at="" + write_offline_source rejected "$pkg_error" + warn "Package REJECTED: $pkg_error" + logger -t suite366-update "offline package rejected: $usb_label ($pkg_error)" 2>/dev/null || true + fi + + # Refresh the app-facing state either way: a rejection must be visible in the + # UI, not just in the journal. + fetch_manifest_online || true + load_offline_source + resolve_target + compute_diffs + if up_to_date; then write_state_json 0; else write_state_json 1; fi +} + # --- check (notify-only) ------------------------------------------------------- notify() { consume_trigger check-requested + if [[ "${UPDATE_SOURCE:-none}" == "none" ]]; then + # Neither source usable. Report it as a failed check rather than "up to + # date" — silently claiming health while blind is how a fleet drifts. + warn "Update check inconclusive: ${online_error:-no source}${usb_error:+ / USB: $usb_error}" + rm -f "$MARKER" + write_state_json 0 + return 0 + fi if up_to_date; then log "Up to date (chart ${cur_chart:-?}, app ${cur_app:-?}, channel ${channel:-?})." rm -f "$MARKER" @@ -288,6 +679,11 @@ do_apply() { consume_trigger apply-requested + if [[ "${UPDATE_SOURCE:-none}" == "none" ]]; then + write_apply_json error "no usable update source: ${online_error:-network unreachable}${usb_error:+ / USB: $usb_error}" + die "Nothing to apply: no reachable channel and no verified offline package." + fi + if up_to_date; then log "Up to date (chart ${cur_chart:-?}, app ${cur_app:-?}) — nothing to apply." rm -f "$MARKER" @@ -302,11 +698,25 @@ do_apply() { write_apply_json running "applying: $summary_line" trap apply_exit_trap EXIT + # Offline source: load every bundled image FIRST, so the helm upgrade and the + # compose recreate below find them locally and never reach for a registry. + if [[ "$UPDATE_SOURCE" == "usb" ]]; then + import_package_images + fi + if [[ "$vllm_diff" == 1 ]]; then log "vLLM image: $cur_vllm -> $want_vllm" [[ -f "$DATA_DIR/llm/.env" ]] || die "$DATA_DIR/llm/.env missing — cannot retarget vLLM image." sed -i "s|^VLLM_IMAGE=.*|VLLM_IMAGE=$want_vllm|" "$DATA_DIR/llm/.env" - if ( cd "$DATA_DIR/llm" && docker compose pull && docker compose up -d ); then + # Offline: the image is already loaded, so `pull` would only fail. Online: + # pull first so a bad tag surfaces before the containers are torn down. + local vllm_ok=0 + if [[ "$UPDATE_SOURCE" == "usb" ]]; then + ( cd "$DATA_DIR/llm" && docker compose up -d ) && vllm_ok=1 + else + ( cd "$DATA_DIR/llm" && docker compose pull && docker compose up -d ) && vllm_ok=1 + fi + if [[ "$vllm_ok" == 1 ]]; then info "vLLM containers recreated." else warn "vLLM image update failed — check: docker logs suite366-vllm-llm" @@ -325,8 +735,19 @@ do_apply() { if [[ "$chart_diff" == 1 || "$app_diff" == 1 ]]; then ensure_appliance_values log "helm upgrade $RELEASE: chart ${cur_chart:-?} -> $want_chart, app ${cur_app:-?} -> ${want_app:-unchanged}" - helm upgrade "$RELEASE" "$CHART_REF" \ - --version "$want_chart" -n "$NAMESPACE" -f "$vals" "${extra_vals[@]}" \ + # Offline: the chart comes from the signed package as a local .tgz, so no + # `--version` (the archive IS the version) and no OCI pull. + local chart_args=() + if [[ "$UPDATE_SOURCE" == "usb" ]]; then + local pkg_charts=( "$OFFLINE_PKG"/chart/*.tgz ) + [[ -f "${pkg_charts[0]:-}" ]] || die "staged package has no chart archive." + chart_args=( "${pkg_charts[0]}" ) + info "chart from package: $(basename "${pkg_charts[0]}")" + else + chart_args=( "$CHART_REF" --version "$want_chart" ) + fi + helm upgrade "$RELEASE" "${chart_args[@]}" \ + -n "$NAMESPACE" -f "$vals" "${extra_vals[@]}" \ --wait --timeout 15m \ || die "helm upgrade failed — roll back with: sudo helm rollback $RELEASE -n $NAMESPACE" fi @@ -335,9 +756,10 @@ do_apply() { kc -n "$NAMESPACE" wait --for=condition=Available deploy --all --timeout=180s \ || warn "Not all deployments became Available — check: sudo k3s kubectl -n $NAMESPACE get pods" - if [[ "$app_diff" == 1 ]]; then + if [[ "$app_diff" == 1 && "$UPDATE_SOURCE" != "usb" ]]; then # sandbox-runner is spawned on demand (not by Helm) — pre-pull it so the - # sandbox works offline after the upgrade. Best-effort. + # sandbox works offline after the upgrade. Best-effort. (An offline package + # ships it, so it was already imported above.) k3s crictl pull "ghcr.io/scriptor-group/suite-366-sandbox-runner:$want_app" >/dev/null 2>&1 \ && info "sandbox-runner:$want_app pre-pulled." \ || warn "sandbox-runner:$want_app pre-pull failed (offline restart may miss it)." @@ -349,15 +771,67 @@ do_apply() { [[ "$app_diff" == 1 ]] && cur_app="$want_app" [[ "$vllm_diff" == 1 ]] && cur_vllm="$want_vllm" chart_diff=0; vllm_diff=0; app_diff=0; summary_line="" + + # Fleet convergence: make sure the app-trigger units exist / are current, and + # refresh this script for the next run. Ordered BEFORE write_state_json so the + # retired offline source is reflected in the state the app reads. + install_units + if [[ "$UPDATE_SOURCE" == "usb" ]]; then + self_update_from_package + retire_staged_package + load_offline_source + else + self_update + fi + + UPDATE_SOURCE=none write_state_json 0 write_apply_json success "update complete (chart $want_chart, app ${cur_app:-?}, channel ${channel:-?})" trap - EXIT log "Update complete (now on chart $want_chart, channel ${channel:-?})." +} - # Fleet convergence: make sure the app-trigger units exist / are current, - # and refresh this script from the channel repo for the next run. - install_units - self_update +# --- Offline package: image import + retirement --------------------------------- +# images/ -> containerd's k8s.io namespace (everything Helm schedules) +# docker-images/ -> the Docker daemon (the vLLM stack runs on compose, not k8s) +import_package_images() { + local tar n=0 + for tar in "$OFFLINE_PKG"/images/*.tar; do + [[ -f "$tar" ]] || continue + log "Importing $(basename "$tar") into containerd" + k3s ctr -n k8s.io images import "$tar" >/dev/null \ + || die "image import failed: $(basename "$tar")" + n=$((n+1)) + done + for tar in "$OFFLINE_PKG"/docker-images/*.tar; do + [[ -f "$tar" ]] || continue + log "Loading $(basename "$tar") into Docker" + docker load -i "$tar" >/dev/null \ + || die "docker load failed: $(basename "$tar")" + n=$((n+1)) + done + info "$n image archive(s) imported from the offline package." +} + +# A package that has been applied must stop advertising itself — and stop +# occupying several GB. Keep the metadata (status `applied`) so the UI can still +# say where the running version came from. +retire_staged_package() { + rm -rf "$OFFLINE_PKG" + write_offline_source applied "" + info "Offline package retired (staged content removed)." +} + +# The updater that a signed package ships is itself covered by SHA256SUMS, so +# this is the ONLY trustworthy way to move update.sh forward on a box with no +# outbound access (see self_update for the online counterpart and its caveat). +self_update_from_package() { + local src="$OFFLINE_PKG/scripts/update.sh" + [[ -s "$src" ]] || return 0 + if bash -n "$src" 2>/dev/null && ! cmp -s "$src" "$DATA_DIR/update.sh"; then + install -m 0700 "$src" "$DATA_DIR/update.sh" + info "update.sh refreshed from the signed package." + fi } # --- install-units ------------------------------------------------------------- @@ -414,16 +888,59 @@ EOF # Refresh this script from the channel repo after a successful apply, so # changes to the update mechanism itself roll out with regular updates (no # per-box SSH). Best-effort, syntax-checked before swapping in. +# +# Trust comes from the SIGNED manifest, not from TLS. channel.json carries +# `updater_sha256`; the manifest's signature covers that field, so a hash match +# means this exact script was published by whoever holds our private key. TLS +# alone would only prove we reached the right host — it says nothing about who +# wrote the file, and the file runs as root on the next apply. +# +# Graduated, same as the manifest check: +# key present + signed manifest + matching hash -> install +# key present, anything else -> REFUSE (fail closed) +# no key installed -> TLS-only, as before self_update() { [[ "$SELF_UPDATE" == "1" ]] || return 0 + local strict=0 + [[ -s "$PACKAGE_PUBLIC_KEY" ]] && strict=1 + + if [[ "$strict" == 1 ]]; then + if [[ "${online_signed:-0}" != "1" ]]; then + warn "not refreshing update.sh: the channel manifest was not signature-verified." + return 0 + fi + if [[ -z "${online_updater_sha:-}" ]]; then + warn "not refreshing update.sh: the signed manifest carries no updater_sha256." + warn " Publish it with tools/sign-channel.sh, or the updater cannot roll forward." + return 0 + fi + fi + local tmp; tmp="$(mktemp)" - if curl -fsSL -m 20 "$SELF_URL" -o "$tmp" && [[ -s "$tmp" ]] && bash -n "$tmp" 2>/dev/null; then - if ! cmp -s "$tmp" "$DATA_DIR/update.sh"; then - install -m 0700 "$tmp" "$DATA_DIR/update.sh" - info "update.sh refreshed from $SELF_URL." + if ! curl -fsSL -m 20 "$SELF_URL" -o "$tmp" || [[ ! -s "$tmp" ]]; then + warn "could not fetch update.sh from $SELF_URL (non-blocking)." + rm -f "$tmp"; return 0 + fi + + if [[ "$strict" == 1 ]]; then + local got + got="$(sha256sum "$tmp" | awk '{print $1}')" + if [[ "$got" != "$online_updater_sha" ]]; then + warn "REFUSING update.sh from $SELF_URL — hash does not match the signed manifest." + warn " expected $online_updater_sha" + warn " got $got" + warn " Either the channel is mid-publish, or someone is serving a different script." + rm -f "$tmp"; return 0 fi - else - warn "could not refresh update.sh from $SELF_URL (non-blocking)." + fi + + if ! bash -n "$tmp" 2>/dev/null; then + warn "fetched update.sh does not parse — keeping the current one." + rm -f "$tmp"; return 0 + fi + if ! cmp -s "$tmp" "$DATA_DIR/update.sh"; then + install -m 0700 "$tmp" "$DATA_DIR/update.sh" + info "update.sh refreshed from $SELF_URL$([[ "$strict" == 1 ]] && printf ' (signature-verified)')." fi rm -f "$tmp" } @@ -438,20 +955,23 @@ require_cluster_tools() { case "$MODE" in check) require_cluster_tools - read_current_state - fetch_manifest + survey notify ;; apply) require_cluster_tools - read_current_state - fetch_manifest + survey do_apply ;; + scan-usb) + have openssl || die "openssl required to verify offline packages." + require_cluster_tools + do_scan_usb "${2:-}" + ;; install-units) install_units ;; *) - die "Unknown mode '$MODE' (use: check | apply | install-units)" + die "Unknown mode '$MODE' (use: check | apply | scan-usb DIR | install-units)" ;; esac diff --git a/values.yaml b/values.yaml index 1662fce..705a75e 100644 --- a/values.yaml +++ b/values.yaml @@ -178,25 +178,44 @@ livekit: issuerName: suite366-local-ca issuerKind: ClusterIssuer -# --- Appliance update bridge (host <-> app) ---------------------------------- -# @DATA_DIR@/updates on the host is shared with the drive-app pod: update.sh -# publishes state.json / apply.json there, and the app drops trigger files -# (check-requested / apply-requested) that systemd .path units pick up to run -# update.sh — this powers the "update available" banner + the update button in -# the admin UI. Uses the chart's generic extraEnv/extraVolumes (no chart -# change). install.sh creates the dir root:1001 mode 0770 (the app container -# runs as uid/gid 1001; k8s does not fsGroup-chown hostPath volumes). +# --- Appliance host <-> app bridges ------------------------------------------ +# Two hostPath dirs are shared with the drive-app pod: +# +# @DATA_DIR@/updates -> /appliance-update (APPLIANCE_UPDATE_DIR) +# update.sh publishes state.json / apply.json there, and the app drops +# trigger files (check-requested / apply-requested) that systemd .path units +# pick up to run update.sh — this powers the "update available" banner + the +# update button in the admin UI. +# +# @DATA_DIR@/support -> /support-access (SUPPORT_ACCESS_DIR) +# Remote-support access toggle, used on RENTED fleet appliances only. The +# host agent publishes state.json (grant + pre-arm state) and the app drops +# grant-requested / revoke-requested / prearm-requested. On a customer-run +# appliance the dir stays empty, state.json never appears, and the app hides +# the feature entirely — so this single values.yaml serves both products. +# +# Both use the chart's generic extraEnv/extraVolumes (no chart change). +# install.sh creates the dirs root:1001 mode 0770 (the app container runs as +# uid/gid 1001; k8s does not fsGroup-chown hostPath volumes). extraEnv: - name: APPLIANCE_UPDATE_DIR value: /appliance-update + - name: SUPPORT_ACCESS_DIR + value: /support-access extraVolumeMounts: - name: appliance-update mountPath: /appliance-update + - name: support-access + mountPath: /support-access extraVolumes: - name: appliance-update hostPath: path: @DATA_DIR@/updates type: DirectoryOrCreate + - name: support-access + hostPath: + path: @DATA_DIR@/support + type: DirectoryOrCreate # Sandbox (code-exec) stack — runs in a dedicated PSS-restricted namespace, # pre-created by install.sh so the GHCR pull secret lands BEFORE sandbox-api