diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3121487..8c0da43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,13 @@ jobs: shell: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check POSIX syntax run: | set -eu sh -n install.sh - sh -n scripts/build-bundle.sh + sh -n scripts/build-bundle.sh scripts/fetch-hvi.sh # The wrappers and uninstaller are generated into the tarball by # build-bundle.sh, so lint the text it will write rather than waiting # to find out on a host. @@ -53,7 +53,7 @@ jobs: - name: shellcheck run: | sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck - shellcheck -s sh install.sh scripts/build-bundle.sh /tmp/brig-uninstall.sh \ + shellcheck -s sh install.sh scripts/build-bundle.sh scripts/fetch-hvi.sh /tmp/brig-uninstall.sh \ /tmp/brig-rootless-setup.sh /tmp/brig-ctl tests/*.sh - name: Script tests @@ -97,7 +97,7 @@ jobs: flag: '--rootless' runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Build a generic-boot bundle (${{ matrix.arch }}, ${{ matrix.rootless }}) env: @@ -146,15 +146,32 @@ jobs: # urunc and urunit are built from pinned commits, and the build # fails if the checkout is anything else. This checks the record: the # commits in pins.env are the ones the script pins. - for c in URUNC URUNIT; do - want="$(sed -n "s/^${c}_REF_DEFAULT=\([0-9a-f]\{40\}\)$/\1/p" scripts/build-bundle.sh)" - [ -n "$want" ] || { echo "no ${c}_REF_DEFAULT in build-bundle.sh"; exit 1; } - grep -E "^${c}_(REF|PINNED)=" "$p" - grep -qx "${c}_REF=$want" "$p" || { echo "pins.env does not carry ${c}_REF=$want"; exit 1; } - grep -qx "${c}_PINNED=true" "$p" || { echo "pins.env does not say ${c}_PINNED=true"; exit 1; } + want="$(sed -n 's/^URUNC_REF_DEFAULT=\([0-9a-f]\{40\}\)$/\1/p' scripts/build-bundle.sh)" + [ -n "$want" ] || { echo "no URUNC_REF_DEFAULT in build-bundle.sh"; exit 1; } + grep -E "^URUNC_(REF|PINNED)=" "$p" + grep -qx "URUNC_REF=$want" "$p" || { echo "pins.env does not carry URUNC_REF=$want"; exit 1; } + grep -qx "URUNC_PINNED=true" "$p" || { echo "pins.env does not say URUNC_PINNED=true"; exit 1; } + # hvi is fetched from the release the script pins; fetch-hvi.sh + # fails the build unless its signature and sums check out. + grep -E "^HVI_(VERSION|COMMIT)=" "$p" + for k in HVI_VERSION HVI_COMMIT; do + want="$(sed -n "s/^$k=\([0-9a-z.-]*\)$/\1/p" scripts/build-bundle.sh)" + [ -n "$want" ] || { echo "no $k in build-bundle.sh"; exit 1; } + grep -qx "$k=$want" "$p" || { echo "pins.env does not carry $k=$want"; exit 1; } done + # urunit has no pin of its own here: the build reads it from the + # pinned urunc commit's packaging/container-initrd/urunit.pin. Read + # that file at the recorded urunc commit and compare. + urunc_repo="$(sed -n 's/^URUNC_REPO=//p' "$p")" + urunc_ref="$(sed -n 's/^URUNC_REF=//p' "$p")" + want="$(curl -fsSL "https://raw.githubusercontent.com/$urunc_repo/$urunc_ref/packaging/container-initrd/urunit.pin" \ + | sed -n 's/^URUNIT_REF=\([0-9a-f]\{40\}\)$/\1/p')" + [ -n "$want" ] || { echo "$urunc_repo@$urunc_ref has no urunit.pin naming a commit"; exit 1; } + grep -E "^URUNIT_(REF|PINNED)=" "$p" + grep -qx "URUNIT_REF=$want" "$p" || { echo "pins.env does not carry URUNIT_REF=$want, urunc's pin"; exit 1; } + grep -qx "URUNIT_PINNED=true" "$p" || { echo "pins.env does not say URUNIT_PINNED=true"; exit 1; } - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: bundle-${{ matrix.arch }}-${{ matrix.rootless }} path: dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3fb3ca3..4173d0b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,7 +59,7 @@ jobs: flag: '--rootless' runs-on: ${{ matrix.runner }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Build the ${{ matrix.arch }} ${{ matrix.rootless }} bundle env: @@ -73,7 +73,7 @@ jobs: --version "${{ needs.version.outputs.version }}" \ --variant generic-boot ${{ matrix.flag }} --out dist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: bundle-${{ matrix.arch }}-${{ matrix.rootless }} path: dist/* @@ -88,11 +88,13 @@ jobs: id-token: write attestations: write steps: - - uses: actions/checkout@v4 - - uses: sigstore/cosign-installer@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + cosign-release: v2.5.3 - name: Collect the per-arch bundles - uses: actions/download-artifact@v4 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: bundle-* path: incoming @@ -121,7 +123,7 @@ jobs: --yes - name: Attest build provenance - uses: actions/attest-build-provenance@v2 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: dist/brig-standalone-*.tar.gz diff --git a/DESIGN.md b/DESIGN.md index 7efc0f0..2c759ff 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -33,18 +33,18 @@ now done and is what `install.sh` and `scripts/build-bundle.sh` produce. fetches nothing else. Because the config files carry the fixed layout, the paths above are not configurable at install time. - **Why three things are built (at release).** Most components are upstream release - artifacts. Three have no upstream release and are compiled in the build: urunc - (from `urunc-dev/urunc` at commit `0b1ef07`, on - `feat/unchanged_containers-exec-fixes`, CGO static, so native per arch under - qemu/binfmt), urunit (from `NOFireAI/urunit` at commit `71bfdee`, on - `urunit_agent`, C static), and the brig `container-initrd`. The two sources are + artifacts, hvi among them (`brig-sh/hvi-vmm` `v0.1.0-rc.1`, cosign-verified). + Three have no upstream release and are compiled in the build: urunc + (from `urunc-dev/urunc` at commit `3c00b69`, on + `feat/linux-hvi-v050`, CGO static, so native per arch under + qemu/binfmt), urunit (from `NOFireAI/urunit` at the commit urunc's + `urunit.pin` names, C static) and the brig `container-initrd`. The sources are pinned to a commit, not to a branch: a branch tip moves with every push, so it is not a release input. The README's build section says how to move a pin. - brig execs into a guest through an in-guest agent (`urunit-agent`) whose - protocol must match the urunc shim, so hull-assets' prebuilt initrd (hull's - agent) is not usable; the build assembles a brig initrd from urunit + - `urunit-agent` (from the same urunc commit as the shim) + busybox + urunc's - `container-init`. The kernel is still fetched, not built: from + brig execs into a guest through the agent in urunit, whose protocol must match + the urunc shim, so hull-assets' prebuilt initrd is not usable; the build + assembles a brig initrd of urunit alone, the early init, supervisor and agent + in one static binary, at the commit the shim's urunc pins. The kernel is still fetched, not built: from `ghcr.io/nofireai/hull-assets` on both arches, by digest, once its cosign signature checks out. Because urunc and the initrd ship in one tarball built together, their agent diff --git a/README.md b/README.md index 75479d6..0cbd7e2 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ $ sha256sum -c checksums.txt --ignore-missing /var/lib/brig/ data/ immutable, unpacked from the tarball: bin/ brig, brigd, urunc, containerd-shim-urunc-v2, containerd, ctr, - runc, nerdctl, cosign, oras, firecracker, cloud-hypervisor, - solo5-hvt, solo5-spt, brig-ctl, brig-pool-up, brig-uninstall.sh + runc, nerdctl, cosign, oras, hvi, firecracker, + cloud-hypervisor, solo5-hvt, solo5-spt, brig-ctl, + brig-pool-up, brig-uninstall.sh libexec/ virtiofsd libexec/cni/ CNI plugins etc/ urunc.toml, containerd.toml, nerdctl.toml, brig-env.sh, @@ -280,8 +281,9 @@ the three that do not: | Component | Source | In the tarball | | --- | --- | --- | | brig, brigd | `brig-sh/brig` release | fetched, verified against its signed `checksums.txt` | -| urunc, containerd-shim-urunc-v2 | built from `urunc-dev/urunc` at commit `0b1ef07` (branch `feat/unchanged_containers-exec-fixes`) | CGO-static, built in a Go container | -| urunit | built from `NOFireAI/urunit` at commit `71bfdee` (branch `urunit_agent`) | C-static; goes into the initrd | +| urunc, containerd-shim-urunc-v2 | built from `urunc-dev/urunc` at commit `3c00b69` (branch `feat/linux-hvi-v050`) | CGO-static, built in a Go container | +| urunit | built from `NOFireAI/urunit` at the commit urunc's `packaging/container-initrd/urunit.pin` names (`48a61bd`, branch `stack/10-acpi-power-button`) | C-static; the whole initrd | +| hvi | `brig-sh/hvi-vmm` release `v0.1.0-rc.1` (commit `65570c7`), the one hull ships on macOS | fetched, musl-static; its `SHA256SUMS` cosign-verified against hvi-vmm's release workflow, the tarball pinned by sha256 | | container-initrd | built from the above | assembled for brig, not fetched | | guest kernel | `ghcr.io/nofireai/hull-assets` | fetched by digest once its cosign signature checks out; the kernel hull and brig boot | | monitors, virtiofsd | `urunc-dev/monitors-build` | fetched, pinned by sha256 | @@ -296,16 +298,28 @@ uninstaller, and packs the whole `/var/lib/brig/data` tree. A branch tip is not a release input. It moves with every push, so two builds of one bundle version would carry different code. So `build-bundle.sh` pins -each of the two to one commit, `URUNC_REF_DEFAULT` and `URUNIT_REF_DEFAULT`. -The build fetches that commit by its SHA, fails if the checkout is anything -else, and records it in `pins.env` as `URUNC_REF` and `URUNIT_REF`. CI checks -that record on every bundle it builds. - -The urunc pin is `feat/unchanged_containers` plus the two `urunit-agent` exec -fixes, urunc-dev/urunc#1059 and urunc-dev/urunc#1060. It also carries a commit -that sizes a container boot's vCPUs from the container's CPU limit, so -`brig run --cpus N` gives the guest N vCPUs. The urunit pin is what -v0.1.0-rc6 to rc8 shipped. +urunc to one commit, `URUNC_REF_DEFAULT`, and takes urunit from the commit that +urunc pins in `packaging/container-initrd/urunit.pin`, so the initrd always +matches the shim that talks to it. The build fetches each commit by its SHA, +fails if the checkout is anything else, and records it in `pins.env` as +`URUNC_REF` and `URUNIT_REF`. CI checks that record on every bundle it builds. +`URUNIT_REPO` and `URUNIT_REF` override the urunit pin for a one-off build. + +The urunc pin is `feat/linux-hvi-v050`: `feat/unchanged_containers` with +the exec fixes, the hvi monitor, urunit's single binary as the initrd, the +overlay-block root (`BRIG_ROOTFS_MODE=overlay-block`) and stop signals handed to +the guest's init. It also sizes a container boot's vCPUs from the container's +CPU limit, so `brig run --cpus N` gives the guest N vCPUs. + +The bundle ships hvi first in the monitor list and writes a `[monitors.hvi]` +section, but it names no default monitor: urunc boots whatever monitor the +`com.urunc.unikernel.hypervisor` annotation asks for, and brig sets that +annotation. A brig that knows hvi on Linux boots hvi unless `BRIG_HYPERVISOR` +names another monitor in the bundle (`cloud-hypervisor` stays installed for +that); an older brig keeps asking for cloud-hypervisor and gets it. The same +holds for the overlay-block root: the bundle's urunc understands +`com.urunc.unikernel.rootfsMode`, and brig sends it when +`BRIG_ROOTFS_MODE=overlay-block` is set. Moving a pin is a reviewed one-line change to the `*_REF_DEFAULT` line, plus its `*_BRANCH_DEFAULT` when the new commit comes from another branch. A one-off @@ -327,22 +341,22 @@ was meant. ### Why the initrd is brig's own, not hull-assets' -brig execs into a guest through an in-guest agent, `urunit-agent`, whose wire -protocol (`pkg/agentproto`) is a contract with the urunc shim. The two agree only -when both are built from the same urunc commit, and hull-assets' prebuilt initrd -carries hull's agent. So `build-bundle.sh` builds a brig initrd with urunc's own -`packaging/container-initrd` tooling, from `urunit`, a `urunit-agent` built from -the same urunc commit as the shim, a static `busybox`, and urunc's -`container-init`. The urunc binary and the initrd's agent ship in one tarball, so -they always match. Only the kernel is fetched rather than built. It is generic, +brig execs into a guest through the exec agent in urunit, whose wire protocol +(`pkg/agentproto`) is a contract with the urunc shim. The two agree when urunit +is the commit the urunc checkout pins, and hull-assets' prebuilt initrd may +carry another. So `build-bundle.sh` builds a brig initrd with urunc's own +`packaging/container-initrd` tooling: `/urunit`, one static binary that is the +early init, the supervisor and the agent, with `/init` and `/vz-init` linked to +it. The urunc binary and the initrd's agent ship in one tarball, so they always +match. Only the kernel is fetched rather than built. It is generic, and comes from hull-assets on both arches, checked against its signature. Two properties of the initrd are checked when the build packs it and again when the runtime boots it, because each fails in a way that does not name itself: `container-initrd` must not be compressed (the runtime appends the container's argv, environment and network identity to a copy of it, and a gzipped one accepts -the append and boots without it), and `busybox` must be built for the target -architecture, or it panics init on its first line. +the append and boots without it), and urunit must be built for the target +architecture, or the kernel cannot run init. The boot assets land in `share/guest/` and the launcher points brig at them through `BRIG_BOOT_ASSETS`, so brig uses the packed assets rather than fetching diff --git a/docs/variants.md b/docs/variants.md index e0c5d28..1a4fb61 100644 --- a/docs/variants.md +++ b/docs/variants.md @@ -45,11 +45,11 @@ by `build-bundle.sh` and packed into the tarball. The installer just unpacks the No urunc release publishes a binary that boots a generic container. So the build compiles one, from `urunc-dev/urunc` at the commit `build-bundle.sh` pins -(`0b1ef07`, on branch `feat/unchanged_containers-exec-fixes`), in a -`golang:1.26.4` container: +(`3c00b69`, on branch `feat/linux-hvi-v050`), in a +`golang:1.26.8` container: ```console -# docker run --rm -v :/app -w /app -e HOME=/tmp golang:1.26.4 \ +# docker run --rm -v :/app -w /app -e HOME=/tmp golang:1.26.8 \ sh -c "git config --global --add safe.directory /app && make static" ``` @@ -85,20 +85,16 @@ boots on macOS. `pins.env` records it as `KERNEL_SOURCE`, by the digest the buil pulled. `share/guest/SHA256SUMS` holds the sha256 of the kernel and the initrd, and each release publishes it as `.boot-assets.sha256`: see the README. -**The initrd is not taken from hull-assets.** brig execs into a guest through an -in-guest agent, `urunit-agent`, whose wire protocol (`pkg/agentproto`) is a -contract with the urunc shim. The two agree only when both are built from the same -urunc commit, and hull-assets' prebuilt initrd carries hull's agent. So the build -assembles a brig initrd, with urunc's -`packaging/container-initrd/build-container-initrd.sh`, from four pieces: - -- `container-init`, urunc's early-userspace script (mounts the shared rootfs, - stages urunit and the agent, `switch_root`s into urunit); -- `urunit`, the tiny C init that becomes PID 1, built static from - `NOFireAI/urunit` at the pinned commit `71bfdee` (branch `urunit_agent`); -- `urunit-agent`, built from the **same** urunc commit that produced the shim, - so the protocol matches by construction; -- a static `busybox`. +**The initrd is not taken from hull-assets.** brig execs into a guest through the +exec agent in urunit, whose wire protocol (`pkg/agentproto`) is a contract with +the urunc shim. The two agree when urunit is the commit the shim's urunc pins in +`packaging/container-initrd/urunit.pin`, and hull-assets' prebuilt initrd may +carry another. So the build assembles a brig initrd, with urunc's +`packaging/container-initrd/build-container-initrd.sh`, from one binary: + +- `/urunit`, built static from `NOFireAI/urunit` at the commit urunc pins. It is + the early init (`/init` and `/vz-init` are links to it), the PID 1 supervisor + and the exec agent. urunc and the initrd ship in one tarball, built together, so their agent commit matches by construction. The installer just fetches and unpacks the tarball; it @@ -111,8 +107,8 @@ runtime boots it, because each fails in a way that does not name itself: argv, environment and network identity to a copy of it, and concatenated cpio works only while both halves are uncompressed. A gzipped one accepts the append and boots without it. -- `busybox` must be built for this architecture. Under the wrong one it exits 127 - on every applet and panics init on the first line. +- `urunit` must be built for this architecture. Under the wrong one the kernel + cannot run init and panics. The annotations that carry the pair are `com.urunc.unikernel.bootKernel` and `com.urunc.unikernel.bootInitrd`, pointing into `share/guest/`. hull takes the @@ -148,9 +144,8 @@ reaps it per container. `build-bundle.sh` would install it as `bin/sidecar`; where its build comes from is unsettled. **hvi is a fifth monitor from a different repo.** The matrix scores hvi beside -firecracker, but hvi is built in `brig-sh/hvi-vmm`, not in the monitors release -the build draws from. Adding it means either a second source in `build-bundle.sh` -or getting hvi into `monitors-build`. +firecracker, but hvi is released by `brig-sh/hvi-vmm`, not by the monitors +release the build draws from. `build-bundle.sh` fetches it as a second source. ## Running the matrix @@ -171,12 +166,14 @@ refuses a stock binary, rather than printing a table of zeroes. 3. Settled: the guest kernel is fetched from `hull-assets`, by the same tags hull uses, and checked against its signature; the runtime is built from `urunc-dev/urunc` at a pinned commit - (`0b1ef07`, on `feat/unchanged_containers-exec-fixes`); and the initrd is - built for brig from `NOFireAI/urunit` at a pinned commit (`71bfdee`, on - `urunit_agent`) plus urunc's own `packaging/container-initrd`, so its agent - matches the shim. Both pins are commits because a branch tip moves with every - push; the README's build section says how to move one. -4. Does hvi join `monitors-build`, or does `build-bundle.sh` learn a second - source? + (`3c00b69`, on `feat/linux-hvi-v050`); and the initrd is built for + brig from `NOFireAI/urunit` at the commit that urunc pins, with urunc's own + `packaging/container-initrd`, so its agent matches the shim. Both are + commits because a branch tip moves with every push; the README's build + section says how to move one. +4. Settled: `build-bundle.sh` fetches hvi from the `brig-sh/hvi-vmm` release + hull ships on macOS (`v0.1.0-rc.1`). `scripts/fetch-hvi.sh` checks the + cosign signature on its `SHA256SUMS`, the pinned sha256 of the tarball and + the commit its manifest names. 5. Is `introspection` a variant at all, or `generic-boot` plus an opt-in flag? Everything it adds is additive, which argues for the flag. diff --git a/scripts/build-bundle.sh b/scripts/build-bundle.sh index 0e56ba5..fcab247 100755 --- a/scripts/build-bundle.sh +++ b/scripts/build-bundle.sh @@ -10,9 +10,9 @@ # private-containerd stack (containerd, ctr, runc, nerdctl, CNI), the microVM # monitors, the urunc runtime, the guest kernel and a brig-built initrd, plus # every config file, systemd unit, wrapper and the uninstaller. Most components -# are upstream release artifacts; three are built here because no release ships -# them: urunc (from urunc-dev/urunc), urunit (from NOFireAI/urunit) and the -# container-initrd (assembled from both). +# are upstream release artifacts, hvi among them. Three are built here because +# no release ships them: urunc (from urunc-dev/urunc), urunit (from +# NOFireAI/urunit) and the container-initrd (assembled from urunc and urunit). # # scripts/build-bundle.sh --arch amd64 --version v0.1.0 --out dist # scripts/build-bundle.sh --arch arm64 --version v0.1.0 --variant generic-boot @@ -22,10 +22,10 @@ # because a node install driven by root never uses any of it, and a user # install cannot work without it. # -# docker is required (the urunc/urunit builds run in containers). Building the -# arm64 pieces on an amd64 runner (or vice versa) needs binfmt/qemu registered, -# because the static urunc binary and urunit are CGO/C and are built native -# inside a target-arch container (docker run --platform). +# docker is required (the urunc and urunit builds run in containers). Building +# the arm64 pieces on an amd64 runner (or vice versa) needs binfmt/qemu +# registered, because the static urunc binary and urunit are CGO/C and are +# built native inside a target-arch container (docker run --platform). # # Produces, under --out: # brig-standalone--linux-.tar.gz (the install tarball) @@ -94,7 +94,7 @@ done # The brig release this bundle carries. On Linux it is the brig a user gets, # so it is pinned like every other component. Bump it with the brig release a # bundle is cut for. BRIG_VERSION=latest still works for a local build. -BRIG_VERSION="${BRIG_VERSION:-v0.4.0}" +BRIG_VERSION="${BRIG_VERSION:-v0.4.1}" BRIG_REPO="${BRIG_REPO:-brig-sh/brig}" # urunc and urunit have no release that carries what brig needs, so they are # built from source, and the source is pinned to one commit each. A branch tip @@ -112,30 +112,58 @@ BRIG_REPO="${BRIG_REPO:-brig-sh/brig}" # urunc: feat/unchanged_containers on top of urunc v0.8.1, plus the two # urunit-agent exec fixes, urunc-dev/urunc#1059 and #1060. It also carries a # commit that sizes a container boot's vCPUs from the container's CPU limit, -# one that boots arm64 guests on cloud-hypervisor's virtio console, and one -# that passes image_type=raw on every --disk (urunc-dev/urunc#1117), which -# cloud-hypervisor v54.0 requires. The agent in the initrd is built from this -# checkout too. +# one that boots arm64 guests on cloud-hypervisor's virtio console, one that +# passes image_type=raw on every --disk (urunc-dev/urunc#1117), which +# cloud-hypervisor v54.0 requires, and the hvi monitor with exec over its agent +# socket. On top of that, feat/linux-hvi-v050: urunit's single binary as +# the whole initrd, the overlay-block root (com.urunc.unikernel.rootfsMode), +# and stop signals handed to the guest init, which it lets power off when the +# agent ends the signal session. It writes urunit.conf as the NUL-terminated +# records of urunc-dev/urunc#1111. The stack sits on urunc main as of +# 89dfe8d, with the bind-mount fixes of urunc-dev/urunc#1150, and presses a +# cloud-hypervisor guest's power button when its init cannot take the signal. URUNC_REPO="${URUNC_REPO:-urunc-dev/urunc}" -URUNC_BRANCH_DEFAULT=feat/unchanged_containers-exec-fixes -URUNC_REF_DEFAULT=0b1ef07229d678e7cbb3f61e12a8efcdd5070a8a -URUNC_GO_IMAGE="${URUNC_GO_IMAGE:-golang:1.26.4}" -# urunit: the init in the initrd. What v0.1.0-rc6 to rc8 shipped. -URUNIT_REPO="${URUNIT_REPO:-NOFireAI/urunit}" -URUNIT_BRANCH_DEFAULT=urunit_agent -URUNIT_REF_DEFAULT=71bfdeefb7bced121c4e75afa97550523af34152 +URUNC_BRANCH_DEFAULT=feat/linux-hvi-v050 +URUNC_REF_DEFAULT=3c00b69ff9ac1b8bb260975cdc63d18a7043558b +URUNC_GO_IMAGE="${URUNC_GO_IMAGE:-golang:1.26.8}" +# urunit: the one binary of the initrd: early init, supervisor and exec agent. +# urunc pins it in packaging/container-initrd/urunit.pin, read after the urunc +# checkout below, so the initrd always matches the shim that talks to it. +# URUNIT_REPO and URUNIT_REF override the pin. URUNIT_REF= builds a branch +# tip, so it needs URUNIT_BRANCH too. +# hvi: the monitor brig boots a generic container on. It is the +# brig-sh/hvi-vmm release hull ships on macOS, static against musl. The tag, +# its commit and the sha256 of each tarball describe one release, so a bump +# changes all four together. scripts/fetch-hvi.sh checks them. +HVI_VERSION=0.1.0-rc.1 +HVI_COMMIT=65570c78e6830ec6166fd5f9079998a47535b09f +HVI_SHA256_LINUX_AMD64=1fc870d4946c08a576fe775b631ed225dc5687611f73c118b14b19dd75469853 +HVI_SHA256_LINUX_ARM64=277c0cfbd9a669f79e9819b35b327a9d815128e181215fa46618acca05bf9acc # A branch given without a ref would otherwise build the pinned commit and # ignore the branch, so ask which one was meant. if [ -n "${URUNC_BRANCH:-}" ] && [ "$URUNC_BRANCH" != "$URUNC_BRANCH_DEFAULT" ] && [ -z "${URUNC_REF+set}" ]; then fatal "URUNC_BRANCH=$URUNC_BRANCH is set but URUNC_REF is not. Set URUNC_REF= to build that branch's tip, or URUNC_REF= to build one commit from it." fi -if [ -n "${URUNIT_BRANCH:-}" ] && [ "$URUNIT_BRANCH" != "$URUNIT_BRANCH_DEFAULT" ] && [ -z "${URUNIT_REF+set}" ]; then +if [ -n "${URUNIT_BRANCH:-}" ] && [ -z "${URUNIT_REF+set}" ]; then fatal "URUNIT_BRANCH=$URUNIT_BRANCH is set but URUNIT_REF is not. Set URUNIT_REF= to build that branch's tip, or URUNIT_REF= to build one commit from it." fi +if [ -n "${URUNIT_REF+set}" ] && [ -z "$URUNIT_REF" ] && [ -z "${URUNIT_BRANCH:-}" ]; then + fatal "URUNIT_REF= builds a branch tip, but URUNIT_BRANCH names no branch. Set URUNIT_BRANCH= too." +fi +if [ -n "${HVI_REF+set}" ] || [ -n "${HVI_BRANCH+set}" ]; then + fatal "hvi is no longer built from source: HVI_REF and HVI_BRANCH do nothing. Move HVI_VERSION and its pins to another release instead." +fi URUNC_BRANCH="${URUNC_BRANCH:-$URUNC_BRANCH_DEFAULT}" URUNC_REF="${URUNC_REF-$URUNC_REF_DEFAULT}" -URUNIT_BRANCH="${URUNIT_BRANCH:-$URUNIT_BRANCH_DEFAULT}" -URUNIT_REF="${URUNIT_REF-$URUNIT_REF_DEFAULT}" +# Empty when urunit comes from urunc's pin: urunit.pin names no branch. +URUNIT_BRANCH="${URUNIT_BRANCH:-}" +# Unset takes urunc's pin, after the urunc checkout. Set and empty builds the +# tip of URUNIT_BRANCH. +URUNIT_FROM_PIN=false +if [ -z "${URUNIT_REF+set}" ]; then + URUNIT_FROM_PIN=true + URUNIT_REF="" +fi # The checkout compares HEAD with the ref, so a ref is a full SHA: a short one, # a tag or a branch name would never match. for r in "URUNC_REF=$URUNC_REF" "URUNIT_REF=$URUNIT_REF"; do @@ -146,9 +174,6 @@ for r in "URUNC_REF=$URUNC_REF" "URUNIT_REF=$URUNIT_REF"; do esac [ "${#v}" -eq 40 ] || fatal "${r%%=*}='$v' is not a commit: pin a full 40-character SHA" done -# Static musl busybox for the initrd, per arch. Extracted from this image so we -# do not depend on busybox.net, which publishes 1.35.0 for x86_64 only. -BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-busybox:1.36.1-musl}" # The monitors and virtiofsd come from one monitors-build release. The release # is tagged vX.Y.Z and names its per-arch tarball after the monitor versions it # carries. monitors-build publishes no checksum file, so each tarball is pinned @@ -177,10 +202,17 @@ ORAS_VERSION="${ORAS_VERSION:-v1.3.4}" COSIGN_VERSION="${COSIGN_VERSION:-v3.1.3}" COSIGN_SHA_LINUX_AMD64="4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71" COSIGN_SHA_LINUX_ARM64="c5d324e091826b0d7a78eb16fef316450b4eb9aaec045611c08ba06f5e73220a" -MONITORS="${MONITORS:-firecracker cloud-hypervisor solo5-hvt solo5-spt}" +# hvi comes from its own release; the rest come from the monitors release. +# stock leaves hvi out: it ships a released urunc, and no urunc release knows +# hvi yet. +if [ "$VARIANT" = "stock" ]; then + MONITORS="${MONITORS:-firecracker cloud-hypervisor solo5-hvt solo5-spt}" +else + MONITORS="${MONITORS:-hvi firecracker cloud-hypervisor solo5-hvt solo5-spt}" +fi ASSETS_REGISTRY="${ASSETS_REGISTRY:-ghcr.io}" ASSETS_REPO="${ASSETS_REPO:-nofireai/hull-assets}" -ASSETS_VERSION="${ASSETS_VERSION:-0.1.9}" +ASSETS_VERSION="${ASSETS_VERSION:-0.1.10}" # Who signs hull-assets: the identity brig checks before it boots a kernel from # there. The build checks the same one before it packs the kernel. ASSETS_SIGNER_DEFAULT='^https://github\.com/NOFireAI/hull-assets/\.github/workflows/build-assets\.yml@refs/heads/main$' @@ -941,7 +973,7 @@ checkout_source() { # and warn when it was a branch tip. pinned_note() { if [ -n "$4" ]; then - info "$1: checked out $2@$5 (pinned; from branch $3)" + info "$1: checked out $2@$5 (pinned${3:+; from branch $3})" else info "WARNING: $1: built $2@$5, the tip of $3 at build time. It is not pinned," info "WARNING: and a rebuild of this bundle version may carry different code." @@ -987,8 +1019,8 @@ fi # ---- urunit + the brig container-initrd (generic-boot / introspection) ------- # brig builds its own initrd rather than shipping hull-assets': it execs into a -# guest through urunit-agent, which has to match the urunc shim's protocol, so -# the agent is built from the same urunc checkout ($src) as the shim above. +# guest through the agent in urunit, which has to match the urunc shim's +# protocol, so urunit is the commit the same urunc checkout ($src) pins. URUNIT_PINNED="" if [ "$VARIANT" = "stock" ]; then # stock boots the unikernel's own init, so there is no urunit to build. @@ -997,6 +1029,25 @@ else command -v docker >/dev/null 2>&1 || fatal "docker is required to build the brig initrd" [ -d "${src:-}" ] || fatal "the urunc checkout is needed to build the initrd (build urunc from source)" usrc="$TMP_DIR/urunit-src" + if [ "$URUNIT_FROM_PIN" = true ]; then + pin="$src/packaging/container-initrd/urunit.pin" + [ -f "$pin" ] || fatal "$URUNC_REPO@$URUNC_REF has no urunit.pin; set URUNIT_REPO and URUNIT_REF" + pin_repo="$(sed -n 's/^URUNIT_REPO=//p' "$pin" | head -1)" + pin_repo="${pin_repo#https://github.com/}" + URUNIT_REPO="${URUNIT_REPO:-${pin_repo%.git}}" + URUNIT_REF="$(sed -n 's/^URUNIT_REF=//p' "$pin" | head -1)" + case "$URUNIT_REF" in + *[!0-9a-f]*|"") fatal "urunit.pin names '$URUNIT_REF', not a commit" ;; + esac + [ "${#URUNIT_REF}" -eq 40 ] || fatal "urunit.pin names '$URUNIT_REF', not a full 40-character SHA" + fi + URUNIT_REPO="${URUNIT_REPO:-NOFireAI/urunit}" + # checkout_source puts the repo after https://github.com/. + case "$URUNIT_REPO" in + */*/*|/*|*/|*[!A-Za-z0-9._/-]*) fatal "URUNIT_REPO='$URUNIT_REPO' is not / on GitHub" ;; + */*) ;; + *) fatal "URUNIT_REPO='$URUNIT_REPO' is not / on GitHub" ;; + esac URUNIT_PINNED=true [ -n "$URUNIT_REF" ] || URUNIT_PINNED=false ut_commit="$(checkout_source "$URUNIT_REPO" "$URUNIT_BRANCH" "$URUNIT_REF" "$usrc")" @@ -1008,20 +1059,13 @@ else || fatal "urunit build failed" [ -s "$usrc/dist/urunit_static" ] || fatal "no urunit_static produced by the build" - # urunc's build-container-initrd.sh would fetch busybox from busybox.net, - # which only publishes 1.35.0 for x86_64 (the aarch64 URL 404s). Supply a - # static busybox for the target arch from the official musl image and pass - # it via BUSYBOX so the script never downloads. - info "extracting a static busybox ($BUSYBOX_IMAGE, linux/$ARCH)" - docker run --rm --platform "linux/$ARCH" "$BUSYBOX_IMAGE" cat /bin/busybox > "$DL/busybox" \ - || fatal "could not extract busybox from $BUSYBOX_IMAGE" - [ -s "$DL/busybox" ] || fatal "extracted busybox is empty" - chmod 0755 "$DL/busybox" - - info "building the brig container-initrd (urunit + urunit-agent + busybox)" + # The single-binary layout: /urunit, with /init and /vz-init linked to it. + # URUNIT hands the script the binary built above, so it neither builds + # urunit itself nor needs a container tool inside this one. + info "building the brig container-initrd (urunit $URUNIT_REF)" docker run --rm --platform "linux/$ARCH" \ - -v "$src":/app -v "$usrc/dist/urunit_static":/urunit-static:ro -v "$DL/busybox":/busybox-in:ro \ - -w /app -e HOME=/tmp -e URUNIT=/urunit-static -e BUSYBOX=/busybox-in -e TARGET_ARCH="$ARCH_UNAME" "$URUNC_GO_IMAGE" \ + -v "$src":/app -v "$usrc/dist/urunit_static":/urunit-static:ro \ + -w /app -e HOME=/tmp -e URUNIT=/urunit-static -e TARGET_ARCH="$ARCH_UNAME" "$URUNC_GO_IMAGE" \ sh -c "apt-get update >/dev/null 2>&1 && apt-get install -y cpio >/dev/null 2>&1 && git config --global --add safe.directory /app && ./packaging/container-initrd/build-container-initrd.sh /app/dist/container-initrd" \ || fatal "container-initrd build failed" [ -s "$src/dist/container-initrd" ] || fatal "no container-initrd produced" @@ -1077,6 +1121,21 @@ c_actual="$(sha256sum "$DL/cosign" | awk '{print $1}')" install -m 0755 "$DL/cosign" "$STAGE/bin/cosign" info " cosign: verified against the pinned hash" +# ---- hvi: its release, checked with the cosign above ------------------------- +HVI_INSTALLED=false +case " $MONITORS " in + *" hvi "*) + case "$ARCH" in + amd64) hvi_sha="$HVI_SHA256_LINUX_AMD64" ;; + arm64) hvi_sha="$HVI_SHA256_LINUX_ARM64" ;; + esac + HVI_VERSION="$HVI_VERSION" HVI_COMMIT="$HVI_COMMIT" HVI_SHA256="$hvi_sha" \ + COSIGN="$STAGE/bin/cosign" sh "$SELF_DIR/fetch-hvi.sh" "$ARCH" "$DL/hvi" \ + || fatal "could not fetch a verified hvi $HVI_VERSION" + HVI_INSTALLED=true + ;; +esac + # ---- the private-stack components -------------------------------------------- case "$ARCH" in amd64) m_sha="$MONITORS_SHA_LINUX_AMD64" ;; @@ -1151,6 +1210,7 @@ tar -xzf "$DL/monitors.tar.gz" -C "$TMP_DIR/monitors" --wildcards '*urunc/bin/*' mon_bin="$TMP_DIR/monitors/urunc/bin" for m in $MONITORS; do case "$m" in + hvi) install -m 0755 "$DL/hvi" "$STAGE/bin/hvi"; continue ;; firecracker|cloud-hypervisor|solo5-hvt|solo5-spt) src_m="$m" ;; qemu) src_m="qemu-system-$ARCH_UNAME" ;; *) fatal "unknown monitor '$m' in the pins" ;; @@ -1238,6 +1298,7 @@ destination = "$LOG_DIR/timestamps.log" CFG for m in $MONITORS; do case "$m" in + hvi) sect=hvi; bin=hvi ;; firecracker) sect=firecracker; bin=firecracker ;; cloud-hypervisor) sect=cloud-hypervisor; bin=cloud-hypervisor ;; solo5-hvt) sect=hvt; bin=solo5-hvt ;; @@ -1464,6 +1525,8 @@ URUNIT_REPO=$([ "$VARIANT" = "stock" ] && echo "" || echo "$URUNIT_REPO") URUNIT_BRANCH=$([ "$VARIANT" = "stock" ] && echo "" || echo "$URUNIT_BRANCH") URUNIT_REF=$URUNIT_REF URUNIT_PINNED=$URUNIT_PINNED +HVI_VERSION=$([ "$HVI_INSTALLED" = true ] && echo "$HVI_VERSION" || echo "") +HVI_COMMIT=$([ "$HVI_INSTALLED" = true ] && echo "$HVI_COMMIT" || echo "") INITRD_SOURCE=$([ "$VARIANT" = "stock" ] && echo "" || echo "built:$URUNC_REF") PREFIX=$PREFIX DATA_DIR=$DATA_DIR diff --git a/scripts/fetch-hvi.sh b/scripts/fetch-hvi.sh new file mode 100755 index 0000000..a3919eb --- /dev/null +++ b/scripts/fetch-hvi.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# +# Fetch the hvi release build-bundle.sh pins, verify it and copy the binary out. +# +# scripts/fetch-hvi.sh +# +# The pins come from the environment: HVI_VERSION (the tag without its v), +# HVI_COMMIT (the full commit the release was built from) and HVI_SHA256 (the +# sha256 of this arch's tarball). COSIGN names the cosign binary, v3 or later. +# +# SHA256SUMS must carry a cosign bundle signed by hvi-vmm's tag-release +# workflow for the pinned tag. The tarball must match both SHA256SUMS and +# HVI_SHA256, and its manifest must name HVI_COMMIT and the sha256 of the hvi +# binary in it. hull's scripts/fetch-hvi.sh checks the macOS tarball of the +# same release the same way. + +set -eu + +die() { + echo "fetch-hvi: $*" >&2 + exit 1 +} + +[ $# -eq 2 ] || die "usage: $0 " +arch="$1" +out="$2" + +: "${HVI_VERSION:?fetch-hvi: HVI_VERSION is not set}" +: "${HVI_COMMIT:?fetch-hvi: HVI_COMMIT is not set}" +: "${HVI_SHA256:?fetch-hvi: HVI_SHA256 is not set}" +cosign="${COSIGN:-cosign}" + +case "$arch" in + amd64) target=x86_64-unknown-linux-musl ;; + arm64) target=aarch64-unknown-linux-musl ;; + *) die "unknown arch '$arch'" ;; +esac +case "$HVI_COMMIT" in + *[!0-9a-f]*) die "HVI_COMMIT=$HVI_COMMIT is not a commit" ;; +esac +[ "${#HVI_COMMIT}" -eq 40 ] || die "HVI_COMMIT=$HVI_COMMIT is not a full 40-character SHA" +case "$HVI_SHA256" in + *[!0-9a-f]*) die "HVI_SHA256=$HVI_SHA256 is not a lowercase sha256" ;; +esac +[ "${#HVI_SHA256}" -eq 64 ] || die "HVI_SHA256=$HVI_SHA256 is not a sha256" + +tag="v$HVI_VERSION" +archive="hvi-$HVI_VERSION-$target.tar.gz" +identity="^https://github\\.com/brig-sh/hvi-vmm/\\.github/workflows/tag-release\\.yml@refs/tags/$(printf '%s' "$tag" | sed 's/\./\\./g')\$" +issuer=https://token.actions.githubusercontent.com +base="https://github.com/brig-sh/hvi-vmm/releases/download/$tag" + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +for f in "$archive" SHA256SUMS SHA256SUMS.cosign.bundle; do + curl -fsSL --retry 3 -o "$tmp/$f" "$base/$f" || die "cannot download $base/$f" +done + +"$cosign" verify-blob \ + --bundle "$tmp/SHA256SUMS.cosign.bundle" \ + --certificate-identity-regexp "$identity" \ + --certificate-oidc-issuer "$issuer" \ + "$tmp/SHA256SUMS" >/dev/null \ + || die "SHA256SUMS of $tag is not signed by brig-sh/hvi-vmm's tag-release workflow" + +listed="$(awk -v f="$archive" '$2 == f || $2 == "*" f { print $1; exit }' "$tmp/SHA256SUMS")" +[ -n "$listed" ] || die "SHA256SUMS lists no $archive" +got="$(sha256sum "$tmp/$archive" | awk '{print $1}')" +[ "$got" = "$listed" ] || die "$archive is $got, SHA256SUMS says $listed" +[ "$got" = "$HVI_SHA256" ] || die "$archive is $got, the pin says $HVI_SHA256" + +mkdir "$tmp/x" +tar -xzf "$tmp/$archive" -C "$tmp/x" hvi manifest.json || die "$archive has no hvi or manifest.json" +# The manifest is flat JSON with one key per line. +rev="$(sed -n 's/^ *"revision": *"\([0-9a-f]*\)".*/\1/p' "$tmp/x/manifest.json")" +[ "$rev" = "$HVI_COMMIT" ] || die "the manifest names commit '$rev', the pin says $HVI_COMMIT" +want="$(sed -n 's/^ *"hvi": *"\([0-9a-f]*\)".*/\1/p' "$tmp/x/manifest.json")" +have="$(sha256sum "$tmp/x/hvi" | awk '{print $1}')" +[ -n "$want" ] || die "the manifest names no sha256 for hvi" +[ "$have" = "$want" ] || die "hvi is $have, the manifest says '$want'" + +install -m 0755 "$tmp/x/hvi" "$out" +echo "fetch-hvi: hvi $HVI_VERSION ($HVI_COMMIT, $target), signature and sums verified" diff --git a/tests/fetch-hvi.sh b/tests/fetch-hvi.sh new file mode 100755 index 0000000..a91ac62 --- /dev/null +++ b/tests/fetch-hvi.sh @@ -0,0 +1,130 @@ +#!/bin/sh +# +# Run scripts/fetch-hvi.sh against a stub curl and cosign, and check that it +# installs hvi only when every check passes. The stub curl serves a release +# built here; the stub cosign accepts the bundle only when it reads "good" and +# the identity is hvi-vmm's tag-release workflow for the tag. + +set -eu + +here="$(cd "$(dirname "$0")/.." && pwd)" + +skip() { echo "SKIP: $*"; exit 0; } +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo "ok - $*"; } + +command -v sha256sum >/dev/null 2>&1 || skip "fetch-hvi.sh needs sha256sum" + +T="$(mktemp -d)" +trap 'rm -rf "$T"' EXIT +mkdir -p "$T/stub" + +version=9.9.9-rc.1 +commit=5555555555555555555555555555555555555555 +archive="hvi-$version-x86_64-unknown-linux-musl.tar.gz" + +sha() { sha256sum "$1" | awk '{print $1}'; } + +# release [rev] [manifest-sha]: build the release in $T/rel. The manifest names +# rev and the sha of hvi unless told otherwise. +release() { + rm -rf "$T/rel" "$T/pkg" + mkdir -p "$T/rel" "$T/pkg" + echo "hvi $version" > "$T/pkg/hvi" + chmod 0755 "$T/pkg/hvi" + cat > "$T/pkg/manifest.json" < "$T/rel/SHA256SUMS" + echo good > "$T/rel/SHA256SUMS.cosign.bundle" +} + +cat > "$T/stub/curl" < "$T/stub/cosign" <<'STUB' +#!/bin/sh +while [ $# -gt 1 ]; do + case "$1" in + --bundle) bundle="$2"; shift 2 ;; + --certificate-identity-regexp) id="$2"; shift 2 ;; + *) shift ;; + esac +done +[ "$id" = '^https://github\.com/brig-sh/hvi-vmm/\.github/workflows/tag-release\.yml@refs/tags/v9\.9\.9-rc\.1$' ] || exit 1 +[ "$(cat "$bundle")" = good ] +STUB +chmod 0755 "$T/stub"/* + +# fetch [VAR=value...]: run fetch-hvi.sh for amd64, output in $T/out.log. +fetch() { + rm -f "$T/hvi" + env PATH="$T/stub:$PATH" COSIGN="$T/stub/cosign" \ + HVI_VERSION=$version HVI_COMMIT=$commit HVI_SHA256="$(sha "$T/rel/$archive")" "$@" \ + sh "$here/scripts/fetch-hvi.sh" amd64 "$T/hvi" > "$T/out.log" 2>&1 +} + +# refused [VAR=value...]: fetch must fail with and install +# nothing. +refused() { + msg="$1"; shift + if fetch "$@"; then cat "$T/out.log" >&2; fail "fetch-hvi.sh passed: $msg"; fi + grep -q "$msg" "$T/out.log" || { cat "$T/out.log" >&2; fail "no '$msg' in the output"; } + [ ! -e "$T/hvi" ] || fail "hvi was installed although: $msg" +} + +# 1. A release that checks out is installed. +release +fetch || { cat "$T/out.log" >&2; fail "a good release was refused"; } +grep -qx "hvi $version" "$T/hvi" || fail "the installed hvi is not the release's" +ok "a signed release that matches its pins is installed" + +# 2. A bundle cosign rejects. +echo forged > "$T/rel/SHA256SUMS.cosign.bundle" +refused "is not signed by brig-sh/hvi-vmm's tag-release workflow" +ok "an unsigned SHA256SUMS is refused" + +# 3. A tarball that SHA256SUMS does not list. +release +echo "x" >> "$T/rel/$archive" +refused "SHA256SUMS says" HVI_SHA256="$(sha "$T/rel/$archive")" +ok "a tarball other than the one SHA256SUMS lists is refused" + +# 4. A signed release whose tarball is not the pinned one. +release +refused "the pin says" HVI_SHA256=0000000000000000000000000000000000000000000000000000000000000000 +ok "a tarball other than the pinned one is refused" + +# 5. A manifest that names another commit. +release 6666666666666666666666666666666666666666 +refused "the manifest names commit '6666666666666666666666666666666666666666'" +ok "a release built from another commit is refused" + +# 6. A binary that is not the one the manifest names. +release "$commit" 7777777777777777777777777777777777777777777777777777777777777777 +refused "the manifest says '7777777777777777777777777777777777777777777777777777777777777777'" +ok "a binary other than the one the manifest names is refused" + +# 7. Pins that are not a commit and a sha are refused before anything is fetched. +release +refused "is not a full 40-character SHA" HVI_COMMIT=65570c7 +refused "is not a sha256" HVI_SHA256=abc +ok "a short commit or sha pin is refused" diff --git a/tests/source-pins.sh b/tests/source-pins.sh index e46f6d8..c40c003 100755 --- a/tests/source-pins.sh +++ b/tests/source-pins.sh @@ -2,8 +2,9 @@ # # Run build-bundle.sh against a stub git and docker, and check which urunc and # urunit commits it asks for. Nothing is fetched: the stub git records the ref -# it was asked to fetch and answers rev-parse with it, the stub docker fakes the -# urunc build and fails the urunit build, which ends the run. +# it was asked to fetch, answers rev-parse with it and plants the urunit.pin +# the build reads from the urunc checkout; the stub docker fakes the urunc +# build and fails the urunit build, which ends the run. # # Needs the tools build-bundle.sh checks for (curl, tar, sha256sum, awk, sed). @@ -18,24 +19,29 @@ ok() { echo "ok - $*"; } command -v sha256sum >/dev/null 2>&1 || skip "build-bundle.sh needs sha256sum" urunc_pin="$(sed -n 's/^URUNC_REF_DEFAULT=\([0-9a-f]*\)$/\1/p' "$here/scripts/build-bundle.sh")" -urunit_pin="$(sed -n 's/^URUNIT_REF_DEFAULT=\([0-9a-f]*\)$/\1/p' "$here/scripts/build-bundle.sh")" urunc_branch="$(sed -n 's/^URUNC_BRANCH_DEFAULT=\(.*\)$/\1/p' "$here/scripts/build-bundle.sh")" [ "${#urunc_pin}" -eq 40 ] || fail "no URUNC_REF_DEFAULT in build-bundle.sh" -[ "${#urunit_pin}" -eq 40 ] || fail "no URUNIT_REF_DEFAULT in build-bundle.sh" tip=1111111111111111111111111111111111111111 +# The urunit commit the stub urunc checkout pins in its urunit.pin. The build +# has no URUNIT_REF_DEFAULT of its own: it reads that file. +urunit_pin=3333333333333333333333333333333333333333 T="$(mktemp -d)" trap 'rm -rf "$T"' EXIT mkdir -p "$T/stub" # git: log every call. fetch remembers what it was asked for, and rev-parse -# HEAD answers with it, or with $STUB_HEAD, or with $tip for a branch. +# HEAD answers with it, or with $STUB_HEAD, or with $tip for a branch. A +# checkout of the urunc source plants urunit.pin, as the real commit carries +# one; STUB_NO_PIN=1 leaves it out, and STUB_PIN_REPO replaces its repo. cat > "$T/stub/git" <> "$T/git.log" +dir=. while [ \$# -gt 0 ]; do case "\$1" in - -C|-c) shift 2 ;; + -C) dir="\$2"; shift 2 ;; + -c) shift 2 ;; *) break ;; esac done @@ -46,11 +52,21 @@ case "\$1" in if [ -n "\${STUB_HEAD:-}" ]; then echo "\$STUB_HEAD" elif [ "\${#f}" -eq 40 ]; then echo "\$f" else echo $tip; fi ;; + checkout) + case "\$dir" in + */urunc-src) + if [ -z "\${STUB_NO_PIN:-}" ]; then + mkdir -p "\$dir/packaging/container-initrd" + printf 'URUNIT_REPO=%s\nURUNIT_REF=%s\n' \ + "\${STUB_PIN_REPO:-https://github.com/NOFireAI/urunit.git}" $urunit_pin \\ + > "\$dir/packaging/container-initrd/urunit.pin" + fi ;; + esac ;; esac exit 0 STUB -# docker: the urunc build writes the two binaries the script copies out; the +# docker: the urunc build writes the binaries the script copies out; the # urunit build fails. cat > "$T/stub/docker" <<'STUB' #!/bin/sh @@ -78,18 +94,21 @@ build() { : > "$T/fetched" # Every run ends in a failure, by design: at the stub urunit build or at # the check under test. - env -u URUNC_REF -u URUNC_BRANCH -u URUNIT_REF -u URUNIT_BRANCH \ + env -u URUNC_REF -u URUNC_BRANCH -u URUNIT_REF -u URUNIT_BRANCH -u HVI_REF -u HVI_BRANCH \ PATH="$T/stub:$PATH" "$@" \ sh "$here/scripts/build-bundle.sh" --arch amd64 --version test --out "$T/dist" \ > "$T/out.log" 2>&1 || true } -# 1. Defaults: both repos are fetched at their pinned commit, by SHA. +# 1. Defaults: urunc is fetched at its pinned commit, by SHA; urunit at the +# commit the urunc checkout's urunit.pin names. build grep -qx "git -C .*/urunc-src fetch -q --depth 1 origin $urunc_pin" "$T/git.log" \ || { cat "$T/git.log" "$T/out.log" >&2; fail "urunc was not fetched at its pin"; } grep -qx "git -C .*/urunit-src fetch -q --depth 1 origin $urunit_pin" "$T/git.log" \ - || { cat "$T/git.log" "$T/out.log" >&2; fail "urunit was not fetched at its pin"; } + || { cat "$T/git.log" "$T/out.log" >&2; fail "urunit was not fetched at urunc's pin"; } +grep -q "urunit: checked out NOFireAI/urunit@$urunit_pin (pinned" "$T/out.log" \ + || { cat "$T/out.log" >&2; fail "the build log does not name the pinned urunit"; } grep -q "urunc: checked out urunc-dev/urunc@$urunc_pin (pinned" "$T/out.log" \ || { cat "$T/out.log" >&2; fail "the build log does not name the pinned urunc"; } if grep -q -e "git clone" -e "WARNING" "$T/git.log" "$T/out.log"; then @@ -97,6 +116,25 @@ if grep -q -e "git clone" -e "WARNING" "$T/git.log" "$T/out.log"; then fi ok "a default build fetches urunc and urunit at their pinned commits" +# 1b. URUNIT_REF overrides urunc's pin for one build, and a urunc commit with no +# urunit.pin is refused rather than guessed. +other=4444444444444444444444444444444444444444 +build URUNIT_REF=$other +grep -qx "git -C .*/urunit-src fetch -q --depth 1 origin $other" "$T/git.log" \ + || { cat "$T/git.log" "$T/out.log" >&2; fail "URUNIT_REF did not override urunc's pin"; } +build STUB_NO_PIN=1 +grep -q "has no urunit.pin" "$T/out.log" \ + || { cat "$T/out.log" >&2; fail "a urunc commit without urunit.pin was not refused"; } +ok "URUNIT_REF overrides the pin, and a urunc without one is refused" + +# 1c. A urunit.pin whose repo is not / on GitHub is refused before +# urunit is fetched. +build STUB_PIN_REPO=git@github.com:NOFireAI/urunit.git +grep -q "URUNIT_REPO='git@github.com:NOFireAI/urunit' is not / on GitHub" "$T/out.log" \ + || { cat "$T/out.log" >&2; fail "a urunit.pin with an ssh URL was not refused"; } +if grep -q "urunit-src" "$T/git.log"; then fail "urunit was fetched from a refused repo"; fi +ok "a urunit.pin repo that is not / is refused" + # 2. A checkout that is not the pinned commit fails the build. build STUB_HEAD=2222222222222222222222222222222222222222 grep -q "checked out 2222222222222222222222222222222222222222, not the pinned $urunc_pin" "$T/out.log" \ @@ -118,6 +156,18 @@ grep -q "URUNC_BRANCH=feat/something-else is set but URUNC_REF is not" "$T/out.l [ ! -s "$T/git.log" ] || fail "git ran for a refused build" ok "a branch without a ref is refused" +# 4b. URUNIT_REF= with no branch to build the tip of is refused, and so is the +# hvi source build that no longer exists. +build URUNIT_REF= +grep -q "URUNIT_REF= builds a branch tip, but URUNIT_BRANCH names no branch" "$T/out.log" \ + || { cat "$T/out.log" >&2; fail "URUNIT_REF= without a branch was not refused"; } +[ ! -s "$T/git.log" ] || fail "git ran for a refused build" +build HVI_REF=24dc2c5f643ae23551ceb158600676c33636a21f +grep -q "hvi is no longer built from source" "$T/out.log" \ + || { cat "$T/out.log" >&2; fail "HVI_REF was not refused"; } +[ ! -s "$T/git.log" ] || fail "git ran for a refused build" +ok "URUNIT_REF= without a branch, and HVI_REF, are refused" + # 5. A ref that is not a full SHA is refused before anything is fetched. build URUNIT_REF=urunit_agent grep -q "URUNIT_REF='urunit_agent' is not a commit" "$T/out.log" \