Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,20 @@ jobs:
fi

# Every tool for every platform, cross-built on linux hosts with the pinned
# zig, one runner a group of tools, the way a release builds them. The
# verify legs build their own platform and run it (`just test`).
# zig, one runner a group of tools, the way a release builds them; and the
# tools nothing cross-compiles (hadolint, in a container), on a runner of
# each linux architecture. The verify legs build their own platform and run
# it (`just test`).
build:
needs: plan
strategy:
fail-fast: false
matrix:
tools: ${{ fromJSON(needs.plan.outputs.groups) }}
name: build (${{ matrix.tools }})
runs-on: ubuntu-24.04
timeout-minutes: 30
include: ${{ fromJSON(needs.plan.outputs.groups) }}
name: build (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
# hadolint is a compiler's work, twice: built, then built again.
timeout-minutes: 60
permissions:
contents: read
steps:
Expand Down
21 changes: 14 additions & 7 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
# per-job permissions, bash everywhere).
#
# A pushed v* tag builds every tool for every platform on linux hosts (zig
# cross-compiles them all), one runner a group of tools, runs each archive's
# cross-compiles them all but hadolint, which a container builds on a runner
# of each linux architecture), one runner a group of tools, runs each archive's
# binaries on its own platform, then attests every archive and checksums.txt
# and attaches them to the release. workflow_dispatch is the dry run: build
# and smoke, no publish.
#
# Nothing here is restored from a cache, and nothing is saved to one: what is
# published is built from the pinned sources by this run alone. ci.yaml keeps
# published is built from the pinned sources by this run alone (hadolint's
# from its pinned image as well, and the alpine packages its lock names). ci.yaml keeps
# zig's runtime libraries between runs; this file must not.
name: release

Expand Down Expand Up @@ -59,10 +61,10 @@ jobs:
strategy:
# Every group or no release: a failed group stops the others.
matrix:
tools: ${{ fromJSON(needs.plan.outputs.groups) }}
name: build (${{ matrix.tools }})
runs-on: ubuntu-24.04
timeout-minutes: 30
include: ${{ fromJSON(needs.plan.outputs.groups) }}
name: build (${{ matrix.name }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 40
permissions:
contents: read
steps:
Expand Down Expand Up @@ -94,7 +96,7 @@ jobs:
run: |
read -r -a tools <<< "$TOOLS"
just build-group "${tools[@]}"
echo "name=${TOOLS// /+}" >> "$GITHUB_OUTPUT"
echo "name=${TOOLS// /+}-${RUNNER_ARCH}" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: dist-${{ steps.group.outputs.name }}
Expand Down Expand Up @@ -147,6 +149,11 @@ jobs:
bash erofs/smoke.sh "unpacked/erofs-${PLATFORM}"
bash xz/smoke.sh "unpacked/xz-${PLATFORM}"
bash graphviz/smoke.sh "unpacked/graphviz-${PLATFORM}"
# hadolint is built for linux alone.
case "${PLATFORM}" in
linux-*) bash hadolint/smoke.sh "unpacked/hadolint-${PLATFORM}" ;;
*) ;;
esac
bash claxon/smoke.sh "unpacked/claxon-${PLATFORM}" "${flac}"
bash symphonia/smoke.sh "unpacked/symphonia-${PLATFORM}" "${flac}"

Expand Down
107 changes: 99 additions & 8 deletions .justfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,21 @@ platforms := "linux-amd64 linux-arm64 darwin-arm64 windows-amd64 windows-arm64"
# a runner of their own. A tool missing here is built by nothing.
build_groups := "flac|microflac|xz|erofs|graphviz|drflac foxenflac|flake cscoreflac|claxon symphonia"

# The tools no runner cross-compiles, in groups as above: each is built on a
# runner of every architecture it has a platform for, for that runner's own
# platform alone (hadolint: GHC compiles for the machine it runs on).
native_groups := "hadolint"

# The FIRST recipe defined here becomes `just`'s default.
lint: do::lint::default do::lint::rust::default do::lint::converge
fix: do::fix::default do::fix::rust::default
# Each tool built for this host and run on it.
test: flac-test drflac-test foxenflac-test microflac-test cscoreflac-test flake-test erofs-test xz-test graphviz-test claxon-test symphonia-test
security: do::security::default

# Build every tool for every platform into $LIMEN_BUILD/dist.
build: (build-group replace(build_groups, "|", " "))
# Build every tool for every platform into $LIMEN_BUILD/dist; of the native
# tools, this host's platform.
build: (build-group replace(build_groups, "|", " ")) (build-group replace(native_groups, "|", " "))

# Build the named tools for every platform into $LIMEN_BUILD/dist.
build-group +tools:
Expand Down Expand Up @@ -66,16 +72,25 @@ rebuild +tools:
done
exit "${failed}"

# build_groups as the JSON array a workflow's matrix reads, one string a group.
# The groups as the JSON array a workflow's matrix reads, one object a build:
# its tools, the runner that builds them, and the name the two make. A group
# of build_groups is one build, on the runner that cross-compiles; a group of
# native_groups is one build a linux architecture, on a runner of it.
build-matrix:
#!/usr/bin/env bash
set -euo pipefail
IFS='|' read -r -a groups <<< "{{ build_groups }}"
IFS='|' read -r -a native <<< "{{ native_groups }}"
printf '['
sep=""
for group in "${groups[@]}"; do
printf '%s"%s"' "${sep}" "${group}"
printf '%s{"tools":"%s","runner":"ubuntu-24.04","name":"%s"}' "${sep}" "${group}" "${group}"
sep=","
done
for group in "${native[@]}"; do
printf '%s{"tools":"%s","runner":"ubuntu-24.04","name":"%s, linux-amd64"}' "${sep}" "${group}" "${group}"
sep=","
printf '%s{"tools":"%s","runner":"ubuntu-24.04-arm","name":"%s, linux-arm64"}' "${sep}" "${group}" "${group}"
done
printf ']\n'

Expand Down Expand Up @@ -593,6 +608,61 @@ _dotnet-rid platform:
*) echo "unknown platform {{ platform }} (one of {{ platforms }})" >&2; exit 1 ;;
esac

# Build hadolint, static on musl, for the linux platform of this host's
# architecture among those given, archived with upstream's licence into
# $LIMEN_BUILD/dist/hadolint-<platform>.tar.gz. The other platforms given are
# passed over: GHC builds for the machine it runs on, so the other linux
# architecture is built on a runner of it, and there is no hadolint here for
# macOS or windows.
hadolint +targets: (_fetch "hadolint")
#!/usr/bin/env bash
set -euo pipefail
case "$(uname -m)" in
x86_64) arch=amd64 ;;
arm64 | aarch64) arch=arm64 ;;
*) echo "hadolint: no build on $(uname -m)" >&2; exit 1 ;;
esac
version=$(limen pins get hadolint version)
src="${LIMEN_BUILD}/src/hadolint-${version}"
mkdir -p "${LIMEN_BUILD}/dist"
set -- {{ targets }}
for platform in "$@"; do
if [ "${platform}" != "linux-${arch}" ]; then
echo "hadolint: ${platform} is not built on this host (linux-${arch} is)"
continue
fi
just _fetch-file "ghc-${platform}"
just _fetch-file "cabal-${platform}"
out="${LIMEN_BUILD}/hadolint/${platform}"
rm -rf "${out}"
HADOLINT_COMMIT="${version}" HADOLINT_URL="$(limen pins get hadolint url)" hadolint/build.sh "${src}" \
"${LIMEN_BUILD}/src/ghc-${platform}-$(limen pins get "ghc-${platform}" version).tar.xz" \
"${LIMEN_BUILD}/src/cabal-${platform}-$(limen pins get "cabal-${platform}" version).tar.xz" \
"${arch}" "${out}/bin"
# Run where it was built, which is always here: no leg of `just test`
# pays for a compiler's build to do it.
hadolint/smoke.sh "${out}/bin"
name="hadolint-${platform}"
stage="${LIMEN_BUILD}/stage/${name}"
rm -rf "${stage}"
mkdir -p "${stage}"
cp "${out}/bin/hadolint" "${out}/bin/NOTICES" "${out}/bin/SOURCE" "${out}/bin/COPYING.LESSER" "${stage}/"
cp "${src}/LICENSE" "${stage}/"
scripts/archive.sh "${LIMEN_BUILD}/stage" "${name}" "${LIMEN_BUILD}/dist"
done

# Build hadolint for this host, where it is a linux one, and run it.
hadolint-test:
#!/usr/bin/env bash
set -euo pipefail
platform=$(just _host-platform)
case "${platform}" in
linux-*) ;;
*) echo "hadolint: built for linux alone, nothing to run on ${platform}"; exit 0 ;;
esac
just hadolint "${platform}"
hadolint/smoke.sh "${LIMEN_BUILD}/hadolint/${platform}/bin"

# The release notes' source section: each tool's upstream release, as pins.yaml
# pins it, the corresponding source of the binaries built from it.
release-notes:
Expand All @@ -615,6 +685,12 @@ release-notes:
echo "- XZ Utils $(limen pins get xz version): $(limen pins get xz url)"
echo "- Graphviz $(limen pins get graphviz version): $(limen pins get graphviz url)"
echo "- expat $(limen pins get expat version): $(limen pins get expat url)"
echo "- hadolint $(limen pins get hadolint version): $(limen pins get hadolint url),"
echo " a GPL-3 binary, statically linked: the SOURCE file in its archive names every component in it at"
echo " its exact version and where its source is (GHC's release, each Hackage package of this repository's"
echo " hadolint/cabal.project.freeze, the alpine packages of hadolint/apk-linked), and this repository's"
echo " hadolint/ at this release's tag builds it again, to the same bytes for as long as Alpine serves the"
echo " packages hadolint/apk.lock names"
echo "- Claxon $(just _crate-version claxon): https://static.crates.io/crates/claxon/claxon-$(just _crate-version claxon).crate"
echo "- Symphonia $(just _crate-version symphonia) and the crates it links, each from crates.io at the version and"
echo " checksum Cargo.lock records: $(just _crate-closure testbin-symphonia | tr '\n' ' ')"
Expand All @@ -623,7 +699,25 @@ release-notes:
# against pins.yaml on every run (a reused build directory keeps the tarball),
# and unpack it into $LIMEN_BUILD/src/<name>-<version>, whatever the top
# directory upstream's tarball has.
_fetch name:
_fetch name: (_fetch-file name)
#!/usr/bin/env bash
set -euo pipefail
version=$(limen pins get {{ name }} version)
url=$(limen pins get {{ name }} url)
case "${url}" in
*.tar.xz) ext=tar.xz ;;
*) ext=tar.gz ;;
esac
dir="${LIMEN_BUILD}/src"
archive="${dir}/{{ name }}-${version}.${ext}"
rm -rf "${dir:?}/{{ name }}-${version}"
mkdir -p "${dir}/{{ name }}-${version}"
tar -xf "${archive}" -C "${dir}/{{ name }}-${version}" --strip-components=1

# Fetch a pinned artifact to $LIMEN_BUILD/src/<name>-<version>.<ext> and
# verify it against pins.yaml on every run, without unpacking it: a toolchain
# is unpacked where it is installed.
_fetch-file name:
#!/usr/bin/env bash
set -euo pipefail
version=$(limen pins get {{ name }} version)
Expand All @@ -643,9 +737,6 @@ _fetch name:
rm -f "${archive}"
exit 1
fi
rm -rf "${dir:?}/{{ name }}-${version}"
mkdir -p "${dir}/{{ name }}-${version}"
tar -xf "${archive}" -C "${dir}/{{ name }}-${version}" --strip-components=1

# The rustc flags that keep the build out of a Rust binary, as
# CARGO_ENCODED_RUSTFLAGS takes them. Its directories: the checkout, cargo's
Expand Down
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ other uses, or covered by any compatibility promise.
| [`erofs/`](erofs/) | `mkfs.erofs`, `fsck.erofs` (not windows) | [erofs/erofs-utils](https://github.com/erofs/erofs-utils), with [lz4](https://github.com/lz4/lz4) |
| [`xz/`](xz/) | `xz` | [tukaani-project/xz](https://github.com/tukaani-project/xz) |
| [`graphviz/`](graphviz/) | `dot` | [graphviz](https://gitlab.com/graphviz/graphviz), with [expat](https://github.com/libexpat/libexpat) |
| [`hadolint/`](hadolint/) | `hadolint` (linux alone; a GPL-3 binary, published to be run by tests and linked by nothing) | [hadolint/hadolint](https://github.com/hadolint/hadolint), at a commit, not a release |
| [`claxon/`](claxon/) | `claxon`, a FLAC decoder on Claxon | [ruuda/claxon](https://github.com/ruuda/claxon) |
| [`symphonia/`](symphonia/) | `symphonia`, a FLAC decoder on Symphonia | [pdeljanov/Symphonia](https://github.com/pdeljanov/Symphonia) |

Expand All @@ -32,7 +33,9 @@ sources, and states the upstream licence its sources, patches and binaries are u
`.hermetic/hermetic.yaml`, and the Rust tools link through it (limen's Rust lanes, the
toolchain pinned by `rust-toolchain.toml`), with the target always explicit: linux (static,
musl) and windows, amd64 and arm64, and darwin arm64, cross-compiled from any of them. No runner
package manager, no container.
package manager, no container. One exception: `hadolint` is a Haskell program, which nothing
here cross-compiles; it is built for linux alone, in a container, on a runner of each
architecture ([`hadolint/`](hadolint/)).
- **Third-party source only.** Everything here builds an upstream project's own sources,
patched only where a platform needs it (`erofs/patches`, for windows); a library that ships
no program gets the smallest one around it that exercises it (`drflac`, `foxenflac`,
Expand All @@ -44,6 +47,8 @@ sources, and states the upstream licence its sources, patches and binaries are u
- **A build is its commit's.** Built again from the same commit, in another directory and at
another time, every archive is the same bytes, so a published one can be checked by building
it: `just build-group <tools>`, then `just rebuild <tools>`, which CI runs for every group.
(`hadolint` for as long as Alpine serves the packages its lock names: after that its build
stops, and says which moved.)
Nothing of the build is recorded. The binaries carry no debug information and no symbols (they
name the build's directories); a Rust binary's paths are remapped and a .NET one's mapped; a
windows executable's time stamp is a hash of its content; and every archive entry has the
Expand Down
58 changes: 58 additions & 0 deletions hadolint/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# hadolint

`hadolint` from [hadolint/hadolint](https://github.com/hadolint/hadolint), built for linux amd64
and arm64, static on musl, from the commit `pins.yaml` pins: a commit of upstream's master, not
a release. A consumer runs it as an oracle against that exact commit. The binary prints the
version of upstream's last release (`Haskell Dockerfile Linter 2.15.1`), which is not the pin:
the pin is the commit, and the archive's `SOURCE` file names it.

There is no hadolint here for macOS or windows. An oracle is needed on one platform, and linux
is the one upstream's static build exists for: one container recipe covers both architectures
and every input of it can be held. A macOS binary would be another build (Apple's toolchain, no
static C library, about twice the time on a runner), and GHC has no compiler for windows arm64.

It is built the way upstream builds its own static release, in an alpine container. Its inputs,
and how each is held:

- `versions.env`: the alpine image, by digest.
- `pins.yaml` (repository root): hadolint's source, GHC (each tarball checked against its
release manager's PGP signature) and cabal-install, one entry an architecture.
- `apk.lock`: every alpine package the container holds once the build's own are installed, the
linker and the assembler as much as the libraries. The image fixes what it ships; these are
installed into it at whatever alpine serves, so the build compares and stops when one moved.
Alpine keeps one version of a package, so a build at an old tag stops then, and says so.
- `apk-linked`: of those, the ones whose code is in the binary (musl, GMP, gcc's runtime).
Measured, not listed: the build reads the linker's map and stops when the packages it took
from are not exactly these.
- `cabal.project.freeze` and `cabal.project.local`: every Haskell package at one version, at
one state of Hackage. Upstream commits no freeze file and resolves on the day of the build;
this one is ours, and the build stops if cabal would resolve anything else.
- `build.sh` starts the container; `in-container.sh` is what runs in it. GHC compiles for the
machine it runs on, so each architecture is built on a runner of it.
- `smoke.sh` runs the binary on a fixed Dockerfile and requires the JSON report of the one rule
it breaks, and the licence files beside it.

Two builds of one commit give the same bytes, while alpine serves the locked packages: the
build job builds it twice and compares.

## Licence

hadolint is GPL-3 (`LICENSE` in the archive), and the binary is statically linked, so the
archive is a GPL work: it is published to be run by tests, and nothing in this organisation
links it. Each archive carries:

- `LICENSE`, the GNU GPL version 3, and `COPYING.LESSER`, the GNU LGPL version 3, under which
GMP is linked in.
- `NOTICES`: the licence files of what is in the binary, taken from the build's own plan (the
libraries the executable depends on, to the end): each Hackage package's licence files from
the top of its source archive, GHC's libraries' from GHC's release, and the texts kept in
`licenses/` by version for what those do not carry: musl, gcc's runtime exception, the
libffi GHC links into its runtime, and the xxHash that GHC's runtime and `hashable` each
compile into themselves, whose terms are in a header. A licence carried only inside a source file is found by reading, as
that one was, not by the build.
- `SOURCE`: the offer of the corresponding source. Every component at its exact version and
where its source is: hadolint at the commit, GHC's release, each Hackage package, each alpine
package with the commit of alpine's aports that built it, and this recipe, which builds it
again, to the same bytes while alpine serves the locked packages.

This directory's own files are under the repository's Apache-2.0 licence.
11 changes: 11 additions & 0 deletions hadolint/apk-linked
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# The alpine packages whose code is in the binary, each at its version: the C
# library, GMP (GHC's integers), and gcc's own runtime (its start files, and
# libgcc). Not a claim: the build reads the link map and stops when the
# packages the linker took from are not exactly these. zlib and libstdc++
# are installed and nothing is taken from them. Each package here has its
# licence in the archive's NOTICES and its source in SOURCE, so a line that
# moves here is read again there.
musl-dev 1.2.6-r3
gmp-static 6.3.0-r4
gcc 15.2.0-r5
libgcc-static 15.2.0-r5
Loading
Loading