diff --git a/.github/actions/install-linux-packages/action.yml b/.github/actions/install-linux-packages/action.yml new file mode 100644 index 0000000..a363639 --- /dev/null +++ b/.github/actions/install-linux-packages/action.yml @@ -0,0 +1,83 @@ +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Install the Linux packages +description: >- + Installs Ubuntu packages from a cache of their archives, and downloads what the + cache lacks eight files at a time, continuing what was cut and trying the next + mirror when one is slow, in 15 minutes at most (see install.sh), because the + Ubuntu mirror of the runners is slow for some of them at times. + +inputs: + packages: + description: The packages, separated by spaces or new lines. + required: true + +runs: + using: composite + steps: + - name: Name the cache + id: names + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + # A cache for each list of packages and each week: a week is long enough + # for many runs to use it, and short enough to stay near the versions + # of the mirror. The newest cache of any earlier week (or what an + # interrupted run saved, see below) starts the cache of a new week, + # which install.sh trims. + hash="$(echo "$PACKAGES" | tr -s '[:space:]' '\n' | sort -u | sha256sum | cut -c1-16)" + echo "prefix=apt-${ImageOS:-ubuntu}-$hash-" >> "$GITHUB_OUTPUT" + echo "key=apt-${ImageOS:-ubuntu}-$hash-$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" + - name: Let the cache write the archives + shell: bash + run: sudo chmod a+rwx /var/cache/apt/archives + - name: Restore the archives + id: cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /var/cache/apt/archives/*.deb + key: ${{ steps.names.outputs.key }} + restore-keys: ${{ steps.names.outputs.prefix }} + - name: Install the packages + id: install + shell: bash + env: + PACKAGES: ${{ inputs.packages }} + run: | + # shellcheck disable=SC2086 # The list is meant to be split. + "$GITHUB_ACTION_PATH/install.sh" $PACKAGES + # The cache is saved by the user of the runner, and one archive that this user + # cannot read fails the whole save (tar did on one archive of Wails, on + # 2026-10-23, and nothing was saved). It also holds when the installation failed. + - name: Let the cache read the archives + if: always() + shell: bash + run: sudo chmod a+r /var/cache/apt/archives/*.deb || true + - name: Save the archives + if: steps.install.outcome == 'success' && steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /var/cache/apt/archives/*.deb + key: ${{ steps.names.outputs.key }} + # A download that did not finish within its time is not lost: the next run + # starts from the archives it got (by the prefix above) instead of from + # nothing, so that a run on a slow mirror gets further each time. + - name: Save what was downloaded + if: always() && steps.install.outcome == 'failure' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /var/cache/apt/archives/*.deb + key: ${{ steps.names.outputs.prefix }}partial-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.github/actions/install-linux-packages/fetch.sh b/.github/actions/install-linux-packages/fetch.sh new file mode 100755 index 0000000..feb43d6 --- /dev/null +++ b/.github/actions/install-linux-packages/fetch.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Makes one archive of a package available for install.sh, which starts several of +# these at the same time: +# +# fetch.sh
... +# +# The hash is the one that apt prints, TYPE:digest (SHA256:... with +# -o Acquire::ForceHash=SHA256). apt checks the hash of what it downloads, but takes +# an archive that is in the directory with its size for a good one, so that the +# archive has to be checked here, before it is moved there. An archive that is +# there already is checked too, and removed when it is wrong, which is what a damaged +# cache would otherwise break the installation with. +# +# The addresses are those of the file on the mirrors, in the order to try them. The +# download goes to partial/ first, where apt keeps what it is downloading, and +# continues from what is there, so that a connection that is cut, or dropped for being +# slow, loses nothing. +# +# A connection that gets less than APT_MIN_SPEED bytes a second (262144 by default) +# for APT_SPEED_TIME seconds (10) is dropped, and the download continues from the next +# mirror: a mirror is slow for some runners and not for others, and the speed of the +# next one is not known before it is tried. When every mirror has been slow in turn, a +# slow download is better than none: the best of them continues, and only a stalled +# connection is dropped. A file that no mirror sends in three rounds, or that has the +# wrong hash three times, is given up. +# +# APT_DEADLINE, in seconds since the epoch, is the time when the download ends. +set -uo pipefail + +[[ $# -ge 5 ]] || { echo "Usage: fetch.sh
..." >&2; exit 2; } + +directory="$1" +size="$2" +name="$3" +hash="$4" +shift 4 +addresses=("$@") +deadline="${APT_DEADLINE:?APT_DEADLINE is the time when the download ends, in seconds since the epoch}" +minimum="${APT_MIN_SPEED:-262144}" +window="${APT_SPEED_TIME:-10}" +final="$directory/$name" +partial="$directory/partial/$name" +error="$(mktemp)" +trap 'rm -f "$error"' EXIT + +# The size of what has been downloaded. +have() { + stat --format %s "$partial" 2> /dev/null || echo 0 +} + +# right : whether the file has the hash that apt printed. +right() { + local tool + case "${hash%%:*}" in + MD5Sum) tool=md5sum ;; + SHA1) tool=sha1sum ;; + SHA256) tool=sha256sum ;; + SHA512) tool=sha512sum ;; + *) return 1 ;; + esac + [[ "$("$tool" "$1" 2> /dev/null | cut -d ' ' -f 1)" == "${hash#*:}" ]] +} + +if [[ -e "$final" ]]; then + if [[ "$(stat --format %s "$final")" -eq "$size" ]] && right "$final"; then + exit 0 + fi + echo "$name: the archive in the directory is not the one that apt expects; removed" + rm -f "$final" +fi + +fruitless=0 +wrong=0 +last="" +while true; do + if [[ "$(have)" -eq "$size" ]]; then + right "$partial" && break + echo "$name: the hash is not $hash; starting again" + rm -f "$partial" + wrong=$((wrong + 1)) + ((wrong < 3)) || { echo "$name: no mirror sends the right file" >&2; exit 1; } + # The mirror that completed it is the last one to try again. + addresses=() + for address in "$@"; do + [[ "$address" == "$last" ]] || addresses+=("$address") + done + [[ -z "$last" ]] || addresses+=("$last") + fi + # More than the file has is not a part of it. + [[ "$(have)" -le "$size" ]] || rm -f "$partial" + start="$(have)" + best="" + best_gain=0 + for address in "${addresses[@]}"; do + before="$(have)" + [[ "$before" -lt "$size" ]] || break + left=$((deadline - $(date +%s))) + ((left > 0)) || exit 1 + last="$address" + speed="$(curl --fail --silent --show-error --location --connect-timeout 10 --max-time "$left" \ + --speed-limit "$minimum" --speed-time "$window" --continue-at - \ + --write-out '%{speed_download}' --output "$partial" "$address" 2> "$error")" + status=$? + # A server that does not continue a download sends all of the file, which curl + # does not take (33): start again. + [[ $status -ne 33 ]] || rm -f "$partial" + gain=$(($(have) - before)) + if ((gain > best_gain)); then + best="$address" + best_gain=$gain + fi + if [[ "$(have)" -ne "$size" ]]; then + host="${address#*://}" + echo "$name: $(have) of $size bytes, ${speed%.*} bytes a second from ${host%%/*}: $(tr '\n' ' ' < "$error")" + fi + done + # All of it came, to be checked at the top. + [[ "$(have)" -ne "$size" ]] || continue + if [[ "$(have)" -gt "$start" ]]; then + fruitless=0 + else + fruitless=$((fruitless + 1)) + ((fruitless < 3)) || { echo "$name: no mirror sends it" >&2; exit 1; } + fi + # Every mirror was slow in turn, or failed: go on with the best of them. + minimum=1000 + if [[ -n "$best" ]]; then + addresses=("$best") + for address in "$@"; do + [[ "$address" == "$best" ]] || addresses+=("$address") + done + fi + sleep 1 +done +mv "$partial" "$final" diff --git a/.github/actions/install-linux-packages/install.sh b/.github/actions/install-linux-packages/install.sh new file mode 100755 index 0000000..c1a438d --- /dev/null +++ b/.github/actions/install-linux-packages/install.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Installs Ubuntu packages on a runner of GitHub Actions in two steps, which the +# action next to this file (action.yml) wraps in a cache: +# +# 1. Downloads the packages that the cache lacks into /var/cache/apt/archives, in +# 15 minutes at most. fetch.sh does it, eight files at a time and the largest +# first, from the mirrors of the runner: it continues what was cut, tries the +# next mirror when one is slow, and checks the hash of every archive, those of +# the cache too, because apt takes an archive of the right size for a good one. +# Then apt fetches what is left. +# 2. Installs them from the archives, which needs no network. +# +# Why not apt alone: the Ubuntu mirror of the runners is slow for some of them. On +# 2026-10-23 a job got 40 KB/s for a connection while other jobs of the same minute +# downloaded 90 MB in seconds, and its 94 MB did not come in 30 minutes (a minute on a +# good day), which a job of 25 or 45 minutes cannot wait for; a file of 25 MB alone +# takes 10 minutes at that speed. apt downloads from one mirror, over one connection, +# one file after the other, however slow it is. +# +# Finally it removes the archives of versions that the package lists no longer +# have, so that the cache does not grow from week to week. +# +# Usage: install.sh ... +# +# Set APT_DOWNLOAD_SECONDS to change the time of the download (900): it has to end +# before the timeout of the job, for the action to save what was downloaded. +# APT_PARALLEL is the number of files that are downloaded at the same time (8; 0 +# leaves all of it to apt). fetch.sh has two more, APT_MIN_SPEED and APT_SPEED_TIME. +set -euo pipefail + +[[ $# -gt 0 ]] || { echo "Usage: install.sh ..." >&2; exit 2; } + +packages=("$@") +seconds="${APT_DOWNLOAD_SECONDS:-900}" +parallel="${APT_PARALLEL:-8}" +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +archives=/var/cache/apt/archives +deadline=$(($(date +%s) + seconds)) + +# A request that gets no data for 30 seconds fails and is tried again, up to 3 +# times; apt waits for 2 minutes by default. The archives stay where they are +# after the installation: a runner has no hook that removes them, but a +# container of Docker has. +apt_options=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 + -o APT::Keep-Downloaded-Packages=true) + +# apt_get