|
3 | 3 | # |
4 | 4 | # EVERY merge to main runs this. Each language's engine id (src/pipeline/run-souffle.sh |
5 | 5 | # --print-engine-id) is a hash of that language's rules alone, so the plan step decides per |
6 | | -# language: an id that already has a release is unchanged and skipped; an id without one is |
7 | | -# built on every platform. A Python-only merge therefore rebuilds Python on all four targets |
8 | | -# and nothing else; a merge touching no rule set builds nothing and finishes in seconds. For |
9 | | -# each language that needs building: |
| 6 | +# language: an id equal to the one recorded in engine/binaries/<lang>/ENGINE_ID is unchanged |
| 7 | +# and skipped; any other id is built on every platform. A Python-only merge therefore rebuilds |
| 8 | +# Python on all four targets and nothing else; a merge touching no rule set builds nothing and |
| 9 | +# finishes in seconds. For each language that needs building: |
10 | 10 | # generate Ubuntu + the pinned Soufflé .deb: emit the program, `souffle -g` it to portable |
11 | 11 | # C++ once, and capture the header tree the C++ compiles against. |
12 | 12 | # build compile that C++ on every platform with the SAME flags the script uses locally |
13 | 13 | # (-std=c++17 -O3, no OpenMP/zlib/sqlite defines — the binary links against nothing |
14 | 14 | # but the C++ runtime). Portable target: no -march=native; macOS universal. |
15 | | -# release one GitHub release per (language, id), tagged engine-<lang>-<id>, holding |
16 | | -# axiom-engine-<lang>-<os>-<arch>[.exe], sha256sum.txt, and the generated .cpp. |
17 | | -# run-souffle.sh computes the same id on the user's machine and fetches the matching asset. |
18 | | -# The id is a function of the repository alone (see engine_id in the script), which is what |
19 | | -# makes "no release for this id" mean exactly "these rules have not been merged and built". |
| 15 | +# publish COMMIT the binaries to main under engine/binaries/<lang>/<platform>/ with the id |
| 16 | +# they were built from in engine/binaries/<lang>/ENGINE_ID — so a checkout carries |
| 17 | +# its own engines and nothing is downloaded — and, as a secondary copy, a GitHub |
| 18 | +# release tagged engine-<lang>-<id> with the same files plus the generated .cpp. |
| 19 | +# run-souffle.sh computes the same id on the user's machine and uses the committed binary only |
| 20 | +# when the ids agree. The id is a function of the repository alone (see engine_id in the |
| 21 | +# script), which is what makes "id differs" mean exactly "these rules were edited after the |
| 22 | +# binary was built". |
20 | 23 | # ───────────────────────────────────────────────────────────────────────────── |
21 | 24 | name: engine-binaries |
22 | 25 |
|
@@ -62,10 +65,11 @@ jobs: |
62 | 65 | for lang in $LANGUAGES; do |
63 | 66 | id="$(bash src/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" |
64 | 67 | echo "$lang: $id" |
65 | | - if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then |
66 | | - echo " unchanged — release engine-$lang-$id exists"; continue |
| 68 | + have="$(tr -d '[:space:]' < "engine/binaries/$lang/ENGINE_ID" 2>/dev/null || true)" |
| 69 | + if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && [ "$have" = "$id" ]; then |
| 70 | + echo " unchanged — engine/binaries/$lang is at this id"; continue |
67 | 71 | fi |
68 | | - echo " changed — no release for this id; building on every platform" |
| 72 | + echo " changed — engine/binaries/$lang is at '${have:-<none>}'; building on every platform" |
69 | 73 | printf '%s' "$id" > "out/$lang.id" |
70 | 74 | langs="$langs \"$lang\"" |
71 | 75 | done |
@@ -184,48 +188,89 @@ jobs: |
184 | 188 | path: axiom-engine-* |
185 | 189 | if-no-files-found: error |
186 | 190 |
|
187 | | - # Per language, and independent of the other languages' builds: `always()` lets this run |
188 | | - # when some build job failed, and the step below publishes a language only if ALL FOUR of |
189 | | - # its binaries exist. A language with a missing platform gets no release — so the next |
190 | | - # merge retries it — while the others publish. |
191 | | - release: |
| 191 | + # One job for every language built: `always()` lets it run when some build failed, and a |
| 192 | + # language is published only if ALL FOUR of its binaries exist — a language with a missing |
| 193 | + # platform is left as it was (the next merge retries it, since its ENGINE_ID still differs) |
| 194 | + # while the others go in. One commit on main carries everything that was rebuilt. |
| 195 | + publish: |
192 | 196 | needs: [generate, build] |
193 | 197 | if: always() && needs.generate.result == 'success' && needs.generate.outputs.langs != '[]' && github.event_name != 'pull_request' |
194 | 198 | runs-on: ubuntu-24.04 |
195 | | - strategy: |
196 | | - matrix: |
197 | | - lang: ${{ fromJson(needs.generate.outputs.langs) }} |
198 | 199 | steps: |
| 200 | + - uses: actions/checkout@v4 |
| 201 | + with: |
| 202 | + ref: main |
199 | 203 | - uses: actions/download-artifact@v4 |
200 | 204 | with: |
201 | 205 | name: generated |
202 | 206 | path: gen |
203 | 207 | - uses: actions/download-artifact@v4 |
204 | 208 | with: |
205 | | - pattern: engine-${{ matrix.lang }}-* |
| 209 | + pattern: engine-* |
206 | 210 | path: bins |
207 | 211 | merge-multiple: true |
208 | 212 |
|
209 | | - - name: Publish release engine-${{ matrix.lang }}-<id> |
| 213 | + - name: Place the binaries under engine/binaries/<lang>/<platform>/ |
| 214 | + id: place |
| 215 | + run: | |
| 216 | + set -e |
| 217 | + echo "artifacts:"; ls -la bins |
| 218 | + published="" |
| 219 | + for lang in $LANGUAGES; do |
| 220 | + [ -f "gen/$lang.id" ] || continue |
| 221 | + id="$(cat "gen/$lang.id")" |
| 222 | + n="$(ls bins/axiom-engine-$lang-* 2>/dev/null | grep -vc '\.sha256$' || true)" |
| 223 | + if [ "$n" != 4 ]; then |
| 224 | + echo "::warning::$lang: only $n of 4 platform binaries were built — engine/binaries/$lang left unchanged; the next merge to main retries" |
| 225 | + continue |
| 226 | + fi |
| 227 | + rm -rf "engine/binaries/$lang" |
| 228 | + for f in bins/axiom-engine-$lang-*; do |
| 229 | + case "$f" in *.sha256) continue;; esac |
| 230 | + name="$(basename "$f")" |
| 231 | + platform="${name#axiom-engine-$lang-}"; platform="${platform%.exe}" |
| 232 | + mkdir -p "engine/binaries/$lang/$platform" |
| 233 | + cp "$f" "engine/binaries/$lang/$platform/$name" |
| 234 | + chmod +x "engine/binaries/$lang/$platform/$name" |
| 235 | + done |
| 236 | + cat bins/axiom-engine-$lang-*.sha256 | sed 's# .*/# #' > "engine/binaries/$lang/sha256sum.txt" |
| 237 | + printf '%s\n' "$id" > "engine/binaries/$lang/ENGINE_ID" |
| 238 | + printf 'Built by the engine-binaries workflow from %s.\nRule-set id: %s (src/pipeline/run-souffle.sh --language %s --print-engine-id).\nrun-souffle.sh uses these only while the rules hash to that id.\n' "${{ github.sha }}" "$id" "$lang" > "engine/binaries/$lang/BUILD.txt" |
| 239 | + published="$published $lang" |
| 240 | + done |
| 241 | + echo "published=${published# }" >> "$GITHUB_OUTPUT" |
| 242 | + git status --short engine/binaries | head -20 |
| 243 | +
|
| 244 | + - name: Commit to main |
| 245 | + if: steps.place.outputs.published != '' |
| 246 | + run: | |
| 247 | + set -e |
| 248 | + git config user.name "engine-binaries" |
| 249 | + git config user.email "engine-binaries@users.noreply.github.com" |
| 250 | + git add -A engine/binaries |
| 251 | + git commit -m "engine: rebuilt binaries for ${{ steps.place.outputs.published }} from ${{ github.sha }}" \ |
| 252 | + -m "Built on linux-x86_64, linux-arm64, darwin-universal and windows-x86_64 by the engine-binaries workflow. Each language's ENGINE_ID is the id of the rules it was compiled from; run-souffle.sh uses the binary only while the checkout's rules hash to that id." |
| 253 | + # main may have moved during the build; rebase our one commit on top and retry. |
| 254 | + for attempt in 1 2 3 4 5; do |
| 255 | + git push origin HEAD:main && exit 0 |
| 256 | + echo "push rejected (attempt $attempt) — rebasing on the current main" |
| 257 | + git fetch origin main && git rebase origin/main |
| 258 | + done |
| 259 | + echo "::error::could not push the binaries to main after 5 attempts"; exit 1 |
| 260 | +
|
| 261 | + - name: Secondary copy — a release per language |
| 262 | + if: steps.place.outputs.published != '' |
210 | 263 | env: |
211 | 264 | GH_TOKEN: ${{ github.token }} |
212 | | - FORCE: ${{ inputs.force }} |
213 | 265 | run: | |
214 | 266 | set -e |
215 | | - lang="${{ matrix.lang }}" |
216 | | - id="$(cat "gen/$lang.id")" |
217 | | - tag="engine-$lang-$id" |
218 | | - n="$(ls bins/axiom-engine-$lang-* 2>/dev/null | grep -vc '\.sha256$')" |
219 | | - if [ "$n" != 4 ]; then |
220 | | - echo "::warning::$lang: only $n of 4 platform binaries were built — not publishing $tag; the next merge to main retries" |
221 | | - exit 0 |
222 | | - fi |
223 | | - cat bins/*.sha256 | sed 's# .*/# #' > bins/sha256sum.txt |
224 | | - rm -f bins/*.sha256 |
225 | | - cp "gen/$lang.cpp" "bins/$lang.cpp" |
226 | | - echo "assets:"; ls -la bins |
227 | | - notes="Prebuilt engine for the **$lang** rule set at id \`$id\` (Soufflé $(cat gen/souffle.version), pinned in src/pipeline/engine.conf), built from ${{ github.sha }}. |
228 | | -
|
229 | | - \`run-souffle.sh --language $lang\` fetches the asset for its platform when \`souffle\` is not installed. \`$lang.cpp\` is the generated program for anyone who prefers to compile it themselves (needs only the Soufflé headers and a C++17 compiler)." |
230 | | - if [ "$FORCE" = "true" ]; then gh release delete "$tag" -y 2>/dev/null || true; fi |
231 | | - gh release create "$tag" --title "engine · $lang · ${id:0:12}" --notes "$notes" --target "${{ github.sha }}" bins/* |
| 267 | + for lang in ${{ steps.place.outputs.published }}; do |
| 268 | + id="$(cat "gen/$lang.id")"; tag="engine-$lang-$id" |
| 269 | + mkdir -p "rel/$lang" |
| 270 | + cp engine/binaries/$lang/*/axiom-engine-* engine/binaries/$lang/sha256sum.txt "rel/$lang/" |
| 271 | + cp "gen/$lang.cpp" "rel/$lang/$lang.cpp" |
| 272 | + gh release delete "$tag" -y 2>/dev/null || true |
| 273 | + gh release create "$tag" --title "engine · $lang · ${id:0:12}" --target main \ |
| 274 | + --notes "Prebuilt engine for the **$lang** rule set at id \`$id\` (Soufflé $(cat gen/souffle.version), pinned in src/pipeline/engine.conf), built from ${{ github.sha }}. The same binaries are committed under engine/binaries/$lang/; \`$lang.cpp\` is the generated program for anyone who prefers to compile it (Soufflé headers + a C++17 compiler)." \ |
| 275 | + rel/$lang/* |
| 276 | + done |
0 commit comments