diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 994360c8..85e02cd2 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -64,3 +64,132 @@ jobs: # base; full history for git-annex ref: refs/pull/${{ inputs.pr }}/merge fetch-depth: 0 + + - name: Find changed decks + id: decks + run: | + # top-level *.html only: decks live at the repository root + DECKS=$(git diff --name-only --diff-filter=AMR HEAD^1 HEAD -- '*.html' \ + | grep -E '^[A-Za-z0-9][A-Za-z0-9._+-]*\.html$' | tr '\n' ' ' || true) + if [ -z "$DECKS" ]; then + echo "::error::Pull request #${{ inputs.pr }} changes no decks; nothing to preview." + exit 1 + fi + echo "Decks to preview: $DECKS" + echo "decks=$DECKS" >> "$GITHUB_OUTPUT" + + - name: Get reveal.js + # not `submodules: true`: 2022-nih-compcore has a relative URL that + # only resolves on datasets.datalad.org + run: git submodule update --init --depth 1 reveal.js reveal.js-mermaid-plugin + + - name: Set up git-annex + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq git-annex + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch --no-tags origin '+refs/heads/git-annex:refs/remotes/origin/git-annex' + git remote add --fetch datasets.datalad.org "$ANNEX_REMOTE_URL" + git annex init --quiet + + - name: Build site + env: + DECKS: ${{ steps.decks.outputs.decks }} + run: | + set -o pipefail + # the builder comes from this workflow's own commit: the pull + # request being previewed need not (and usually does not) have it + git fetch --no-tags --quiet origin "$GITHUB_SHA" + git show "$GITHUB_SHA:tools/build_preview.py" > "$RUNNER_TEMP/build_preview.py" + read -ra DECKS <<< "$DECKS" + python3 "$RUNNER_TEMP/build_preview.py" preview-site "${DECKS[@]}" | tee "$RUNNER_TEMP/build.log" + if grep -q '^MISSING: ' "$RUNNER_TEMP/build.log"; then + { + echo "### Files not available for the preview" + echo + echo "Not on any reachable annex remote yet (e.g. not pushed to datasets.datalad.org), or referenced but nonexistent:" + echo + sed -n 's/^MISSING: \(.*\)/- `\1`/p' "$RUNNER_TEMP/build.log" + } >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload site + uses: actions/upload-artifact@v7 + with: + name: preview-site + path: preview-site/ + if-no-files-found: error + include-hidden-files: true + retention-days: 1 + + deploy-preview: + needs: build-preview + # also run for `remove`, where the build is skipped + if: ${{ !cancelled() && (inputs.action == 'remove' || needs.build-preview.result == 'success') }} + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Download site + if: inputs.action == 'deploy' + uses: actions/download-artifact@v8 + with: + name: preview-site + path: preview-site + + - name: Deploy preview + id: preview + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: preview-site + preview-branch: gh-pages + action: ${{ inputs.action }} + pr-number: ${{ inputs.pr }} + deploy-commit-message: Deploy preview for PR ${{ inputs.pr }} + remove-commit-message: Remove preview for PR ${{ inputs.pr }} + # The action's own comment looks for the pull request in the + # triggering event, which `workflow_dispatch` does not carry. + comment: false + + - name: Compose preview comment + if: inputs.action == 'deploy' + env: + PREVIEW_URL: ${{ steps.preview.outputs.preview-url }} + HEAD_SHA: ${{ needs.build-preview.outputs.head-sha }} + DECKS: ${{ needs.build-preview.outputs.decks }} + DEPLOYMENTS_URL: ${{ github.server_url }}/${{ github.repository }}/deployments + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + { + echo ":rocket: Preview of $HEAD_SHA:" + echo + for deck in $DECKS; do + echo "- [$deck](${PREVIEW_URL%/}/$deck)" + done + echo + echo "It will be live once the [GitHub Pages deployment]($DEPLOYMENTS_URL) finishes." + echo "Images whose git-annex content is not on datasets.datalad.org yet are listed in the [build summary]($RUN_URL)." + echo "Re-run the [Preview workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/preview.yml) to update it." + } > "$RUNNER_TEMP/comment.md" + + - name: Comment preview link + if: inputs.action == 'deploy' + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: pr-preview + number: ${{ inputs.pr }} + path: ${{ runner.temp }}/comment.md + + - name: Comment preview removal + if: inputs.action == 'remove' + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: pr-preview + number: ${{ inputs.pr }} + message: | + Preview removed. diff --git a/tools/build_preview.py b/tools/build_preview.py new file mode 100755 index 00000000..cd07fb54 --- /dev/null +++ b/tools/build_preview.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Assemble a minimal static site with only the given decks and what they use. + +Usage (from the repository root): tools/build_preview.py OUTDIR DECK.html [DECK.html ...] + +Starting from the decks, follows local references (src/href/data-src/ +data-background*/data-markdown attributes, markdown links and CSS url()) +through HTML, CSS and Markdown files, and copies every referenced file into +OUTDIR under its repository path. Files whose git-annex content is not +present are fetched with `git annex get` first (when git-annex is +available). Files that could not be obtained are reported, not fatal, so a +deck still previews with whatever content is available. + +Used by .github/workflows/preview.yml, but also works locally. +""" + +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + +# The repository to build from is the current directory, not the one this +# script lives in: the Preview workflow runs it on another checkout. +REPO = Path.cwd() + +# Files that are scanned for further references +SCANNED_SUFFIXES = {".html", ".htm", ".css", ".md"} + +# Each pattern captures (kind, quote, reference) +REF_PATTERNS = [ + # HTML attributes, e.g. src="..", data-src='..', data-background-image=".." + re.compile( + r"""\b(src|href|data-src|poster|data-markdown|data-background(?:-image|-video|-iframe)?)""" + r"""\s*=\s*(["'])(.+?)\2""", + re.I, + ), + # markdown images: ![alt](target "title") + re.compile(r"""!\[[^\]]*\]\(\s*]+)"""), + # CSS url(...) + re.compile(r"""url\(\s*()(["']?)([^"')]+)\2\s*\)""", re.I), +] + +# href is mostly used for hyperlinks (to other decks, PDFs, ...), which are +# not needed to render the deck; follow it only for stylesheets. +HREF_FOLLOWED_SUFFIXES = {".css"} + + +def local_target(ref, referrer): + """Return the repository-relative path `ref` points to, or None.""" + ref = ref.strip() + if not ref or ref.startswith(("#", "/", "data:", "{", "$")): + return None + parts = urlsplit(ref) + if parts.scheme or parts.netloc or not parts.path: + return None + target = (REPO / referrer).parent / unquote(parts.path) + target = Path(os.path.normpath(target)) + try: + rel = target.relative_to(REPO) + except ValueError: + return None # points outside of the repository + if rel.parts[0] == ".git": + return None + return rel + + +def refs_in(path): + try: + text = (REPO / path).read_text(encoding="utf-8", errors="replace") + except OSError: + return + for pattern in REF_PATTERNS: + for m in pattern.finditer(text): + kind, _, ref = m.groups() + target = local_target(ref, path) + if target is None: + continue + if kind.lower() == "href" and target.suffix.lower() not in HREF_FOLLOWED_SUFFIXES: + continue + yield target + + +def collect(decks): + """Walk references starting from the decks; return the set of files.""" + files, seen = set(), set() + queue = list(decks) + while queue: + rel = queue.pop() + if rel in seen: + continue + seen.add(rel) + full = REPO / rel + # an annexed file without content is a broken symlink: keep it so its + # content can be fetched, but there is nothing to scan in it yet + if full.is_dir() and not full.is_symlink(): + continue + if not os.path.lexists(full): + files.add(rel) # reported as missing later + continue + files.add(rel) + if rel.suffix.lower() in SCANNED_SUFFIXES: + queue.extend(refs_in(rel)) + return files + + +def annex_get(paths): + if not paths or not shutil.which("git-annex"): + return + # one call for all; failures for individual files are reported afterwards + subprocess.run( + ["git", "annex", "get", "--"] + [str(p) for p in sorted(paths)], + cwd=REPO, + check=False, + ) + + +def main(outdir, decks): + outdir = Path(outdir).resolve() + decks = [Path(d) for d in decks] + files = collect(decks) + + annex_get([f for f in files if (REPO / f).is_symlink() and not (REPO / f).exists()]) + + copied, missing = [], [] + for rel in sorted(files): + src = REPO / rel + if not src.exists(): # nonexistent, or annexed without content + missing.append(rel) + continue + dst = outdir / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) # dereferences annex symlinks + copied.append(rel) + + print(f"Copied {len(copied)} file(s) for {len(decks)} deck(s) into {outdir}") + for rel in missing: + print(f"MISSING: {rel}") + return missing + + +if __name__ == "__main__": + if len(sys.argv) < 3: + sys.exit(__doc__) + missing = main(sys.argv[1], sys.argv[2:]) + if os.environ.get("GITHUB_ACTIONS"): + for rel in missing: + print(f"::warning file={rel}::Not available for the preview: {rel}")