Add on-demand GitHub Pages previews of decks changed in pull requests

.github/workflows/preview.yml

@@ -64,3 +64,132 @@ jobs:
           # base; full history for git-annex
           ref: refs/pull/${{ inputs.pr }}/merge
           fetch-depth: 0
+
+      - name: Find changed decks
+        id: decks
+        run: |
+          # top-level *.html only: decks live at the repository root
+          DECKS=$(git diff --name-only --diff-filter=AMR HEAD^1 HEAD -- '*.html' \
+            | grep -E '^[A-Za-z0-9][A-Za-z0-9._+-]*\.html$' | tr '\n' ' ' || true)
+          if [ -z "$DECKS" ]; then
+            echo "::error::Pull request #${{ inputs.pr }} changes no decks; nothing to preview."
+            exit 1
+          fi
+          echo "Decks to preview: $DECKS"
+          echo "decks=$DECKS" >> "$GITHUB_OUTPUT"
+
+      - name: Get reveal.js
+        # not `submodules: true`: 2022-nih-compcore has a relative URL that
+        # only resolves on datasets.datalad.org
+        run: git submodule update --init --depth 1 reveal.js reveal.js-mermaid-plugin
+
+      - name: Set up git-annex
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq git-annex
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git fetch --no-tags origin '+refs/heads/git-annex:refs/remotes/origin/git-annex'
+          git remote add --fetch datasets.datalad.org "$ANNEX_REMOTE_URL"
+          git annex init --quiet
+
+      - name: Build site
+        env:
+          DECKS: ${{ steps.decks.outputs.decks }}
+        run: |
+          set -o pipefail
+          # the builder comes from this workflow's own commit: the pull
+          # request being previewed need not (and usually does not) have it
+          git fetch --no-tags --quiet origin "$GITHUB_SHA"
+          git show "$GITHUB_SHA:tools/build_preview.py" > "$RUNNER_TEMP/build_preview.py"
+          read -ra DECKS <<< "$DECKS"
+          python3 "$RUNNER_TEMP/build_preview.py" preview-site "${DECKS[@]}" | tee "$RUNNER_TEMP/build.log"
+          if grep -q '^MISSING: ' "$RUNNER_TEMP/build.log"; then
+            {
+              echo "### Files not available for the preview"
+              echo
+              echo "Not on any reachable annex remote yet (e.g. not pushed to datasets.datalad.org), or referenced but nonexistent:"
+              echo
+              sed -n 's/^MISSING: \(.*\)/- `\1`/p' "$RUNNER_TEMP/build.log"
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload site
+        uses: actions/upload-artifact@v7
+        with:
+          name: preview-site
+          path: preview-site/
+          if-no-files-found: error
+          include-hidden-files: true
+          retention-days: 1
+
+  deploy-preview:
+    needs: build-preview
+    # also run for `remove`, where the build is skipped
+    if: ${{ !cancelled() && (inputs.action == 'remove' || needs.build-preview.result == 'success') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Download site
+        if: inputs.action == 'deploy'
+        uses: actions/download-artifact@v8
+        with:
+          name: preview-site
+          path: preview-site
+
+      - name: Deploy preview
+        id: preview
+        uses: rossjrw/pr-preview-action@v1
+        with:
+          source-dir: preview-site
+          preview-branch: gh-pages
+          action: ${{ inputs.action }}
+          pr-number: ${{ inputs.pr }}
+          deploy-commit-message: Deploy preview for PR ${{ inputs.pr }}
+          remove-commit-message: Remove preview for PR ${{ inputs.pr }}
+          # The action's own comment looks for the pull request in the
+          # triggering event, which `workflow_dispatch` does not carry.
+          comment: false
+
+      - name: Compose preview comment
+        if: inputs.action == 'deploy'
+        env:
+          PREVIEW_URL: ${{ steps.preview.outputs.preview-url }}
+          HEAD_SHA: ${{ needs.build-preview.outputs.head-sha }}
+          DECKS: ${{ needs.build-preview.outputs.decks }}
+          DEPLOYMENTS_URL: ${{ github.server_url }}/${{ github.repository }}/deployments
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          {
+            echo ":rocket: Preview of $HEAD_SHA:"
+            echo
+            for deck in $DECKS; do
+              echo "- [$deck](${PREVIEW_URL%/}/$deck)"
+            done
+            echo
+            echo "It will be live once the [GitHub Pages deployment]($DEPLOYMENTS_URL) finishes."
+            echo "Images whose git-annex content is not on datasets.datalad.org yet are listed in the [build summary]($RUN_URL)."
+            echo "Re-run the [Preview workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/preview.yml) to update it."
+          } > "$RUNNER_TEMP/comment.md"
+
+      - name: Comment preview link
+        if: inputs.action == 'deploy'
+        uses: marocchino/sticky-pull-request-comment@v3
+        with:
+          header: pr-preview
+          number: ${{ inputs.pr }}
+          path: ${{ runner.temp }}/comment.md
+
+      - name: Comment preview removal
+        if: inputs.action == 'remove'
+        uses: marocchino/sticky-pull-request-comment@v3
+        with:
+          header: pr-preview
+          number: ${{ inputs.pr }}
+          message: |
+            Preview removed.

tools/build_preview.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Assemble a minimal static site with only the given decks and what they use.
+
+Usage (from the repository root): tools/build_preview.py OUTDIR DECK.html [DECK.html ...]
+
+Starting from the decks, follows local references (src/href/data-src/
+data-background*/data-markdown attributes, markdown links and CSS url())
+through HTML, CSS and Markdown files, and copies every referenced file into
+OUTDIR under its repository path. Files whose git-annex content is not
+present are fetched with `git annex get` first (when git-annex is
+available). Files that could not be obtained are reported, not fatal, so a
+deck still previews with whatever content is available.
+
+Used by .github/workflows/preview.yml, but also works locally.
+"""
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from urllib.parse import unquote, urlsplit
+
+# The repository to build from is the current directory, not the one this
+# script lives in: the Preview workflow runs it on another checkout.
+REPO = Path.cwd()
+
+# Files that are scanned for further references
+SCANNED_SUFFIXES = {".html", ".htm", ".css", ".md"}
+
+# Each pattern captures (kind, quote, reference)
+REF_PATTERNS = [
+    # HTML attributes, e.g. src="..", data-src='..', data-background-image=".."
+    re.compile(
+        r"""\b(src|href|data-src|poster|data-markdown|data-background(?:-image|-video|-iframe)?)"""
+        r"""\s*=\s*(["'])(.+?)\2""",
+        re.I,
+    ),
+    # markdown images: ![alt](target "title")
+    re.compile(r"""!\[[^\]]*\]\(\s*<?()()([^)\s>]+)"""),
+    # CSS url(...)
+    re.compile(r"""url\(\s*()(["']?)([^"')]+)\2\s*\)""", re.I),
+]
+
+# href is mostly used for hyperlinks (to other decks, PDFs, ...), which are
+# not needed to render the deck; follow it only for stylesheets.
+HREF_FOLLOWED_SUFFIXES = {".css"}
+
+
+def local_target(ref, referrer):
+    """Return the repository-relative path `ref` points to, or None."""
+    ref = ref.strip()
+    if not ref or ref.startswith(("#", "/", "data:", "{", "$")):
+        return None
+    parts = urlsplit(ref)
+    if parts.scheme or parts.netloc or not parts.path:
+        return None
+    target = (REPO / referrer).parent / unquote(parts.path)
+    target = Path(os.path.normpath(target))
+    try:
+        rel = target.relative_to(REPO)
+    except ValueError:
+        return None  # points outside of the repository
+    if rel.parts[0] == ".git":
+        return None
+    return rel
+
+
+def refs_in(path):
+    try:
+        text = (REPO / path).read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return
+    for pattern in REF_PATTERNS:
+        for m in pattern.finditer(text):
+            kind, _, ref = m.groups()
+            target = local_target(ref, path)
+            if target is None:
+                continue
+            if kind.lower() == "href" and target.suffix.lower() not in HREF_FOLLOWED_SUFFIXES:
+                continue
+            yield target
+
+
+def collect(decks):
+    """Walk references starting from the decks; return the set of files."""
+    files, seen = set(), set()
+    queue = list(decks)
+    while queue:
+        rel = queue.pop()
+        if rel in seen:
+            continue
+        seen.add(rel)
+        full = REPO / rel
+        # an annexed file without content is a broken symlink: keep it so its
+        # content can be fetched, but there is nothing to scan in it yet
+        if full.is_dir() and not full.is_symlink():
+            continue
+        if not os.path.lexists(full):
+            files.add(rel)  # reported as missing later
+            continue
+        files.add(rel)
+        if rel.suffix.lower() in SCANNED_SUFFIXES:
+            queue.extend(refs_in(rel))
+    return files
+
+
+def annex_get(paths):
+    if not paths or not shutil.which("git-annex"):
+        return
+    # one call for all; failures for individual files are reported afterwards
+    subprocess.run(
+        ["git", "annex", "get", "--"] + [str(p) for p in sorted(paths)],
+        cwd=REPO,
+        check=False,
+    )
+
+
+def main(outdir, decks):
+    outdir = Path(outdir).resolve()
+    decks = [Path(d) for d in decks]
+    files = collect(decks)
+
+    annex_get([f for f in files if (REPO / f).is_symlink() and not (REPO / f).exists()])
+
+    copied, missing = [], []
+    for rel in sorted(files):
+        src = REPO / rel
+        if not src.exists():  # nonexistent, or annexed without content
+            missing.append(rel)
+            continue
+        dst = outdir / rel
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, dst)  # dereferences annex symlinks
+        copied.append(rel)
+
+    print(f"Copied {len(copied)} file(s) for {len(decks)} deck(s) into {outdir}")
+    for rel in missing:
+        print(f"MISSING: {rel}")
+    return missing
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        sys.exit(__doc__)
+    missing = main(sys.argv[1], sys.argv[2:])
+    if os.environ.get("GITHUB_ACTIONS"):
+        for rel in missing:
+            print(f"::warning file={rel}::Not available for the preview: {rel}")