Add structured CI phase and fixture telemetry

.github/ci/measure.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Run one CI phase and append a portable, non-gating JSON record."""
+
+import argparse
+import datetime
+import json
+import os
+import platform
+import resource
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--job", required=True)
+    parser.add_argument("--phase", required=True)
+    parser.add_argument("--log")
+    parser.add_argument("--annotation", action="append", default=[])
+    parser.add_argument("command", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    command = args.command[1:] if args.command[:1] == ["--"] else args.command
+    if not command:
+        parser.error("a command is required after --")
+
+    output = Path(args.output)
+    output.mkdir(parents=True, exist_ok=True)
+    annotations = dict(item.split("=", 1) for item in args.annotation)
+    started_at = datetime.datetime.now(datetime.timezone.utc)
+    started = time.monotonic_ns()
+    usage_before = resource.getrusage(resource.RUSAGE_CHILDREN)
+
+    log_handle = open(args.log, "wb") if args.log else None
+    process = None
+    exit_code = 127
+    launch_error = None
+    try:
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        assert process.stdout is not None
+        for chunk in iter(lambda: process.stdout.read1(65536), b""):
+            sys.stdout.buffer.write(chunk)
+            sys.stdout.buffer.flush()
+            if log_handle:
+                log_handle.write(chunk)
+        exit_code = process.wait()
+    except KeyboardInterrupt:
+        if process is not None:
+            process.terminate()
+            process.wait()
+        exit_code = 130
+    except OSError as error:
+        launch_error = f"{type(error).__name__}: {error}"
+        print(launch_error, file=sys.stderr)
+    finally:
+        if log_handle:
+            log_handle.close()
+
+    usage_after = resource.getrusage(resource.RUSAGE_CHILDREN)
+    duration = (time.monotonic_ns() - started) / 1_000_000_000
+    # ru_maxrss is KiB on Linux and bytes on macOS.
+    max_rss_kib = usage_after.ru_maxrss / 1024 if sys.platform == "darwin" else usage_after.ru_maxrss
+    record = {
+        "schema_version": 1,
+        "job": args.job,
+        "phase": args.phase,
+        "started_at": started_at.isoformat(),
+        "duration_seconds": duration,
+        "exit_code": exit_code,
+        "user_cpu_seconds": usage_after.ru_utime - usage_before.ru_utime,
+        "system_cpu_seconds": usage_after.ru_stime - usage_before.ru_stime,
+        "max_child_rss_kib": max_rss_kib,
+        "command": command,
+        "process_id": process.pid if process is not None else None,
+        "launch_error": launch_error,
+        "runner_os": os.environ.get("RUNNER_OS", platform.system()),
+        "runner_arch": os.environ.get("RUNNER_ARCH", platform.machine()),
+        "github_sha": os.environ.get("GITHUB_SHA"),
+        "github_run_id": os.environ.get("GITHUB_RUN_ID"),
+        "julia_version": os.environ.get("JULIA_VERSION"),
+        "annotations": annotations,
+    }
+    with (output / "phases.jsonl").open("a", encoding="utf-8") as stream:
+        json.dump(record, stream, separators=(",", ":"))
+        stream.write("\n")
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/ci/summarize_parallel_tests.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Convert ParallelTestRunner's stable verbose table into structured JSON."""
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+ROW = re.compile(
+    r"^(?P<fixture>.+?)\s+\((?P<worker>\d+)\)\s*│\s*"
+    r"(?P<test>[\d.]+)\s*│\s*(?P<init>[\d.]+)\s*│\s*"
+    r"(?P<compile>[\d.]+)\s*│\s*(?P<gc>[\d.]+)\s*│\s*"
+    r"(?P<gc_percent>[\d.]+)\s*│\s*(?P<allocation>[\d.]+)\s*│\s*"
+    r"(?P<rss>[\d.]+)\s*│\s*$"
+)
+ANSI = re.compile(r"\x1b\[[0-9;]*m")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("log")
+    parser.add_argument("output")
+    parser.add_argument("--job", required=True)
+    args = parser.parse_args()
+    fixtures = []
+    for raw_line in Path(args.log).read_text(errors="replace").splitlines():
+        match = ROW.match(ANSI.sub("", raw_line))
+        if not match:
+            continue
+        values = match.groupdict()
+        test_time = float(values["test"])
+        compile_percent = float(values["compile"])
+        fixtures.append(
+            {
+                "fixture": values["fixture"].strip(),
+                "worker": int(values["worker"]),
+                "test_seconds": test_time,
+                "initialization_seconds": float(values["init"]),
+                "compile_percent": compile_percent,
+                "compile_share_seconds": test_time * compile_percent / 100,
+                "execution_share_seconds": test_time * (1 - compile_percent / 100),
+                "gc_seconds": float(values["gc"]),
+                "gc_percent": float(values["gc_percent"]),
+                "allocated_megabytes": float(values["allocation"]),
+                "rss_megabytes": float(values["rss"]),
+                "exceeds_50_seconds": test_time > 50,
+            }
+        )
+    result = {
+        "schema_version": 1,
+        "job": args.job,
+        "fixture_count": len(fixtures),
+        "fixtures": fixtures,
+        "totals": {
+            "test_seconds": sum(item["test_seconds"] for item in fixtures),
+            "initialization_seconds": sum(item["initialization_seconds"] for item in fixtures),
+            "compile_share_seconds": sum(item["compile_share_seconds"] for item in fixtures),
+            "execution_share_seconds": sum(item["execution_share_seconds"] for item in fixtures),
+            "allocated_megabytes": sum(item["allocated_megabytes"] for item in fixtures),
+            "peak_rss_megabytes": max((item["rss_megabytes"] for item in fixtures), default=0),
+        },
+    }
+    Path(args.output).write_text(json.dumps(result, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -53,8 +53,29 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
-      - name: Package tests
-        run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=2", "--verbose"])'
+        id: cache
+      - name: Package environment and tests
+        env:
+          LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job package)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])'
+      - name: Structure ParallelTestRunner totals
+        if: always()
+        run: |
+          test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job package
+      - name: Upload package telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-package-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error
 
   scientific:
     needs: changes
@@ -68,8 +89,25 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
-      - name: Scientific witnesses
-        run: julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("test/scientific_witnesses/runtests.jl")'
+        id: cache
+      - name: Scientific environment and witnesses
+        env:
+          LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job scientific)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=test/scientific_witnesses --startup-file=no -e 'using LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/scientific.log" -- julia --project=test/scientific_witnesses --startup-file=no -e 'include("test/scientific_witnesses/runtests.jl")'
+      - name: Upload scientific telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-scientific-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error
 
   macos-smoke:
     if: github.event_name == 'pull_request'
@@ -79,8 +117,28 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
+        id: cache
       - name: Public API and KernelAbstractions smoke tests
-        run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=1", "test_public_api", "test_direct_pointwise_stage"])'
+        env:
+          LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-smoke)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=1", "--verbose", "test_public_api", "test_direct_pointwise_stage"])'
+      - name: Structure ParallelTestRunner totals
+        if: always()
+        run: test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job macos-smoke
+      - name: Upload macOS smoke telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-macos-smoke-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error
 
   macos-package:
     if: github.event_name != 'pull_request'
@@ -90,8 +148,28 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
+        id: cache
       - name: Complete macOS package tests
-        run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=2"])'
+        env:
+          LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-package)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])'
+      - name: Structure ParallelTestRunner totals
+        if: always()
+        run: test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job macos-package
+      - name: Upload macOS package telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-macos-package-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error
 
   docs:
     if: github.event_name == 'pull_request'
@@ -101,8 +179,23 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
+        id: cache
       - name: Documentation
-        run: julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("docs/make.jl")'
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job docs)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")'
+      - name: Upload documentation telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-docs-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error
 
   metal:
     needs: changes
@@ -116,5 +209,22 @@ jobs:
       - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3
         with: {version: '${{ env.JULIA_VERSION }}'}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
+        id: cache
       - name: Metal qualification
-        run: julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("test/metal/runtests.jl")'
+        env:
+          LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job metal)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=test/metal --startup-file=no -e 'using KernelAbstractions, LocalMath, Metal'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/metal.log" -- julia --project=test/metal --startup-file=no -e 'include("test/metal/runtests.jl")'
+      - name: Upload Metal telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-metal-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error

.github/workflows/docs.yml

@@ -26,8 +26,23 @@ jobs:
         with:
           version: ${{ env.JULIA_VERSION }}
       - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3
+        id: cache
       - name: Build and publish documentation
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           LOCALMATH_DEPLOY_DOCS: ${{ github.event_name == 'push' || github.ref == 'refs/heads/main' }}
-        run: julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("docs/make.jl")'
+        run: |
+          mkdir -p "$RUNNER_TEMP/localmath-telemetry"
+          measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job documentation-publish)
+          "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass'
+          "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()'
+          "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()'
+          "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath'
+          "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")'
+      - name: Upload documentation telemetry
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: telemetry-documentation-publish-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/localmath-telemetry
+          if-no-files-found: error