From cebb5a99387e1cea3742dd00856e87699ca0bcfd Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Fri, 18 Sep 2026 19:56:11 -0400 Subject: [PATCH 1/2] Add structured CI phase telemetry --- .github/ci/measure.py | 84 +++++++++++++++++ .github/ci/summarize_parallel_tests.py | 68 +++++++++++++ .github/workflows/ci.yml | 126 +++++++++++++++++++++++-- .github/workflows/docs.yml | 17 +++- test/metal/runtests.jl | 17 +++- test/runtests.jl | 6 +- test/scientific_witnesses/runtests.jl | 31 +++--- test/telemetry_support.jl | 38 ++++++++ 8 files changed, 363 insertions(+), 24 deletions(-) create mode 100644 .github/ci/measure.py create mode 100644 .github/ci/summarize_parallel_tests.py create mode 100644 test/telemetry_support.jl diff --git a/.github/ci/measure.py b/.github/ci/measure.py new file mode 100644 index 0000000..df424aa --- /dev/null +++ b/.github/ci/measure.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Run one CI phase and append a portable, non-gating JSON record.""" + +import argparse +import datetime +import json +import os +import platform +import resource +import subprocess +import sys +import time +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("--job", required=True) + parser.add_argument("--phase", required=True) + parser.add_argument("--log") + parser.add_argument("--annotation", action="append", default=[]) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("a command is required after --") + + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + annotations = dict(item.split("=", 1) for item in args.annotation) + started_at = datetime.datetime.now(datetime.timezone.utc) + started = time.monotonic_ns() + usage_before = resource.getrusage(resource.RUSAGE_CHILDREN) + + log_handle = open(args.log, "wb") if args.log else None + try: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + assert process.stdout is not None + for chunk in iter(lambda: process.stdout.read1(65536), b""): + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + if log_handle: + log_handle.write(chunk) + exit_code = process.wait() + except KeyboardInterrupt: + process.terminate() + process.wait() + exit_code = 130 + finally: + if log_handle: + log_handle.close() + + usage_after = resource.getrusage(resource.RUSAGE_CHILDREN) + duration = (time.monotonic_ns() - started) / 1_000_000_000 + # ru_maxrss is KiB on Linux and bytes on macOS. + max_rss_kib = usage_after.ru_maxrss / 1024 if sys.platform == "darwin" else usage_after.ru_maxrss + record = { + "schema_version": 1, + "job": args.job, + "phase": args.phase, + "started_at": started_at.isoformat(), + "duration_seconds": duration, + "exit_code": exit_code, + "user_cpu_seconds": usage_after.ru_utime - usage_before.ru_utime, + "system_cpu_seconds": usage_after.ru_stime - usage_before.ru_stime, + "max_child_rss_kib": max_rss_kib, + "command": command, + "process_id": process.pid, + "runner_os": os.environ.get("RUNNER_OS", platform.system()), + "runner_arch": os.environ.get("RUNNER_ARCH", platform.machine()), + "github_sha": os.environ.get("GITHUB_SHA"), + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + "julia_version": os.environ.get("JULIA_VERSION"), + "annotations": annotations, + } + with (output / "phases.jsonl").open("a", encoding="utf-8") as stream: + json.dump(record, stream, separators=(",", ":")) + stream.write("\n") + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/ci/summarize_parallel_tests.py b/.github/ci/summarize_parallel_tests.py new file mode 100644 index 0000000..7bf4d83 --- /dev/null +++ b/.github/ci/summarize_parallel_tests.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Convert ParallelTestRunner's stable verbose table into structured JSON.""" + +import argparse +import json +import re +from pathlib import Path + + +ROW = re.compile( + r"^(?P.+?)\s+\((?P\d+)\)\s*│\s*" + r"(?P[\d.]+)\s*│\s*(?P[\d.]+)\s*│\s*" + r"(?P[\d.]+)\s*│\s*(?P[\d.]+)\s*│\s*" + r"(?P[\d.]+)\s*│\s*(?P[\d.]+)\s*│\s*" + r"(?P[\d.]+)\s*│\s*$" +) +ANSI = re.compile(r"\x1b\[[0-9;]*m") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("log") + parser.add_argument("output") + parser.add_argument("--job", required=True) + args = parser.parse_args() + fixtures = [] + for raw_line in Path(args.log).read_text(errors="replace").splitlines(): + match = ROW.match(ANSI.sub("", raw_line)) + if not match: + continue + values = match.groupdict() + test_time = float(values["test"]) + compile_percent = float(values["compile"]) + fixtures.append( + { + "fixture": values["fixture"].strip(), + "worker": int(values["worker"]), + "test_seconds": test_time, + "initialization_seconds": float(values["init"]), + "compile_percent": compile_percent, + "compile_share_seconds": test_time * compile_percent / 100, + "execution_share_seconds": test_time * (1 - compile_percent / 100), + "gc_seconds": float(values["gc"]), + "gc_percent": float(values["gc_percent"]), + "allocated_megabytes": float(values["allocation"]), + "rss_megabytes": float(values["rss"]), + "exceeds_50_seconds": test_time > 50, + } + ) + result = { + "schema_version": 1, + "job": args.job, + "fixture_count": len(fixtures), + "fixtures": fixtures, + "totals": { + "test_seconds": sum(item["test_seconds"] for item in fixtures), + "initialization_seconds": sum(item["initialization_seconds"] for item in fixtures), + "compile_share_seconds": sum(item["compile_share_seconds"] for item in fixtures), + "execution_share_seconds": sum(item["execution_share_seconds"] for item in fixtures), + "allocated_megabytes": sum(item["allocated_megabytes"] for item in fixtures), + "peak_rss_megabytes": max((item["rss_megabytes"] for item in fixtures), default=0), + }, + } + Path(args.output).write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f4367c..33f45ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,8 +53,29 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 - - name: Package tests - run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=2", "--verbose"])' + id: cache + - name: Package environment and tests + env: + LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job package) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])' + - name: Structure ParallelTestRunner totals + if: always() + run: | + test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job package + - name: Upload package telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-package-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error scientific: needs: changes @@ -68,8 +89,25 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 - - name: Scientific witnesses - run: julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("test/scientific_witnesses/runtests.jl")' + id: cache + - name: Scientific environment and witnesses + env: + LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job scientific) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=test/scientific_witnesses --startup-file=no -e 'using LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/scientific.log" -- julia --project=test/scientific_witnesses --startup-file=no -e 'include("test/scientific_witnesses/runtests.jl")' + - name: Upload scientific telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-scientific-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error macos-smoke: if: github.event_name == 'pull_request' @@ -79,8 +117,28 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 + id: cache - name: Public API and KernelAbstractions smoke tests - run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=1", "test_public_api", "test_direct_pointwise_stage"])' + env: + LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-smoke) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=1", "--verbose", "test_public_api", "test_direct_pointwise_stage"])' + - name: Structure ParallelTestRunner totals + if: always() + run: test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job macos-smoke + - name: Upload macOS smoke telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-macos-smoke-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error macos-package: if: github.event_name != 'pull_request' @@ -90,8 +148,28 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 + id: cache - name: Complete macOS package tests - run: julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate(); Pkg.test(; test_args=["--jobs=2"])' + env: + LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-package) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])' + - name: Structure ParallelTestRunner totals + if: always() + run: test ! -f "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" || python3 .github/ci/summarize_parallel_tests.py "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" "$RUNNER_TEMP/localmath-telemetry/parallel-tests.json" --job macos-package + - name: Upload macOS package telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-macos-package-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error docs: if: github.event_name == 'pull_request' @@ -101,8 +179,23 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 + id: cache - name: Documentation - run: julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("docs/make.jl")' + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job docs) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")' + - name: Upload documentation telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-docs-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error metal: needs: changes @@ -116,5 +209,22 @@ jobs: - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3 with: {version: '${{ env.JULIA_VERSION }}'} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 + id: cache - name: Metal qualification - run: julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("test/metal/runtests.jl")' + env: + LOCALMATH_CI_TELEMETRY: ${{ runner.temp }}/localmath-telemetry/fixtures + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job metal) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=test/metal --startup-file=no -e 'using KernelAbstractions, LocalMath, Metal' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/metal.log" -- julia --project=test/metal --startup-file=no -e 'include("test/metal/runtests.jl")' + - name: Upload Metal telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-metal-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1bb245c..0eb6295 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,8 +26,23 @@ jobs: with: version: ${{ env.JULIA_VERSION }} - uses: julia-actions/cache@a45e8fa8be21c18a06b7177052533149e61e9b38 # v3 + id: cache - name: Build and publish documentation env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} LOCALMATH_DEPLOY_DOCS: ${{ github.event_name == 'push' || github.ref == 'refs/heads/main' }} - run: julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate(); include("docs/make.jl")' + run: | + mkdir -p "$RUNNER_TEMP/localmath-telemetry" + measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job documentation-publish) + "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' + "${measure[@]}" --phase instantiate -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()' + "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath' + "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")' + - name: Upload documentation telemetry + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: telemetry-documentation-publish-${{ github.run_attempt }} + path: ${{ runner.temp }}/localmath-telemetry + if-no-files-found: error diff --git a/test/metal/runtests.jl b/test/metal/runtests.jl index 8313a57..cf4ff8c 100644 --- a/test/metal/runtests.jl +++ b/test/metal/runtests.jl @@ -4,6 +4,8 @@ using Metal using Statistics using Test +include(joinpath(@__DIR__, "..", "telemetry_support.jl")) + Metal.functional() || error("the selected Metal witness is not functional") Metal.allowscalar(false) @@ -33,9 +35,14 @@ const LOCALMATH_METAL_WITNESSES = ( @test discovered == Set(LOCALMATH_METAL_WITNESSES) end -foreach(include, LOCALMATH_METAL_WITNESSES) +for (ordinal, witness) in enumerate(LOCALMATH_METAL_WITNESSES) + LocalMathTestTelemetry.include_fixture( + @__MODULE__, joinpath(@__DIR__, witness), witness; + kind = "metal_fixture", ordinal, + ) +end -for witness in ( +for (ordinal, witness) in enumerate(( "lbm_d2q9.jl", "lattice_spring.jl", "matrix_free_fem.jl", @@ -47,8 +54,12 @@ for witness in ( "ordered_pgs_3d.jl", "ordered_stoichiometry.jl", "localmath_authoring.jl", + )) + LocalMathTestTelemetry.include_fixture( + @__MODULE__, + joinpath(@__DIR__, "..", "scientific_witnesses", witness), witness; + kind = "metal_scientific_fixture", ordinal, ) - include(joinpath(@__DIR__, "..", "scientific_witnesses", witness)) end @testset "LocalMath cross-domain Metal witnesses" begin diff --git a/test/runtests.jl b/test/runtests.jl index eca528e..1ccfc9e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -125,7 +125,10 @@ for test_file in LOCALMATH_INCLUDED_TESTS test_name = first(splitext(test_file)) LOCALMATH_TEST_SUITE[test_name] = quote include($(joinpath(@__DIR__, "support.jl"))) - include($(joinpath(@__DIR__, test_file))) + LocalMathTestTelemetry.include_fixture( + @__MODULE__, $(joinpath(@__DIR__, test_file)), $test_name; + kind = "parallel_fixture", + ) end end @@ -133,6 +136,7 @@ const LOCALMATH_TEST_INITIALIZATION = quote using Test import KernelAbstractions import LocalMath + include($(joinpath(@__DIR__, "telemetry_support.jl"))) end ParallelTestRunner.runtests( diff --git a/test/scientific_witnesses/runtests.jl b/test/scientific_witnesses/runtests.jl index a1150ed..797f7bf 100644 --- a/test/scientific_witnesses/runtests.jl +++ b/test/scientific_witnesses/runtests.jl @@ -1,14 +1,23 @@ -include("lbm_d2q9.jl") -include("lattice_spring.jl") -include("matrix_free_fem.jl") -include("zbuffer.jl") -include("compacted_dem_contacts.jl") -include("compacted_active_fem.jl") -include("compacted_particle_cells.jl") -include("ordered_rsa.jl") -include("ordered_pgs_3d.jl") -include("ordered_stoichiometry.jl") -include("localmath_authoring.jl") +include(joinpath(@__DIR__, "..", "telemetry_support.jl")) + +for (ordinal, witness) in enumerate(( + "lbm_d2q9.jl", + "lattice_spring.jl", + "matrix_free_fem.jl", + "zbuffer.jl", + "compacted_dem_contacts.jl", + "compacted_active_fem.jl", + "compacted_particle_cells.jl", + "ordered_rsa.jl", + "ordered_pgs_3d.jl", + "ordered_stoichiometry.jl", + "localmath_authoring.jl", + )) + LocalMathTestTelemetry.include_fixture( + @__MODULE__, joinpath(@__DIR__, witness), witness; + kind = "scientific_fixture", ordinal, + ) +end using Test diff --git a/test/telemetry_support.jl b/test/telemetry_support.jl new file mode 100644 index 0000000..d22c701 --- /dev/null +++ b/test/telemetry_support.jl @@ -0,0 +1,38 @@ +module LocalMathTestTelemetry + +using TOML + +function include_fixture( + target::Module, path::AbstractString, label::AbstractString; + kind = "fixture", ordinal::Integer = 0, + ) + telemetry_directory = get(ENV, "LOCALMATH_CI_TELEMETRY", "") + isempty(telemetry_directory) && return Base.include(target, path) + + result = @timed Base.include(target, path) + mkpath(telemetry_directory) + safe_label = replace(label, r"[^A-Za-z0-9_.-]" => "_") + output = joinpath( + telemetry_directory, + "$(kind)-$(getpid())-$(safe_label).toml", + ) + open(output, "w") do stream + TOML.print(stream, Dict( + "schema_version" => 1, + "kind" => kind, + "fixture" => label, + "ordinal" => ordinal, + "first_family" => ordinal == 1, + "process_id" => getpid(), + "elapsed_seconds" => result.time, + "compile_seconds" => result.compile_time, + "recompile_seconds" => result.recompile_time, + "gc_seconds" => result.gctime, + "allocated_bytes" => result.bytes, + "rss_bytes" => Sys.maxrss(), + )) + end + return result.value +end + +end From 1ffa87e89a5cf5ad0cd56a9f9bcbb769c58b8735 Mon Sep 17 00:00:00 2001 From: PraneethMerugu Date: Fri, 18 Sep 2026 20:03:38 -0400 Subject: [PATCH 2/2] Separate dependency resolution from precompile telemetry --- .github/ci/measure.py | 14 +++++++++++--- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/docs.yml | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/ci/measure.py b/.github/ci/measure.py index df424aa..d50e259 100644 --- a/.github/ci/measure.py +++ b/.github/ci/measure.py @@ -34,6 +34,9 @@ def main() -> int: usage_before = resource.getrusage(resource.RUSAGE_CHILDREN) log_handle = open(args.log, "wb") if args.log else None + process = None + exit_code = 127 + launch_error = None try: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert process.stdout is not None @@ -44,9 +47,13 @@ def main() -> int: log_handle.write(chunk) exit_code = process.wait() except KeyboardInterrupt: - process.terminate() - process.wait() + if process is not None: + process.terminate() + process.wait() exit_code = 130 + except OSError as error: + launch_error = f"{type(error).__name__}: {error}" + print(launch_error, file=sys.stderr) finally: if log_handle: log_handle.close() @@ -66,7 +73,8 @@ def main() -> int: "system_cpu_seconds": usage_after.ru_stime - usage_before.ru_stime, "max_child_rss_kib": max_rss_kib, "command": command, - "process_id": process.pid, + "process_id": process.pid if process is not None else None, + "launch_error": launch_error, "runner_os": os.environ.get("RUNNER_OS", platform.system()), "runner_arch": os.environ.get("RUNNER_ARCH", platform.machine()), "github_sha": os.environ.get("GITHUB_SHA"), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33f45ab..54ac2f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job package) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])' @@ -97,7 +97,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job scientific) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=test/scientific_witnesses --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=test/scientific_witnesses --startup-file=no -e 'using LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/scientific.log" -- julia --project=test/scientific_witnesses --startup-file=no -e 'include("test/scientific_witnesses/runtests.jl")' @@ -125,7 +125,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-smoke) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=1", "--verbose", "test_public_api", "test_direct_pointwise_stage"])' @@ -156,7 +156,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job macos-package) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=. --startup-file=no -e 'using Pkg; Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=. --startup-file=no -e 'using LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/parallel-tests.log" -- julia --project=. --startup-file=no -e 'using Pkg; Pkg.test(; test_args=["--jobs=2", "--verbose"])' @@ -185,7 +185,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job docs) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")' @@ -217,7 +217,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry/fixtures" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job metal) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=test/metal --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=test/metal --startup-file=no -e 'using KernelAbstractions, LocalMath, Metal' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/metal.log" -- julia --project=test/metal --startup-file=no -e 'include("test/metal/runtests.jl")' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0eb6295..e072eeb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,7 +35,7 @@ jobs: mkdir -p "$RUNNER_TEMP/localmath-telemetry" measure=(python3 .github/ci/measure.py --output "$RUNNER_TEMP/localmath-telemetry" --job documentation-publish) "${measure[@]}" --phase cache-observation --annotation cache_hit=${{ steps.cache.outputs.cache-hit }} -- python3 -c 'pass' - "${measure[@]}" --phase instantiate -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + "${measure[@]}" --phase instantiate -- env JULIA_PKG_PRECOMPILE_AUTO=0 julia --project=docs --startup-file=no -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' "${measure[@]}" --phase precompile -- julia --project=docs --startup-file=no -e 'using Pkg; Pkg.precompile()' "${measure[@]}" --phase import -- julia --project=docs --startup-file=no -e 'using Documenter, LocalMath' "${measure[@]}" --phase suite --log "$RUNNER_TEMP/localmath-telemetry/docs.log" -- julia --project=docs --startup-file=no -e 'include("docs/make.jl")'