From 92d1b26aacda6c38c79dd8768cb166fd4f9fffad Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Wed, 4 Nov 2026 10:50:54 -0800 Subject: [PATCH] Prepare scoped CI for the standalone CUTLASS package Allow project-owned workflow rows so package CI changes select their own coverage without changing the shared matrix. Register the CUTLASS jobs and ownership before the package lands, leaving its workflow rows optional. Select compile-time benchmarks from their actual inputs while preserving coverage for public headers and shared build infrastructure. --- .../actions/workflow-build/build-workflow.py | 68 ++++++++-- .../workflows/ci-workflow-pull-request.yml | 14 ++ ci/inspect_compile_time_changes.py | 67 ++++++++++ ci/matrix.yaml | 8 ++ ci/project_files_and_dependencies.yaml | 18 +++ ci/test/inspect_changes/core_dirty.output | 2 +- .../cuda_coop_cutlass.dirty_files | 1 + .../inspect_changes/cuda_coop_cutlass.output | 2 + .../cuda_coop_python_license.output | 2 +- .../inspect_changes/libcudacxx_both.output | 2 +- .../libcudacxx_public_only.output | 2 +- .../inspect_changes/libcudacxx_thrust.output | 2 +- ci/test/test_inspect_compile_time_changes.py | 126 ++++++++++++++++++ ci/test/test_project_workflows.py | 111 +++++++++++++++ .../ci/references/matrix_yaml.rst | 15 +++ 15 files changed, 427 insertions(+), 13 deletions(-) create mode 100644 ci/inspect_compile_time_changes.py create mode 100644 ci/test/inspect_changes/cuda_coop_cutlass.dirty_files create mode 100644 ci/test/inspect_changes/cuda_coop_cutlass.output create mode 100644 ci/test/test_inspect_compile_time_changes.py create mode 100644 ci/test/test_project_workflows.py diff --git a/.github/actions/workflow-build/build-workflow.py b/.github/actions/workflow-build/build-workflow.py index fb50174fccce..bd7a69b83eb6 100755 --- a/.github/actions/workflow-build/build-workflow.py +++ b/.github/actions/workflow-build/build-workflow.py @@ -62,6 +62,7 @@ import re import struct import sys +from pathlib import Path import yaml @@ -1060,21 +1061,22 @@ def natural_sort_key(key): return workflow_dispatch_groups -def find_workflow_line_number(workflow_name): +def find_workflow_line_number(workflow_name, filename=None): + filename = filename or matrix_yaml["filename"] regex = re.compile(f"^( )*{workflow_name}:", re.IGNORECASE) line_number = 0 - with open(matrix_yaml["filename"], "r") as f: + with open(filename, "r") as f: for line in f: line_number += 1 if regex.match(line): return line_number raise Exception( - f"Workflow '{workflow_name}' not found in {matrix_yaml['filename]']} (could not match regex: {regex})" + f"Workflow '{workflow_name}' not found in {filename} (could not match regex: {regex})" ) -def get_matrix_job_origin(matrix_job, workflow_name, workflow_location): - filename = matrix_yaml["filename"] +def get_matrix_job_origin(matrix_job, workflow_name, workflow_location, filename=None): + filename = filename or matrix_yaml["filename"] original_matrix_job = json.dumps(matrix_job, indent=None, separators=(", ", ": ")) original_matrix_job = original_matrix_job.replace('"', "") return { @@ -1350,9 +1352,9 @@ def parse_workflow_matrix_jobs(workflow_name, filter_projects=None): workflow_location = ( f"{matrix_yaml['filename']}:{workflow_line_number} (job {idx + 1})" ) - matrix_job["origin"] = get_matrix_job_origin( - matrix_job, workflow_name, workflow_location - ) + matrix_job["origin"] = matrix_yaml.get("project_workflow_origins", {}).get( + (workflow_name, idx) + ) or get_matrix_job_origin(matrix_job, workflow_name, workflow_location) # Fill in default values, explode lists. matrix_jobs = preprocess_matrix_jobs(matrix_jobs, is_exclusion_matrix) @@ -1548,6 +1550,53 @@ def print_devcontainer_info(args): print(json.dumps(devcontainer_json, indent=2)) +def load_project_workflows(matrix, matrix_file): + """Append optional project-owned workflow rows and retain their source locations.""" + allowed_workflows = { + "pull_request", + "pull_request_lite", + "pull_request_fork", + "nightly", + "weekly", + } + origins = {} + for project, config in matrix["projects"].items(): + workflows_file = config.get("workflows_file") + if workflows_file is None: + continue + path = Path(matrix_file).parent / workflows_file + if not path.exists(): + continue + workflows = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(workflows, dict): + raise ValueError(f"{path}: expected a mapping of workflows to job lists") + for workflow_name, jobs in workflows.items(): + if ( + workflow_name not in allowed_workflows + or workflow_name not in matrix["workflows"] + ): + raise ValueError( + f"{path}: unsupported project workflow '{workflow_name}'" + ) + if not isinstance(jobs, list): + raise ValueError( + f"{path}: workflow '{workflow_name}' must contain a job list" + ) + existing_jobs = matrix["workflows"][workflow_name] or [] + line_number = find_workflow_line_number(workflow_name, str(path)) + for idx, job in enumerate(jobs): + if not isinstance(job, dict) or job.get("project") != project: + raise ValueError( + f"{path}: every job must select only project '{project}'" + ) + location = f"{path}:{line_number} (job {idx + 1})" + origins[(workflow_name, len(existing_jobs) + idx)] = ( + get_matrix_job_origin(job, workflow_name, location, str(path)) + ) + matrix["workflows"][workflow_name] = existing_jobs + jobs + return origins + + def preprocess_matrix_yaml(matrix): # Numeric YAML keys lose the spelling of dotted versions. if not all(isinstance(version, str) for version in matrix["ctk_versions"]): @@ -1616,6 +1665,9 @@ def main(): matrix_yaml = yaml.safe_load(f) matrix_yaml = preprocess_matrix_yaml(matrix_yaml) matrix_yaml["filename"] = args.matrix_file + matrix_yaml["project_workflow_origins"] = load_project_workflows( + matrix_yaml, args.matrix_file + ) if args.workflows: print_gha_workflow(args) diff --git a/.github/workflows/ci-workflow-pull-request.yml b/.github/workflows/ci-workflow-pull-request.yml index ef342ab2f8a1..d3e9a91c625f 100644 --- a/.github/workflows/ci-workflow-pull-request.yml +++ b/.github/workflows/ci-workflow-pull-request.yml @@ -77,6 +77,11 @@ jobs: echo "pr_number=${{ fromJSON(steps.get-pr-info.outputs.pr-info).number }}" | tee -a "${GITHUB_OUTPUT}" git fetch origin "${BASE_SHA}" -q echo "merge_base_sha=$(git merge-base "${BASE_SHA}" "${GITHUB_SHA}")" | tee -a "${GITHUB_OUTPUT}" + - name: Inspect compile-time benchmark inputs + id: inspect-compile-time-changes + env: + BASE_SHA: ${{ steps.export-pr-info.outputs.base_sha }} + run: python3 ci/inspect_compile_time_changes.py --refs "${BASE_SHA}" "${GITHUB_SHA}" - name: Build benchmark dispatch matrix id: build-bench-matrix run: | @@ -109,6 +114,15 @@ jobs: exit 0 fi + if [[ "${{ steps.inspect-compile-time-changes.outputs.compile_time_changed }}" != "true" ]]; then + echo "Public headers and compile-time benchmark inputs are unchanged; skipping dispatch." + { + echo "compile_time_enabled=false" + echo 'compile_time_matrix={"include":[]}' + } >> "${GITHUB_OUTPUT}" + exit 0 + fi + compile_time_matrix_json="$(./ci/compile_time/parse_matrix.py "ci/matrix.yaml" --workflow pull_request)" compile_time_enabled="$( jq -r 'if (.include // []) == [] then "false" else "true" end' \ diff --git a/ci/inspect_compile_time_changes.py b/ci/inspect_compile_time_changes.py new file mode 100644 index 000000000000..8b742e06172f --- /dev/null +++ b/ci/inspect_compile_time_changes.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Select PR compile-time benchmarks from their inputs before allocating runners.""" + +from __future__ import annotations + +from typing import Sequence + +import inspect_changes + +# The benchmark builds public headers, not Python packages or C++ library tests. +# Reuse their ownership from the matrix classifier. +PUBLIC_PROJECTS = ("libcudacxx_public", "cub_public", "thrust_public") +BENCHMARK_INPUTS = ( + r"ci/compile_time/", + r"ci/build_compile_time_bench\.sh$", + r"\.github/workflows/compile-time-bench\.yml$", +) + + +def compile_time_changed( + config: inspect_changes.Config, + dirty_files: Sequence[str], + ignored_files: Sequence[str], +) -> bool: + # Include benchmark inputs ignored by the main matrix, while continuing to + # ignore documentation-only edits. + all_files = [ + path for path in (*dirty_files, *ignored_files) if not path.endswith(".md") + ] + inputs = inspect_changes.compile_patterns(BENCHMARK_INPUTS) + benchmark_changed = any( + inspect_changes.matches_any(inputs, path) for path in all_files + ) + project_files = inspect_changes.build_project_dirty_map(config, dirty_files) + + # Preserve the classifier's conservative fallback for unknown infrastructure. + infrastructure_changed = bool(project_files[inspect_changes.CORE_PROJECT_KEY]) + public_changed = any(project_files[key] for key in PUBLIC_PROJECTS) + # Package config, install rules, and header-test generation live beside the + # public headers. Their changes also affect compile-time data. + cmake_inputs = inspect_changes.compile_patterns( + ( + r"(libcudacxx|cub|thrust|cudax)/(cmake/|CMakeLists\.txt$)", + r"cccl-version\.json$", + ) + ) + cmake_changed = any( + inspect_changes.matches_any(cmake_inputs, path) for path in all_files + ) + return ( + infrastructure_changed or public_changed or cmake_changed or benchmark_changed + ) + + +def main(argv: Sequence[str] | None = None) -> int: + args = inspect_changes.parse_args(argv) + config = inspect_changes.load_config(inspect_changes.CONFIG_PATH) + dirty_files, ignored_files = inspect_changes.collect_dirty_files(args, config) + enabled = compile_time_changed(config, dirty_files, ignored_files) + inspect_changes.write_output("compile_time_changed", str(enabled).lower()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/matrix.yaml b/ci/matrix.yaml index f49b76bd9602..149a1b0e0cbc 100644 --- a/ci/matrix.yaml +++ b/ci/matrix.yaml @@ -864,6 +864,11 @@ jobs: test_py_coop_numba_mlir_compile: { name: "Compile cuda.coop Numba-CUDA-MLIR (GPU hidden)", gpu: false, needs: 'build', force_producer_ctk: "pybuild", invoke: { prefix: 'test', args: '-stage numba-mlir-compile'} } test_py_coop_numba_mlir_runtime: { name: "Test cuda.coop Numba-CUDA-MLIR (L4)", gpu: true, needs: 'build', force_producer_ctk: "pybuild", invoke: { prefix: 'test', args: '-stage numba-mlir-runtime'} } + # Standalone CUTLASS compiler tests build and qualify the runtime wheel. + test_py_coop_cutlass_host: { name: "Test cuda.coop.cutlass (host)", gpu: false, invoke: { prefix: 'test', args: '-stage host'} } + test_py_coop_cutlass_compile: { name: "Compile cuda.coop.cutlass (GPU hidden)", gpu: false, invoke: { prefix: 'test', args: '-stage compile'} } + test_py_coop_cutlass_runtime: { name: "Test cuda.coop.cutlass", gpu: true, needs: 'test_py_coop_cutlass_compile', invoke: { prefix: 'test', args: '-stage runtime'} } + # Run jobs for 'target' project (ci/util/build_and_test_targets.sh): run_cpu: { gpu: false } run_gpu: { gpu: true } @@ -948,6 +953,9 @@ projects: test: ['test_py_compute_minimal'] cuda_coop_python: name: "Python (cuda.coop)" + cuda_coop_cutlass: + name: "Python (cuda.coop.cutlass)" + workflows_file: ../python/cuda_coop_cutlass/ci/matrix.yaml cccl_c_parallel: name: 'CCCL C Parallel' stds: [20] diff --git a/ci/project_files_and_dependencies.yaml b/ci/project_files_and_dependencies.yaml index 33243ebf39dc..ca435c2dd667 100644 --- a/ci/project_files_and_dependencies.yaml +++ b/ci/project_files_and_dependencies.yaml @@ -199,6 +199,23 @@ projects: - "^pyproject\\.toml$" exclude_regexes: [] + cuda_coop_cutlass: + name: "cuda.coop.cutlass" + matrix_project: "cuda_coop_cutlass" + lite_dependencies: [libcudacxx_public, cub_public, thrust_public] + full_dependencies: [] + include_regexes: + - 'python/cuda_coop_cutlass/' + - 'ci/(build|test)_cuda_coop_cutlass\.sh$' + - 'ci/windows/build_cuda_coop_cutlass\.ps1$' + - 'ci/cuda_coop_cutlass_requirements\.txt$' + - 'ci/validate_cuda_coop_cutlass_artifacts\.py$' + - 'ci/test/test_cuda_coop_cutlass_ci\.py$' + - '^LICENSE$' + - '^cub/LICENSE\.TXT$' + - '^libcudacxx/LICENSE\.TXT$' + - '^thrust/LICENSE$' + packaging: name: "CCCL Packaging" matrix_project: "packaging" @@ -213,6 +230,7 @@ projects: - cudax_internal - cudax_public full_dependencies: [] + exclude_project_files: [cuda_coop_cutlass] include_regexes: - "examples/" - "test/cmake/" diff --git a/ci/test/inspect_changes/core_dirty.output b/ci/test/inspect_changes/core_dirty.output index cac815153ad5..873425c2c9d6 100644 --- a/ci/test/inspect_changes/core_dirty.output +++ b/ci/test/inspect_changes/core_dirty.output @@ -1,2 +1,2 @@ -FULL_BUILD=libcudacxx cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python packaging stdpar nvbench_helper nvrtcc tidy +FULL_BUILD=libcudacxx cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python cuda_coop_cutlass packaging stdpar nvbench_helper nvrtcc tidy LITE_BUILD= diff --git a/ci/test/inspect_changes/cuda_coop_cutlass.dirty_files b/ci/test/inspect_changes/cuda_coop_cutlass.dirty_files new file mode 100644 index 000000000000..241c8e59f109 --- /dev/null +++ b/ci/test/inspect_changes/cuda_coop_cutlass.dirty_files @@ -0,0 +1 @@ +python/cuda_coop_cutlass/cuda/coop/cutlass/_compiler/_runtime.py diff --git a/ci/test/inspect_changes/cuda_coop_cutlass.output b/ci/test/inspect_changes/cuda_coop_cutlass.output new file mode 100644 index 000000000000..8b78b4850de9 --- /dev/null +++ b/ci/test/inspect_changes/cuda_coop_cutlass.output @@ -0,0 +1,2 @@ +FULL_BUILD=cuda_coop_cutlass +LITE_BUILD= diff --git a/ci/test/inspect_changes/cuda_coop_python_license.output b/ci/test/inspect_changes/cuda_coop_python_license.output index 25b5a5aa9040..b4388ace4bf8 100644 --- a/ci/test/inspect_changes/cuda_coop_python_license.output +++ b/ci/test/inspect_changes/cuda_coop_python_license.output @@ -1,2 +1,2 @@ -FULL_BUILD=cuda_coop_python +FULL_BUILD=cuda_coop_python cuda_coop_cutlass LITE_BUILD= diff --git a/ci/test/inspect_changes/libcudacxx_both.output b/ci/test/inspect_changes/libcudacxx_both.output index 1cacd88a3835..5496888b3b76 100644 --- a/ci/test/inspect_changes/libcudacxx_both.output +++ b/ci/test/inspect_changes/libcudacxx_both.output @@ -1,2 +1,2 @@ FULL_BUILD=libcudacxx tidy -LITE_BUILD=cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python packaging stdpar nvbench_helper +LITE_BUILD=cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python cuda_coop_cutlass packaging stdpar nvbench_helper diff --git a/ci/test/inspect_changes/libcudacxx_public_only.output b/ci/test/inspect_changes/libcudacxx_public_only.output index 1cacd88a3835..5496888b3b76 100644 --- a/ci/test/inspect_changes/libcudacxx_public_only.output +++ b/ci/test/inspect_changes/libcudacxx_public_only.output @@ -1,2 +1,2 @@ FULL_BUILD=libcudacxx tidy -LITE_BUILD=cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python packaging stdpar nvbench_helper +LITE_BUILD=cub thrust cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python cuda_coop_cutlass packaging stdpar nvbench_helper diff --git a/ci/test/inspect_changes/libcudacxx_thrust.output b/ci/test/inspect_changes/libcudacxx_thrust.output index 0ad3962639bb..222485192299 100644 --- a/ci/test/inspect_changes/libcudacxx_thrust.output +++ b/ci/test/inspect_changes/libcudacxx_thrust.output @@ -1,2 +1,2 @@ FULL_BUILD=libcudacxx thrust tidy -LITE_BUILD=cub cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python packaging stdpar nvbench_helper +LITE_BUILD=cub cudax cccl_c_parallel cccl_c_parallel_v2 python_v2 python_tsan cccl_c_stf python cuda_coop_python cuda_coop_cutlass packaging stdpar nvbench_helper diff --git a/ci/test/test_inspect_compile_time_changes.py b/ci/test/test_inspect_compile_time_changes.py new file mode 100644 index 000000000000..4af0bc95c941 --- /dev/null +++ b/ci/test/test_inspect_compile_time_changes.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run with python3 -m pytest ci/test/test_inspect_compile_time_changes.py.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import inspect_changes # noqa: E402 +from inspect_compile_time_changes import compile_time_changed # noqa: E402 + + +def benchmark_selected(paths): + config = inspect_changes.load_config(inspect_changes.CONFIG_PATH) + ignored = inspect_changes.compile_patterns(config.ignore_regexes) + dirty_files = [ + path for path in paths if not inspect_changes.matches_any(ignored, path) + ] + ignored_files = [ + path for path in paths if inspect_changes.matches_any(ignored, path) + ] + return compile_time_changed(config, dirty_files, ignored_files) + + +@pytest.mark.parametrize( + "path", + [ + "python/cuda_coop_cutlass/cuda/coop/cutlass/_thread_data.py", + "python/cuda_coop_cutlass/pyproject.toml", + "python/cuda_coop_cutlass/ci/matrix.yaml", + "ci/test_cuda_coop_cutlass.sh", + "python/cuda_cccl/cuda/compute/_bindings.py", + "python/cuda_stf/cuda/stf/_experimental.py", + "docs/python/coop_cutlass/programming_guide.rst", + "cub/test/catch2_test_block_scan.cu", + "ci/compile_time/README.md", + "cudax/include/cuda/experimental/device", + ], +) +def test_unrelated_changes_do_not_allocate_benchmark_runners(path): + assert not benchmark_selected([path]) + + +@pytest.mark.parametrize( + "path", + [ + "cub/cub/block/block_scan.cuh", + "thrust/thrust/sort.h", + "libcudacxx/include/cuda/std/tuple", + "cub/cmake/CubBuildCompilerTargets.cmake", + "thrust/CMakeLists.txt", + "libcudacxx/cmake/LibcudacxxPublicHeaderTesting.cmake", + "cccl-version.json", + "CMakePresets.json", + "ci/matrix.yaml", + "ci/project_files_and_dependencies.yaml", + ".devcontainer/launch.sh", + "ci/future_shared_infrastructure.py", + ], +) +def test_shared_headers_and_build_inputs_select_the_benchmark(path): + assert benchmark_selected([path]) + + +@pytest.mark.parametrize( + "path", + [ + "ci/compile_time/summarize_events.py", + "ci/build_compile_time_bench.sh", + ".github/workflows/compile-time-bench.yml", + ], +) +def test_benchmark_inputs_select_the_benchmark(path): + assert benchmark_selected([path]) + + +@pytest.mark.parametrize( + ("earlier_path", "expected"), + [ + ("python/cuda_coop_cutlass/pyproject.toml", "false"), + ("ci/compile_time/parse_matrix.py", "true"), + ], +) +def test_refs_use_the_pr_merge_base_and_write_github_outputs( + tmp_path, monkeypatch, earlier_path, expected +): + # Upstream-only changes must not add jobs to a PR; earlier commits in the PR + # must still count even when its final commit changes an unrelated file. + def git(*args): + return subprocess.check_output(["git", *args], cwd=tmp_path, text=True).strip() + + def commit_file(path, contents): + target = tmp_path / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(contents) + git("add", path) + git("-c", "commit.gpgsign=false", "commit", "-qm", "fixture") + return git("rev-parse", "HEAD") + + git("init", "-q") + git("remote", "add", "origin", str(tmp_path)) + git("config", "user.email", "test@example.com") + git("config", "user.name", "CI routing test") + base = commit_file("README.md", "base\n") + upstream = commit_file("cub/cub/block/block_scan.cuh", "upstream only\n") + git("checkout", "--detach", base) + commit_file(earlier_path, "earlier PR commit\n") + head = commit_file( + "python/cuda_coop_cutlass/cuda/coop/cutlass/_thread_data.py", + "latest PR commit\n", + ) + + monkeypatch.setattr(inspect_changes, "REPO_ROOT", tmp_path) + output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + from inspect_compile_time_changes import main + + assert main(["--refs", upstream, head]) == 0 + assert dict(line.split("=", 1) for line in output.read_text().splitlines()) == { + "compile_time_changed": expected, + } diff --git a/ci/test/test_project_workflows.py b/ci/test/test_project_workflows.py new file mode 100644 index 000000000000..7ba1d992fc39 --- /dev/null +++ b/ci/test/test_project_workflows.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise project-owned workflow rows through the CI matrix generator.""" + +import json +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +GENERATOR = ROOT / ".github/actions/workflow-build/build-workflow.py" +load_project_workflows = runpy.run_path(str(GENERATOR))["load_project_workflows"] + + +@pytest.fixture +def matrix(tmp_path): + config = yaml.safe_load((ROOT / "ci/matrix.yaml").read_text()) + config["projects"]["fragment_test"] = { + "name": "Fragment test", + "workflows_file": "../package/workflows.yaml", + } + config["workflows"]["pull_request"] = [ + {"project": "cub", "jobs": ["build"], "ctk": "13.X", "cxx": "gcc13", "std": 17} + ] + matrix_path = tmp_path / "ci/matrix.yaml" + matrix_path.parent.mkdir() + return config, matrix_path + + +def generate(config, matrix_path): + matrix_path.write_text(yaml.safe_dump(config, sort_keys=False)) + subprocess.run( + [ + sys.executable, + str(GENERATOR), + str(matrix_path), + "--workflows", + "pull_request", + ], + cwd=matrix_path.parent, + check=True, + capture_output=True, + text=True, + ) + return json.loads((matrix_path.parent / "workflow/workflow.json").read_text()) + + +def write_fragment(matrix_path, workflows): + path = matrix_path.parent.parent / "package/workflows.yaml" + path.parent.mkdir(exist_ok=True) + path.write_text(yaml.safe_dump(workflows, sort_keys=False)) + return path + + +def test_missing_project_workflows_preserve_existing_jobs(matrix): + config, matrix_path = matrix + actual = generate(config, matrix_path) + del config["projects"]["fragment_test"]["workflows_file"] + assert actual == generate(config, matrix_path) + + +def test_project_workflows_append_jobs_with_their_source_location(matrix): + config, matrix_path = matrix + path = write_fragment( + matrix_path, + { + "pull_request": [ + { + "project": "fragment_test", + "jobs": ["build"], + "ctk": "13.X", + "cxx": "gcc13", + "std": 17, + } + ] + }, + ) + workflow = generate(config, matrix_path) + jobs = [job for group in workflow.values() for job in group.get("standalone", [])] + assert {job["origin"]["matrix_job"]["project"] for job in jobs} == { + "cub", + "fragment_test", + } + fragment_job = next( + job for job in jobs if job["origin"]["matrix_job"]["project"] == "fragment_test" + ) + assert Path(fragment_job["origin"]["filename"]).resolve() == path.resolve() + + +@pytest.mark.parametrize( + "workflows", + [ + {"override": []}, + {"exclude": []}, + {"python-wheels": []}, + {"unknown": []}, + {"pull_request": [{"project": "cub"}]}, + {"pull_request": [{"project": ["fragment_test", "cub"]}]}, + ], +) +def test_project_workflows_cannot_change_other_projects_or_dispatch(matrix, workflows): + config, matrix_path = matrix + write_fragment(matrix_path, workflows) + with pytest.raises( + ValueError, match="unsupported project workflow|must select only project" + ): + load_project_workflows(config, matrix_path) diff --git a/docs/infrastructure/ci/references/matrix_yaml.rst b/docs/infrastructure/ci/references/matrix_yaml.rst index fc4d68825451..3d8296ac23b3 100644 --- a/docs/infrastructure/ci/references/matrix_yaml.rst +++ b/docs/infrastructure/ci/references/matrix_yaml.rst @@ -68,6 +68,21 @@ Workflow types * - ``devcontainers`` - Image-generation matrix. Entries catalog currently available devcontainer configs and map to no real jobs. +Project workflow files +---------------------- + +A project may declare ``workflows_file`` in its ``projects`` entry. The path is +relative to ``ci/matrix.yaml`` and contains a mapping of workflow names to job +lists. Its rows are appended to the main matrix and must select that project +with a scalar ``project`` value. Supported names are ``pull_request``, +``pull_request_lite``, ``pull_request_fork``, ``nightly``, and ``weekly``; +each must already exist in the main matrix. + +An absent file contributes no rows, allowing CI definitions to precede the +package that supplies its jobs. Keep the file under a path owned by the project +in ``ci/project_files_and_dependencies.yaml`` so its edits select that project's +coverage. Shared job definitions and toolchain settings remain in the main matrix. + Job entry format ----------------