From 3fd77cfb5ff466021591e70fab23efd30ea7b975 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Mon, 2 Nov 2026 18:11:16 +0100 Subject: [PATCH 1/2] perf(repo-policy-sync): reuse shared Bazel dependency downloads --- .github/workflows/repo-policy-sync.yml | 95 +++++++++++++++++++++++++- repo_policy_sync/README.md | 21 ++++++ repo_policy_sync/src/cache.py | 46 +++++++++++++ repo_policy_sync/src/engine.py | 2 + repo_policy_sync/src/github.py | 2 + repo_policy_sync/tests/test_cache.py | 81 ++++++++++++++++++++++ 6 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 repo_policy_sync/src/cache.py create mode 100644 repo_policy_sync/tests/test_cache.py diff --git a/.github/workflows/repo-policy-sync.yml b/.github/workflows/repo-policy-sync.yml index a39e5f7..93f4e4d 100644 --- a/.github/workflows/repo-policy-sync.yml +++ b/.github/workflows/repo-policy-sync.yml @@ -65,11 +65,70 @@ jobs: exit 1 fi - apply: + cache: if: >- (github.event_name == 'schedule' && github.event.schedule == '30 0 * * *') || (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply') runs-on: ubuntu-24.04 + # A broken upstream download should not hold the apply job for hours. + timeout-minutes: 20 + permissions: + contents: read + # The cache action deletes old archives after uploading a replacement. + actions: write + env: + # This workflow chooses its cache seed. Other organizations can use their + # own reference integration without changing the policy-sync tool. + REFERENCE_REPOSITORY: eclipse-score/reference_integration + GH_TOKEN: ${{ secrets.SCORE_BOT_CLASSIC_PAT }} + steps: + - name: Checkout cache seed + uses: actions/checkout@v7.0.0 + with: + repository: ${{ env.REFERENCE_REPOSITORY }} + token: ${{ secrets.SCORE_BOT_CLASSIC_PAT }} + # The cache action compares the reference integration's lockfile with + # its previous commit to decide whether to rebuild the download cache. + fetch-depth: 2 + - name: Checkout tools version + uses: actions/checkout@v7.0.0 + with: + path: _policy-sync-tools + - name: Select Bazel version and Git authentication + run: | + # Warm dependencies with the Bazel version used by tools, including + # its security updates, rather than the seed's possibly older version. + cp _policy-sync-tools/.bazelversion .bazelversion + gh auth setup-git --hostname github.com + - name: Setup shared download caches + uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0 + with: + disk-cache-key: repo-policy-sync + # Keep the action's automatic repository-cache restore/save decisions: + # this checkout and its lockfile describe exactly the cache we build. + repository-cache-restore: auto + repository-cache-save: auto + # Persist dependency downloads and Bazel executables. Build results + # and unpacked dependencies stay on the runner for this job only. + disk-cache-restore: "false" + disk-cache-save: "false" + external-cache-restore: "false" + external-cache-save: "false" + enable-profiling: "false" + report-cache-hits: "false" + report-test-cache-hits: "false" + - name: Populate dependency downloads + run: bazel mod deps + + apply: + needs: cache + # Keep applying policies if cache warm-up fails; an older cache or direct + # downloads can still be used. The cache job's failure remains visible. + if: >- + !cancelled() && ( + (github.event_name == 'schedule' && github.event.schedule == '30 0 * * *') || + (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply')) + runs-on: ubuntu-24.04 env: MODE: apply GH_TOKEN: ${{ secrets.SCORE_BOT_CLASSIC_PAT }} @@ -92,6 +151,40 @@ jobs: fi - name: Configure Git authentication for apply mode run: gh auth setup-git --hostname github.com + - name: Restore shared download caches + uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0 + with: + disk-cache-key: repo-policy-sync + # Apply starts from the reference integration's downloads. Only the + # cache job saves them, so additions from other repositories do not + # accumulate in the persisted cache. + repository-cache-restore: "true" + repository-cache-save: "false" + bazelisk-cache-save: "false" + disk-cache-restore: "false" + disk-cache-save: "false" + external-cache-restore: "false" + external-cache-save: "false" + # Commands run with a restricted environment, without the extra state + # needed by profiling and cache-hit reporting. + enable-profiling: "false" + report-cache-hits: "false" + report-test-cache-hits: "false" + - name: Configure isolated Bazel caches + run: | + # Commands and hooks get fresh homes to keep runner credentials and + # personal settings out. Import only cache settings into those homes. + # Keep unpacked dependencies for this job so later policies reuse them. + policy_bazelrc="$RUNNER_TEMP/repo-policy-sync.bazelrc" + { + printf 'startup --output_user_root="%s"\n' "$RUNNER_TEMP/repo-policy-sync-bazel" + printf 'import "%s"\n' "$RUNNER_TEMP/setup-bazel-cache.bazelrc" + } > "$policy_bazelrc" + { + printf 'REPO_POLICY_SYNC_BAZELRC=%s\n' "$policy_bazelrc" + # Bazelisk downloads Bazel itself; the seed job saves this directory. + printf 'BAZELISK_HOME=%s\n' "$HOME/.cache/bazelisk" + } >> "$GITHUB_ENV" - name: Run policy sync in apply mode id: sync run: >- diff --git a/repo_policy_sync/README.md b/repo_policy_sync/README.md index 6495950..c21ac23 100644 --- a/repo_policy_sync/README.md +++ b/repo_policy_sync/README.md @@ -129,6 +129,27 @@ synchronization. Policy evaluation and apply work, including policy follow-up commands such as Bazel, run across independent repositories with `--policy-workers N`. +Follow-up commands and pre-commit hooks use temporary home directories. To +share Bazel caches in CI, set `REPO_POLICY_SYNC_BAZELRC` to a trusted rc file +containing cache settings and optionally `BAZELISK_HOME` to the Bazelisk download +cache. The selected rc is imported into each temporary home's `.bazelrc`, +including for Bazel 8. Other user configuration and credentials are not +inherited. + +Before apply, a separate job warms one shared Bazel repository download cache +and the Bazelisk executable cache using `eclipse-score/reference_integration`. +The seed repository is chosen in the workflow, not in the tool configuration. +`setup-bazel-cache` uses the seed's lockfile for its automatic cache reset and +save decisions. The apply job only restores caches, so its additional downloads +cannot overwrite the seed. Build results and extracted dependencies are not +persisted. A job-local `output_user_root` lets later policies and hook retries +reuse each repository's extracted dependencies within the apply job. + +A failed cache warm-up does not prevent apply from running; it can restore an +older cache or download missing dependencies itself. The +cache job still makes the workflow fail. A failed warm-up with cold caches +does not save new caches, following the cache action's failure policy. + To rebuild a policy PR from the current default branch, use the guarded recreate operation with exactly one repository and policy: diff --git a/repo_policy_sync/src/cache.py b/repo_policy_sync/src/cache.py new file mode 100644 index 0000000..efc8801 --- /dev/null +++ b/repo_policy_sync/src/cache.py @@ -0,0 +1,46 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Let commands with temporary homes reuse the configured Bazel caches.""" + +import os +from pathlib import Path +import shlex + + +def configure_bazel_cache(home: Path, environment: dict[str, str]) -> None: + """Make the configured caches available inside a command's temporary home. + + Policy commands and pre-commit hooks get a fresh HOME for each invocation. + Without explicit cache paths, they would download dependencies into that + home and lose them when the command finishes. + + REPO_POLICY_SYNC_BAZELRC selects a file containing the shared cache paths. + Import it from the temporary ~/.bazelrc so Bazel finds it even when called + by a hook. This also works with Bazel 8, which does not read the BAZELRC + environment variable. The runner's own ~/.bazelrc is not copied. + """ + + cache_rc = os.environ.get("REPO_POLICY_SYNC_BAZELRC") + if cache_rc: + # A relative path refers to the directory where policy sync started, + # not the target repository where the command will run. Use `import` + # so Bazel reports a missing file instead of silently running uncached. + path = Path(cache_rc).resolve() + (home / ".bazelrc").write_text(f"import {shlex.quote(str(path))}\n") + + bazelisk_home = os.environ.get("BAZELISK_HOME") + if bazelisk_home: + # Bazelisk caches the Bazel executable separately from dependencies. + # Preserve its download directory when replacing HOME. + environment["BAZELISK_HOME"] = str(Path(bazelisk_home).resolve()) diff --git a/repo_policy_sync/src/engine.py b/repo_policy_sync/src/engine.py index d352534..bc2e6d1 100644 --- a/repo_policy_sync/src/engine.py +++ b/repo_policy_sync/src/engine.py @@ -29,6 +29,7 @@ parse_bazel_version, starlark_call_ranges, ) +from .cache import configure_bazel_cache from .errors import CommandError, RepoPolicySyncError, redact_sensitive_text from .github import GitHubResolver from .models import Change, EnsureOperation, Evaluation, Policy @@ -168,6 +169,7 @@ def _run_after_apply_command(root: Path, command: tuple[str, ...]) -> None: with tempfile.TemporaryDirectory( prefix="repo-policy-sync-after-apply-" ) as home: + configure_bazel_cache(Path(home), environment) environment.update( { "HOME": home, diff --git a/repo_policy_sync/src/github.py b/repo_policy_sync/src/github.py index b8716ac..880a5fc 100644 --- a/repo_policy_sync/src/github.py +++ b/repo_policy_sync/src/github.py @@ -25,6 +25,7 @@ from pathlib import Path from threading import RLock +from .cache import configure_bazel_cache from .errors import CommandError, RepoPolicySyncError, redact_sensitive_text from .models import Change, Policy, policy_branch_slug @@ -498,6 +499,7 @@ def run_pre_commit( if key in _PRE_COMMIT_ENVIRONMENT_KEYS or key.startswith("LC_") } with tempfile.TemporaryDirectory(prefix=f"{TOOL_SLUG}-pre-commit-") as home: + configure_bazel_cache(Path(home), environment) # The normal apply workflow authenticates nested Git fetches by # installing a narrowly scoped `url.*.insteadOf` rewrite in the # runner's global Git configuration. Keep that mechanism diff --git a/repo_policy_sync/tests/test_cache.py b/repo_policy_sync/tests/test_cache.py new file mode 100644 index 0000000..3ceb858 --- /dev/null +++ b/repo_policy_sync/tests/test_cache.py @@ -0,0 +1,81 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +from pathlib import Path +import shlex + +import pytest + +from repo_policy_sync.src.engine import _run_after_apply_command +from repo_policy_sync.src.github import GitHubCli + + +@pytest.mark.parametrize("cache_enabled", [False, True]) +def test_follow_up_commands_and_hooks_share_only_explicit_cache_configuration( + tmp_path: Path, monkeypatch, cache_enabled: bool +) -> None: + """Policy commands and hooks find the shared caches from fresh homes. + + Both must import the selected cache configuration without copying the + runner's personal Bazel configuration or passing its GitHub token. + """ + + user_home = tmp_path / "user-home" + user_home.mkdir() + (user_home / ".bazelrc").write_text("build --remote_header=secret\n") + cache_rc = tmp_path / "cache configuration.bazelrc" + cache_rc.write_text(f'common --repository_cache="{tmp_path / "downloads"}"\n') + bazelisk_home = tmp_path / "bazelisk" + monkeypatch.setenv("HOME", str(user_home)) + monkeypatch.setenv("BAZELRC", str(user_home / ".bazelrc")) + monkeypatch.setenv("GH_TOKEN", "secret") + monkeypatch.delenv("REPO_POLICY_SYNC_BAZELRC", raising=False) + monkeypatch.delenv("BAZELISK_HOME", raising=False) + if cache_enabled: + # Cache paths are relative to where policy sync starts, even though + # the commands themselves run in a different repository directory. + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("REPO_POLICY_SYNC_BAZELRC", cache_rc.name) + monkeypatch.setenv("BAZELISK_HOME", bazelisk_home.name) + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / ".pre-commit-config.yaml").write_text("repos: []\n") + homes: list[Path] = [] + + def observe(command, *, cwd, env, **kwargs): + assert cwd == checkout + assert "GH_TOKEN" not in env + assert "BAZELRC" not in env + home = Path(env["HOME"]) + assert home != user_home + homes.append(home) + rc = home / ".bazelrc" + if cache_enabled: + assert shlex.split(rc.read_text()) == ["import", str(cache_rc)] + assert env["BAZELISK_HOME"] == str(bazelisk_home) + else: + assert not rc.exists() + assert "BAZELISK_HOME" not in env + return "" + + monkeypatch.setattr("repo_policy_sync.src.engine.subprocess.run", observe) + monkeypatch.setattr(GitHubCli, "_run", staticmethod(observe)) + monkeypatch.setattr( + "repo_policy_sync.src.github._copy_global_git_url_rewrites", lambda home: None + ) + + _run_after_apply_command(checkout, ("bazel", "mod", "deps")) + assert GitHubCli().run_pre_commit(checkout=checkout) + + assert len(set(homes)) == 2 + assert all(not home.exists() for home in homes) From 6a7673ff8bc4fb949d0a0e9451d1c2b7c73372fa Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Mon, 2 Nov 2026 18:16:45 +0100 Subject: [PATCH 2/2] docs(repo-policy-sync): clarify hook Git credential access --- .github/workflows/repo-policy-sync.yml | 6 ++++-- repo_policy_sync/README.md | 7 +++++-- repo_policy_sync/tests/test_cache.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/repo-policy-sync.yml b/.github/workflows/repo-policy-sync.yml index 93f4e4d..c6a7d90 100644 --- a/.github/workflows/repo-policy-sync.yml +++ b/.github/workflows/repo-policy-sync.yml @@ -172,8 +172,10 @@ jobs: report-test-cache-hits: "false" - name: Configure isolated Bazel caches run: | - # Commands and hooks get fresh homes to keep runner credentials and - # personal settings out. Import only cache settings into those homes. + # Commands and hooks get fresh homes without personal Bazel settings + # or token environment variables. Pre-commit separately receives Git + # URL rewrites, which can contain credentials for dependency downloads. + # Import the shared Bazel cache settings into each temporary home. # Keep unpacked dependencies for this job so later policies reuse them. policy_bazelrc="$RUNNER_TEMP/repo-policy-sync.bazelrc" { diff --git a/repo_policy_sync/README.md b/repo_policy_sync/README.md index c21ac23..586ad73 100644 --- a/repo_policy_sync/README.md +++ b/repo_policy_sync/README.md @@ -133,8 +133,11 @@ Follow-up commands and pre-commit hooks use temporary home directories. To share Bazel caches in CI, set `REPO_POLICY_SYNC_BAZELRC` to a trusted rc file containing cache settings and optionally `BAZELISK_HOME` to the Bazelisk download cache. The selected rc is imported into each temporary home's `.bazelrc`, -including for Bazel 8. Other user configuration and credentials are not -inherited. +including for Bazel 8. Personal Bazel settings and token environment variables +such as `GH_TOKEN` and `GITHUB_TOKEN` are not inherited. Pre-commit hooks do +receive global Git `url.*.insteadOf` rewrites for authenticated dependency +downloads. Those URLs can contain credentials, so hooks are not isolated from +Git credentials. Before apply, a separate job warms one shared Bazel repository download cache and the Bazelisk executable cache using `eclipse-score/reference_integration`. diff --git a/repo_policy_sync/tests/test_cache.py b/repo_policy_sync/tests/test_cache.py index 3ceb858..d737a23 100644 --- a/repo_policy_sync/tests/test_cache.py +++ b/repo_policy_sync/tests/test_cache.py @@ -27,7 +27,7 @@ def test_follow_up_commands_and_hooks_share_only_explicit_cache_configuration( """Policy commands and hooks find the shared caches from fresh homes. Both must import the selected cache configuration without copying the - runner's personal Bazel configuration or passing its GitHub token. + runner's personal Bazel configuration or passing GitHub token variables. """ user_home = tmp_path / "user-home"