From 78ec483013c9fbf191c4b18d95a124214dec6cbe Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 14:36:35 +0100 Subject: [PATCH 01/11] release: automate checksum-pinned archive installation --- .github/workflows/release.yml | 102 ++++++++- BUILD.bazel | 1 + CHANGELOG.md | 13 ++ CONTRIBUTING.md | 29 ++- README.md | 77 ++++--- docs/Configuration_Reference.md | 4 +- docs/Installation_Reference.md | 54 ++++- docs/Language_Onboarding.md | 185 ++++++++-------- docs/Maintainers.md | 6 +- examples/README.md | 8 +- .../go/tools/dd_topt_go_bootstrap/BUILD.bazel | 10 +- .../go/tools/dd_topt_go_bootstrap/archive.go | 201 ++++++++++++++++++ .../dd_topt_go_bootstrap/archive_test.go | 170 +++++++++++++++ modules/go/tools/dd_topt_go_bootstrap/main.go | 64 ++++-- .../references/bzlmod-onboarding.md | 33 +-- .../references/bzlmod-onboarding.md | 21 +- .../references/bzlmod-onboarding.md | 21 +- tools/dev/BUILD.bazel | 1 + tools/dev/build_release.py | 191 +++++++++++++++++ .../integration/run_bzlmod_go_integration.sh | 17 ++ .../run_workspace_go_integration.sh | 17 ++ tools/tests/python/BUILD.bazel | 3 + tools/tests/python/test_release_tools.py | 186 ++++++++++++++++ 23 files changed, 1231 insertions(+), 183 deletions(-) create mode 100644 modules/go/tools/dd_topt_go_bootstrap/archive.go create mode 100644 modules/go/tools/dd_topt_go_bootstrap/archive_test.go create mode 100644 tools/dev/build_release.py create mode 100644 tools/tests/python/test_release_tools.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b90015be..2a71fc13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,11 +111,11 @@ jobs: shell: bash run: ./tools/tests/integration/run_mock_server_tests.sh - - name: Release checklist reminder + - name: Release validation completed shell: bash run: | echo "Release validation completed." - echo "Before publishing, follow CONTRIBUTING.md release runbook steps." + echo "Tag runs prepare a draft only after every release gate succeeds." release-hermetic: timeout-minutes: 45 @@ -355,3 +355,101 @@ jobs: if: runner.os == 'Windows' shell: pwsh run: ./tools/tests/integration/run_mock_server_tests.ps1 + + release-archive: + timeout-minutes: 45 + runs-on: ubuntu-latest + steps: + - name: Checkout the release revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Test release packaging + run: python3 -m unittest tools.tests.python.test_release_tools + + - name: Build the release archive and installation notes + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + args=() + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + args+=(--tag "${GITHUB_REF#refs/tags/}") + fi + python3 tools/dev/check_module_versions.py + python3 tools/dev/check_release_archive_contents.py + python3 tools/dev/build_release.py --ref "$GITHUB_SHA" --output "$RUNNER_TEMP/release" "${args[@]}" + (cd "$RUNNER_TEMP/release" && sha256sum --check SHA256SUMS) + + - name: Test the packaged Go consumer in Bzlmod and WORKSPACE + timeout-minutes: 35 + shell: bash + run: | + set -euo pipefail + RTO_RELEASE_ARCHIVE="$RUNNER_TEMP/release/$(python3 -c 'import json, os; print(json.load(open(os.environ["RUNNER_TEMP"] + "/release/release.json"))["archive"])')" + RTO_RELEASE_PREFIX="$(python3 -c 'import json, os; print(json.load(open(os.environ["RUNNER_TEMP"] + "/release/release.json"))["strip_prefix"])')" + export RTO_RELEASE_ARCHIVE RTO_RELEASE_PREFIX + INTEGRATION_SCENARIO_MODE=release bash tools/tests/integration/run_bzlmod_go_integration.sh + INTEGRATION_SCENARIO_MODE=release bash tools/tests/integration/run_workspace_go_integration.sh + + - name: Save the validated release bundle + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-bundle + path: ${{ runner.temp }}/release/ + if-no-files-found: error + retention-days: 14 + + release-draft: + # A manual validation run never publishes, creates a tag, or edits a release. + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'DataDog/rules_test_optimization' + needs: + - release-validation + - release-hermetic + - release-shared-validation + - release-docs-links + - release-platform-smoke + - release-archive + timeout-minutes: 5 + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download the validated bundle from this run + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: release-bundle + path: release + + - name: Create a draft with changelog, checksums, and install snippets + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + cd release + sha256sum --check SHA256SUMS + python3 - <<'PY' + import json, os + metadata = json.load(open("release.json")) + assert metadata["commit"] == os.environ["GITHUB_SHA"], "bundle revision differs from validated checkout" + assert "refs/tags/" + metadata["tag"] == os.environ["GITHUB_REF"], "bundle tag mismatch" + PY + test "$(gh api "repos/$GH_REPO/commits/${GITHUB_REF#refs/tags/}" --jq .sha)" = "$GITHUB_SHA" + # No --clobber or edit path: an existing release is left untouched. + gh release create "${GITHUB_REF#refs/tags/}" \ + --draft --verify-tag --title "${GITHUB_REF#refs/tags/}" \ + --notes-file RELEASE_NOTES.md \ + ./*.tar.gz SHA256SUMS release.json INSTALL.md RELEASE_NOTES.md diff --git a/BUILD.bazel b/BUILD.bazel index a01ab9f3..e2c2ea35 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,5 +7,6 @@ # Root package exists to expose repository-level utility files to tests. exports_files([ ".bazelversion", + ".github/workflows/release.yml", "bazelw", ]) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5190f07f..797c9d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,19 @@ versioning. ## [Unreleased] +### Added + +- Release bundles with SHA256 checksums, archive-based installation snippets + for core/companions/Go forks, and notes generated from this changelog. Tag + workflows prepare a draft only after all release validations pass; manual + branch runs produce a downloadable preview without publishing. + +### Changed + +- Recommend release archives for installation. Go Bzlmod bootstrap now infers + and preserves checksum-pinned archive overrides and mirrors without requiring + a Git override for the Orchestrion-enabled fork. + ## [1.3.0] - 2026-10-21 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01ccfc69..2119e2b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,7 +305,34 @@ This product includes software developed at Datadog - CI release gate: - Trigger `.github/workflows/release.yml` via `workflow_dispatch` (or push a version tag) before publication. - - The workflow now always executes the full validation suite for release safety. + - Both paths execute the full validation suite and test the packaged archive + with real Go consumers in Bzlmod and WORKSPACE. A branch dispatch saves the + bundle as a workflow artifact but never creates a tag or release. + - A version-tag push prepares a **draft** GitHub release only after every + release job passes. It does not publish the draft or submit BCR changes. + Review the generated notes/assets, then publish explicitly. + - The draft contains `rules_test_optimization-X.Y.Z.tar.gz`, `SHA256SUMS`, + `release.json`, `INSTALL.md`, and `RELEASE_NOTES.md`. Notes come from the + matching `CHANGELOG.md` section. Installation blocks cover core, all six + companions, and every maintained Go fork, with one archive and checksum. + - The archive contains only the committed revision, including executable + modes. Packaging uses a fixed gzip header, so repeated builds of the same + revision with the same toolchain produce the same bytes. The workflow + verifies its checksums before attaching the exact validated bundle. + - An existing release is not edited or overwritten. If draft creation fails + partway through, inspect the existing draft and its assets before deciding + how to resume; rerunning must not replace published checksums. + +- Local bundle preview (Python 3.11+ and Git; no publication): + + ```sh + python3 tools/dev/build_release.py --ref HEAD --tag vX.Y.Z --output /path/to/empty-directory + ``` + + The expected tag must match `MODULE.bazel` and have a nonempty changelog + section. Omit `--tag` for a branch validation preview. Uncommitted edits are + not included. Do not relabel an older release archive or regenerate a + published asset in place. - Version alignment: - Keep root `MODULE.bazel` and all companion `modules/*/MODULE.bazel` versions aligned. diff --git a/README.md b/README.md index c7157074..58577aee 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This product includes software developed at Datadog This repository provides Bazel integrations that fetch Datadog Test Optimization metadata during module/repository resolution and materialize JSON files for use in your build. It also generates public filegroups so consumers can depend on stable labels instead of wiring files manually. -> Release status: this branch prepares `1.3.0`; it is not published yet. Bazel Central Registry publication is also pending. Use `git_override` (Bzlmod) or commit-pinned `git_repository` / `http_archive` (WORKSPACE), with the same commit for core and companions. The historical `v1.2.0` pins below still refer to that release, not this candidate. +> `v1.3.0` is published. For releases with an `INSTALL.md` asset, copy its checksum-pinned `archive_override` (Bzlmod) or `http_archive` (WORKSPACE) blocks. Keep core, companions, and the Go fork on the same archive. `v1.3.0` predates those assets; its commit-based installation remains supported. See [release installation](docs/Installation_Reference.md#option-a-install-from-a-release-archive-recommended). ## Onboarding paths @@ -327,6 +327,12 @@ check as the dev helper. ## Quickstart by scenario +Replace ``, ``, and +`` with the values from the release's `INSTALL.md`. Match the +Datadog `bazel_dep` versions to that release; the snippets below use `1.3.0` +API examples. For `v1.3.0` or an unreleased commit without these assets, use +the [Git alternative](docs/Installation_Reference.md#option-b-install-from-a-git-commit). + ### Bzlmod + core only (any language) Use this when you do not need `dd_topt_go_test`: @@ -334,10 +340,11 @@ Use this when you do not need `dd_topt_go_test`: ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) test_optimization_sync = use_extension( @@ -425,18 +432,19 @@ setup. ```bzl bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-go", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-go", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/go", + urls = [""], + sha256 = "", + strip_prefix = "/modules/go", ) bazel_dep(name = "rules_go", version = "0.60.0") ``` @@ -607,18 +615,19 @@ Orchestrion build flag. ```bzl bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-python", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-python", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/python", + urls = [""], + sha256 = "", + strip_prefix = "/modules/python", ) python_topt = use_extension( @@ -756,11 +765,11 @@ absent. ```bzl bazel_dep(name = "datadog-rules-test-optimization-java", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-java", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/java", + urls = [""], + sha256 = "", + strip_prefix = "/modules/java", ) ``` @@ -818,11 +827,11 @@ not include `srcs` by default and the extra files cost runfiles size. bazel_dep(name = "aspect_rules_js", version = "3.0.0-rc5") bazel_dep(name = "rules_nodejs", version = "6.7.3") bazel_dep(name = "datadog-rules-test-optimization-nodejs", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-nodejs", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/nodejs", + urls = [""], + sha256 = "", + strip_prefix = "/modules/nodejs", ) node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") @@ -851,11 +860,11 @@ dd_topt_nodejs_test( ```bzl bazel_dep(name = "rules_dotnet", version = "0.21.5") bazel_dep(name = "datadog-rules-test-optimization-dotnet", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-dotnet", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/dotnet", + urls = [""], + sha256 = "", + strip_prefix = "/modules/dotnet", ) dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") @@ -906,11 +915,11 @@ def dotnet_csharp_test_adapter(name, data = None, env = None, **kwargs): ```bzl bazel_dep(name = "rules_ruby", version = "0.21.1") bazel_dep(name = "datadog-rules-test-optimization-ruby", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-ruby", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/ruby", + urls = [""], + sha256 = "", + strip_prefix = "/modules/ruby", ) ruby = use_extension("@rules_ruby//ruby:extensions.bzl", "ruby") diff --git a/docs/Configuration_Reference.md b/docs/Configuration_Reference.md index 1fc94096..deea9e22 100644 --- a/docs/Configuration_Reference.md +++ b/docs/Configuration_Reference.md @@ -41,12 +41,12 @@ local files and managed blocks; it never edits `WORKSPACE` itself: | `--plain-wrapper-name` | `dd_go_test` | Central local wrapper function name generated by the wrapper template | | `--optimized-wrapper-name` | `dd_topt_go_test` | Compatibility alias for the central wrapper; not a separate enablement path | | `--datadog-fetch` | `git` | Fetch mode for `datadog-rules-test-optimization-go`: `git` or `archive` | -| `--rules-go-fetch` | `git` | Fetch mode for the Orchestrion-enabled `rules_go` fork: `git` or `archive` | +| `--rules-go-fetch` | `git` | Fetch mode for the Orchestrion-enabled fork: `git` or `archive`. Bzlmod infers existing archive overrides unless an explicit Git/source flag is supplied. | | `--rules-go-upstream` | `default` | Datadog-managed upstream support line. Omit this to preserve the repository default; the current default resolves to `v0_60_0` | | `--rules-go-variant` | `base` | Published fork variant. Only `base` is supported. | | `--rules-go-repo-name` | `io_bazel_rules_go` | Bazel repository name that consumers use for `rules_go` | | `--rto-commit` | empty | Published rules_test_optimization commit used in generated WORKSPACE snippets. Required when either fetch mode is `git` | -| `--rto-archive-url` | empty | Archive URL used when either fetch mode is `archive` | +| `--rto-archive-url` | empty | Datadog source archive URL for WORKSPACE archive mode or explicit Bzlmod fork wiring | | `--rto-archive-sha256` | empty | Archive SHA256 used when either fetch mode is `archive` | | `--rto-archive-prefix` | empty | Archive root prefix used when either fetch mode is `archive` | | `--rto-archive-type` | `tar.gz` | Archive type passed to `http_archive` | diff --git a/docs/Installation_Reference.md b/docs/Installation_Reference.md index 61665897..81c50916 100644 --- a/docs/Installation_Reference.md +++ b/docs/Installation_Reference.md @@ -14,12 +14,54 @@ and multi-service onboarding, use [`docs/Language_Onboarding.md`](Language_Onboa ## Bzlmod installation -### Option A: install from Git commit (recommended until BCR publication) +### Option A: install from a release archive (recommended) -The snippets below use the `1.3.0` candidate metadata. Pin a commit containing -that version for both core and companions. The historical `v1.2.0` archive -tuple further below belongs to the previous release and must not be relabeled -as `1.3.0`. +Download `INSTALL.md`, `release.json`, and `SHA256SUMS` from the chosen +[GitHub release](https://github.com/DataDog/rules_test_optimization/releases). +`INSTALL.md` contains complete dependency blocks with the release version, +archive URL, SHA256, and each module's `strip_prefix`. Copy the core block and +only the companion blocks you need. These overrides do not depend on BCR +publication. + +In `MODULE.bazel`, use `archive_override`, which uses Bazel's HTTP downloader +and repository cache. In WORKSPACE, use `http_archive`. Reuse the same archive +and SHA256 for core, companions, and the selected Go fork. Each companion uses +`/modules/`; a Go fork uses +`/third_party/rgo//base`. + +The following is the shape of a core Bzlmod install. Replace every placeholder +with the tuple from one release; do not copy a checksum from another version. + +```bzl +bazel_dep(name = "datadog-rules-test-optimization", version = "") +archive_override( + module_name = "datadog-rules-test-optimization", + urls = [""], + sha256 = "", + strip_prefix = "", +) +``` + +For Go, the bootstrap can infer the archive from a literal core/Go companion +override or preserve an existing supported `rules_go` archive override. It +keeps URL mirrors, SHA256, and the selected fork. A compatible manual fork +override moves into the managed block, so there is only one override per +module. Custom fork patches or computed Starlark values are not silently +rewritten. For computed dependency values, supply +`--rules-go-fetch=archive --rto-archive-url=... --rto-archive-sha256=... --rto-archive-prefix=...`. +`--rules-go-fetch=git` retains explicit Git-mode bootstrapping. + +Release bundles are prepared by the [release workflow](../CONTRIBUTING.md#release-runbook-core--companion-modules). +GitHub's automatic source downloads are distinct from the uploaded, checksum-pinned +release asset. `v1.3.0` predates bundle automation and has no `INSTALL.md` asset; +use the Git recipe below for that version. The historical `v1.2.0` archive +tuple further below still belongs to `v1.2.0`. + +### Option B: install from a Git commit + +Use this for an unreleased change, an older release without bundle assets, or +a private repository that requires Git authentication. Pin the same published +commit for core and companions. The example module version is `1.3.0`. In your `MODULE.bazel`: @@ -161,7 +203,7 @@ the `rules_test_optimization` Git history, add `--verify-main-reachable` for the same `origin/main` reachability check as the dev helper. Existing unmanaged files are preserved unless `--force` is passed. -### Option B: local development overrides +### Option C: local development overrides ```bzl local_path_override( diff --git a/docs/Language_Onboarding.md b/docs/Language_Onboarding.md index 2bf1b148..abee05bc 100644 --- a/docs/Language_Onboarding.md +++ b/docs/Language_Onboarding.md @@ -11,6 +11,11 @@ This product includes software developed at Datadog This document is organized so each language owner can read one section and wire their runtime without having to reconstruct the rest of the repository. +For released dependencies, copy the archive URL, SHA256, prefix, and module +version from the release's `INSTALL.md`. The Bzlmod examples use placeholders +for that tuple. Releases without this asset and development snapshots can use +the [Git alternative](Installation_Reference.md#option-b-install-from-a-git-commit). + Terminology used here: - Single-service: one Datadog service for the workspace or runtime slice @@ -205,18 +210,19 @@ workspaces. ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-go", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-go", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/go", + urls = [""], + sha256 = "", + strip_prefix = "/modules/go", ) bazel_dep(name = "rules_go", version = "0.60.0") @@ -542,18 +548,19 @@ service with `runtime_name = "go"`. ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-go", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-go", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/go", + urls = [""], + sha256 = "", + strip_prefix = "/modules/go", ) bazel_dep(name = "rules_go", version = "0.60.0") @@ -618,18 +625,19 @@ dd_test_optimization_targets( ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-python", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-python", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/python", + urls = [""], + sha256 = "", + strip_prefix = "/modules/python", ) topt = use_extension( @@ -886,18 +894,19 @@ runtime metadata. ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-python", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-python", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/python", + urls = [""], + sha256 = "", + strip_prefix = "/modules/python", ) topt = use_extension( @@ -961,18 +970,19 @@ dd_test_optimization_targets( ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-java", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-java", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/java", + urls = [""], + sha256 = "", + strip_prefix = "/modules/java", ) topt = use_extension( @@ -1039,18 +1049,19 @@ dd_topt_java_test( ```bzl # MODULE.bazel bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-java", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-java", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/java", + urls = [""], + sha256 = "", + strip_prefix = "/modules/java", ) topt = use_extension( @@ -1127,18 +1138,19 @@ dd_payload_uploader( bazel_dep(name = "aspect_rules_js", version = "3.0.0-rc5") bazel_dep(name = "rules_nodejs", version = "6.7.3") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-nodejs", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-nodejs", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/nodejs", + urls = [""], + sha256 = "", + strip_prefix = "/modules/nodejs", ) node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") @@ -1200,18 +1212,19 @@ dd_topt_nodejs_test( bazel_dep(name = "aspect_rules_js", version = "3.0.0-rc5") bazel_dep(name = "rules_nodejs", version = "6.7.3") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-nodejs", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-nodejs", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/nodejs", + urls = [""], + sha256 = "", + strip_prefix = "/modules/nodejs", ) node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node") @@ -1306,18 +1319,19 @@ def dotnet_csharp_test_adapter(name, data = None, env = None, **kwargs): # MODULE.bazel bazel_dep(name = "rules_dotnet", version = "0.21.5") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-dotnet", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-dotnet", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/dotnet", + urls = [""], + sha256 = "", + strip_prefix = "/modules/dotnet", ) dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") @@ -1381,18 +1395,19 @@ dd_topt_dotnet_test( # MODULE.bazel bazel_dep(name = "rules_dotnet", version = "0.21.5") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-dotnet", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-dotnet", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/dotnet", + urls = [""], + sha256 = "", + strip_prefix = "/modules/dotnet", ) dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") @@ -1470,18 +1485,19 @@ dd_payload_uploader( # MODULE.bazel bazel_dep(name = "rules_ruby", version = "0.21.1") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-ruby", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-ruby", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/ruby", + urls = [""], + sha256 = "", + strip_prefix = "/modules/ruby", ) ruby = use_extension("@rules_ruby//ruby:extensions.bzl", "ruby") @@ -1545,18 +1561,19 @@ dd_topt_ruby_test( # MODULE.bazel bazel_dep(name = "rules_ruby", version = "0.21.1") bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-ruby", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-ruby", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/ruby", + urls = [""], + sha256 = "", + strip_prefix = "/modules/ruby", ) ruby = use_extension("@rules_ruby//ruby:extensions.bzl", "ruby") diff --git a/docs/Maintainers.md b/docs/Maintainers.md index 67bcc8e6..aa0b960f 100644 --- a/docs/Maintainers.md +++ b/docs/Maintainers.md @@ -357,8 +357,10 @@ Planned next slices: ## Publication checklist (when BCR publication is enabled) -This repository currently uses pre-publication install paths in README -(`git_override` / commit pin). When publishing to BCR, update and verify: +Release archives support direct installation without BCR. See the +[release runbook](../CONTRIBUTING.md#release-runbook-core--companion-modules) +for bundle generation and draft-release gates. BCR publication is a separate +operation. When publishing to BCR, update and verify: - Module entries for: - `datadog-rules-test-optimization` diff --git a/examples/README.md b/examples/README.md index 2b1a0d1f..029abf02 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,9 +19,11 @@ repository, use `./bazelw` for local development convenience. ## Prerequisites -- Until BCR publication, install with `git_override(...)` as shown in the root `README.md`. - If you are consuming a published release, keep the `bazel_dep(...)` lines and - omit override blocks. +- For a released version, copy the `archive_override(...)` blocks from its + `INSTALL.md` asset. See [release installation](../docs/Installation_Reference.md#option-a-install-from-a-release-archive-recommended). + The commit-based blocks in this document also support development snapshots + and older releases without bundle assets. Omit overrides only when the + selected modules are available from your configured registry. - Add module dependencies before `use_extension(...)`: - `bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0")` - `bazel_dep(name = "datadog-rules-test-optimization-go", version = "1.3.0")` for Go macro usage diff --git a/modules/go/tools/dd_topt_go_bootstrap/BUILD.bazel b/modules/go/tools/dd_topt_go_bootstrap/BUILD.bazel index c3b27844..f0513924 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/BUILD.bazel +++ b/modules/go/tools/dd_topt_go_bootstrap/BUILD.bazel @@ -8,7 +8,10 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "bootstrap_lib", - srcs = ["main.go"], + srcs = [ + "archive.go", + "main.go", + ], importpath = "github.com/DataDog/rules_test_optimization/modules/go/tools/dd_topt_go_bootstrap", visibility = ["//visibility:public"], deps = ["//tools/onboardingpins"], @@ -16,6 +19,9 @@ go_library( go_test( name = "bootstrap_test", - srcs = ["main_test.go"], + srcs = [ + "archive_test.go", + "main_test.go", + ], embed = [":bootstrap_lib"], ) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive.go b/modules/go/tools/dd_topt_go_bootstrap/archive.go new file mode 100644 index 00000000..bab2c57b --- /dev/null +++ b/modules/go/tools/dd_topt_go_bootstrap/archive.go @@ -0,0 +1,201 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under +// the Apache 2.0 License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +package main + +import ( + "errors" + "fmt" + "path" + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/DataDog/rules_test_optimization/modules/go/tools/onboardingpins" +) + +// moduleArchive describes the whole Datadog source archive, not just the fork. +// Keep mirrors and the checksum together when deriving a rules_go subdirectory. +type moduleArchive struct { + urls []string + sha256 string + prefix string + archiveType string +} + +func (a moduleArchive) validate() error { + if len(a.urls) == 0 || !regexp.MustCompile(`^[a-fA-F0-9]{64}$`).MatchString(a.sha256) { + return errors.New("archive override requires literal non-empty urls and a 64-character sha256; use --rto-archive-url, --rto-archive-sha256 and --rto-archive-prefix for computed MODULE values") + } + for _, url := range a.urls { + if strings.TrimSpace(url) == "" { + return errors.New("archive override contains an empty URL") + } + } + if a.prefix == "" || path.IsAbs(a.prefix) || path.Clean(a.prefix) != a.prefix || a.prefix == ".." || strings.HasPrefix(a.prefix, "../") { + return errors.New("archive override requires a non-empty relative strip_prefix without parent traversal") + } + return nil +} + +func (a moduleArchive) render(forkPrefix string) string { + var result strings.Builder + result.WriteString("archive_override(\n module_name = \"rules_go\",\n urls = [\n") + for _, url := range a.urls { + fmt.Fprintf(&result, " %q,\n", url) + } + fmt.Fprintf(&result, " ],\n sha256 = %q,\n strip_prefix = %q,\n", a.sha256, a.prefix+"/"+forkPrefix) + if a.archiveType != "" { + fmt.Fprintf(&result, " type = %q,\n", a.archiveType) + } + result.WriteString(")\n") + return result.String() +} + +// configureModuleArchive prefers an existing rules_go archive, then a core/Go +// archive. Explicit Git flags keep their previous meaning. No URL is fetched. +func configureModuleArchive(cfg *config, content string) error { + if cfg.rulesGoFetchSet && cfg.rulesGoFetch == "git" && cfg.rtoArchiveURL != "" { + return errors.New("--rules-go-fetch=git conflicts with explicit --rto-archive-* flags for Bzlmod") + } + if cfg.rtoArchiveURL != "" || cfg.rulesGoFetch == "archive" { + archive := &moduleArchive{urls: []string{cfg.rtoArchiveURL}, sha256: cfg.rtoArchiveSHA256, prefix: cfg.rtoArchivePrefix, archiveType: cfg.rtoArchiveType} + if err := archive.validate(); err != nil { + return err + } + cfg.rulesGoArchive = archive + return nil + } + if cfg.rulesGoCommitSet || cfg.rulesGoRemoteSet || cfg.rulesGoFetchSet { + return nil + } + if remote, commit := inferManagedRulesGoOverride(content); remote != "" && commit != "" { + cfg.rulesGoRemote, cfg.rulesGoCommit = remote, commit + return nil + } + for _, name := range []string{"rules_go", "datadog-rules-test-optimization-go", "datadog-rules-test-optimization"} { + archive, err := findModuleArchive(content, name) + if err != nil { + return err + } + if archive == nil { + continue + } + if name == "rules_go" { + index := strings.Index(archive.prefix, "/third_party/rgo/") + if index < 0 { + return errors.New("rules_go archive strip_prefix is not a supported Datadog fork path") + } + upstream, variant, err := onboardingpins.RulesGoSelectionForStripPrefix(archive.prefix[index+1:]) + if err != nil { + return err + } + if !cfg.rulesGoUpstreamSet && !cfg.rulesGoVariantSet { + cfg.rulesGoUpstream, cfg.rulesGoVariant = upstream, variant + } + fork, err := rulesGoStripPrefix(*cfg) + if err != nil { + return err + } + if archive.prefix[index+1:] != fork { + return errors.New("existing rules_go archive selects another upstream; update its override before changing the bootstrap selection") + } + archive.prefix = archive.prefix[:index] + } else { + if name == "datadog-rules-test-optimization-go" { + if !strings.HasSuffix(archive.prefix, "/modules/go") { + return errors.New("Go companion archive strip_prefix must end with /modules/go; pass explicit archive flags for a custom layout") + } + archive.prefix = strings.TrimSuffix(archive.prefix, "/modules/go") + } + } + cfg.rulesGoArchive = archive + return nil + } + return nil +} + +func moduleOverridePresent(content, module string) bool { + for _, block := range topLevelCallBlocks(stripStarlarkLineComments(content)) { + if strings.HasSuffix(block.function, "_override") && literalAttribute(block.text, "module_name") == module { + return true + } + } + return false +} + +func moduleArchiveCompatible(content string, cfg config) (bool, error) { + existing, err := findModuleArchive(content, "rules_go") + if err != nil || existing == nil { + return false, err + } + fork, err := rulesGoStripPrefix(cfg) + if err != nil { + return false, err + } + wanted := cfg.rulesGoArchive + return reflect.DeepEqual(existing.urls, wanted.urls) && strings.EqualFold(existing.sha256, wanted.sha256) && existing.prefix == wanted.prefix+"/"+fork && existing.archiveType == wanted.archiveType, nil +} + +// These are literal-only readers, like the existing bootstrap's Git readers. +// Never guess the result of a Starlark expression or silently drop a fork patch. +func findModuleArchive(content, module string) (*moduleArchive, error) { + for _, block := range topLevelCallBlocks(stripStarlarkLineComments(content)) { + if block.function != "archive_override" || literalAttribute(block.text, "module_name") != module { + continue + } + if module == "rules_go" { + // Attribute-looking text in URLs is data, not a keyword argument. + withoutStrings := regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`).ReplaceAllString(block.text, `""`) + for _, attr := range regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*=`).FindAllStringSubmatch(withoutStrings, -1) { + switch attr[1] { + case "module_name", "urls", "sha256", "strip_prefix", "type": + default: + return nil, fmt.Errorf("cannot manage rules_go archive attribute %q; keep custom patches/authentication wiring outside bootstrap", attr[1]) + } + } + } + archive := &moduleArchive{sha256: literalAttribute(block.text, "sha256"), prefix: literalAttribute(block.text, "strip_prefix"), archiveType: literalAttribute(block.text, "type")} + match := regexp.MustCompile(`(?s)\burls\s*=\s*\[([^\]]*)\]`).FindStringSubmatch(block.text) + if len(match) == 2 { + remaining := match[1] + for _, literal := range regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`).FindAllString(remaining, -1) { + value, err := unquoteStarlarkLiteral(literal) + if err != nil { + return nil, err + } + archive.urls = append(archive.urls, value) + remaining = strings.Replace(remaining, literal, "", 1) + } + if strings.Trim(strings.TrimSpace(remaining), ", \t\r\n") != "" { + return nil, errors.New("archive urls must be a literal list; pass explicit --rto-archive-* flags for computed values") + } + } + if err := archive.validate(); err != nil { + return nil, fmt.Errorf("%s: %w", module, err) + } + return archive, nil + } + return nil, nil +} + +func literalAttribute(block, name string) string { + pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(name) + `\s*=\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*[,)]`) + match := pattern.FindStringSubmatch(block) + if len(match) != 2 { + return "" + } + value, _ := unquoteStarlarkLiteral(match[1]) + return value +} + +func unquoteStarlarkLiteral(value string) (string, error) { + if strings.HasPrefix(value, "'") { + value = `"` + strings.ReplaceAll(strings.ReplaceAll(value[1:len(value)-1], `\'`, `'`), `"`, `\"`) + `"` + } + return strconv.Unquote(value) +} diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go new file mode 100644 index 00000000..daf222e3 --- /dev/null +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -0,0 +1,170 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under +// the Apache 2.0 License. +// +// This product includes software developed at Datadog +// (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func archiveTestOverride(module, prefix string) string { + return fmt.Sprintf(`archive_override( + module_name = %q, + urls = ["https://mirror.example/source.tar.gz", "https://github.com/DataDog/rules_test_optimization/releases/download/v1.3.0/source.tar.gz"], + sha256 = %q, + strip_prefix = %q, +) +`, module, strings.Repeat("a", 64), prefix) +} + +func TestArchiveBootstrapSourcesAndIdempotence(t *testing.T) { + for _, module := range []string{"datadog-rules-test-optimization", "datadog-rules-test-optimization-go", "rules_go"} { + t.Run(module, func(t *testing.T) { + prefix := "rules_test_optimization-1.3.0" + if module == "datadog-rules-test-optimization-go" { + prefix += "/modules/go" + } else if module == "rules_go" { + prefix += "/third_party/rgo/v0_60_0/base" + } + input := "module(name = \"consumer\")\n" + archiveTestOverride(module, prefix) + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file, rulesGoVariant: "base", orchestrionVersion: defaultOrchestrionVersion, ddTraceGoVersion: defaultDDTraceGoVersion} + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + first, _ := os.ReadFile(file) + if strings.Contains(string(first), "git_override(") || strings.Count(string(first), `module_name = "rules_go"`) != 1 { + t.Fatalf("expected exactly one archive fork override, no Git:\n%s", first) + } + for _, wanted := range []string{"https://mirror.example/source.tar.gz", strings.Repeat("a", 64), "rules_test_optimization-1.3.0/third_party/rgo/v0_60_0/base"} { + if !strings.Contains(string(first), wanted) { + t.Fatalf("missing %q:\n%s", wanted, first) + } + } + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + second, _ := os.ReadFile(file) + if string(first) != string(second) { + t.Fatalf("bootstrap is not idempotent:\n%s\nthen:\n%s", first, second) + } + }) + } +} + +func TestArchiveBootstrapRejectsInvalidInputsWithoutWriting(t *testing.T) { + valid := archiveTestOverride("datadog-rules-test-optimization", "release") + for name, input := range map[string]string{ + "missing checksum": strings.Replace(valid, strings.Repeat("a", 64), "", 1), + "malformed checksum": strings.Replace(valid, strings.Repeat("a", 64), "not-a-sha", 1), + "computed URLs": strings.Replace(valid, `"https://mirror.example/source.tar.gz"`, "MIRROR_URL", 1), + "computed prefix": strings.Replace(valid, `strip_prefix = "release"`, `strip_prefix = PREFIX`, 1), + "parent traversal": strings.Replace(valid, `strip_prefix = "release"`, `strip_prefix = "../release"`, 1), + "fork patches": strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(", "archive_override(\n patches = [\"//:custom.patch\"],", 1), + } { + t.Run(name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + if err := patchModuleFile(config{moduleFile: file, rulesGoVariant: "base"}); err == nil { + t.Fatal("expected actionable validation error") + } + after, _ := os.ReadFile(file) + if string(after) != input { + t.Fatal("invalid input changed MODULE.bazel") + } + }) + } +} + +func TestArchiveBootstrapExplicitFlagsAndGitEscapeHatch(t *testing.T) { + cfg := config{rulesGoFetch: "archive", rtoArchiveURL: "file:///tmp/source.tar.gz", rtoArchiveSHA256: strings.Repeat("b", 64), rtoArchivePrefix: "release"} + if err := configureModuleArchive(&cfg, ""); err != nil || cfg.rulesGoArchive == nil { + t.Fatalf("explicit archive flags: %v", err) + } + for _, cfg := range []config{{rulesGoCommitSet: true}, {rulesGoRemoteSet: true}, {rulesGoFetch: "git", rulesGoFetchSet: true}} { + if err := configureModuleArchive(&cfg, archiveTestOverride("datadog-rules-test-optimization", "release")); err != nil || cfg.rulesGoArchive != nil { + t.Fatalf("explicit Git flags should disable archive inference: %v", err) + } + } +} + +func TestArchiveBootstrapPreservesSupportedUpstreams(t *testing.T) { + for _, upstream := range []string{"v0_60_0", "v0_61_1", "v0_62_0", "v0_63_0"} { + t.Run(upstream, func(t *testing.T) { + input := managedBlockStart + "\n" + archiveTestOverride("rules_go", "release/third_party/rgo/"+upstream+"/base") + managedBlockEnd + cfg := config{} + if err := hydrateManagedRulesGoVariant(&cfg, input); err != nil || cfg.rulesGoUpstream != upstream { + t.Fatalf("hydrate archive selection: cfg=%+v err=%v", cfg, err) + } + if err := configureModuleArchive(&cfg, input); err != nil { + t.Fatal(err) + } + block, err := managedModuleBlockWithError(cfg) + if err != nil || !strings.Contains(block, "release/third_party/rgo/"+upstream+"/base") { + t.Fatalf("preserve archive fork: %v\n%s", err, block) + } + }) + } +} + +func TestArchiveBootstrapCommentsAndLiteralFormatting(t *testing.T) { + input := "# " + strings.ReplaceAll(archiveTestOverride("rules_go", "wrong"), "\n", "\n# ") + "\n" + strings.ReplaceAll(archiveTestOverride("datadog-rules-test-optimization", "release"), `"`, `'`) + cfg := config{} + if err := configureModuleArchive(&cfg, input); err != nil || cfg.rulesGoArchive == nil || cfg.rulesGoArchive.prefix != "release" { + t.Fatalf("comments/single quotes: %v %+v", err, cfg.rulesGoArchive) + } +} + +func TestArchiveBootstrapSingleQuotedManualForkAndURLParameters(t *testing.T) { + input := "module(name = \"consumer\")\n" + strings.ReplaceAll(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), `"`, `'`) + input = strings.ReplaceAll(input, "source.tar.gz", "source.tar.gz?version=1") + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + if err := patchModuleFile(config{moduleFile: file}); err != nil { + t.Fatal(err) + } + result, _ := os.ReadFile(file) + if strings.Count(string(result), "archive_override(") != 1 || !strings.Contains(string(result), "?version=1") { + t.Fatalf("manual fork should move into managed block, preserving URL parameters:\n%s", result) + } +} + +func TestReplaceManagedSectionPreservesSurroundingTextOnRerun(t *testing.T) { + for _, suffix := range []string{"", "\n", "\n\n# Other settings\n"} { + block := "# BEGIN\nowned = True\n# END\n" + input := "# Existing header\n" + block + suffix + first, err := replaceManagedSection(input, "# BEGIN", "# END", block) + if err != nil || first != input { + t.Fatalf("managed newline replacement changed surrounding text: %v\n%q -> %q", err, input, first) + } + } +} + +func TestArchiveBootstrapDoesNotReplaceAnExistingGitFork(t *testing.T) { + input := archiveTestOverride("datadog-rules-test-optimization", "release") + managedBlockStart + ` +git_override( + module_name = "rules_go", + remote = "https://example.com/owned-fork.git", + commit = "existing-published-commit", + strip_prefix = "third_party/rgo/v0_60_0/base", +) +` + managedBlockEnd + cfg := config{} + if err := configureModuleArchive(&cfg, input); err != nil || cfg.rulesGoArchive != nil || cfg.rulesGoCommit != "existing-published-commit" { + t.Fatalf("existing Git fork must win over a core archive: %+v, %v", cfg, err) + } +} diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index 82703d0c..69fd34c7 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -234,6 +234,8 @@ type config struct { rulesGoCommit string datadogFetch string rulesGoFetch string + rulesGoFetchSet bool + rulesGoArchive *moduleArchive rulesGoRepoName string rtoCommit string rtoArchiveURL string @@ -332,13 +334,13 @@ func parseFlags() config { flag.StringVar(&cfg.rulesGoUpstream, "rules-go-upstream", "default", "Datadog-managed rules_go upstream support id, for example v0_60_0") flag.StringVar(&cfg.rulesGoVariant, "rules-go-variant", defaultRulesGoVariant, "rules_go Orchestrion variant to use: base") flag.StringVar(&cfg.datadogFetch, "datadog-fetch", defaultDatadogFetch, "WORKSPACE snippet fetch mode for Datadog companion repositories: git or archive") - flag.StringVar(&cfg.rulesGoFetch, "rules-go-fetch", defaultRulesGoFetch, "WORKSPACE snippet fetch mode for rules_go: git or archive") + flag.StringVar(&cfg.rulesGoFetch, "rules-go-fetch", defaultRulesGoFetch, "rules_go fetch mode: git or archive; Bzlmod preserves existing archive overrides by default") flag.StringVar(&cfg.rulesGoRepoName, "rules-go-repo-name", defaultRulesGoRepoName, "WORKSPACE snippet repository name for the rules_go fork") flag.StringVar(&cfg.rtoCommit, "rto-commit", "", "WORKSPACE snippet commit for Datadog repositories; falls back to --rules-go-commit when omitted") - flag.StringVar(&cfg.rtoArchiveURL, "rto-archive-url", "", "WORKSPACE snippet archive URL for archive fetch mode") - flag.StringVar(&cfg.rtoArchiveSHA256, "rto-archive-sha256", "", "WORKSPACE snippet archive SHA256 for archive fetch mode") - flag.StringVar(&cfg.rtoArchivePrefix, "rto-archive-prefix", "", "WORKSPACE snippet archive root prefix for archive fetch mode") - flag.StringVar(&cfg.rtoArchiveType, "rto-archive-type", "tar.gz", "WORKSPACE snippet archive type for archive fetch mode") + flag.StringVar(&cfg.rtoArchiveURL, "rto-archive-url", "", "Datadog source archive URL for Bzlmod or WORKSPACE archive mode") + flag.StringVar(&cfg.rtoArchiveSHA256, "rto-archive-sha256", "", "Datadog source archive SHA256 for archive mode") + flag.StringVar(&cfg.rtoArchivePrefix, "rto-archive-prefix", "", "Datadog source archive root prefix for archive mode") + flag.StringVar(&cfg.rtoArchiveType, "rto-archive-type", "tar.gz", "Datadog source archive type for archive mode") flag.Parse() flag.Visit(func(f *flag.Flag) { if f.Name == "dd-trace-go-version" { @@ -350,6 +352,9 @@ func parseFlags() config { if f.Name == "rules-go-commit" { cfg.rulesGoCommitSet = true } + if f.Name == "rules-go-fetch" { + cfg.rulesGoFetchSet = true + } if f.Name == "rules-go-upstream" { cfg.rulesGoUpstreamSet = true } @@ -815,6 +820,9 @@ func patchModuleFile(cfg config) error { return fmt.Errorf("read MODULE.bazel: %w", err) } text := string(content) + if err := configureModuleArchive(&cfg, text); err != nil { + return err + } if cfg.guided { if err := validateGuidedPrerequisites(text); err != nil { @@ -830,7 +838,7 @@ func patchModuleFile(cfg config) error { } } - if !cfg.rulesGoCommitSet { + if cfg.rulesGoArchive == nil && !cfg.rulesGoCommitSet { if remote, commit := inferManagedRulesGoOverride(text); remote != "" && commit != "" { if !cfg.rulesGoRemoteSet { cfg.rulesGoRemote = remote @@ -839,24 +847,34 @@ func patchModuleFile(cfg config) error { } } - if !cfg.rulesGoRemoteSet && !cfg.rulesGoCommitSet && cfg.rulesGoCommit == "" { + if cfg.rulesGoArchive == nil && !cfg.rulesGoRemoteSet && !cfg.rulesGoCommitSet && cfg.rulesGoCommit == "" { if remote, commit := inferDatadogRepoOverride(text); remote != "" && commit != "" { cfg.rulesGoRemote = remote cfg.rulesGoCommit = commit } } - if strings.TrimSpace(cfg.rulesGoCommit) == "" { + if cfg.rulesGoArchive == nil && strings.TrimSpace(cfg.rulesGoCommit) == "" { return errors.New("rules_go fork commit is required; add a git_override for datadog-rules-test-optimization-go/datadog-rules-test-optimization or pass --rules-go-commit explicitly") } - if !strings.Contains(text, managedBlockStart) && strings.Contains(text, `module_name = "rules_go"`) { + if !strings.Contains(text, managedBlockStart) && moduleOverridePresent(text, "rules_go") { compatible, err := rulesGoOverrideCompatible(text, cfg) if err != nil { return err } if !compatible { - return errors.New("MODULE.bazel already contains an incompatible rules_go git_override; update it manually before running the Datadog bootstrap") + return errors.New("MODULE.bazel already contains an incompatible rules_go override; update it manually before running the Datadog bootstrap") + } + if cfg.rulesGoArchive != nil { + // Move the compatible fork override into the managed block rather + // than emitting a second override for the same module. + for _, block := range topLevelCallBlocks(text) { + if block.function == "archive_override" && literalAttribute(stripStarlarkLineComments(block.text), "module_name") == "rules_go" { + text = text[:block.start] + text[block.end:] + break + } + } } } @@ -899,14 +917,18 @@ func managedModuleBlockWithError(cfg config) (string, error) { if err != nil { return "", err } - return fmt.Sprintf(`%s -git_override( + override := fmt.Sprintf(`git_override( module_name = "rules_go", remote = "%s", commit = "%s", strip_prefix = "%s", ) - +`, cfg.rulesGoRemote, cfg.rulesGoCommit, stripPrefix) + if cfg.rulesGoArchive != nil { + override = cfg.rulesGoArchive.render(stripPrefix) + } + return fmt.Sprintf(`%s +%s %s orchestrion = use_extension("@rules_go//go:extensions.bzl", "orchestrion") orchestrion.from_source( @@ -916,7 +938,7 @@ orchestrion.from_source( ) use_repo(orchestrion, "rules_go_orchestrion_tool") %s -`, managedBlockStart, cfg.rulesGoRemote, cfg.rulesGoCommit, stripPrefix, managedGoSDKBlock(cfg), cfg.orchestrionVersion, managedTracerConfigBlock(cfg), managedOrchestrionGoSDKAttrs(cfg), managedBlockEnd), nil +`, managedBlockStart, override, managedGoSDKBlock(cfg), cfg.orchestrionVersion, managedTracerConfigBlock(cfg), managedOrchestrionGoSDKAttrs(cfg), managedBlockEnd), nil } func managedGoSDKBlock(cfg config) string { @@ -2140,6 +2162,9 @@ func replaceManagedSection(content, startMarker, endMarker, block string) (strin switch { case start >= 0 && end >= 0 && end > start: end += len(endMarker) + if strings.HasSuffix(block, "\n") && strings.HasPrefix(content[end:], "\n") { + end++ + } return content[:start] + block + content[end:], nil case start >= 0 || end >= 0: return "", fmt.Errorf("existing Datadog managed block is malformed: %s / %s", startMarker, endMarker) @@ -2347,6 +2372,9 @@ func moduleUsesRepo(content, repoName string) bool { } func rulesGoOverrideCompatible(content string, cfg config) (bool, error) { + if cfg.rulesGoArchive != nil { + return moduleArchiveCompatible(content, cfg) + } overridePattern := regexp.MustCompile(`(?s)git_override\(\s*module_name\s*=\s*"rules_go"(.*?)\n\)`) remotePattern := regexp.MustCompile(`remote\s*=\s*"([^"]+)"`) commitPattern := regexp.MustCompile(`commit\s*=\s*"([^"]+)"`) @@ -2986,7 +3014,13 @@ func hydrateManagedRulesGoVariant(cfg *config, content string) error { if len(match) != 2 { return nil } - upstream, variant, err := onboardingpins.RulesGoSelectionForStripPrefix(match[1]) + prefix := match[1] + if strings.Contains(content[start:end], "archive_override(") { + if index := strings.Index(prefix, "third_party/rgo/"); index > 0 { + prefix = prefix[index:] + } + } + upstream, variant, err := onboardingpins.RulesGoSelectionForStripPrefix(prefix) if err != nil { return err } diff --git a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md index 53eecbf1..e176738a 100644 --- a/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md +++ b/tools/agent-skills/go-test-optimization-onboarding/references/bzlmod-onboarding.md @@ -13,22 +13,26 @@ Go companion module through Bzlmod. ## Module Wiring -Add core and Go companion dependencies: +Copy core and Go companion dependencies from the release's `INSTALL.md`. +Replace the archive placeholders below with that tuple and match the Datadog +module versions to the release. For releases without bundle assets or for +development commits, use the Git alternative in `docs/Installation_Reference.md`. ```bzl bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-go", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-go", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/go", + urls = [""], + sha256 = "", + strip_prefix = "/modules/go", ) bazel_dep(name = "rules_go", version = "0.60.0") @@ -43,8 +47,9 @@ common:test-optimization --repo_env=DD_TEST_OPTIMIZATION_ENABLED=1 The optimized target transition enables Orchestrion. The config remains the only user-facing metadata switch; no global Orchestrion flag is required. -Use a commit that is reachable from `origin/main`. Do not publish branch-only -commits in consumer snippets because squash merges can make them disappear. +When using the Git alternative, use a commit reachable from `origin/main`. +Do not publish branch-only commits in consumer snippets because squash merges +can make them disappear. The `rules_go` version must match the selected Datadog-managed fork support line. The current default support line is `rules_go_upstream = "v0_60_0"`, which uses `rules_go` `0.60.0`. @@ -58,11 +63,11 @@ Orchestrion extension wiring that bootstrap normally writes. Plain upstream Manual Bzlmod wiring must include this block: ```bzl -git_override( +archive_override( module_name = "rules_go", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "third_party/rgo/v0_60_0/base", + urls = [""], + sha256 = "", + strip_prefix = "/third_party/rgo/v0_60_0/base", ) test_optimization_go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") diff --git a/tools/agent-skills/java-test-optimization-onboarding/references/bzlmod-onboarding.md b/tools/agent-skills/java-test-optimization-onboarding/references/bzlmod-onboarding.md index afa66383..c7429576 100644 --- a/tools/agent-skills/java-test-optimization-onboarding/references/bzlmod-onboarding.md +++ b/tools/agent-skills/java-test-optimization-onboarding/references/bzlmod-onboarding.md @@ -12,23 +12,26 @@ Use this path when the consumer repository uses `MODULE.bazel`. ## Dependency Wiring -Add the core module and Java companion module. Until Bazel Central Registry -publication exists, pin both with `git_override` to the same published commit: +Add core and Java companion blocks from the release's `INSTALL.md`, with +the same archive URL and SHA256. Replace the placeholders below and match the +module versions to the release. For older releases without bundle assets or +development commits, use the Git alternative in `docs/Installation_Reference.md`. ```bzl bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-java", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-java", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/java", + urls = [""], + sha256 = "", + strip_prefix = "/modules/java", ) ``` diff --git a/tools/agent-skills/python-test-optimization-onboarding/references/bzlmod-onboarding.md b/tools/agent-skills/python-test-optimization-onboarding/references/bzlmod-onboarding.md index 5c7c50d3..e7f9a403 100644 --- a/tools/agent-skills/python-test-optimization-onboarding/references/bzlmod-onboarding.md +++ b/tools/agent-skills/python-test-optimization-onboarding/references/bzlmod-onboarding.md @@ -12,23 +12,26 @@ Use this path when the consumer repository uses `MODULE.bazel`. ## Dependency Wiring -Add the core module and Python companion module. Until Bazel Central Registry -publication exists, pin both with `git_override` to the same published commit: +Add core and Python companion blocks from the release's `INSTALL.md`, with +the same archive URL and SHA256. Replace the placeholders below and match the +module versions to the release. For older releases without bundle assets or +development commits, use the Git alternative in `docs/Installation_Reference.md`. ```bzl bazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", + urls = [""], + sha256 = "", + strip_prefix = "", ) bazel_dep(name = "datadog-rules-test-optimization-python", version = "1.3.0") -git_override( +archive_override( module_name = "datadog-rules-test-optimization-python", - remote = "https://github.com/DataDog/rules_test_optimization.git", - commit = "", - strip_prefix = "modules/python", + urls = [""], + sha256 = "", + strip_prefix = "/modules/python", ) ``` diff --git a/tools/dev/BUILD.bazel b/tools/dev/BUILD.bazel index 7422dd98..7474b82d 100644 --- a/tools/dev/BUILD.bazel +++ b/tools/dev/BUILD.bazel @@ -9,6 +9,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") exports_files( [ "benchmark_go_orchestrion_perf.py", + "build_release.py", "check_bazelversion_sync.py", "check_module_versions.py", "check_release_archive_contents.py", diff --git a/tools/dev/build_release.py b/tools/dev/build_release.py new file mode 100644 index 00000000..bef8c977 --- /dev/null +++ b/tools/dev/build_release.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Build a checksum-pinned release bundle from one committed Git revision. + +This command is offline and never creates a tag or publishes a release. The same +bundle is used by release validation, the draft release, and consumer installs. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import tarfile +import tempfile + + +LANGUAGES = ("go", "python", "java", "nodejs", "dotnet", "ruby") +REPOSITORY = "DataDog/rules_test_optimization" + + +def git(repo: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True).strip() + + +def module_version(text: str) -> str: + match = re.search(r"\bmodule\s*\([^)]*\bversion\s*=\s*\"(\d+\.\d+\.\d+)\"", text) + if not match: + raise ValueError("expected module(version = \"X.Y.Z\", ...)") + return match[1] + + +def archive_metadata(archive: Path, prefix: str, version: str, commit: str) -> dict: + """Check every module/fork prefix against the actual packaged source.""" + with tarfile.open(archive, "r:gz") as source: + def read(path: str) -> str: + handle = source.extractfile(f"{prefix}/{path}") + if handle is None: + raise ValueError(f"archive is missing {path}") + return handle.read().decode("utf-8") + + modules = {"datadog-rules-test-optimization": prefix} + for language in LANGUAGES: + path = f"modules/{language}" + text = read(f"{path}/MODULE.bazel") + if module_version(text) != version: + raise ValueError(f"{path}/MODULE.bazel version does not match {version}") + dependency = re.search( + r'bazel_dep\(\s*name\s*=\s*"datadog-rules-test-optimization"\s*,\s*version\s*=\s*"([^"]+)"', text + ) + if not dependency or dependency[1] != version: + raise ValueError(f"{path}/MODULE.bazel core dependency does not match {version}") + modules[f"datadog-rules-test-optimization-{language}"] = f"{prefix}/{path}" + + registry = json.loads(read("third_party/rules_go_orchestrion/registry.json")) + forks = {} + for upstream, entry in registry["upstreams"].items(): + path = entry["variants"]["base"]["tree_path"] + if PurePosixPath(path).is_absolute() or ".." in PurePosixPath(path).parts: + raise ValueError(f"unsafe rules_go tree path: {path}") + read(f"{path}/MODULE.bazel") + forks[upstream] = { + "version": entry["rules_go_version"], + "strip_prefix": f"{prefix}/{path}", + } + if registry["default_upstream"] not in forks: + raise ValueError("default rules_go upstream has no packaged base fork") + + with archive.open("rb") as handle: + digest = hashlib.file_digest(handle, "sha256").hexdigest() + return { + "version": version, + "tag": f"v{version}", + "commit": commit, + "archive": archive.name, + "url": f"https://github.com/{REPOSITORY}/releases/download/v{version}/{archive.name}", + "sha256": digest, + "strip_prefix": prefix, + "modules": modules, + "rules_go": forks, + "default_rules_go_upstream": registry["default_upstream"], + } + + +def archive_call(metadata: dict, name: str, prefix: str, *, bzlmod: bool) -> str: + function, attribute = ("archive_override", "module_name") if bzlmod else ("http_archive", "name") + return ( + f"{function}(\n" + f" {attribute} = {json.dumps(name)},\n" + f" urls = [{json.dumps(metadata['url'])}],\n" + f" sha256 = {json.dumps(metadata['sha256'])},\n" + f" strip_prefix = {json.dumps(prefix)},\n" + ")\n" + ) + + +def installation_notes(metadata: dict) -> str: + lines = [ + f"# Install v{metadata['version']}", + "", + f"Source commit: `{metadata['commit']}`.", + "", + "Use the same archive and checksum for core, companions, and the Go fork.", + "These snippets install dependencies; follow the README for runtime/toolchain setup.", + "For Bzlmod, add the core block and only the companions you use.", + "For Go, also add one of the rules_go blocks below before running the bootstrap.", + "WORKSPACE users should use the public language repository helpers after installing core.", + ] + for bzlmod in (True, False): + lines.extend(["", "## Bzlmod" if bzlmod else "## WORKSPACE", ""]) + if not bzlmod: + lines.extend(['```bzl', 'load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")', '```', ""]) + for name, prefix in metadata["modules"].items(): + lines.extend([f"### {name}", "", "```bzl"]) + if bzlmod: + lines.append(f'bazel_dep(name = "{name}", version = "{metadata["version"]}")') + lines.extend([archive_call(metadata, name, prefix, bzlmod=bzlmod).rstrip(), "```", ""]) + lines.extend(["### Go fork (choose one supported upstream)", ""]) + for upstream, fork in metadata["rules_go"].items(): + lines.extend([f"#### {upstream}", "", "```bzl"]) + if bzlmod: + lines.append(f'bazel_dep(name = "rules_go", version = "{fork["version"]}")') + lines.extend([archive_call(metadata, "rules_go" if bzlmod else "io_bazel_rules_go", fork["strip_prefix"], bzlmod=bzlmod).rstrip(), "```", ""]) + return "\n".join(lines) + "\n" + + +def changelog_section(text: str, version: str) -> str: + match = re.search(rf"^## \[{re.escape(version)}\][^\n]*\n(.*?)(?=^## |\Z)", text, re.M | re.S) + if not match or not match[1].strip(): + raise ValueError(f"CHANGELOG.md has no release notes for {version}") + return match[1].strip() + + +def build(repo: Path, ref: str, output: Path, tag: str | None = None) -> dict: + commit = git(repo, "rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}") + version = module_version(git(repo, "show", f"{commit}:MODULE.bazel")) + if tag is not None and tag != f"v{version}": + raise ValueError(f"release tag mismatch: {tag} does not match v{version}") + changelog = changelog_section(git(repo, "show", f"{commit}:CHANGELOG.md"), version) + if output.exists() and any(output.iterdir()): + raise ValueError(f"output directory must be empty: {output}") + output.mkdir(parents=True, exist_ok=True) + prefix = f"rules_test_optimization-{version}" + archive = output / f"{prefix}.tar.gz" + with tempfile.TemporaryDirectory(prefix="rto-release-source-") as temporary: + tar = Path(temporary) / "source.tar" + git(repo, "archive", "--format=tar", f"--prefix={prefix}/", f"--output={tar}", commit) + with tar.open("rb") as source, archive.open("wb") as target: + with gzip.GzipFile(filename="", mode="wb", fileobj=target, mtime=0) as compressed: + shutil.copyfileobj(source, compressed) + metadata = archive_metadata(archive, prefix, version, commit) + (output / "release.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + installation = installation_notes(metadata) + (output / "INSTALL.md").write_text(installation, encoding="utf-8") + (output / "RELEASE_NOTES.md").write_text( + f"# v{version}\n\n{changelog}\n\n{installation}", encoding="utf-8" + ) + checksums = [] + for path in sorted(output.iterdir()): + with path.open("rb") as handle: + checksums.append(f"{hashlib.file_digest(handle, 'sha256').hexdigest()} {path.name}\n") + (output / "SHA256SUMS").write_text("".join(checksums), encoding="utf-8") + return metadata + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--ref", default="HEAD") + parser.add_argument("--tag", help="Expected release tag; a mismatch aborts before packaging") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + metadata = build(args.repo, args.ref, args.output, args.tag) + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"release bundle: {error}\n") + print(json.dumps(metadata, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index 48911f81..e3218144 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -625,6 +625,17 @@ run_bep_freshness_scenario() { } create_fixture_archive() { + if [[ -n "${RTO_RELEASE_ARCHIVE:-}" ]]; then + if [[ ! -f "$RTO_RELEASE_ARCHIVE" || ! "${RTO_RELEASE_PREFIX:-}" =~ ^rules_test_optimization-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "error: release archive mode requires an existing RTO_RELEASE_ARCHIVE and versioned RTO_RELEASE_PREFIX" >&2 + exit 1 + fi + ARCHIVE_PATH="$RTO_RELEASE_ARCHIVE" + ARCHIVE_NAME="$RTO_RELEASE_PREFIX" + ARCHIVE_SHA256="$(sha256_file "$ARCHIVE_PATH")" + ARCHIVE_URL="$(file_uri "$ARCHIVE_PATH")" + return + fi local root_dir="$ARCHIVE_ROOT/$ARCHIVE_NAME" rm -rf "$ARCHIVE_ROOT" @@ -1641,6 +1652,12 @@ if [[ "$INTEGRATION_SCENARIO_MODE" == "measure" ]]; then exit 0 fi +if [[ "$INTEGRATION_SCENARIO_MODE" == "release" ]]; then + run_fixture + assert_go_integration_metadata_requests + exit 0 +fi + if [[ "$INTEGRATION_SCENARIO_MODE" != "full" ]]; then echo "error: unsupported INTEGRATION_SCENARIO_MODE=$INTEGRATION_SCENARIO_MODE" >&2 exit 1 diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index d3d972b2..017dee58 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -601,6 +601,17 @@ run_bep_freshness_scenario() { } create_fixture_archive() { + if [[ -n "${RTO_RELEASE_ARCHIVE:-}" ]]; then + if [[ ! -f "$RTO_RELEASE_ARCHIVE" || ! "${RTO_RELEASE_PREFIX:-}" =~ ^rules_test_optimization-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "error: release archive mode requires an existing RTO_RELEASE_ARCHIVE and versioned RTO_RELEASE_PREFIX" >&2 + exit 1 + fi + ARCHIVE_PATH="$RTO_RELEASE_ARCHIVE" + ARCHIVE_NAME="$RTO_RELEASE_PREFIX" + ARCHIVE_SHA256="$(sha256_file "$ARCHIVE_PATH")" + ARCHIVE_URL="$(file_uri "$ARCHIVE_PATH")" + return + fi local root_dir="$ARCHIVE_ROOT/$ARCHIVE_NAME" rm -rf "$ARCHIVE_ROOT" @@ -1825,6 +1836,12 @@ if [[ "$INTEGRATION_SCENARIO_MODE" == "measure" ]]; then exit 0 fi +if [[ "$INTEGRATION_SCENARIO_MODE" == "release" ]]; then + run_positive_fixture "archive" + assert_go_integration_metadata_requests + exit 0 +fi + if [[ "$INTEGRATION_SCENARIO_MODE" != "full" ]]; then echo "error: unsupported INTEGRATION_SCENARIO_MODE=$INTEGRATION_SCENARIO_MODE" >&2 exit 1 diff --git a/tools/tests/python/BUILD.bazel b/tools/tests/python/BUILD.bazel index 8f85d2d1..091c3567 100644 --- a/tools/tests/python/BUILD.bazel +++ b/tools/tests/python/BUILD.bazel @@ -26,6 +26,7 @@ sh_test( "fixtures/bep_snake_case_remote_cached.ndjson", "test_log_level_tools.py", "test_python_tools.py", + "test_release_tools.py", "test_rules_go_profile_tools.py", "test_rules_go_variant_tools.py", "test_uploader_application_tools.py", @@ -44,6 +45,7 @@ sh_test( "test_uploader_transport_tools.py", "test_uploader_worker_pool_tools.py", "uploader_test_support.py", + "//:.github/workflows/release.yml", "//modules/go/tools/onboardingpins:rules_go_forks_gen.go", "//third_party/rules_go_orchestrion:bzlmod_runtime_profile", "//third_party/rules_go_orchestrion:registry.json", @@ -62,6 +64,7 @@ sh_test( "//tools/core:uploader_python_launcher.sh.tpl", "//tools/core:uploader_python_runtime", "//tools/core:validate_payload_schema.py", + "//tools/dev:build_release.py", "//tools/dev:check_bazelversion_sync.py", "//tools/dev:check_module_versions.py", "//tools/dev:check_release_archive_contents.py", diff --git a/tools/tests/python/test_release_tools.py b/tools/tests/python/test_release_tools.py new file mode 100644 index 00000000..95d315a0 --- /dev/null +++ b/tools/tests/python/test_release_tools.py @@ -0,0 +1,186 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under +# the Apache 2.0 License. +# +# This product includes software developed at Datadog +# (https://www.datadoghq.com/) Copyright 2025-Present Datadog, Inc. + +"""Offline regression tests for the exact bytes and snippets shipped in releases.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +try: + from test_rules_go_variant_tools import _load_module, _runfile +except ModuleNotFoundError: + from tools.tests.python.test_rules_go_variant_tools import _load_module, _runfile + +RELEASE = _load_module("build_release", "tools/dev/build_release.py") + + +class ReleaseBundleTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="rto-release-test-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.repo = self.root / "repo" + self.repo.mkdir() + self.env = { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "Release test", + "GIT_AUTHOR_EMAIL": "release-test@example.invalid", + "GIT_COMMITTER_NAME": "Release test", + "GIT_COMMITTER_EMAIL": "release-test@example.invalid", + } + for name in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"): + self.env.pop(name, None) + self.git("init", "--quiet") + self.write("MODULE.bazel", 'module(name = "datadog-rules-test-optimization", version = "1.3.0")\n') + for language in RELEASE.LANGUAGES: + self.write(f"modules/{language}/MODULE.bazel", f'module(name = "datadog-rules-test-optimization-{language}", version = "1.3.0")\nbazel_dep(name = "datadog-rules-test-optimization", version = "1.3.0")\n') + registry = {"default_upstream": "v0_60_0", "upstreams": {}} + for upstream in ("v0_60_0", "v0_61_1", "v0_62_0", "v0_63_0"): + path = f"third_party/rgo/{upstream}/base" + registry["upstreams"][upstream] = {"rules_go_version": upstream[1:].replace("_", "."), "variants": {"base": {"tree_path": path}}} + self.write(f"{path}/MODULE.bazel", 'module(name = "rules_go", version = "0.60.0")\n') + self.write("third_party/rules_go_orchestrion/registry.json", json.dumps(registry)) + self.write("CHANGELOG.md", "## [Unreleased]\n\nNot part of this release.\n\n## [1.3.0] - 2026-10-21\n\n### Fixed\n\n- The actual changelog.\n\n## [1.2.0]\n\nOld notes.\n") + self.write("bazelw", "#!/bin/sh\nexit 0\n") + (self.repo / "bazelw").chmod(0o755) + self.commit() + + def git(self, *args): + return subprocess.check_output(["git", "-C", str(self.repo), "-c", "commit.gpgsign=false", "-c", "core.autocrlf=false", *args], env=self.env, text=True).strip() + + def write(self, path, content): + target = self.repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + def commit(self): + self.git("add", ".") + self.git("commit", "--quiet", "-m", "Release fixture") + + def build(self, name="dist", **kwargs): + return RELEASE.build(self.repo, "HEAD", self.root / name, **kwargs) + + def test_archive_is_reproducible_and_only_contains_the_commit(self): + self.write("untracked-secret", "must not ship") + self.write("bazelw", "uncommitted edit must not ship\n") + first = self.build("one") + second = self.build("two") + self.assertEqual(first, second) + for path in (self.root / "one").iterdir(): + self.assertEqual(path.read_bytes(), (self.root / "two" / path.name).read_bytes()) + with tarfile.open(self.root / "one" / first["archive"]) as archive: + names = archive.getnames() + self.assertNotIn(f'{first["strip_prefix"]}/untracked-secret', names) + launcher = archive.getmember(f'{first["strip_prefix"]}/bazelw') + if os.name != "nt": + self.assertTrue(launcher.mode & 0o111) + self.assertEqual(archive.extractfile(launcher).read(), b"#!/bin/sh\nexit 0\n") + + def test_all_module_and_supported_fork_prefixes_exist_in_archive(self): + metadata = self.build() + self.assertEqual(len(metadata["modules"]), 7) + self.assertEqual(len(metadata["rules_go"]), 4) + with tarfile.open(self.root / "dist" / metadata["archive"]) as archive: + for prefix in metadata["modules"].values(): + archive.getmember(f"{prefix}/MODULE.bazel") + for fork in metadata["rules_go"].values(): + archive.getmember(f'{fork["strip_prefix"]}/MODULE.bazel') + + def test_checksum_manifest_covers_every_other_asset(self): + self.build() + output = self.root / "dist" + checked = set() + for line in (output / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + self.assertEqual(hashlib.sha256((output / name).read_bytes()).hexdigest(), digest) + checked.add(name) + self.assertEqual(checked, {path.name for path in output.iterdir()} - {"SHA256SUMS"}) + + def test_notes_use_only_current_changelog_and_exact_install_pins(self): + metadata = self.build() + output = self.root / "dist" + notes = (output / "RELEASE_NOTES.md").read_text() + self.assertIn("The actual changelog.", notes) + self.assertNotIn("Not part of this release.", notes) + self.assertNotIn("Old notes.", notes) + self.assertNotIn("git_override(", notes) + self.assertIn("archive_override(", notes) + self.assertIn("http_archive(", notes) + self.assertIn(metadata["url"], notes) + self.assertIn(metadata["sha256"], notes) + self.assertIn(metadata["commit"], notes) + self.assertNotIn(" Date: Wed, 28 Oct 2026 14:48:25 +0100 Subject: [PATCH 02/11] test: align archive consumers with the current Bazel baseline --- README.md | 4 ++-- tools/tests/integration/run_bzlmod_go_integration.sh | 1 + tools/tests/integration/run_workspace_go_integration.sh | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 58577aee..10cee134 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ the default `dd-trace-go` version, and the default Orchestrion version. Published GitHub pins use `tar.gz`; other archive types must be mirrored and documented outside this helper. -Historical `v1.2.0` GitHub pins (not the `1.3.0` candidate): +Historical `v1.2.0` GitHub pins (not `v1.3.0`): ```bash RTO_COMMIT="69953536d4ef1252c8181c267d16c61263f0aa4c" @@ -1369,7 +1369,7 @@ For a generic wrapper pattern, see [Other languages (without companion macro)](# | Bazel | `8.8.2` | Repository baseline (`.bazelversion`) and primary CI lanes | | rules_go (Go users) | `0.60.0` / `rules_go_upstream = "v0_60_0"` | README examples use the default support line; importpath inference requires `0.51.0+` | | Go toolchain (example) | `1.25.0` | Consumer repositories may use another supported version | -| Module versions | `1.3.0` candidate | Not published yet; use commit pin/override install paths | +| Module versions | `1.3.0` | Published; use release archives when available, or the Git alternative | - **Bazel 8.8.2 (repo baseline)** - Matches `.bazelversion` and primary CI lanes - **WORKSPACE compatibility lane** - CI intentionally validates `--noenable_bzlmod --enable_workspace` on Bazel `8.8.2` during migration away from WORKSPACE mode in Bazel 9+ diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index e3218144..a20e516b 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -658,6 +658,7 @@ write_fixture_bazelrc() { local rules_go_repo="$2" cat > "$ws_dir/.bazelrc" < "$ws_dir/.bazelrc" < Date: Wed, 28 Oct 2026 14:49:02 +0100 Subject: [PATCH 03/11] test: retain the inherited consumer compatibility configuration --- tools/tests/integration/run_bzlmod_go_integration.sh | 1 - tools/tests/integration/run_workspace_go_integration.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index a20e516b..e3218144 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -658,7 +658,6 @@ write_fixture_bazelrc() { local rules_go_repo="$2" cat > "$ws_dir/.bazelrc" < "$ws_dir/.bazelrc" < Date: Wed, 28 Oct 2026 15:01:25 +0100 Subject: [PATCH 04/11] test: isolate staged BEP copies from Bazel output symlinks --- tools/tests/BUILD.bazel | 1 + .../integration/run_bzlmod_go_integration.sh | 4 +- .../run_workspace_go_integration.sh | 4 +- tools/tests/python/BUILD.bazel | 1 + tools/tests/python/test_release_tools.py | 46 +++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/tools/tests/BUILD.bazel b/tools/tests/BUILD.bazel index a4c0260e..3b2240e0 100644 --- a/tools/tests/BUILD.bazel +++ b/tools/tests/BUILD.bazel @@ -19,6 +19,7 @@ exports_files( [ "example_stub_repo.bzl", "integration/mock_dd_server.py", + "integration/run_bzlmod_go_integration.sh", "integration/run_workspace_go_integration.sh", ], visibility = ["//visibility:public"], diff --git a/tools/tests/integration/run_bzlmod_go_integration.sh b/tools/tests/integration/run_bzlmod_go_integration.sh index e3218144..e6feaad1 100755 --- a/tools/tests/integration/run_bzlmod_go_integration.sh +++ b/tools/tests/integration/run_bzlmod_go_integration.sh @@ -438,7 +438,9 @@ simulate_bep_artifact_only_outputs() { exit 1 fi rm -rf "$artifact_root" "$simulated_testlogs" - cp -R "$ws_dir/bazel-testlogs" "$simulated_testlogs" + # bazel-testlogs is a symlink. Copy its contents so staging cannot mutate + # Bazel's read-only outputs through that symlink. + cp -RL "$ws_dir/bazel-testlogs" "$simulated_testlogs" mkdir -p "$(dirname "$staged_source")" cp -R "$original_source" "$staged_source" chmod -R u+w "$simulated_testlogs" "$artifact_root" 2>/dev/null || true diff --git a/tools/tests/integration/run_workspace_go_integration.sh b/tools/tests/integration/run_workspace_go_integration.sh index 017dee58..8e6b764b 100755 --- a/tools/tests/integration/run_workspace_go_integration.sh +++ b/tools/tests/integration/run_workspace_go_integration.sh @@ -423,7 +423,9 @@ simulate_bep_artifact_only_outputs() { exit 1 fi rm -rf "$artifact_root" "$simulated_testlogs" - cp -R "$ws_dir/bazel-testlogs" "$simulated_testlogs" + # bazel-testlogs is a symlink. Copy its contents so staging cannot mutate + # Bazel's read-only outputs through that symlink. + cp -RL "$ws_dir/bazel-testlogs" "$simulated_testlogs" mkdir -p "$(dirname "$staged_source")" cp -R "$original_source" "$staged_source" chmod -R u+w "$simulated_testlogs" "$artifact_root" 2>/dev/null || true diff --git a/tools/tests/python/BUILD.bazel b/tools/tests/python/BUILD.bazel index 091c3567..f9166ff4 100644 --- a/tools/tests/python/BUILD.bazel +++ b/tools/tests/python/BUILD.bazel @@ -82,6 +82,7 @@ sh_test( "//tools/test_optimization:create_support_bundle.py", "//tools/test_optimization:render_report_summary.py", "//tools/tests:integration/mock_dd_server.py", + "//tools/tests:integration/run_bzlmod_go_integration.sh", "//tools/tests:integration/run_workspace_go_integration.sh", "//tools/tests/python/fixtures/generated_uploader", ], diff --git a/tools/tests/python/test_release_tools.py b/tools/tests/python/test_release_tools.py index 95d315a0..b759e7b9 100644 --- a/tools/tests/python/test_release_tools.py +++ b/tools/tests/python/test_release_tools.py @@ -10,7 +10,9 @@ import json import os from pathlib import Path +import shutil import subprocess +import sys import tarfile import tempfile import unittest @@ -24,6 +26,50 @@ RELEASE = _load_module("build_release", "tools/dev/build_release.py") +@unittest.skipIf(sys.platform == "win32" or not shutil.which("bash"), "requires POSIX symlinks and Bash") +class ReleaseConsumerHarnessTests(unittest.TestCase): + def test_bep_simulation_copies_outputs_without_mutating_bazel_testlogs(self): + for mode in ("bzlmod", "workspace"): + harness = _runfile(f"tools/tests/integration/run_{mode}_go_integration.sh").read_text(encoding="utf-8") + function = harness.split("simulate_bep_artifact_only_outputs() {", 1)[1].split("\n}\n", 1)[0] + for use_symlink in (False, True): + with self.subTest(mode=mode, use_symlink=use_symlink), tempfile.TemporaryDirectory(prefix="rto-bep-copy-") as temporary: + root = Path(temporary) + workspace = root / "workspace" + workspace.mkdir() + (workspace / ".topt").mkdir() + testlogs = root / "testlogs" if use_symlink else workspace / "bazel-testlogs" + output = testlogs / "app/hello_test/test.outputs" + output.mkdir(parents=True) + payload = output / "payload.json" + payload.write_text("{}", encoding="utf-8") + payload.chmod(0o444) + output.chmod(0o555) + if use_symlink: + (workspace / "bazel-testlogs").symlink_to(testlogs, target_is_directory=True) + bep = root / "fresh.bep.json" + bep.write_text(json.dumps({"testResult": {"testActionOutput": [{"name": "test.outputs", "uri": output.as_uri()}]}}) + "\n", encoding="utf-8") + try: + result = subprocess.run( + ["bash", "-c", "set -euo pipefail\nsimulate_bep_artifact_only_outputs() {" + function + '\n}\nsimulate_bep_artifact_only_outputs "$1" "$2"', "bep-copy", str(workspace), str(bep)], + env={**os.environ, "PYTHON": sys.executable}, text=True, capture_output=True, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + simulated = workspace / ".topt/simulated-local-testlogs" + self.assertFalse(simulated.is_symlink()) + self.assertFalse((simulated / "app/hello_test/test.outputs").exists()) + self.assertEqual(payload.read_text(encoding="utf-8"), "{}") + self.assertEqual(output.stat().st_mode & 0o777, 0o555) + self.assertEqual(payload.stat().st_mode & 0o777, 0o444) + staged = workspace / ".topt/simulated-remote-artifacts/app/hello_test/test.outputs" + self.assertEqual((staged / "payload.json").read_text(encoding="utf-8"), "{}") + event = json.loads(bep.read_text(encoding="utf-8")) + self.assertEqual(event["testResult"]["testActionOutput"][0]["uri"], staged.resolve().as_uri()) + finally: + output.chmod(0o755) + payload.chmod(0o644) + + class ReleaseBundleTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="rto-release-test-") From bf8939e71ca24f3eda86c4b54a28dc6630142640 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 15:40:35 +0100 Subject: [PATCH 05/11] test: write release fixtures with stable LF line endings --- tools/tests/python/test_release_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/python/test_release_tools.py b/tools/tests/python/test_release_tools.py index b759e7b9..01b025c6 100644 --- a/tools/tests/python/test_release_tools.py +++ b/tools/tests/python/test_release_tools.py @@ -109,7 +109,7 @@ def git(self, *args): def write(self, path, content): target = self.repo / path target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content, encoding="utf-8") + target.write_text(content, encoding="utf-8", newline="\n") def commit(self): self.git("add", ".") From 08e6c60f153c095aff04a29d6206cb4a96694cf2 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 15:54:05 +0100 Subject: [PATCH 06/11] go: preserve archive expressions and MODULE declaration boundaries --- .../go/tools/dd_topt_go_bootstrap/archive.go | 2 +- .../dd_topt_go_bootstrap/archive_test.go | 45 ++++++++++++++++ modules/go/tools/dd_topt_go_bootstrap/main.go | 53 +++++++++++++++++-- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive.go b/modules/go/tools/dd_topt_go_bootstrap/archive.go index bab2c57b..c971cd52 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive.go @@ -160,7 +160,7 @@ func findModuleArchive(content, module string) (*moduleArchive, error) { } } archive := &moduleArchive{sha256: literalAttribute(block.text, "sha256"), prefix: literalAttribute(block.text, "strip_prefix"), archiveType: literalAttribute(block.text, "type")} - match := regexp.MustCompile(`(?s)\burls\s*=\s*\[([^\]]*)\]`).FindStringSubmatch(block.text) + match := regexp.MustCompile(`(?s)\burls\s*=\s*\[([^\]]*)\]\s*[,)]`).FindStringSubmatch(block.text) if len(match) == 2 { remaining := match[1] for _, literal := range regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`).FindAllString(remaining, -1) { diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go index daf222e3..41831681 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -68,6 +68,8 @@ func TestArchiveBootstrapRejectsInvalidInputsWithoutWriting(t *testing.T) { "missing checksum": strings.Replace(valid, strings.Repeat("a", 64), "", 1), "malformed checksum": strings.Replace(valid, strings.Repeat("a", 64), "not-a-sha", 1), "computed URLs": strings.Replace(valid, `"https://mirror.example/source.tar.gz"`, "MIRROR_URL", 1), + "conditional URLs": strings.Replace(valid, "],\n", "] if USE_MIRROR else [\"https://fallback.example/source.tar.gz\"],\n", 1), + "concatenated URLs": strings.Replace(valid, "],\n", "] + [\"https://fallback.example/source.tar.gz\"],\n", 1), "computed prefix": strings.Replace(valid, `strip_prefix = "release"`, `strip_prefix = PREFIX`, 1), "parent traversal": strings.Replace(valid, `strip_prefix = "release"`, `strip_prefix = "../release"`, 1), "fork patches": strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(", "archive_override(\n patches = [\"//:custom.patch\"],", 1), @@ -88,6 +90,49 @@ func TestArchiveBootstrapRejectsInvalidInputsWithoutWriting(t *testing.T) { } } +func TestArchiveBootstrapPreservesDeclarationsAroundCommentedOverrides(t *testing.T) { + for _, comment := range []string{"# example: foo(", "# example: foo)", "# Unicode: héllo( and an unmatched quote \""} { + t.Run(comment, func(t *testing.T) { + prefix := "# Existing header: é\nmodule(name = \"consumer\")\n" + suffix := "\nbazel_dep(name = \"keep_me\", version = \"1.0.0\")\n" + override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n "+comment+"\n", 1) + input := prefix + override + suffix + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file} + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + first, _ := os.ReadFile(file) + if !strings.HasPrefix(string(first), prefix) || !strings.Contains(string(first), suffix) || strings.Count(string(first), "archive_override(") != 1 { + t.Fatalf("moving an override changed surrounding declarations:\n%s", first) + } + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + second, _ := os.ReadFile(file) + if string(first) != string(second) { + t.Fatal("commented override is not idempotent") + } + }) + } +} + +func TestTopLevelCallSpansIgnoreQuotedAndCommentedDelimiters(t *testing.T) { + for _, value := range []string{`"https://mirror.example/file)#asset"`, "\"\"\"first )\nsecond (\nend\"\"\"", `'escaped \' ) # text'`} { + input := "# λ before a call(\n" + "archive_override(\n urls = [" + value + "], # ) ( \"\n)\n" + "bazel_dep(name = \"keep\", version = \"1\")\n" + blocks := topLevelCallBlocks(input) + if len(blocks) != 2 || blocks[0].function != "archive_override" || blocks[1].function != "bazel_dep" { + t.Fatalf("incorrect call spans for %q: %#v", value, blocks) + } + if input[blocks[0].start:blocks[0].end] != "archive_override(\n urls = ["+value+"], # ) ( \"\n)\n" { + t.Fatalf("scanner did not preserve original byte offsets for %q", value) + } + } +} + func TestArchiveBootstrapExplicitFlagsAndGitEscapeHatch(t *testing.T) { cfg := config{rulesGoFetch: "archive", rtoArchiveURL: "file:///tmp/source.tar.gz", rtoArchiveSHA256: strings.Repeat("b", 64), rtoArchivePrefix: "release"} if err := configureModuleArchive(&cfg, ""); err != nil || cfg.rulesGoArchive == nil { diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index 69fd34c7..8c7d8068 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -2872,26 +2872,27 @@ type callBlock struct { func topLevelCallBlocks(content string) []callBlock { lines := splitLinesPreserveNewline(content) + syntaxLines := splitLinesPreserveNewline(maskStarlarkStringsAndComments(content)) startPattern := regexp.MustCompile(`^\s*(?:([A-Za-z_][A-Za-z0-9_]*)\s*=\s*)?([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?)\(`) offset := 0 blocks := []callBlock{} for i := 0; i < len(lines); i++ { line := lines[i] - match := startPattern.FindStringSubmatch(line) + match := startPattern.FindStringSubmatch(syntaxLines[i]) if match == nil { offset += len(line) continue } start := offset - depth := strings.Count(line, "(") - strings.Count(line, ")") + depth := strings.Count(syntaxLines[i], "(") - strings.Count(syntaxLines[i], ")") offset += len(line) j := i for depth > 0 && j+1 < len(lines) { j++ line = lines[j] - depth += strings.Count(line, "(") - depth -= strings.Count(line, ")") + depth += strings.Count(syntaxLines[j], "(") + depth -= strings.Count(syntaxLines[j], ")") offset += len(line) } blocks = append(blocks, callBlock{ @@ -2912,6 +2913,50 @@ func topLevelCallBlocks(content string) []callBlock { return blocks } +// Mask non-syntax bytes when finding call boundaries, without moving offsets in +// the original source. A parenthesis in a comment, URL, or docstring is data. +func maskStarlarkStringsAndComments(content string) string { + masked := []byte(content) + blank := func(start, end int) { + for i := start; i < end; i++ { + if masked[i] != '\n' { + masked[i] = ' ' + } + } + } + for i := 0; i < len(content); { + start := i + if content[i] == '#' { + for i < len(content) && content[i] != '\n' { + i++ + } + blank(start, i) + continue + } + if content[i] != '\'' && content[i] != '"' { + i++ + continue + } + delimiter := content[i : i+1] + if triple := strings.Repeat(delimiter, 3); strings.HasPrefix(content[i:], triple) { + delimiter = triple + } + i += len(delimiter) + for i < len(content) { + if content[i] == '\\' && i+1 < len(content) { + i += 2 + } else if strings.HasPrefix(content[i:], delimiter) { + i += len(delimiter) + break + } else { + i++ + } + } + blank(start, i) + } + return string(masked) +} + type tracerConfig struct { shared string perModule map[string]string From 6cfb831da65239f4c5eadafa731896f2c6447e18 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 16:12:16 +0100 Subject: [PATCH 07/11] release: make archive line endings independent of host Git config --- tools/dev/build_release.py | 11 ++++++----- tools/tests/python/test_release_tools.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tools/dev/build_release.py b/tools/dev/build_release.py index bef8c977..f5deb08f 100644 --- a/tools/dev/build_release.py +++ b/tools/dev/build_release.py @@ -154,22 +154,23 @@ def build(repo: Path, ref: str, output: Path, tag: str | None = None) -> dict: archive = output / f"{prefix}.tar.gz" with tempfile.TemporaryDirectory(prefix="rto-release-source-") as temporary: tar = Path(temporary) / "source.tar" - git(repo, "archive", "--format=tar", f"--prefix={prefix}/", f"--output={tar}", commit) + # Host checkout preferences must not turn executable scripts into CRLF. + git(repo, "-c", "core.autocrlf=false", "-c", "core.eol=lf", "archive", "--format=tar", f"--prefix={prefix}/", f"--output={tar}", commit) with tar.open("rb") as source, archive.open("wb") as target: with gzip.GzipFile(filename="", mode="wb", fileobj=target, mtime=0) as compressed: shutil.copyfileobj(source, compressed) metadata = archive_metadata(archive, prefix, version, commit) - (output / "release.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + (output / "release.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8", newline="\n") installation = installation_notes(metadata) - (output / "INSTALL.md").write_text(installation, encoding="utf-8") + (output / "INSTALL.md").write_text(installation, encoding="utf-8", newline="\n") (output / "RELEASE_NOTES.md").write_text( - f"# v{version}\n\n{changelog}\n\n{installation}", encoding="utf-8" + f"# v{version}\n\n{changelog}\n\n{installation}", encoding="utf-8", newline="\n" ) checksums = [] for path in sorted(output.iterdir()): with path.open("rb") as handle: checksums.append(f"{hashlib.file_digest(handle, 'sha256').hexdigest()} {path.name}\n") - (output / "SHA256SUMS").write_text("".join(checksums), encoding="utf-8") + (output / "SHA256SUMS").write_text("".join(checksums), encoding="utf-8", newline="\n") return metadata diff --git a/tools/tests/python/test_release_tools.py b/tools/tests/python/test_release_tools.py index 01b025c6..c58ecd86 100644 --- a/tools/tests/python/test_release_tools.py +++ b/tools/tests/python/test_release_tools.py @@ -144,6 +144,18 @@ def test_all_module_and_supported_fork_prefixes_exist_in_archive(self): for fork in metadata["rules_go"].values(): archive.getmember(f'{fork["strip_prefix"]}/MODULE.bazel') + def test_archive_ignores_host_line_ending_preferences(self): + original = self.build("original") + self.git("config", "core.autocrlf", "true") + self.git("config", "core.eol", "crlf") + metadata = self.build() + self.assertEqual(original, metadata) + for path in (self.root / "original").iterdir(): + self.assertEqual(path.read_bytes(), (self.root / "dist" / path.name).read_bytes()) + with tarfile.open(self.root / "dist" / metadata["archive"]) as archive: + launcher = archive.extractfile(f'{metadata["strip_prefix"]}/bazelw') + self.assertEqual(launcher.read(), b"#!/bin/sh\nexit 0\n") + def test_checksum_manifest_covers_every_other_asset(self): self.build() output = self.root / "dist" @@ -153,6 +165,8 @@ def test_checksum_manifest_covers_every_other_asset(self): self.assertEqual(hashlib.sha256((output / name).read_bytes()).hexdigest(), digest) checked.add(name) self.assertEqual(checked, {path.name for path in output.iterdir()} - {"SHA256SUMS"}) + for name in ("INSTALL.md", "RELEASE_NOTES.md", "release.json", "SHA256SUMS"): + self.assertNotIn(b"\r\n", (output / name).read_bytes(), name) def test_notes_use_only_current_changelog_and_exact_install_pins(self): metadata = self.build() From 94ce05ed6230b5644a57a813128f7638b1d45350 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 16:26:52 +0100 Subject: [PATCH 08/11] test: exercise computed archive URLs at the inference boundary --- .../tools/dd_topt_go_bootstrap/archive_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go index 41831681..3d151277 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -90,6 +90,22 @@ func TestArchiveBootstrapRejectsInvalidInputsWithoutWriting(t *testing.T) { } } +func TestArchiveInferenceRejectsComputedURLLists(t *testing.T) { + const module = "datadog-rules-test-optimization" + valid := archiveTestOverride(module, "release") + if archive, err := findModuleArchive(valid, module); err != nil || archive == nil || len(archive.urls) != 2 { + t.Fatalf("expected literal URL-list inference to succeed: archive=%+v err=%v", archive, err) + } + for _, suffix := range []string{` if USE_MIRROR else ["https://fallback.example/source.tar.gz"]`, ` + ["https://fallback.example/source.tar.gz"]`} { + t.Run(suffix, func(t *testing.T) { + input := strings.Replace(valid, "],\n", "]"+suffix+",\n", 1) + if archive, err := findModuleArchive(input, module); err == nil || archive != nil { + t.Fatalf("computed URL list was accepted: archive=%+v err=%v", archive, err) + } + }) + } +} + func TestArchiveBootstrapPreservesDeclarationsAroundCommentedOverrides(t *testing.T) { for _, comment := range []string{"# example: foo(", "# example: foo)", "# Unicode: héllo( and an unmatched quote \""} { t.Run(comment, func(t *testing.T) { From 21b2b90f2e01f43ebdf78d78572102be0af010c1 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 19:21:29 +0100 Subject: [PATCH 09/11] release: preserve archive wiring during bootstrap --- .../go/tools/dd_topt_go_bootstrap/archive.go | 37 +++++++-- .../dd_topt_go_bootstrap/archive_test.go | 77 +++++++++++++++++++ tools/dev/build_release.py | 4 + tools/tests/python/test_release_tools.py | 14 ++++ 4 files changed, 124 insertions(+), 8 deletions(-) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive.go b/modules/go/tools/dd_topt_go_bootstrap/archive.go index c971cd52..4d83a651 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive.go @@ -63,6 +63,15 @@ func configureModuleArchive(cfg *config, content string) error { return errors.New("--rules-go-fetch=git conflicts with explicit --rto-archive-* flags for Bzlmod") } if cfg.rtoArchiveURL != "" || cfg.rulesGoFetch == "archive" { + // Explicit pins can replace computed values, but must not erase patches + // from a previously managed fork when its block is rewritten. + for _, block := range topLevelCallBlocks(stripStarlarkLineComments(content)) { + if block.function == "archive_override" && literalAttribute(block.text, "module_name") == "rules_go" { + if err := validateRulesGoArchiveAttributes(block.text); err != nil { + return err + } + } + } archive := &moduleArchive{urls: []string{cfg.rtoArchiveURL}, sha256: cfg.rtoArchiveSHA256, prefix: cfg.rtoArchivePrefix, archiveType: cfg.rtoArchiveType} if err := archive.validate(); err != nil { return err @@ -141,6 +150,18 @@ func moduleArchiveCompatible(content string, cfg config) (bool, error) { return reflect.DeepEqual(existing.urls, wanted.urls) && strings.EqualFold(existing.sha256, wanted.sha256) && existing.prefix == wanted.prefix+"/"+fork && existing.archiveType == wanted.archiveType, nil } +func validateRulesGoArchiveAttributes(block string) error { + // Attribute-looking text inside strings and comments is not an argument. + for _, attr := range regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*=`).FindAllStringSubmatch(maskStarlarkStringsAndComments(block), -1) { + switch attr[1] { + case "module_name", "urls", "sha256", "strip_prefix", "type": + default: + return fmt.Errorf("cannot manage rules_go archive attribute %q; keep custom patches/authentication wiring outside bootstrap", attr[1]) + } + } + return nil +} + // These are literal-only readers, like the existing bootstrap's Git readers. // Never guess the result of a Starlark expression or silently drop a fork patch. func findModuleArchive(content, module string) (*moduleArchive, error) { @@ -149,17 +170,17 @@ func findModuleArchive(content, module string) (*moduleArchive, error) { continue } if module == "rules_go" { - // Attribute-looking text in URLs is data, not a keyword argument. - withoutStrings := regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`).ReplaceAllString(block.text, `""`) - for _, attr := range regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*=`).FindAllStringSubmatch(withoutStrings, -1) { - switch attr[1] { - case "module_name", "urls", "sha256", "strip_prefix", "type": - default: - return nil, fmt.Errorf("cannot manage rules_go archive attribute %q; keep custom patches/authentication wiring outside bootstrap", attr[1]) - } + if err := validateRulesGoArchiveAttributes(block.text); err != nil { + return nil, err } } archive := &moduleArchive{sha256: literalAttribute(block.text, "sha256"), prefix: literalAttribute(block.text, "strip_prefix"), archiveType: literalAttribute(block.text, "type")} + if loc := regexp.MustCompile(`\btype\s*=`).FindStringIndex(maskStarlarkStringsAndComments(block.text)); loc != nil { + value := strings.TrimSpace(block.text[loc[1]:]) + if !regexp.MustCompile(`^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*[,)]`).MatchString(value) { + return nil, errors.New("archive type must be a literal string; pass explicit --rto-archive-* flags for computed values") + } + } match := regexp.MustCompile(`(?s)\burls\s*=\s*\[([^\]]*)\]\s*[,)]`).FindStringSubmatch(block.text) if len(match) == 2 { remaining := match[1] diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go index 3d151277..5461d6c7 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -106,6 +106,83 @@ func TestArchiveInferenceRejectsComputedURLLists(t *testing.T) { } } +func TestArchiveBootstrapRejectsComputedTypesWithoutWriting(t *testing.T) { + for _, module := range []string{"datadog-rules-test-optimization", "datadog-rules-test-optimization-go", "rules_go"} { + prefix := "release" + if module == "datadog-rules-test-optimization-go" { + prefix += "/modules/go" + } else if module == "rules_go" { + prefix += "/third_party/rgo/v0_60_0/base" + } + for _, value := range []string{`ARCHIVE_TYPE`, `"tar." + EXTENSION`, `"tar.gz" if USE_GZIP else "zip"`} { + t.Run(module+"/"+value, func(t *testing.T) { + input := strings.Replace(archiveTestOverride(module, prefix), "archive_override(\n", "archive_override(\n type = "+value+",\n", 1) + if archive, err := findModuleArchive(input, module); err == nil || archive != nil { + t.Errorf("computed archive type was accepted: archive=%+v err=%v", archive, err) + } + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte("module(name = \"consumer\")\n"+input), 0o644); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(file) + if err := patchModuleFile(config{moduleFile: file}); err == nil || !strings.Contains(err.Error(), "archive type must be a literal string") { + t.Errorf("expected actionable computed-type error, got %v", err) + } + after, _ := os.ReadFile(file) + if string(after) != string(before) { + t.Fatal("computed archive type changed MODULE.bazel") + } + }) + } + } +} + +func TestExplicitArchiveFlagsRejectManagedForkPatchesWithoutWriting(t *testing.T) { + for _, attr := range []string{`patches = ["//:custom.patch"]`, `patch_cmds = ["echo patch"]`, `patch_args = ["-p1"]`} { + t.Run(attr, func(t *testing.T) { + override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n "+attr+",\n", 1) + input := "module(name = \"consumer\")\n" + managedBlockStart + "\n" + override + managedBlockEnd + "\n" + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file, rulesGoFetch: "archive", rtoArchiveURL: "https://mirror.example/source", rtoArchiveSHA256: strings.Repeat("b", 64), rtoArchivePrefix: "release", rtoArchiveType: "tar.gz"} + if err := patchModuleFile(cfg); err == nil || !strings.Contains(err.Error(), "cannot manage rules_go archive attribute") { + t.Errorf("explicit archive flags accepted fork patch attributes: %v", err) + } + after, _ := os.ReadFile(file) + if string(after) != input { + t.Fatal("explicit archive flags deleted custom fork patches") + } + }) + } +} + +func TestArchiveBootstrapPreservesLiteralTypesAndExplicitComputedEscapeHatch(t *testing.T) { + for _, value := range []string{`"tar.gz"`, `'tar.gz'`, `ARCHIVE_TYPE`} { + t.Run(value, func(t *testing.T) { + override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n type = "+value+",\n", 1) + override = strings.ReplaceAll(override, "source.tar.gz", "source?type=asset") + input := "module(name = \"consumer\")\n" + managedBlockStart + "\n" + override + managedBlockEnd + "\nbazel_dep(name = \"keep_me\", version = \"1.0.0\")\n" + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file} + if value == "ARCHIVE_TYPE" { + cfg.rulesGoFetch, cfg.rtoArchiveURL, cfg.rtoArchiveSHA256, cfg.rtoArchivePrefix, cfg.rtoArchiveType = "archive", "https://mirror.example/source", strings.Repeat("b", 64), "release", "tar.gz" + } + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + result, _ := os.ReadFile(file) + if !strings.Contains(string(result), `type = "tar.gz"`) || !strings.Contains(string(result), `bazel_dep(name = "keep_me", version = "1.0.0")`) { + t.Fatalf("archive type or unrelated declaration lost:\n%s", result) + } + }) + } +} + func TestArchiveBootstrapPreservesDeclarationsAroundCommentedOverrides(t *testing.T) { for _, comment := range []string{"# example: foo(", "# example: foo)", "# Unicode: héllo( and an unmatched quote \""} { t.Run(comment, func(t *testing.T) { diff --git a/tools/dev/build_release.py b/tools/dev/build_release.py index f5deb08f..efafb0cd 100644 --- a/tools/dev/build_release.py +++ b/tools/dev/build_release.py @@ -94,12 +94,16 @@ def read(path: str) -> str: def archive_call(metadata: dict, name: str, prefix: str, *, bzlmod: bool) -> str: function, attribute = ("archive_override", "module_name") if bzlmod else ("http_archive", "name") + repository_mapping = "" + if not bzlmod and name == "datadog-rules-test-optimization-go": + repository_mapping = ' repo_mapping = {"@rules_go": "@io_bazel_rules_go"},\n' return ( f"{function}(\n" f" {attribute} = {json.dumps(name)},\n" f" urls = [{json.dumps(metadata['url'])}],\n" f" sha256 = {json.dumps(metadata['sha256'])},\n" f" strip_prefix = {json.dumps(prefix)},\n" + f"{repository_mapping}" ")\n" ) diff --git a/tools/tests/python/test_release_tools.py b/tools/tests/python/test_release_tools.py index c58ecd86..39d7a060 100644 --- a/tools/tests/python/test_release_tools.py +++ b/tools/tests/python/test_release_tools.py @@ -6,6 +6,7 @@ """Offline regression tests for the exact bytes and snippets shipped in releases.""" +import ast import hashlib import json import os @@ -183,6 +184,19 @@ def test_notes_use_only_current_changelog_and_exact_install_pins(self): self.assertIn(metadata["commit"], notes) self.assertNotIn(" Date: Wed, 28 Oct 2026 19:34:20 +0100 Subject: [PATCH 10/11] go: validate only outer archive override attributes --- .../go/tools/dd_topt_go_bootstrap/archive.go | 22 +++++++++++++++++-- .../dd_topt_go_bootstrap/archive_test.go | 8 +++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive.go b/modules/go/tools/dd_topt_go_bootstrap/archive.go index 4d83a651..77579c64 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive.go @@ -151,8 +151,26 @@ func moduleArchiveCompatible(content string, cfg config) (bool, error) { } func validateRulesGoArchiveAttributes(block string) error { - // Attribute-looking text inside strings and comments is not an argument. - for _, attr := range regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*=`).FindAllStringSubmatch(maskStarlarkStringsAndComments(block), -1) { + // Only outer call arguments belong to archive_override. Explicit pins may + // replace computed values containing their own calls and keyword arguments. + syntax := []byte(maskStarlarkStringsAndComments(block)) + depth := 0 + for i, char := range syntax { + switch char { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + default: + if depth > 1 { + syntax[i] = ' ' + } + } + } + if regexp.MustCompile(`(?:\(|,)\s*\*\*`).Match(syntax) { + return errors.New("cannot manage rules_go archive attribute expansion; keep custom patches/authentication wiring outside bootstrap") + } + for _, attr := range regexp.MustCompile(`(?:\(|,)\s*([A-Za-z_][A-Za-z0-9_]*)\s*=`).FindAllStringSubmatch(string(syntax), -1) { switch attr[1] { case "module_name", "urls", "sha256", "strip_prefix", "type": default: diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go index 5461d6c7..02a7cd93 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -114,7 +114,7 @@ func TestArchiveBootstrapRejectsComputedTypesWithoutWriting(t *testing.T) { } else if module == "rules_go" { prefix += "/third_party/rgo/v0_60_0/base" } - for _, value := range []string{`ARCHIVE_TYPE`, `"tar." + EXTENSION`, `"tar.gz" if USE_GZIP else "zip"`} { + for _, value := range []string{`ARCHIVE_TYPE`, `"tar." + EXTENSION`, `"tar.gz" if USE_GZIP else "zip"`, `archive_type(format = "tar.gz")`} { t.Run(module+"/"+value, func(t *testing.T) { input := strings.Replace(archiveTestOverride(module, prefix), "archive_override(\n", "archive_override(\n type = "+value+",\n", 1) if archive, err := findModuleArchive(input, module); err == nil || archive != nil { @@ -138,7 +138,7 @@ func TestArchiveBootstrapRejectsComputedTypesWithoutWriting(t *testing.T) { } func TestExplicitArchiveFlagsRejectManagedForkPatchesWithoutWriting(t *testing.T) { - for _, attr := range []string{`patches = ["//:custom.patch"]`, `patch_cmds = ["echo patch"]`, `patch_args = ["-p1"]`} { + for _, attr := range []string{`patches = ["//:custom.patch"]`, `patch_cmds = ["echo patch"]`, `patch_args = ["-p1"]`, `**{"patches": ["//:custom.patch"]}`} { t.Run(attr, func(t *testing.T) { override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n "+attr+",\n", 1) input := "module(name = \"consumer\")\n" + managedBlockStart + "\n" + override + managedBlockEnd + "\n" @@ -159,7 +159,7 @@ func TestExplicitArchiveFlagsRejectManagedForkPatchesWithoutWriting(t *testing.T } func TestArchiveBootstrapPreservesLiteralTypesAndExplicitComputedEscapeHatch(t *testing.T) { - for _, value := range []string{`"tar.gz"`, `'tar.gz'`, `ARCHIVE_TYPE`} { + for _, value := range []string{`"tar.gz"`, `'tar.gz'`, `ARCHIVE_TYPE`, `archive_type(format = "tar.gz")`, `archive_type(options = ["tar.gz"])`, `archive_type(options = {"format": "tar.gz"})`, `"tar.gz" if FORMAT == "gzip" else "zip"`} { t.Run(value, func(t *testing.T) { override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n type = "+value+",\n", 1) override = strings.ReplaceAll(override, "source.tar.gz", "source?type=asset") @@ -169,7 +169,7 @@ func TestArchiveBootstrapPreservesLiteralTypesAndExplicitComputedEscapeHatch(t * t.Fatal(err) } cfg := config{moduleFile: file} - if value == "ARCHIVE_TYPE" { + if value != `"tar.gz"` && value != `'tar.gz'` { cfg.rulesGoFetch, cfg.rtoArchiveURL, cfg.rtoArchiveSHA256, cfg.rtoArchivePrefix, cfg.rtoArchiveType = "archive", "https://mirror.example/source", strings.Repeat("b", 64), "release", "tar.gz" } if err := patchModuleFile(cfg); err != nil { From 75a9b7bfd34db25a1814c34b9a13a477ce5ad34c Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Wed, 28 Oct 2026 21:43:27 +0100 Subject: [PATCH 11/11] fix: honor archive pins across bootstrap parsing --- .../go/tools/dd_topt_go_bootstrap/archive.go | 88 ++++++++--- .../dd_topt_go_bootstrap/archive_test.go | 143 ++++++++++++++++++ modules/go/tools/dd_topt_go_bootstrap/main.go | 8 +- 3 files changed, 216 insertions(+), 23 deletions(-) diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive.go b/modules/go/tools/dd_topt_go_bootstrap/archive.go index 77579c64..be40f992 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive.go @@ -139,6 +139,18 @@ func moduleOverridePresent(content, module string) bool { func moduleArchiveCompatible(content string, cfg config) (bool, error) { existing, err := findModuleArchive(content, "rules_go") + if err != nil && (cfg.rtoArchiveURL != "" || cfg.rulesGoFetch == "archive") { + // Complete explicit pins can replace computed values, but never discard + // fork patches, authentication, or other unsupported archive attributes. + for _, block := range topLevelCallBlocks(stripStarlarkLineComments(content)) { + if block.function == "archive_override" && literalAttribute(block.text, "module_name") == "rules_go" { + if err := validateRulesGoArchiveAttributes(block.text); err != nil { + return false, err + } + return true, nil + } + } + } if err != nil || existing == nil { return false, err } @@ -193,26 +205,35 @@ func findModuleArchive(content, module string) (*moduleArchive, error) { } } archive := &moduleArchive{sha256: literalAttribute(block.text, "sha256"), prefix: literalAttribute(block.text, "strip_prefix"), archiveType: literalAttribute(block.text, "type")} - if loc := regexp.MustCompile(`\btype\s*=`).FindStringIndex(maskStarlarkStringsAndComments(block.text)); loc != nil { - value := strings.TrimSpace(block.text[loc[1]:]) - if !regexp.MustCompile(`^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*[,)]`).MatchString(value) { + if value, present := callAttributeValue(block.text, "type"); present { + if !regexp.MustCompile(`^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')$`).MatchString(value) { return nil, errors.New("archive type must be a literal string; pass explicit --rto-archive-* flags for computed values") } } - match := regexp.MustCompile(`(?s)\burls\s*=\s*\[([^\]]*)\]\s*[,)]`).FindStringSubmatch(block.text) - if len(match) == 2 { - remaining := match[1] - for _, literal := range regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'`).FindAllString(remaining, -1) { - value, err := unquoteStarlarkLiteral(literal) - if err != nil { - return nil, err - } - archive.urls = append(archive.urls, value) - remaining = strings.Replace(remaining, literal, "", 1) + value, present := callAttributeValue(block.text, "urls") + if !present || !strings.HasPrefix(value, "[") || !strings.HasSuffix(value, "]") { + return nil, errors.New("archive urls must be a literal list; pass explicit --rto-archive-* flags for computed values") + } + remaining := strings.TrimSpace(value[1 : len(value)-1]) + literalPattern := regexp.MustCompile(`^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')`) + for remaining != "" { + literal := literalPattern.FindString(remaining) + if literal == "" { + return nil, errors.New("archive urls must be a literal list; pass explicit --rto-archive-* flags for computed values") + } + url, err := unquoteStarlarkLiteral(literal) + if err != nil { + return nil, err } - if strings.Trim(strings.TrimSpace(remaining), ", \t\r\n") != "" { + archive.urls = append(archive.urls, url) + remaining = strings.TrimSpace(remaining[len(literal):]) + if remaining == "" { + break + } + if remaining[0] != ',' { return nil, errors.New("archive urls must be a literal list; pass explicit --rto-archive-* flags for computed values") } + remaining = strings.TrimSpace(remaining[1:]) } if err := archive.validate(); err != nil { return nil, fmt.Errorf("%s: %w", module, err) @@ -223,13 +244,42 @@ func findModuleArchive(content, module string) (*moduleArchive, error) { } func literalAttribute(block, name string) string { - pattern := regexp.MustCompile(`\b` + regexp.QuoteMeta(name) + `\s*=\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*[,)]`) - match := pattern.FindStringSubmatch(block) - if len(match) != 2 { + value, present := callAttributeValue(block, name) + if !present || !regexp.MustCompile(`^("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')$`).MatchString(value) { return "" } - value, _ := unquoteStarlarkLiteral(match[1]) - return value + literal, _ := unquoteStarlarkLiteral(value) + return literal +} + +// Read one complete outer call argument, never a nested keyword or a fragment +// of a computed expression. The masked view preserves original byte offsets. +func callAttributeValue(block, name string) (string, bool) { + syntax := maskStarlarkStringsAndComments(block) + opening := strings.IndexByte(syntax, '(') + if opening < 0 { + return "", false + } + attribute := regexp.MustCompile(`^\s*` + regexp.QuoteMeta(name) + `\s*=`) + depth, start := 1, opening+1 + for i := start; i < len(syntax); i++ { + switch syntax[i] { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + } + if (depth == 1 && syntax[i] == ',') || depth == 0 { + if match := attribute.FindStringIndex(syntax[start:i]); match != nil { + return strings.TrimSpace(stripStarlarkLineComments(block[start+match[1] : i])), true + } + start = i + 1 + if depth == 0 { + break + } + } + } + return "", false } func unquoteStarlarkLiteral(value string) (string, error) { diff --git a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go index 02a7cd93..83b969e5 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/archive_test.go +++ b/modules/go/tools/dd_topt_go_bootstrap/archive_test.go @@ -62,6 +62,98 @@ func TestArchiveBootstrapSourcesAndIdempotence(t *testing.T) { } } +func TestArchiveBootstrapPreservesQuotedURLBrackets(t *testing.T) { + for _, quote := range []string{`"`, `'`} { + for _, url := range []string{ + "https://[2001:db8::1]/source.tar.gz", + "https://mirror.example/source.tar.gz?part=[first,second]", + } { + t.Run(quote+url, func(t *testing.T) { + input := "module(name = \"consumer\")\n" + strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), `"https://mirror.example/source.tar.gz"`, quote+url+quote, 1) + archive, err := findModuleArchive(input, "rules_go") + if err != nil || archive == nil || len(archive.urls) != 2 || archive.urls[0] != url { + t.Fatalf("literal URL brackets changed archive inference: archive=%+v err=%v", archive, err) + } + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file} + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(file) + if err != nil || !strings.Contains(string(first), url) { + t.Fatalf("URL lost during migration: %v\n%s", err, first) + } + if err := patchModuleFile(cfg); err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(file) + if err != nil || string(first) != string(second) { + t.Fatalf("URL bracket migration is not idempotent: %v", err) + } + }) + } + } +} + +func TestRunArchiveExplicitPinsReplaceComputedValues(t *testing.T) { + t.Setenv("GOPROXY", "off") + t.Setenv("GOSUMDB", "off") + for _, managed := range []bool{false, true} { + for name, edit := range map[string][2]string{ + "computed prefix": {`strip_prefix = "release/third_party/rgo/v0_60_0/base"`, `strip_prefix = "release/" + "third_party/rgo/v0_60_0/base"`}, + "computed type": {"archive_override(\n", "archive_override(\n type = ARCHIVE_TYPE,\n"}, + } { + t.Run(fmt.Sprintf("managed=%t/%s", managed, name), func(t *testing.T) { + dir := t.TempDir() + override := strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), edit[0], edit[1], 1) + if managed { + override = managedBlockStart + "\n" + override + managedBlockEnd + "\n" + } + const preserved = "bazel_dep(name = \"keep_me\", version = \"1.0.0\")\n" + input := "module(name = \"consumer\")\n" + override + preserved + moduleFile := filepath.Join(dir, "MODULE.bazel") + if err := os.WriteFile(moduleFile, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.com/consumer\n\ngo 1.25.0\n"), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{ + workspaceDir: dir, goModSync: "off", datadogFetch: defaultDatadogFetch, + rulesGoFetch: "archive", rulesGoVariant: defaultRulesGoVariant, + orchestrionVersion: defaultOrchestrionVersion, ddTraceGoVersion: defaultDDTraceGoVersion, + rtoArchiveURL: "https://mirror.example/replacement", rtoArchiveSHA256: strings.Repeat("b", 64), + rtoArchivePrefix: "replacement", rtoArchiveType: "tar.gz", + } + if err := run(cfg); err != nil { + t.Fatalf("explicit archive pins could not replace computed values: %v", err) + } + first, err := os.ReadFile(moduleFile) + if err != nil { + t.Fatal(err) + } + archive, err := findModuleArchive(string(first), "rules_go") + if err != nil || archive == nil || len(archive.urls) != 1 || archive.urls[0] != cfg.rtoArchiveURL || archive.sha256 != cfg.rtoArchiveSHA256 || archive.prefix != "replacement/third_party/rgo/v0_60_0/base" || archive.archiveType != "tar.gz" { + t.Fatalf("explicit pins were not preserved: archive=%+v err=%v\n%s", archive, err, first) + } + if strings.Count(string(first), `module_name = "rules_go"`) != 1 || !strings.Contains(string(first), preserved) || strings.Contains(string(first), "ARCHIVE_TYPE") { + t.Fatalf("migration damaged surrounding declarations or duplicated the override:\n%s", first) + } + if err := run(cfg); err != nil { + t.Fatalf("rerun failed: %v", err) + } + second, err := os.ReadFile(moduleFile) + if err != nil || string(first) != string(second) { + t.Fatalf("explicit-pin migration is not idempotent: %v", err) + } + }) + } + } +} + func TestArchiveBootstrapRejectsInvalidInputsWithoutWriting(t *testing.T) { valid := archiveTestOverride("datadog-rules-test-optimization", "release") for name, input := range map[string]string{ @@ -106,6 +198,57 @@ func TestArchiveInferenceRejectsComputedURLLists(t *testing.T) { } } +func TestArchiveInferenceRejectsNestedComputedURLs(t *testing.T) { + for _, value := range []string{ + `mirror_urls(urls = ["https://mirror.example/source.tar.gz"])`, + `dict(urls = ["https://mirror.example/source.tar.gz"])["urls"]`, + } { + t.Run(value, func(t *testing.T) { + input := archiveTestOverride("datadog-rules-test-optimization", "release") + input = strings.Replace(input, `urls = ["https://mirror.example/source.tar.gz", "https://github.com/DataDog/rules_test_optimization/releases/download/v1.3.0/source.tar.gz"]`, "urls = "+value, 1) + if archive, err := findModuleArchive(input, "datadog-rules-test-optimization"); err == nil || archive != nil { + t.Fatalf("nested computed URL list was accepted: archive=%+v err=%v", archive, err) + } + }) + } +} + +func TestExplicitArchivePinsRejectUnmanagedForkAttributesWithoutWriting(t *testing.T) { + for _, attr := range []string{`patches = ["//:custom.patch"]`, `auth_patterns = {"mirror.example": "Bearer "}`, `**{"patches": ["//:custom.patch"]}`} { + t.Run(attr, func(t *testing.T) { + input := "module(name = \"consumer\")\n" + strings.Replace(archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base"), "archive_override(\n", "archive_override(\n type = ARCHIVE_TYPE,\n "+attr+",\n", 1) + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file, rulesGoFetch: "archive", rtoArchiveURL: "https://mirror.example/source", rtoArchiveSHA256: strings.Repeat("b", 64), rtoArchivePrefix: "release", rtoArchiveType: "tar.gz"} + if err := patchModuleFile(cfg); err == nil || !strings.Contains(err.Error(), "cannot manage rules_go archive attribute") { + t.Errorf("explicit pins accepted unsupported fork attributes: %v", err) + } + after, err := os.ReadFile(file) + if err != nil || string(after) != input { + t.Fatalf("unsupported fork attributes changed MODULE.bazel: %v", err) + } + }) + } +} + +func TestExplicitArchivePinsDoNotReplaceDifferentLiteralFork(t *testing.T) { + input := "module(name = \"consumer\")\n" + archiveTestOverride("rules_go", "release/third_party/rgo/v0_60_0/base") + file := filepath.Join(t.TempDir(), "MODULE.bazel") + if err := os.WriteFile(file, []byte(input), 0o644); err != nil { + t.Fatal(err) + } + cfg := config{moduleFile: file, rulesGoFetch: "archive", rtoArchiveURL: "https://mirror.example/replacement.tar.gz", rtoArchiveSHA256: strings.Repeat("b", 64), rtoArchivePrefix: "replacement"} + if err := patchModuleFile(cfg); err == nil || !strings.Contains(err.Error(), "incompatible rules_go override") { + t.Fatalf("different literal fork was accepted: %v", err) + } + after, err := os.ReadFile(file) + if err != nil || string(after) != input { + t.Fatalf("different literal fork changed MODULE.bazel: %v", err) + } +} + func TestArchiveBootstrapRejectsComputedTypesWithoutWriting(t *testing.T) { for _, module := range []string{"datadog-rules-test-optimization", "datadog-rules-test-optimization-go", "rules_go"} { prefix := "release" diff --git a/modules/go/tools/dd_topt_go_bootstrap/main.go b/modules/go/tools/dd_topt_go_bootstrap/main.go index 8c7d8068..ecdc1cb2 100644 --- a/modules/go/tools/dd_topt_go_bootstrap/main.go +++ b/modules/go/tools/dd_topt_go_bootstrap/main.go @@ -3054,12 +3054,12 @@ func hydrateManagedRulesGoVariant(cfg *config, content string) error { if start < 0 || end < 0 || end <= start { return nil } - stripPrefixPattern := regexp.MustCompile(`strip_prefix\s*=\s*"([^"]+)"`) - match := stripPrefixPattern.FindStringSubmatch(content[start:end]) - if len(match) != 2 { + prefix := literalAttribute(content[start:end], "strip_prefix") + if prefix == "" { + // Computed archive values are validated or replaced by explicit pins + // in configureModuleArchive. Do not infer a fork from part of a value. return nil } - prefix := match[1] if strings.Contains(content[start:end], "archive_override(") { if index := strings.Index(prefix, "third_party/rgo/"); index > 0 { prefix = prefix[index:]