diff --git a/.github/actions/node-test-fixture/action.yml b/.github/actions/node-test-fixture/action.yml new file mode 100644 index 000000000..8904fd374 --- /dev/null +++ b/.github/actions/node-test-fixture/action.yml @@ -0,0 +1,48 @@ +name: Locked Node test fixture +description: Reuse installed test dependencies only for an identical lockfile and runtime. +inputs: + directory: + description: Absolute fixture directory. + required: true + dependencies: + description: Space-separated npm dependencies, or empty for a supplied lockfile. + default: '' +runs: + using: composite + steps: + - name: Resolve the fixture and identify its complete dependency graph + id: lock + shell: bash + env: + FIXTURE_DIRECTORY: ${{ inputs.directory }} + FIXTURE_DEPENDENCIES: ${{ inputs.dependencies }} + run: | + set -euo pipefail + mkdir -p "$FIXTURE_DIRECTORY" + cd "$FIXTURE_DIRECTORY" + if [ -n "$FIXTURE_DEPENDENCIES" ]; then + printf '%s\n' '{"name":"ddtest-ci-fixture","version":"1.0.0","private":true}' > package.json + read -ra dependencies <<< "$FIXTURE_DEPENDENCIES" + npm install --package-lock-only --ignore-scripts --save-exact --no-audit --no-fund "${dependencies[@]}" + fi + lock=$(sha256sum "$FIXTURE_DIRECTORY/package-lock.json" | cut -d ' ' -f 1) + echo "key=$(node --version)-$(npm --version)-$lock" >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + id: modules + with: + path: ${{ inputs.directory }}/node_modules + key: ${{ runner.os }}-${{ runner.arch }}-node-test-fixture-${{ steps.lock.outputs.key }} + - name: Install a missing fixture from its lockfile + if: steps.modules.outputs.cache-hit != 'true' + shell: bash + env: + FIXTURE_DIRECTORY: ${{ inputs.directory }} + run: | + cd "$FIXTURE_DIRECTORY" + npm ci --no-audit --no-fund + # Save the prepared dependencies before tests can write framework caches. + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + if: steps.modules.outputs.cache-hit != 'true' + with: + path: ${{ inputs.directory }}/node_modules + key: ${{ runner.os }}-${{ runner.arch }}-node-test-fixture-${{ steps.lock.outputs.key }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b26d2efa0..4262483f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,31 @@ name: CI on: + workflow_call: push: branches: - "**" jobs: + test-binary: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 + with: + go-version-file: go.mod + - name: Build the CLI once for the public integration journeys + env: + CGO_ENABLED: '0' + run: | + mkdir -p "$RUNNER_TEMP/test-binary" + go build -o "$RUNNER_TEMP/test-binary/ddtest" . + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary/ddtest + if-no-files-found: error + test-support: runs-on: ubuntu-latest steps: @@ -61,11 +81,13 @@ jobs: shard: [1, 2, 3] env: DATADOG_API_KEY_CONFIGURED: ${{ secrets.DATADOG_API_KEY != '' }} + ORCHESTRION_VERSION: 5df280060f05cbf2863d43748667fcf9cd18c448 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go + id: go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -97,9 +119,13 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + ~/go/bin/orchestrion + # Cache entries are immutable: a new revision must save fresh compiler + # output, including the SDK install and instrumented test builds. + key: ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-go-${{ hashFiles('go.mod') }}- + ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ runner.arch }}-go-orchestrion-v2-${{ steps.go.outputs.go-version }}-${{ env.ORCHESTRION_VERSION }}-${{ hashFiles('go.mod') }}- - name: Download dependencies run: go mod download @@ -135,9 +161,12 @@ jobs: uses: datadog/test-visibility-github-action@7ce545f228901e4ac4ef0b45d946120f1c4124e0 # v3.3.0 with: languages: go + # The compiler cache above already owns modules and Orchestrion. + # Avoid restoring the action's overlapping module cache a second time. + cache: false # Pin upstream Go 1.27 compatibility fixes until they reach a stable # Orchestrion release. - go-tracer-version: 5df280060f05cbf2863d43748667fcf9cd18c448 + go-tracer-version: ${{ env.ORCHESTRION_VERSION }} api_key: ${{ secrets.DATADOG_API_KEY }} site: datadoghq.com service: ddtest @@ -206,6 +235,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go + id: go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -238,9 +268,10 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-go-ddtest-instrument-${{ hashFiles('go.mod') }}- + ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-${{ runner.arch }}-go-ddtest-instrument-v2-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}- - name: Download dependencies run: go mod download @@ -619,6 +650,7 @@ jobs: args: --timeout=5m jest-testdrive-workflow: + needs: test-binary name: jest-testdrive-workflow (${{ matrix.node }}, ${{ matrix.jest }}, ${{ matrix.group }} ${{ matrix.shard }}) runs-on: ubuntu-latest strategy: @@ -640,16 +672,25 @@ jobs: - {node: '20', jest: '29.7.0', tracer: '5.132.0', typescript: 'ts-jest@29 typescript@5 @types/jest@29', group: configurations, shard: '1/1', tests: 'TestPublicJest(Configurations|TwoPackageConfiguration|LoaderConfiguration)'} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: ${{ matrix.node }} - - name: Install real Jest and tracer - run: | - npm install --prefix "${RUNNER_TEMP}/jest-testdrive" jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 - npm ls --prefix "${RUNNER_TEMP}/jest-testdrive" --depth=0 + - name: Reuse the locked Jest and tracer fixture + uses: ./.github/actions/node-test-fixture + with: + directory: ${{ runner.temp }}/jest-testdrive + dependencies: jest@${{ matrix.jest }} dd-trace@${{ matrix.tracer }} ${{ matrix.typescript }} typescript@5 - name: Validate paired outcomes, all feature probes and cleanup env: DDTEST_JEST_NODE_MODULES: ${{ runner.temp }}/jest-testdrive/node_modules @@ -668,6 +709,7 @@ jobs: testdrive-language-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 25 strategy: @@ -676,6 +718,14 @@ jobs: framework: [pytest] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -695,6 +745,7 @@ jobs: path: ${{ runner.temp }}/ddtest-workflow-reports ruby-testdrive: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 40 strategy: @@ -714,6 +765,14 @@ jobs: - name: latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -753,6 +812,7 @@ jobs: retention-days: 7 java-testdrive: + needs: test-binary name: java-testdrive (${{ matrix.name }}, ${{ matrix.java }}, ${{ matrix.tracer }}, ${{ matrix.shard }}) runs-on: ubuntu-latest timeout-minutes: 20 @@ -772,6 +832,14 @@ jobs: tracer: latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -813,6 +881,7 @@ jobs: retention-days: 7 testdrive-vitest-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -829,6 +898,14 @@ jobs: tracer: '6.21.0' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -847,10 +924,19 @@ jobs: run: go test -v ./internal/testdrive -run '^TestPublicVitestRunnerTestdrive$' -count=1 -timeout=15m onboard-apply-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -873,6 +959,7 @@ jobs: run: DDTEST_ACTIONLINT="$(go env GOPATH)/bin/actionlint" go test -v ./internal/testdrive -run '^TestPublic(OnboardApply|MochaApply|WebdriverIOApply|CucumberApply)$' -count=1 -timeout=8m testdrive-mocha-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 12 strategy: @@ -882,6 +969,14 @@ jobs: shard: ['1/2', '2/2'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -907,7 +1002,7 @@ jobs: run: echo "version=$(npm view @playwright/test version)" >> "$GITHUB_OUTPUT" testdrive-playwright-workflow: - needs: playwright-latest + needs: [test-binary, playwright-latest] runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -923,6 +1018,14 @@ jobs: PLAYWRIGHT_VERSION: ${{ matrix.playwright == 'latest' && needs.playwright-latest.outputs.version || matrix.playwright }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -946,6 +1049,7 @@ jobs: run: go test -v ./internal/testdrive -run '^(TestPublicPlaywrightTestdrive|TestPlaywrightDiscoveryOwnsImportSideEffects|TestPlaywrightDiscoveryKeepsConfiguredReportsOutOfTheCheckout)$' -count=1 -timeout=12m testdrive-cypress-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -957,6 +1061,14 @@ jobs: shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod @@ -985,49 +1097,78 @@ jobs: DDTEST_TEST_SHARD: ${{ matrix.shard }} run: xvfb-run -a go test -v ./internal/testdrive -run '^(TestPublicCypressTestdrive|TestCypressTypeScriptWrapperTranspilesToCommonJS)$' -failfast -count=1 -timeout=12m + # Resolve floating dependencies before expanding the matrix. Identical aliases + # share locked fixtures and jobs within this comprehensive run. + cucumber-matrix: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24' + - name: Resolve versions, lock fixtures and select distinct configurations + id: plan + run: | + python3 scripts/ci_cucumber_matrix.py --output "$RUNNER_TEMP/cucumber-fixtures" \ + --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures + if-no-files-found: error + retention-days: 14 + testdrive-cucumber-workflow: + name: testdrive-cucumber-workflow (${{ matrix.configuration.cucumber }}, ${{ matrix.configuration.shard }}) + needs: [cucumber-matrix, test-binary] runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: - # 10.9.0 and 13.3.0 run every scenario across shards; 8.11.1 the floor check. - include: - - {cucumber: '8.11.1', shard: '1/1'} - - {cucumber: '10.9.0', shard: '1/4'} - - {cucumber: '10.9.0', shard: '2/4'} - - {cucumber: '10.9.0', shard: '3/4'} - - {cucumber: '10.9.0', shard: '4/4'} - - {cucumber: '13.3.0', shard: '1/4'} - - {cucumber: '13.3.0', shard: '2/4'} - - {cucumber: '13.3.0', shard: '3/4'} - - {cucumber: '13.3.0', shard: '4/4'} - - {cucumber: 'latest', shard: '1/4'} - - {cucumber: 'latest', shard: '2/4'} - - {cucumber: 'latest', shard: '3/4'} - - {cucumber: 'latest', shard: '4/4'} + configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - node-version: '24' + node-version: ${{ matrix.configuration.node }} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: ci-cucumber-fixtures + path: ${{ runner.temp }}/cucumber-fixtures - name: Install Cucumber testdrive fixture dependencies run: | - npm install --prefix "$RUNNER_TEMP/cucumber-testdrive" "@cucumber/cucumber@${{ matrix.cucumber }}" nyc@17.1.0 c8@12.0.0 dd-trace@6.20.1 @istanbuljs/esm-loader-hook@0.3.0 tsx@4.23.15 --save-exact --no-audit --no-fund - npm ls --prefix "$RUNNER_TEMP/cucumber-testdrive" --depth=0 + test "$(npm --version)" = '${{ matrix.configuration.npm }}' + echo 'Cucumber ${{ matrix.configuration.version }} covers: ${{ matrix.configuration.aliases }}' + mkdir -p "$RUNNER_TEMP/cucumber-testdrive" + cp "$RUNNER_TEMP/cucumber-fixtures/${{ matrix.configuration.fixture }}"/package*.json "$RUNNER_TEMP/cucumber-testdrive/" + - name: Reuse the locked Cucumber fixture + uses: ./.github/actions/node-test-fixture + with: + directory: ${{ runner.temp }}/cucumber-testdrive - name: Validate native scenarios, matching failures and source coverage env: DDTEST_CUCUMBER_NODE_MODULES: ${{ runner.temp }}/cucumber-testdrive/node_modules - DDTEST_TEST_SHARD: ${{ matrix.shard }} + DDTEST_TEST_SHARD: ${{ matrix.configuration.shard }} run: | - test_filter='^TestPublicCucumberTestdrive$' - if [ '${{ matrix.cucumber }}' = '8.11.1' ]; then test_filter='^TestPublicCucumberTestdrive/passing$'; fi - go test -v ./internal/testdrive -run "$test_filter" -failfast -count=1 -timeout=12m + go test -v ./internal/testdrive -run '${{ matrix.configuration.tests }}' -failfast -count=1 -timeout=12m testdrive-webdriverio-workflow: + needs: test-binary runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -1048,6 +1189,14 @@ jobs: - {package-manager: npm, wdio: 'latest', tracer: 'latest', shard: '4/4'} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: integration-cli + path: ${{ runner.temp }}/test-binary + - name: Make the shared CLI executable + run: | + chmod +x "$RUNNER_TEMP/test-binary/ddtest" + echo "DDTEST_TEST_BINARY=$RUNNER_TEMP/test-binary/ddtest" >> "$GITHUB_ENV" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v5 with: go-version-file: go.mod diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 770e77fb6..155053f97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,6 +43,14 @@ jobs: GH_TOKEN: ${{ steps.octo-sts.outputs.token }} run: bash .github/scripts/release.sh validate + ci: + needs: authorize + uses: ./.github/workflows/ci.yml + permissions: + contents: read + actions: read + secrets: inherit + # Before a release, the Go instrumentation suite runs every dd-trace-go and # Orchestrion version that ddtest instrument supports, as the module proxy # has them now, so that no release ships without them passing. @@ -57,7 +65,7 @@ jobs: secrets: inherit release: - needs: compatibility + needs: [ci, compatibility] if: github.repository == 'DataDog/ddtest' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: diff --git a/docs/ci-performance.md b/docs/ci-performance.md new file mode 100644 index 000000000..9cef13736 --- /dev/null +++ b/docs/ci-performance.md @@ -0,0 +1,74 @@ +# CI fixture reuse + +PR branches, main and releases use the same CI workflow and the same onboard and +testdrive scenarios. Both existing Go instrumentation paths and their coverage +remain enabled. Datadog Test Impact Analysis owns test skipping; this workflow +does not maintain a separate changed-file test selector. + +## Go test discovery and compiler caches + +The Go shard selector runs `go run ./scripts/testnames`. The helper asks +`go list -race -json ./...` for the files selected by the current platform and +build tags, then uses Go's parser and example discovery to list tests, examples +and fuzz targets. Discovery builds only the small helper; it does not build +every test executable or run their initializers and `TestMain` functions. + +The native Orchestrion and `ddtest instrument` jobs retain separate compiler +caches. Keys include OS, architecture, Go version, dependencies and the source +revision; the Orchestrion cache also includes its pinned version and executable. +A new revision restores compatible previous output and saves updated compiler +artifacts, rather than repeatedly restoring an immutable dependency-only cache. +The Datadog action uses this cache instead of restoring another overlapping +module cache. Its configuration script still runs to resolve compatible SDK +versions and set up the current job's instrumentation and reporting. + +## Shared CLI executable + +The `test-binary` job builds one static executable for the Linux public CLI journeys, +including jobs in browser containers. +Every framework job downloads that run's `integration-cli` artifact and sets +`DDTEST_TEST_BINARY` to its absolute path. The Go test helper uses that executable +across temporary customer checkouts. Outside CI, it builds the CLI once per test +process; the test process's isolated temporary directory owns its cleanup. + +## Locked Node fixtures + +The local `node-test-fixture` action prepares a lockfile and reuses installed +dependencies only when the OS, architecture, exact Node/npm versions and complete +lockfile match. Cache misses run `npm ci`; there are no partial restore keys. +Dependencies are saved before tests run, keeping runtime-generated caches out of +the reusable fixture. +Jest matrix jobs and Cucumber workers use this action. Browser fixtures with +additional external installation state retain their existing setup. + +The Cucumber planner resolves `latest` once before expanding the matrix and +creates one lockfile per resolved version. Labels share a job only when the +complete dependency graph, runtime, test filter and shard are identical. All +distinct compatibility cases remain. When `latest` resolves to pinned `13.3.0`, +13 requested lanes become 9 executions; a distinct latest version keeps its lanes. + +Inspect the resolved configurations with Node 24 and npm registry access: + +```sh +python3 scripts/ci_cucumber_matrix.py --output /tmp/cucumber-ci-fixtures \ + --summary /tmp/cucumber-ci-matrix.md +``` + +The output directory must not exist. Inspect `matrix.json`, the summary and the +version directories' lockfiles; remove the directory afterwards. + +## Python prerequisites + +The Python version-check subprocess disables Datadog reporting and telemetry; +it only imports dependencies and checks their versions. The actual testdrive +journeys keep their original reporting, local intake, configuration variants and +six feature assertions. No validation scenario is removed or moved to a release +only workflow. + +## Test helper isolation + +Crash-recovery helpers run with their own CI reporting disabled, while their +parent tests keep reporting and checking process ownership and cleanup. Jenkins +allocation budgets run in an isolated helper so background SDK allocations do +not count against the parser. The original memory limits remain, and the parent +also exercises every behavioral assertion to retain instrumentation coverage. diff --git a/internal/onboard/ci_provider_jenkins_test.go b/internal/onboard/ci_provider_jenkins_test.go index 73359deab..eb176af22 100644 --- a/internal/onboard/ci_provider_jenkins_test.go +++ b/internal/onboard/ci_provider_jenkins_test.go @@ -11,6 +11,7 @@ import ( "errors" "maps" "os" + "os/exec" "path/filepath" "runtime" "slices" @@ -1388,6 +1389,18 @@ func TestJenkinsRubyVersionManagersKeepTheStatedRuntime(t *testing.T) { // cells of all of a Jenkinsfile's matrices, and a value that interpolates // others. func TestJenkinsExpansionsAreBounded(t *testing.T) { + isolated := os.Getenv("DDTEST_JENKINS_ALLOCATION_HELPER") == "1" + if isolated { + require.Equal(t, "false", os.Getenv("DD_CIVISIBILITY_ENABLED")) + } else { + // TotalAlloc is process-wide. Measure without background SDK work, + // then also run the behavioral assertions here to retain the + // instrumented parent's coverage of the parser's bounds. + command := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestJenkinsExpansionsAreBounded$") + command.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_JENKINS_ALLOCATION_HELPER=1") + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + } values := func(n int) string { var items []string for i := range n { @@ -1400,13 +1413,17 @@ func TestJenkinsExpansionsAreBounded(t *testing.T) { } huge := "pipeline {\n agent any\n stages {\n" + matrix("M", " axis { name 'A'; values "+values(200)+" }\n axis { name 'B'; values "+values(1000)+" }\n", "stage('t') { steps { sh 'npm test' } }") + " }\n}\n" var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) + if isolated { + runtime.GC() + runtime.ReadMemStats(&before) + } workflow, err := readJenkinsfile(t.TempDir(), "Jenkinsfile", huge) - runtime.ReadMemStats(&after) + if isolated { + runtime.ReadMemStats(&after) + require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(8<<20), "200,000 cells are never built") + } require.NoError(t, err) require.Equal(t, []string{"UNRESOLVED Jenkinsfile M: ddtest cannot expand its matrix (it has more than 200 cells), so it cannot tell which stages it runs"}, jenkinsRuns(workflow.Jobs["M"])) - require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(8<<20), "200,000 cells are never built") // Nested matrices multiply: past the Jenkinsfile's budget, each is // unresolved. @@ -1432,15 +1449,19 @@ func TestJenkinsExpansionsAreBounded(t *testing.T) { doubling += "def v" + strconv.Itoa(i) + " = \"${v" + strconv.Itoa(i-1) + "}${v" + strconv.Itoa(i-1) + "}\"\n" } doubling += "node { stage('T') { sh \"echo ${v30}\"\nsh 'npm test' } }\n" - runtime.GC() - runtime.ReadMemStats(&before) + if isolated { + runtime.GC() + runtime.ReadMemStats(&before) + } jobs := jenkinsJobs(t, doubling) - runtime.ReadMemStats(&after) + if isolated { + runtime.ReadMemStats(&after) + require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(16<<20), "a 16 GiB string is never built") + } runs := jenkinsRuns(jobs["T"]) require.Len(t, runs, 2) require.Contains(t, runs[0], "UNRESOLVED Jenkinsfile T: its sh script reads ${v30}, which only the Jenkins build knows") require.Equal(t, "npm test", runs[1]) - require.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(16<<20), "a 16 GiB string is never built") // Nested stages repeat their names in the jobs inside: a long one is // named by its line. diff --git a/internal/releaseworkflow/ci_shards_test.go b/internal/releaseworkflow/ci_shards_test.go index 15d7d94e8..026f91102 100644 --- a/internal/releaseworkflow/ci_shards_test.go +++ b/internal/releaseworkflow/ci_shards_test.go @@ -14,13 +14,82 @@ import ( type ciTestJob struct { If string Needs any + Uses string + Env map[string]string Strategy struct { FailFast bool `yaml:"fail-fast"` Matrix struct{ Shard []any } } Steps []struct { Name, Run, Uses string + ID string `yaml:"id"` Env map[string]string + With map[string]string + } +} + +func TestCompilerCachesRefreshAndKeepInstrumentationSeparate(t *testing.T) { + jobs := ciTestJobs(t) + keys := make(map[string]string) + for _, name := range []string{"test-shards", "test-ddtest-instrument-shards"} { + job := jobs[name] + for _, step := range job.Steps { + if step.Name == "Set up Go" { + require.Equal(t, "go", step.ID) + require.Equal(t, "false", step.With["cache"]) + } + if step.Name == "Cache Go modules" { + key := step.With["key"] + for _, input := range []string{"runner.os", "runner.arch", "steps.go.outputs.go-version", "github.sha"} { + require.Contains(t, key, "${{ "+input+" }}", name) + } + require.Contains(t, step.With["path"], "~/.cache/go-build") + require.NotEmpty(t, step.With["restore-keys"]) + require.Equal(t, strings.TrimSuffix(key, "${{ github.sha }}"), strings.Split(strings.TrimSpace(step.With["restore-keys"]), "\n")[0]) + keys[name] = key + } + if strings.HasPrefix(step.Uses, "datadog/test-visibility-github-action@") { + require.Equal(t, "false", step.With["cache"], "avoid a second overlapping module-cache restore") + require.Equal(t, "${{ env.ORCHESTRION_VERSION }}", step.With["go-tracer-version"]) + require.NotEmpty(t, job.Env["ORCHESTRION_VERSION"]) + } + } + require.NotEmpty(t, keys[name]) + } + require.NotEqual(t, keys["test-shards"], keys["test-ddtest-instrument-shards"]) +} + +func TestReleaseUsesTheSameCIAndRetainsTheInstrumentationGate(t *testing.T) { + data, err := os.ReadFile("../../.github/workflows/release.yml") + require.NoError(t, err) + var workflow struct{ Jobs map[string]ciTestJob } + require.NoError(t, yaml.Unmarshal(data, &workflow)) + require.Equal(t, "./.github/workflows/ci.yml", workflow.Jobs["ci"].Uses) + require.Equal(t, "./.github/workflows/go-instrumentation.yml", workflow.Jobs["compatibility"].Uses) + require.Equal(t, []any{"ci", "compatibility"}, workflow.Jobs["release"].Needs) +} + +func TestPublicCLIJobsShareOneExecutable(t *testing.T) { + jobs := ciTestJobs(t) + for _, name := range []string{"jest-testdrive-workflow", "testdrive-language-workflow", "ruby-testdrive", + "java-testdrive", "testdrive-vitest-workflow", "onboard-apply-workflow", "testdrive-mocha-workflow", + "testdrive-playwright-workflow", "testdrive-cypress-workflow", "testdrive-cucumber-workflow", "testdrive-webdriverio-workflow"} { + job := jobs[name] + switch dependencies := job.Needs.(type) { + case string: + require.Equal(t, "test-binary", dependencies, name) + case []any: + require.Contains(t, dependencies, "test-binary", name) + default: + t.Fatalf("%s has no shared binary dependency", name) + } + artifact, executable := false, false + for _, step := range job.Steps { + artifact = artifact || strings.HasPrefix(step.Uses, "actions/download-artifact@") + executable = executable || strings.Contains(step.Run, "DDTEST_TEST_BINARY=") + } + require.True(t, artifact, name) + require.True(t, executable, name) } } diff --git a/internal/releaseworkflow/node_fixture_test.go b/internal/releaseworkflow/node_fixture_test.go new file mode 100644 index 000000000..db63d8b1c --- /dev/null +++ b/internal/releaseworkflow/node_fixture_test.go @@ -0,0 +1,70 @@ +package releaseworkflow + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +func TestNodeFixtureCacheUsesTheWholeLockfileAndExactRuntime(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture preparation runs in Ubuntu bash") + } + if _, err := exec.LookPath("sha256sum"); err != nil { + t.Skip("fixture preparation requires sha256sum") + } + data, err := os.ReadFile("../../.github/actions/node-test-fixture/action.yml") + require.NoError(t, err) + var action struct { + Runs struct { + Steps []struct { + ID, Run, If, Uses string + With map[string]string + } + } + } + require.NoError(t, yaml.Unmarshal(data, &action)) + steps := action.Runs.Steps + require.Len(t, steps, 4) + require.Contains(t, steps[1].With["key"], "runner.os") + require.Contains(t, steps[1].With["key"], "runner.arch") + require.Empty(t, steps[1].With["restore-keys"]) + require.Equal(t, "steps.modules.outputs.cache-hit != 'true'", steps[2].If) + require.Contains(t, steps[2].Run, "npm ci") + require.Equal(t, steps[1].With["key"], steps[3].With["key"]) + require.Equal(t, steps[1].With["path"], steps[3].With["path"]) + require.Equal(t, steps[2].If, steps[3].If) + tools, fixture := t.TempDir(), t.TempDir() + lock := filepath.Join(t.TempDir(), "lock.json") + require.NoError(t, os.WriteFile(lock, []byte(`{"dependencies":{"nested":"1.0.0"}}`), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(tools, "node"), []byte("#!/bin/sh\necho \"$FIXTURE_NODE\"\n"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(tools, "npm"), []byte("#!/bin/sh\nif [ \"$1\" = install ]; then cp \"$FIXTURE_LOCK\" package-lock.json; else echo \"$FIXTURE_NPM\"; fi\n"), 0755)) + dependencies := "example@1.0.0" + key := func(node, npm string) string { + t.Helper() + output := filepath.Join(t.TempDir(), "output") + cmd := exec.CommandContext(t.Context(), "bash", "-eu", "-c", steps[0].Run) + cmd.Env = append(os.Environ(), "PATH="+tools+string(os.PathListSeparator)+os.Getenv("PATH"), + "FIXTURE_DIRECTORY="+fixture, "FIXTURE_DEPENDENCIES="+dependencies, "FIXTURE_LOCK="+lock, + "FIXTURE_NODE="+node, "FIXTURE_NPM="+npm, "GITHUB_OUTPUT="+output) + result, err := cmd.CombinedOutput() + require.NoError(t, err, string(result)) + data, err := os.ReadFile(output) + require.NoError(t, err) + return string(data) + } + original := key("24.0.0", "11.0.0") + require.Equal(t, original, key("24.0.0", "11.0.0")) + require.NotEqual(t, original, key("24.0.1", "11.0.0")) + require.NotEqual(t, original, key("24.0.0", "11.0.1")) + require.NoError(t, os.WriteFile(lock, []byte(`{"dependencies":{"nested":"1.0.1"}}`), 0600)) + dependencies = "" + require.Equal(t, original, key("24.0.0", "11.0.0"), "a supplied lockfile must not be resolved again") + dependencies = "example@1.0.0" + require.NotEqual(t, original, key("24.0.0", "11.0.0")) +} diff --git a/internal/releaseworkflow/release_commit_test.go b/internal/releaseworkflow/release_commit_test.go index 64aa27dad..909477c0b 100644 --- a/internal/releaseworkflow/release_commit_test.go +++ b/internal/releaseworkflow/release_commit_test.go @@ -22,9 +22,12 @@ func TestReleaseCommit(t *testing.T) { } git := func(dir string, args ...string) string { t.Helper() - out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).Output() + command := exec.Command("git", append([]string{"-C", dir}, args...)...) + var diagnostic bytes.Buffer + command.Stderr = &diagnostic + out, err := command.Output() if err != nil { - t.Fatalf("git %s: %v", strings.Join(args, " "), err) + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, diagnostic.String()) } return strings.TrimSpace(string(out)) } diff --git a/internal/testdrive/apply_cucumber_integration_test.go b/internal/testdrive/apply_cucumber_integration_test.go index b8797f3a9..c8f918fca 100644 --- a/internal/testdrive/apply_cucumber_integration_test.go +++ b/internal/testdrive/apply_cucumber_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicCucumberApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicCucumberApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_mocha_integration_test.go b/internal/testdrive/apply_mocha_integration_test.go index 236e09e75..221657ec9 100644 --- a/internal/testdrive/apply_mocha_integration_test.go +++ b/internal/testdrive/apply_mocha_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicMochaApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicMochaApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_webdriverio_integration_test.go b/internal/testdrive/apply_webdriverio_integration_test.go index 2fc0e7ba2..7e16327f0 100644 --- a/internal/testdrive/apply_webdriverio_integration_test.go +++ b/internal/testdrive/apply_webdriverio_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -34,7 +35,8 @@ func TestPublicWebdriverIOApply(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -45,8 +47,6 @@ func TestPublicWebdriverIOApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/apply_workflow_integration_test.go b/internal/testdrive/apply_workflow_integration_test.go index 918c43b45..586e4318d 100644 --- a/internal/testdrive/apply_workflow_integration_test.go +++ b/internal/testdrive/apply_workflow_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,7 +23,8 @@ func TestPublicOnboardApply(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -33,8 +35,6 @@ func TestPublicOnboardApply(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/configurations_integration_test.go b/internal/testdrive/configurations_integration_test.go index 8eee4a411..0e91ab393 100644 --- a/internal/testdrive/configurations_integration_test.go +++ b/internal/testdrive/configurations_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -40,9 +41,8 @@ func TestPublicJestConfigurations(t *testing.T) { cmd.Env = environment return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte root, err := filepath.EvalSymlinks(t.TempDir()) require.NoError(t, err) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) diff --git a/internal/testdrive/coordinator_workflow_integration_test.go b/internal/testdrive/coordinator_workflow_integration_test.go index 727fb436b..f8bff6201 100644 --- a/internal/testdrive/coordinator_workflow_integration_test.go +++ b/internal/testdrive/coordinator_workflow_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,11 +29,8 @@ func TestPublicCoordinatorJestFactsAndFreshReport(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - build := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") - build.Dir = "../.." - data, err := build.CombinedOutput() - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte root := workspace.CanonicalRoot(t.TempDir()) require.NoError(t, workspace.Copy(ctx, filepath.Dir(modules), root, skipIgnoredOutput)) write := func(name, body string) { diff --git a/internal/testdrive/cucumber_workflow_integration_test.go b/internal/testdrive/cucumber_workflow_integration_test.go index d8cff6df5..0daa00acf 100644 --- a/internal/testdrive/cucumber_workflow_integration_test.go +++ b/internal/testdrive/cucumber_workflow_integration_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -32,7 +33,8 @@ func TestPublicCucumberTestdrive(t *testing.T) { require.NoError(t, json.Unmarshal(manifest, &installed)) ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte var hookFile string newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) @@ -48,8 +50,6 @@ func TestPublicCucumberTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range cucumberWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { if _, err := os.Stat(filepath.Join(modules, "tsx")); mode == "source loader" && err != nil { diff --git a/internal/testdrive/cypress_workflow_integration_test.go b/internal/testdrive/cypress_workflow_integration_test.go index 13c3a3667..df414fa83 100644 --- a/internal/testdrive/cypress_workflow_integration_test.go +++ b/internal/testdrive/cypress_workflow_integration_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -27,7 +28,8 @@ func TestPublicCypressTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 28*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -41,8 +43,6 @@ func TestPublicCypressTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range cypressWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/forwarded_configuration_integration_test.go b/internal/testdrive/forwarded_configuration_integration_test.go index 71d3d84e4..f6e862108 100644 --- a/internal/testdrive/forwarded_configuration_integration_test.go +++ b/internal/testdrive/forwarded_configuration_integration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -59,9 +60,8 @@ func TestPublicJestForwardedConfiguration(t *testing.T) { cmd.Env = append(cmd.Env, "NODE_OPTIONS=") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, flags := range [][]string{{}, {"--command", "npm test"}} { diff --git a/internal/testdrive/frameworks_integration_test.go b/internal/testdrive/frameworks_integration_test.go index 7b978b8e9..81dbf119f 100644 --- a/internal/testdrive/frameworks_integration_test.go +++ b/internal/testdrive/frameworks_integration_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -26,8 +27,7 @@ func TestPublicFrameworkTestdrives(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 20*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) fixtures := []struct { name, manifest, command string files map[string]string diff --git a/internal/testdrive/java_validation_integration_test.go b/internal/testdrive/java_validation_integration_test.go index 461dd70e9..d0ae330c1 100644 --- a/internal/testdrive/java_validation_integration_test.go +++ b/internal/testdrive/java_validation_integration_test.go @@ -19,6 +19,7 @@ import ( "time" shard "github.com/DataDog/ddtest/internal/testdrive" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -32,8 +33,7 @@ func TestJavaValidationIntegration(t *testing.T) { tracer := cmp.Or(os.Getenv("DDTEST_JAVA_TRACER_VERSION"), "1.67.1") ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) type javaReport struct { Success bool diff --git a/internal/testdrive/jest_workflow_integration_test.go b/internal/testdrive/jest_workflow_integration_test.go index 2fb1efc05..34e549671 100644 --- a/internal/testdrive/jest_workflow_integration_test.go +++ b/internal/testdrive/jest_workflow_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -25,11 +26,8 @@ func TestPublicJestTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 12*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - build := exec.CommandContext(ctx, "go", "build", "-o", binary, ".") - build.Dir = "../.." - output, err := build.CombinedOutput() - require.NoError(t, err, string(output)) + binary := testutil.DDTest(t) + var output []byte _, tsErr := os.Stat(filepath.Join(modules, "ts-jest")) for _, mode := range testdrive.JestWorkflowScenarios(t) { if strings.HasPrefix(mode, "TypeScript") && tsErr != nil { diff --git a/internal/testdrive/languages_integration_test.go b/internal/testdrive/languages_integration_test.go index fc9422ff6..3f0711195 100644 --- a/internal/testdrive/languages_integration_test.go +++ b/internal/testdrive/languages_integration_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,8 +29,7 @@ func TestPublicPythonRubyTestdrives(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 18*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) for _, fixture := range []struct { name, command string files map[string]string diff --git a/internal/testdrive/loader_configuration_integration_test.go b/internal/testdrive/loader_configuration_integration_test.go index 011bc474e..52b7b3f60 100644 --- a/internal/testdrive/loader_configuration_integration_test.go +++ b/internal/testdrive/loader_configuration_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -65,9 +66,8 @@ Module._resolveFilename=function(request,...args){return entries[request]||origi cmd.Env = append(cmd.Env, "NODE_OPTIONS=") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) + var data []byte data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, flags := range [][]string{{"--check-only"}, {}} { diff --git a/internal/testdrive/mocha_workflow_integration_test.go b/internal/testdrive/mocha_workflow_integration_test.go index 867bf0814..4f0a97b98 100644 --- a/internal/testdrive/mocha_workflow_integration_test.go +++ b/internal/testdrive/mocha_workflow_integration_test.go @@ -3,6 +3,7 @@ package testdrive import ( "context" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" "os" "os/exec" @@ -21,7 +22,8 @@ func TestPublicMochaTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 8*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -32,8 +34,6 @@ func TestPublicMochaTestdrive(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range mochaWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/package_configuration_integration_test.go b/internal/testdrive/package_configuration_integration_test.go index 0e4aec123..cac59f437 100644 --- a/internal/testdrive/package_configuration_integration_test.go +++ b/internal/testdrive/package_configuration_integration_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/kballard/go-shellquote" "github.com/stretchr/testify/require" "os" @@ -82,9 +83,7 @@ func TestPublicJestPackageConfiguration(t *testing.T) { cmd.Env = append(cmd.Env, "NODE_OPTIONS=--conditions=development") return cmd.CombinedOutput() } - binary := filepath.Join(t.TempDir(), "ddtest") - data, err = command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) data, err = command(root, "git", "init", "-q") require.NoError(t, err, string(data)) for _, args := range [][]string{{"--all", "--check-only"}, {"--all"}} { @@ -256,9 +255,7 @@ func TestPublicJestTwoPackageConfiguration(t *testing.T) { } data, err := command(root, "git", "init", "-q") require.NoError(t, err, string(data)) - binary := filepath.Join(t.TempDir(), "ddtest") - data, err = command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(data)) + binary := testutil.DDTest(t) for _, args := range [][]string{{"--all", "--check-only"}, {"--all"}} { data, err = command(root, binary, append([]string{"testdrive", "--framework", "jest", "--yes"}, args...)...) require.NoError(t, err, string(data)) diff --git a/internal/testdrive/playwright_workflow_integration_test.go b/internal/testdrive/playwright_workflow_integration_test.go index df0b3f738..4e165e5b0 100644 --- a/internal/testdrive/playwright_workflow_integration_test.go +++ b/internal/testdrive/playwright_workflow_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/ext" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -28,7 +29,8 @@ func TestPublicPlaywrightTestdrive(t *testing.T) { require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 15*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte command := func(root, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -39,8 +41,6 @@ func TestPublicPlaywrightTestdrive(t *testing.T) { } return cmd.CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) for _, mode := range playwrightWorkflowScenarios.selected(t) { t.Run(mode, func(t *testing.T) { root, err := filepath.EvalSymlinks(t.TempDir()) diff --git a/internal/testdrive/python_feedback_test.go b/internal/testdrive/python_feedback_test.go index e05dd9d10..22dfbe9bb 100644 --- a/internal/testdrive/python_feedback_test.go +++ b/internal/testdrive/python_feedback_test.go @@ -80,6 +80,26 @@ func TestPythonValidationProbesRespectTimeout(t *testing.T) { } } +func TestPythonPrerequisiteProbeDoesNotStartDatadogReporting(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX executable fixture") + } + directory := t.TempDir() + result := filepath.Join(directory, "probe-environment") + python := filepath.Join(directory, "python") + require.NoError(t, os.WriteFile(python, []byte("#!/bin/sh\nprintf '%s %s %s' \"$DD_INSTRUMENTATION_TELEMETRY_ENABLED\" \"$DD_TRACE_ENABLED\" \"$DD_CIVISIBILITY_ENABLED\" > \"$PROBE_ENVIRONMENT\"\n"), 0755)) + t.Setenv("PROBE_ENVIRONMENT", result) + t.Setenv("DDTEST_PYTEST_PYTHON", python) + for _, key := range []string{"DD_INSTRUMENTATION_TELEMETRY_ENABLED", "DD_TRACE_ENABLED", "DD_CIVISIBILITY_ENABLED"} { + t.Setenv(key, "true") + } + require.Equal(t, python, feedbackPython(t)) + data, err := os.ReadFile(result) + require.NoError(t, err) + require.Equal(t, "false false false", string(data)) + require.Equal(t, "true", os.Getenv("DD_CIVISIBILITY_ENABLED")) +} + func feedbackPython(t *testing.T) string { t.Helper() python := os.Getenv("DDTEST_PYTEST_PYTHON") @@ -89,7 +109,11 @@ func feedbackPython(t *testing.T) string { if python == "" { t.Skip("Python required") } - if output, err := exec.CommandContext(t.Context(), python, "-c", `import pytest, ddtrace; from packaging.version import Version; assert Version(ddtrace.__version__) >= Version("4.11.0") and Version(pytest.__version__) >= Version("7.2")`).CombinedOutput(); err != nil { + probe := exec.CommandContext(t.Context(), python, "-c", `import pytest, ddtrace; from packaging.version import Version; assert Version(ddtrace.__version__) >= Version("4.11.0") and Version(pytest.__version__) >= Version("7.2")`) + // This process only checks installed versions. The real validation processes + // still exercise telemetry and all feature controls against the local intake. + probe.Env = append(os.Environ(), "DD_INSTRUMENTATION_TELEMETRY_ENABLED=false", "DD_TRACE_ENABLED=false", "DD_CIVISIBILITY_ENABLED=false") + if output, err := probe.CombinedOutput(); err != nil { if os.Getenv("DDTEST_REQUIRE_PROJECT_RUNTIMES") == "1" || os.Getenv("DDTEST_PYTEST_PYTHON") != "" { t.Fatalf("pytest and ddtrace required: %s: %v", output, err) } diff --git a/internal/testdrive/ruby_fork_integration_test.go b/internal/testdrive/ruby_fork_integration_test.go index 017c91279..3b28c8144 100644 --- a/internal/testdrive/ruby_fork_integration_test.go +++ b/internal/testdrive/ruby_fork_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -44,8 +45,7 @@ func TestRubyMinitestForkValidationIntegration(t *testing.T) { integrationFile(t, root, ".github/workflows/test.yml", "jobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: ruby/setup-ruby@v1\n with: {ruby-version: '"+installed.Ruby+"'}\n - uses: datadog/test-visibility-github-action@v3\n with: {languages: ruby, api_key: '${{ secrets.DD_API_KEY }}', ruby-tracer-version: '"+installed.Gems["datadog-ci"]+"'}\n - run: bundle exec rake test\n env:\n RUBYOPT: '${{ env.RUBYOPT }} -r./ci/datadog_test_optimization.rb'\n") integrationCommand(t, ctx, root, nil, "bundle", "lock", "--local") integrationCommand(t, ctx, root, nil, "git", "add", ".") - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) env := []string{"DD_GIT_REPOSITORY_URL=https://example.com/ddtest-isolation-fixture.git", "DD_GIT_COMMIT_SHA=" + strings.Repeat("1", 40), "RUBYOPT=-W0"} protocol := rubyProtocolCommand(t, ctx, root, env, binary, "testdrive", "--framework", "minitest", "--all", "--agent", "--yes", "--tracer-version", installed.Gems["datadog-ci"]) var state struct{ Status string } diff --git a/internal/testdrive/ruby_nested_integration_test.go b/internal/testdrive/ruby_nested_integration_test.go index 957b5e0bb..92cb5833b 100644 --- a/internal/testdrive/ruby_nested_integration_test.go +++ b/internal/testdrive/ruby_nested_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -23,8 +24,7 @@ func TestRubyNestedWorkingDirectoryIntegration(t *testing.T) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby:RUBY_VERSION,minitest:Gem::Specification.find_by_name("minitest").version.to_s,rake:Gem::Specification.find_by_name("rake").version.to_s,tracer:Gem::Specification.find_by_name("datadog-ci").version.to_s)`) var gems struct{ Ruby, Minitest, Rake, Tracer string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() retainIntegrationReports(t, root, "minitest-nested-review") integrationCommand(t, ctx, root, nil, "git", "init", "-q") diff --git a/internal/testdrive/ruby_receipt_integration_test.go b/internal/testdrive/ruby_receipt_integration_test.go index a21319220..547a9286a 100644 --- a/internal/testdrive/ruby_receipt_integration_test.go +++ b/internal/testdrive/ruby_receipt_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -22,8 +23,7 @@ func TestRubyReceiptReviewIntegration(t *testing.T) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby:RUBY_VERSION,minitest:Gem::Specification.find_by_name("minitest").version.to_s,rake:Gem::Specification.find_by_name("rake").version.to_s,tracer:Gem::Specification.find_by_name("datadog-ci").version.to_s)`) var gems struct{ Ruby, Minitest, Rake, Tracer string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() retainIntegrationReports(t, root, "minitest-receipts") integrationCommand(t, ctx, root, nil, "git", "init", "-q") diff --git a/internal/testdrive/ruby_validation_integration_test.go b/internal/testdrive/ruby_validation_integration_test.go index 4d182f942..c797d3a3a 100644 --- a/internal/testdrive/ruby_validation_integration_test.go +++ b/internal/testdrive/ruby_validation_integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -29,8 +30,7 @@ func runRubyValidationIntegration(t *testing.T, frameworks ...string) { metadata := integrationCommand(t, ctx, ".", nil, "ruby", "-rjson", "-e", `puts JSON.generate(ruby: RUBY_VERSION, rspec: Gem::Specification.find_by_name("rspec-core").version.to_s, minitest: Gem::Specification.find_by_name("minitest").version.to_s, tracer: Gem::Specification.find_by_name("datadog-ci").version.to_s, rake: Gem::Specification.find_by_name("rake").version.to_s)`) var gems struct{ Ruby, Rspec, Minitest, Tracer, Rake string } require.NoError(t, json.Unmarshal([]byte(metadata), &gems)) - binary := filepath.Join(t.TempDir(), "ddtest") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) for _, framework := range frameworks { t.Run(framework, func(t *testing.T) { root := filepath.Join(t.TempDir(), "project space") diff --git a/internal/testdrive/service_owner_test.go b/internal/testdrive/service_owner_test.go index 4366a7de8..4ed3fdfb0 100644 --- a/internal/testdrive/service_owner_test.go +++ b/internal/testdrive/service_owner_test.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/ddtest/internal/framework" "github.com/DataDog/ddtest/internal/onboard" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,6 +29,7 @@ type crashedService struct { func TestDeclaredServiceStopsWhenCoordinatorIsKilled(t *testing.T) { if root := os.Getenv("DDTEST_SERVICE_CRASH_ROOT"); root != "" { + require.Equal(t, "false", os.Getenv("DD_CIVISIBILITY_ENABLED"), "the helper must not start a CI test session") for _, key := range []string{"TMPDIR", "TMP", "TEMP"} { t.Setenv(key, os.Getenv("DDTEST_SERVICE_CRASH_TMP")) } @@ -69,15 +71,20 @@ func TestDeclaredServiceStopsWhenCoordinatorIsKilled(t *testing.T) { defer func() { _ = canary.Close() }() binary, err := os.Executable() require.NoError(t, err) + t.Setenv("DD_CIVISIBILITY_ENABLED", "true") // Exercise helper isolation even in a plain local test run. command := exec.Command(binary, "-test.run=^TestDeclaredServiceStopsWhenCoordinatorIsKilled$") - command.Env = append(os.Environ(), "DDTEST_SERVICE_CRASH_ROOT="+root, "DDTEST_SERVICE_CRASH_TMP="+temporary) + command.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_SERVICE_CRASH_ROOT="+root, "DDTEST_SERVICE_CRASH_TMP="+temporary) var diagnostic bytes.Buffer command.Stdout = &diagnostic command.Stderr = &diagnostic require.NoError(t, command.Start()) t.Cleanup(func() { _ = command.Process.Kill() }) ready := filepath.Join(root, "ready.json") - require.Eventually(t, func() bool { _, err := os.Stat(ready); return err == nil }, 10*time.Second, 20*time.Millisecond) + if !assert.Eventually(t, func() bool { _, err := os.Stat(ready); return err == nil }, 10*time.Second, 20*time.Millisecond) { + _ = command.Process.Kill() + _ = command.Wait() + t.Fatalf("service helper did not become ready:\n%s", diagnostic.String()) + } data, err := os.ReadFile(ready) require.NoError(t, err) var state crashedService diff --git a/internal/testdrive/temp_main_test.go b/internal/testdrive/temp_main_test.go index ea9bb08d3..e5e978ae5 100644 --- a/internal/testdrive/temp_main_test.go +++ b/internal/testdrive/temp_main_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DataDog/ddtest/internal/onboard" + "github.com/DataDog/ddtest/internal/testutil" ) func TestMain(m *testing.M) { @@ -21,6 +22,7 @@ func TestMain(m *testing.M) { os.Exit(1) } _ = os.Setenv("DDTEST_ISOLATED_TEST_TMP", "1") + testutil.SetBinaryDirectory(root) for _, key := range []string{"TMPDIR", "TMP", "TEMP"} { _ = os.Setenv(key, root) } diff --git a/internal/testdrive/vitest_runner_integration_test.go b/internal/testdrive/vitest_runner_integration_test.go index fffcf04c8..905331edb 100644 --- a/internal/testdrive/vitest_runner_integration_test.go +++ b/internal/testdrive/vitest_runner_integration_test.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/DataDog/ddtest/internal/testutil" "os" "os/exec" "path/filepath" @@ -26,8 +27,7 @@ func TestPublicVitestRunnerTestdrive(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest-under-test") - integrationCommand(t, ctx, "../..", nil, "go", "build", "-o", binary, ".") + binary := testutil.DDTest(t) root := t.TempDir() integrationCommand(t, ctx, root, nil, "git", "init", "-q") vitest, tracer := os.Getenv("DDTEST_VITEST_VERSION"), os.Getenv("DDTEST_VITEST_TRACER_VERSION") diff --git a/internal/testdrive/webdriverio_workflow_integration_test.go b/internal/testdrive/webdriverio_workflow_integration_test.go index edf987897..93a514a0a 100644 --- a/internal/testdrive/webdriverio_workflow_integration_test.go +++ b/internal/testdrive/webdriverio_workflow_integration_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/DataDog/ddtest/internal/testdrive/workspace" + "github.com/DataDog/ddtest/internal/testutil" "github.com/stretchr/testify/require" ) @@ -44,7 +45,8 @@ func TestPublicWebdriverIOTestdrive(t *testing.T) { } ctx, cancel := context.WithTimeout(t.Context(), 24*time.Minute) defer cancel() - binary := filepath.Join(t.TempDir(), "ddtest") + binary := testutil.DDTest(t) + var output []byte newCommand := func(root, name string, args ...string) *exec.Cmd { cmd := exec.CommandContext(ctx, name, args...) cmd.Dir = root @@ -58,8 +60,6 @@ func TestPublicWebdriverIOTestdrive(t *testing.T) { command := func(root, name string, args ...string) ([]byte, error) { return newCommand(root, name, args...).CombinedOutput() } - output, err := command("../..", "go", "build", "-o", binary, ".") - require.NoError(t, err, string(output)) // Process sockets live under /tmp, not TMPDIR; collect the folders that exist // before a run so later checks only judge the ones the run created. processFolders := func() map[string]bool { diff --git a/internal/testutil/binary.go b/internal/testutil/binary.go new file mode 100644 index 000000000..f0e71946d --- /dev/null +++ b/internal/testutil/binary.go @@ -0,0 +1,72 @@ +// Package testutil shares expensive fixtures between the CLI integration tests. +package testutil + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" +) + +type binaryCache struct { + once sync.Once + directory string + path string + err error +} + +var compiled = &binaryCache{directory: os.TempDir()} + +// SetBinaryDirectory gives the shared executable a process-owned directory. +// Call from TestMain before running tests that override their temporary paths. +func SetBinaryDirectory(directory string) { + compiled = &binaryCache{directory: directory} +} + +// Capture the package directory before customer fixtures use t.Chdir. +var repositoryRoot, repositoryRootError = filepath.Abs("../..") + +// DDTest uses the current CI run's binary, or builds it once per test process. +// TestMain's isolated temporary directory owns the locally built executable. +func DDTest(t testing.TB) string { + t.Helper() + if path := os.Getenv("DDTEST_TEST_BINARY"); path != "" { + info, err := os.Stat(path) + if !filepath.IsAbs(path) || err != nil || !info.Mode().IsRegular() { + t.Fatalf("DDTEST_TEST_BINARY must name an existing absolute executable: %q", path) + } + return path + } + compiled.once.Do(func() { compiled.path, compiled.err = buildDDTest(t.Context(), compiled.directory) }) + if compiled.err != nil { + t.Fatal(compiled.err) + } + return compiled.path +} + +func buildDDTest(ctx context.Context, parent string) (string, error) { + if repositoryRootError != nil { + return "", repositoryRootError + } + directory, err := os.MkdirTemp(parent, "ddtest-binary-") + if err != nil { + return "", err + } + name := "ddtest" + if runtime.GOOS == "windows" { + name += ".exe" + } + path := filepath.Join(directory, name) + command := exec.CommandContext(ctx, "go", "build", "-o", path, ".") + command.Dir = repositoryRoot + command.Env = append(os.Environ(), "CGO_ENABLED=0") + if output, err := command.CombinedOutput(); err != nil { + _ = os.RemoveAll(directory) + return "", fmt.Errorf("build integration CLI: %w\n%s", err, output) + } + return path, nil +} diff --git a/internal/testutil/binary_test.go b/internal/testutil/binary_test.go new file mode 100644 index 000000000..3aea65939 --- /dev/null +++ b/internal/testutil/binary_test.go @@ -0,0 +1,52 @@ +package testutil + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDDTestBuildIsSharedAcrossWorkingDirectories(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX fake Go executable") + } + previous := compiled + t.Cleanup(func() { compiled = previous }) + t.Setenv("DDTEST_TEST_BINARY", "") + directory := t.TempDir() + SetBinaryDirectory(directory) + firstTemporaryDirectory := t.TempDir() + t.Setenv("TMPDIR", firstTemporaryDirectory) + log := filepath.Join(directory, "builds") + t.Setenv("BUILD_LOG", log) + goCommand := "#!/bin/sh\nprintf '%s\\n' \"$PWD\" >> \"$BUILD_LOG\"\nprintf '#!/bin/sh\\nexit 0\\n' > \"$3\"\nchmod +x \"$3\"\n" + require.NoError(t, os.WriteFile(filepath.Join(directory, "go"), []byte(goCommand), 0755)) + t.Setenv("PATH", directory+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Chdir(t.TempDir()) + first := DDTest(t) + t.Chdir(directory) + require.NoError(t, os.RemoveAll(firstTemporaryDirectory), "simulate the first journey's cleanup") + t.Setenv("TMPDIR", directory) + t.Chdir(t.TempDir()) + require.Equal(t, first, DDTest(t)) + data, err := os.ReadFile(log) + require.NoError(t, err) + require.Len(t, strings.Split(strings.TrimSpace(string(data)), "\n"), 1) + require.FileExists(t, filepath.Join(strings.TrimSpace(string(data)), "go.mod")) + info, err := os.Stat(first) + require.NoError(t, err) + require.True(t, info.Mode().IsRegular()) +} + +func TestDDTestUsesTheProvidedCIExecutable(t *testing.T) { + path := filepath.Join(t.TempDir(), "ddtest") + require.NoError(t, os.WriteFile(path, []byte("fixture"), 0755)) + t.Setenv("DDTEST_TEST_BINARY", path) + t.Setenv("PATH", t.TempDir()) // A provided executable never needs Go installed. + t.Chdir(t.TempDir()) + require.Equal(t, path, DDTest(t)) +} diff --git a/scripts/ci_cucumber_matrix.py b/scripts/ci_cucumber_matrix.py new file mode 100644 index 000000000..0ebefebd1 --- /dev/null +++ b/scripts/ci_cucumber_matrix.py @@ -0,0 +1,116 @@ +"""Resolve Cucumber fixtures once and deduplicate identical CI configurations.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + + +LANES = [("8.11.1", 1, "^TestPublicCucumberTestdrive/passing$"), + ("10.9.0", 4, "^TestPublicCucumberTestdrive$"), + ("13.3.0", 4, "^TestPublicCucumberTestdrive$"), + ("latest", 4, "^TestPublicCucumberTestdrive$")] +DEPENDENCIES = {"nyc": "17.1.0", "c8": "12.0.0", "dd-trace": "6.20.1", + "@istanbuljs/esm-loader-hook": "0.3.0", "tsx": "4.23.15"} +VERSION = re.compile(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?") + + +def execute(command, cwd=None): + return subprocess.run(command, cwd=cwd, check=True, capture_output=True, + text=True, timeout=240).stdout.strip() + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def resolve_version(requested): + version = (requested if VERSION.fullmatch(requested) else + json.loads(execute(["npm", "view", f"@cucumber/cucumber@{requested}", "version", "--json"]))) + if not isinstance(version, str) or not VERSION.fullmatch(version): + raise ValueError(f"Cucumber {requested}: expected one exact version, got {version!r}") + return version + + +def lock_fixture(output, version): + directory = output / version + directory.mkdir() + dependencies = {**DEPENDENCIES, "@cucumber/cucumber": version} + manifest = {"name": "ddtest-cucumber-ci-fixture", "version": "1.0.0", + "private": True, "dependencies": dependencies} + (directory / "package.json").write_text(json.dumps(manifest, indent=2) + "\n") + execute(["npm", "install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund"], + cwd=directory) + lock = json.loads((directory / "package-lock.json").read_text()) + for package, expected in dependencies.items(): + if lock.get("packages", {}).get(f"node_modules/{package}", {}).get("version") != expected: + raise ValueError(f"Cucumber {version}: lockfile did not retain {package}@{expected}") + return fingerprint(lock) + + +def deduplicate(rows): + selected = {} + for row in rows: + # Labels are aliases; every execution input, including the complete + # lockfile, runtime, test filter and shard, participates in the key. + key = fingerprint({name: value for name, value in row.items() if name != "cucumber"}) + if key not in selected: + selected[key] = {**row, "aliases": [row["cucumber"]]} + else: + selected[key]["aliases"].append(row["cucumber"]) + return [{**row, "aliases": ", ".join(row["aliases"])} for row in selected.values()] + + +def build_plan(output): + output.mkdir(parents=True) # Refuse stale files from an earlier resolution. + node, npm = execute(["node", "--version"]).removeprefix("v"), execute(["npm", "--version"]) + if not VERSION.fullmatch(node) or not VERSION.fullmatch(npm): + raise ValueError("expected exact Node and npm versions") + fixtures, rows = {}, [] + for requested, total, tests in LANES: + version = resolve_version(requested) + if version not in fixtures: + fixtures[version] = lock_fixture(output, version) + for shard in range(1, total + 1): + rows.append({"cucumber": requested, "version": version, "fixture": version, + "lock": fixtures[version], "node": node, "npm": npm, + "tests": tests, "shard": f"{shard}/{total}"}) + return {"requested_jobs": len(rows), "matrix": {"include": deduplicate(rows)}} + + +def summary(plan): + rows = plan["matrix"]["include"] + lines = ["## Resolved Cucumber configurations", "", + f"{plan['requested_jobs']} requested lanes → {len(rows)} distinct jobs.", "", + "Matching aliases share a dependency lockfile and execution. Every distinct", + "runtime, dependency graph, test selection and shard still runs.", "", + "| Requested labels | Resolved Cucumber | Shard |", "| --- | --- | --- |"] + lines += [f"| {row['aliases']} | {row['version']} | {row['shard']} |" for row in rows] + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="new directory for fixtures and matrix.json") + parser.add_argument("--github-output", type=Path) + parser.add_argument("--summary", type=Path) + args = parser.parse_args() + try: + plan = build_plan(args.output) + (args.output / "matrix.json").write_text(json.dumps(plan, indent=2) + "\n") + if args.github_output: + with args.github_output.open("a") as output: + output.write("matrix=" + json.dumps(plan["matrix"], separators=(",", ":")) + "\n") + if args.summary: + args.summary.write_text(summary(plan)) + print(summary(plan), end="") + except subprocess.CalledProcessError as error: + parser.exit(1, error.stderr or str(error)) + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + parser.exit(1, str(error) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/go_test_shards.py b/scripts/go_test_shards.py index b2009c57b..41755e96b 100644 --- a/scripts/go_test_shards.py +++ b/scripts/go_test_shards.py @@ -19,8 +19,9 @@ def parse_shard(value): def discover_tests(): - # Match CI's build selection, including tests guarded by the race build tag. - result = subprocess.run(["go", "test", "-race", "-list", ".", "./..."], + # Go selects race-enabled source files; the helper parses declarations + # without compiling every test executable or running package initializers. + result = subprocess.run(["go", "run", "./scripts/testnames"], check=True, capture_output=True, text=True) names = sorted({line for line in result.stdout.splitlines() if re.fullmatch(r"(?:Test|Example|Fuzz)\w*", line)}) diff --git a/scripts/test_ci_cucumber_matrix.py b/scripts/test_ci_cucumber_matrix.py new file mode 100644 index 000000000..ebf36938c --- /dev/null +++ b/scripts/test_ci_cucumber_matrix.py @@ -0,0 +1,156 @@ +"""Keep every distinct configuration while eliminating resolved aliases.""" + +import contextlib +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import ci_cucumber_matrix as planner + + +class MatrixTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.output = Path(directory.name) / "fixtures" + self.latest = "13.3.0" + self.commands = [] + + def execute(self, command, cwd=None): + self.commands.append(command) + if command == ["node", "--version"]: + return "v24.14.1" + if command == ["npm", "--version"]: + return "11.11.0" + if command[:2] == ["npm", "view"]: + return json.dumps(self.latest) + self.assertEqual(command, ["npm", "install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund"]) + manifest = json.loads((cwd / "package.json").read_text()) + lock = {"lockfileVersion": 3, "packages": { + f"node_modules/{name}": {"version": version} + for name, version in manifest["dependencies"].items()}} + lock["packages"]["node_modules/transitive"] = {"version": "1.2.3", "integrity": "sha512-fixture"} + (cwd / "package-lock.json").write_text(json.dumps(lock)) + return "" + + def build(self): + with patch.object(planner, "execute", side_effect=self.execute): + return planner.build_plan(self.output) + + def test_equal_latest_and_pin_share_locks_and_jobs_but_keep_every_shard(self): + plan = self.build() + rows = plan["matrix"]["include"] + self.assertEqual(plan["requested_jobs"], 13) + self.assertEqual(len(rows), 9) + self.assertEqual(sum(len(row["aliases"].split(", ")) for row in rows), 13) + shared = [row for row in rows if row["version"] == "13.3.0"] + self.assertEqual({row["shard"] for row in shared}, {"1/4", "2/4", "3/4", "4/4"}) + self.assertTrue(all(row["aliases"] == "13.3.0, latest" for row in shared)) + self.assertEqual({path.name for path in self.output.iterdir()}, {"8.11.1", "10.9.0", "13.3.0"}) + self.assertEqual(sum(command[:2] == ["npm", "view"] for command in self.commands), 1) + for row in rows: + lock = json.loads((self.output / row["fixture"] / "package-lock.json").read_text()) + self.assertEqual(row["lock"], planner.fingerprint(lock)) + self.assertEqual((row["node"], row["npm"]), ("24.14.1", "11.11.0")) + + def test_a_new_latest_release_retains_all_thirteen_jobs(self): + self.latest = "13.4.0" + rows = self.build()["matrix"]["include"] + self.assertEqual(len(rows), 13) + self.assertEqual(len([row for row in rows if row["version"] == "13.4.0"]), 4) + self.assertTrue(all(row["aliases"] == row["cucumber"] for row in rows)) + + def test_version_equality_never_collapses_floor_and_full_scenarios(self): + self.latest = "8.11.1" + rows = self.build()["matrix"]["include"] + self.assertEqual(len(rows), 13) + floor = [row for row in rows if row["cucumber"] == "8.11.1"] + latest = [row for row in rows if row["cucumber"] == "latest"] + self.assertEqual(floor[0]["tests"], "^TestPublicCucumberTestdrive/passing$") + self.assertTrue(all(row["tests"] == "^TestPublicCucumberTestdrive$" for row in latest)) + + def test_runtime_graph_filter_and_shard_differences_cannot_be_deduplicated(self): + row = self.build()["matrix"]["include"][5] + row = {key: value for key, value in row.items() if key != "aliases"} + alias = {**row, "cucumber": "latest"} + self.assertEqual(len(planner.deduplicate([row, alias])), 1) + for key in ("node", "npm", "lock", "tests", "shard"): + with self.subTest(key=key): + self.assertEqual(len(planner.deduplicate([row, {**alias, key: "different"}])), 2) + lock = json.loads((self.output / row["fixture"] / "package-lock.json").read_text()) + lock["packages"]["node_modules/transitive"]["version"] = "1.2.4" + self.assertNotEqual(planner.fingerprint(lock), row["lock"]) + + def test_ambiguous_or_invalid_registry_versions_fail(self): + for version in (["13.3.0", "13.4.0"], None, "latest", "../unsafe"): + self.latest = version + with self.subTest(version=version), patch.object(planner, "execute", side_effect=self.execute): + with self.assertRaises(ValueError): + planner.resolve_version("latest") + + def test_lock_resolution_failure_never_publishes_a_partial_matrix(self): + github_output = self.output.parent / "github-output" + command = ["planner", "--output", str(self.output), "--github-output", str(github_output)] + with patch.object(sys, "argv", command), patch.object(planner, "execute", side_effect=[ + "v24.14.1", "11.11.0", subprocess.CalledProcessError(1, ["npm"], stderr="registry unavailable")]): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + planner.main() + self.assertEqual(error.exception.code, 1) + self.assertFalse(github_output.exists()) + self.assertFalse((self.output / "matrix.json").exists()) + + def test_success_publishes_the_same_complete_matrix_to_github_and_the_artifact(self): + github_output = self.output.parent / "github-output" + github_output.write_text("existing=value\n") + summary = self.output.parent / "summary.md" + command = ["planner", "--output", str(self.output), "--github-output", str(github_output), + "--summary", str(summary)] + with patch.object(sys, "argv", command), patch.object(planner, "execute", side_effect=self.execute): + with contextlib.redirect_stdout(io.StringIO()): + planner.main() + lines = github_output.read_text().splitlines() + self.assertEqual(lines[0], "existing=value") + self.assertEqual(len(lines), 2) + key, value = lines[1].split("=", 1) + self.assertEqual(key, "matrix") + artifact = json.loads((self.output / "matrix.json").read_text()) + self.assertEqual(json.loads(value), artifact["matrix"]) + self.assertEqual(len(artifact["matrix"]["include"]), 9) + self.assertIn("13 requested lanes → 9 distinct jobs", summary.read_text()) + + def test_an_incomplete_or_wrong_lockfile_fails(self): + self.output.mkdir() + with patch.object(planner, "execute") as execute: + execute.side_effect = lambda command, cwd: (cwd / "package-lock.json").write_text('{"packages":{}}') + with self.assertRaisesRegex(ValueError, "lockfile did not retain"): + planner.lock_fixture(self.output, "13.3.0") + + def test_existing_output_is_not_reused(self): + self.build() + with self.assertRaises(FileExistsError): + self.build() + + +class WorkflowTests(unittest.TestCase): + def test_workers_use_the_planned_runtime_and_lockfile_without_resolving_again(self): + workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/ci.yml").read_text() + worker = workflow.split(" testdrive-cucumber-workflow:\n", 1)[1].split("\n testdrive-webdriverio-workflow:", 1)[0] + self.assertIn("configuration: ${{ fromJSON(needs.cucumber-matrix.outputs.matrix).include }}", worker) + self.assertIn("node-version: ${{ matrix.configuration.node }}", worker) + self.assertIn('test "$(npm --version)" =', worker) + self.assertIn("name: ci-cucumber-fixtures", worker) + self.assertIn("uses: ./.github/actions/node-test-fixture", worker) + self.assertIn("directory: ${{ runner.temp }}/cucumber-testdrive", worker) + self.assertNotIn("dependencies:", worker) # Consume the supplied lockfile. + self.assertNotIn("npm install", worker) + self.assertNotIn("npm view", worker) + self.assertIn("-run '${{ matrix.configuration.tests }}'", worker) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_go_test_shards.py b/scripts/test_go_test_shards.py index 2728e52fa..61b4834c5 100644 --- a/scripts/test_go_test_shards.py +++ b/scripts/test_go_test_shards.py @@ -22,7 +22,7 @@ def test_discovery_uses_go_build_selection_and_ignores_diagnostics_and_benchmark result = subprocess.CompletedProcess([], 0, "TestCase\nTestCase\nExample\nFuzzSeed\nBenchmarkOther\nok\tpkg\n") with patch.object(subprocess, "run", return_value=result) as run: self.assertEqual(shards.discover_tests(), ["Example", "FuzzSeed", "TestCase"]) - self.assertEqual(run.call_args.args[0], ["go", "test", "-race", "-list", ".", "./..."]) + self.assertEqual(run.call_args.args[0], ["go", "run", "./scripts/testnames"]) def test_failed_or_empty_discovery_cannot_silently_pass(self): with patch.object(subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["go"])): diff --git a/scripts/testnames/main.go b/scripts/testnames/main.go new file mode 100644 index 000000000..e7e6ec2ea --- /dev/null +++ b/scripts/testnames/main.go @@ -0,0 +1,132 @@ +// Command testnames lists the root module's race-enabled tests without building +// test executables or running package initializers and TestMain. +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/token" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "unicode" + "unicode/utf8" +) + +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { + return true + } + r, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(r) +} + +// Match cmd/go's syntactic signature check, including aliased and dot imports. +func hasArgument(fn *ast.FuncDecl, kind string) bool { + if fn.Type.Results.NumFields() != 0 || fn.Type.Params.NumFields() != 1 || fn.Type.TypeParams.NumFields() != 0 { + return false + } + pointer, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr) + if !ok { + return false + } + switch argument := pointer.X.(type) { + case *ast.Ident: + return argument.Name == kind + case *ast.SelectorExpr: + return argument.Sel.Name == kind + } + return false +} + +func fileTests(path string) ([]string, error) { + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + return nil, err + } + var names []string + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok || fn.Recv != nil { + continue + } + name := fn.Name.Name + if name == "TestMain" && hasArgument(fn, "M") { + continue + } + kind := "" + switch { + case isTest(name, "Test"): + kind = "T" + case isTest(name, "Fuzz"): + kind = "F" + } + if kind != "" { + if !hasArgument(fn, kind) { + return nil, fmt.Errorf("%s: invalid signature for %s", path, name) + } + names = append(names, name) + } + } + for _, example := range doc.Examples(file) { + if example.Output != "" || example.EmptyOutput { + names = append(names, "Example"+example.Name) + } + } + return names, nil +} + +func discover(ctx context.Context, directory string) ([]string, error) { + // Let Go select files for the current platform, tags, cgo and race mode. + command := exec.CommandContext(ctx, "go", "list", "-race", "-json", "./...") + command.Dir = directory + command.Stderr = os.Stderr + output, err := command.Output() + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(output)) + var names []string + for { + var pkg struct { + Dir string + TestGoFiles, XTestGoFiles []string + } + if err := decoder.Decode(&pkg); err == io.EOF { + break + } else if err != nil { + return nil, err + } + for _, path := range append(pkg.TestGoFiles, pkg.XTestGoFiles...) { + selected, err := fileTests(filepath.Join(pkg.Dir, path)) + if err != nil { + return nil, err + } + names = append(names, selected...) + } + } + slices.Sort(names) + return slices.Compact(names), nil +} + +func main() { + names, err := discover(context.Background(), "") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, name := range names { + fmt.Println(name) + } +} diff --git a/scripts/testnames/main_test.go b/scripts/testnames/main_test.go new file mode 100644 index 000000000..85e3e9d93 --- /dev/null +++ b/scripts/testnames/main_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func fixture(t *testing.T, files map[string]string) string { + t.Helper() + t.Setenv("GOFLAGS", "") // The reference fixture must use the native compiler. + t.Setenv("GOWORK", "off") + root := t.TempDir() + files["go.mod"] = "module example.com/inventory\n\ngo 1.24\n" + for path, source := range files { + require.NoError(t, os.WriteFile(filepath.Join(root, path), []byte(source), 0600)) + } + return root +} + +func TestSourceInventoryMatchesGoTest(t *testing.T) { + otherOS := "windows" + if runtime.GOOS == otherOS { + otherOS = "linux" + } + root := fixture(t, map[string]string{ + "inventory.go": "package inventory\nfunc Value() {}\nfunc WithoutOutput() {}\ntype Widget struct{}\n", + "inventory_test.go": `package inventory +import ("fmt"; "os"; tst "testing") +func TestMain(m *tst.M) { os.Exit(m.Run()) } +func Test(t *tst.T) {}; func TestSecond(t *tst.T) {} +func TestÉ(t *tst.T) {} +func Testlowercase(t *tst.T) {} +func (Widget) TestMethod(t *tst.T) {} +func BenchmarkIgnored(b *tst.B) {} +func FuzzValue(f *tst.F) { f.Add("seed"); f.Fuzz(func(t *tst.T, s string) {}) } +func ExampleValue() { + fmt.Println("value") + // Output: value +} +func ExampleValue_empty() { + // Output: +} +func ExampleWithoutOutput() {} +var decoy = "func TestNotADeclaration(t *testing.T) {}" +`, + "external_test.go": `package inventory_test +import . "testing" +func TestExternal(t *T) {} +func TestMain(t *T) {} +`, + "race_test.go": "//go:build race\n\npackage inventory\nimport \"testing\"\nfunc TestRace(t *testing.T) {}\n", + "native_test.go": "//go:build !race\n\npackage inventory\nimport \"testing\"\nfunc TestNotRace(t *testing.T) {}\n", + "tagged_test.go": "//go:build ddtest_extra_fixture\n\npackage inventory\nimport \"testing\"\nfunc TestTagged(t *testing.T) {}\n", + "platform_" + otherOS + "_test.go": "package inventory\nimport \"testing\"\nfunc TestOtherOS(t *testing.T) {}\n", + }) + for _, tags := range []string{"", "-tags=ddtest_extra_fixture"} { + t.Run(tags, func(t *testing.T) { + t.Setenv("GOFLAGS", tags) + names, err := discover(t.Context(), root) + require.NoError(t, err) + // Vet warns about the intentionally ignored Testlowercase declaration. + command := exec.CommandContext(t.Context(), "go", "test", "-vet=off", "-race", "-list", ".", "./...") + command.Dir = root + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + var compiled []string + for _, line := range strings.Split(string(output), "\n") { + if len(strings.Fields(line)) == 1 && (strings.HasPrefix(line, "Test") || strings.HasPrefix(line, "Fuzz") || strings.HasPrefix(line, "Example")) { + compiled = append(compiled, line) + } + } + slices.Sort(compiled) + require.Equal(t, compiled, names) + require.Contains(t, names, "TestRace") + require.Contains(t, names, "TestMain") + require.Contains(t, names, "ExampleValue_empty") + require.NotContains(t, names, "TestNotRace") + require.NotContains(t, names, "TestOtherOS") + require.Equal(t, tags != "", slices.Contains(names, "TestTagged")) + }) + } +} + +func TestDiscoveryDoesNotBuildOrRunTheTestBinary(t *testing.T) { + root := fixture(t, map[string]string{ + "inventory.go": "package inventory\nfunc broken() { notDefined() }\nfunc init() { panic(\"initializer ran\") }\n", + "inventory_test.go": "package inventory\nimport \"testing\"\nfunc TestMain(m *testing.M) { panic(\"TestMain ran\") }\nfunc TestSelected(t *testing.T) {}\n", + }) + names, err := discover(t.Context(), root) + require.NoError(t, err) + require.Equal(t, []string{"TestSelected"}, names) +} + +func TestInvalidTestDeclarationsFailDiscovery(t *testing.T) { + for _, declaration := range []string{ + "func TestMissingArgument() {}", + "func TestWrongArgument(t *testing.B) {}", + "func TestResult(t *testing.T) int { return 0 }", + "func TestGeneric[T any](t *testing.T) {}", + "func FuzzWrongArgument(t *testing.T) {}", + } { + t.Run(declaration, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "invalid_test.go") + require.NoError(t, os.WriteFile(path, []byte("package inventory\nimport \"testing\"\n"+declaration), 0600)) + _, err := fileTests(path) + require.ErrorContains(t, err, "invalid signature") + }) + } +} + +func TestGoListFailureCannotProduceAnEmptySuccessfulInventory(t *testing.T) { + root := fixture(t, map[string]string{"inventory.go": "package inventory\nimport \"unterminated\n"}) + _, err := discover(t.Context(), root) + require.Error(t, err) +} diff --git a/skills/ddtest-onboarding-eval/scripts/test_review.py b/skills/ddtest-onboarding-eval/scripts/test_review.py index 71a64f9e4..cd580088f 100644 --- a/skills/ddtest-onboarding-eval/scripts/test_review.py +++ b/skills/ddtest-onboarding-eval/scripts/test_review.py @@ -190,22 +190,30 @@ def test_malformed_nested_report_still_finalizes_evidence_and_cleanup(self): self.assertIn('Invalid report fields', manifest['collection_errors'][0]) def test_ref_outside_branches_and_tags_is_fetched(self): - source = self.case.root / 'source.git' def git(*args): return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() - git('clone', '--bare', '--depth=1', '--single-branch', '--no-tags', '--', - Path(self.case.source).as_uri(), str(source)) - expected = git('-C', str(source), 'rev-parse', 'HEAD') - branch = git('-C', str(source), 'symbolic-ref', 'HEAD') - git('-C', str(source), 'update-ref', 'refs/pull/1/head', expected) - # CI has a depth-one checkout. Remove branch reachability without - # requiring a parent commit or creating an unsigned fixture commit. - git('-C', str(source), 'update-ref', '-d', branch) - self.case.args.repo = str(source) - self.case.args.ref = 'refs/pull/1/head' - code, manifest = self.case.trial() - self.assertEqual(code, 0, manifest) - self.assertEqual(manifest['commit'], expected) + for detached in (False, True): + with self.subTest(detached=detached): + source = self.case.root / f'source-{detached}.git' + git('clone', '--bare', '--depth=1', '--single-branch', '--no-tags', '--', + Path(self.case.source).as_uri(), str(source)) + expected = git('-C', str(source), 'rev-parse', 'HEAD') + branch = 'refs/heads/juan-fernandez/evaluator-fixture' + git('-C', str(source), 'update-ref', branch, expected) + git('-C', str(source), 'symbolic-ref', 'HEAD', branch) + if detached: + git('-C', str(source), 'update-ref', '--no-deref', 'HEAD', expected) + git('-C', str(source), 'update-ref', 'refs/pull/1/head', expected) + # A PR checkout can have detached HEAD. Remove every branch + # without requiring symbolic HEAD, a parent or a new commit. + for ref in git('-C', str(source), 'for-each-ref', '--format=%(refname)', 'refs/heads').splitlines(): + git('-C', str(source), 'update-ref', '-d', ref) + self.case.args.repo = str(source) + self.case.args.ref = 'refs/pull/1/head' + self.case.args.output = self.case.root / f'evidence-{detached}' + code, manifest = self.case.trial() + self.assertEqual(code, 0, manifest) + self.assertEqual(manifest['commit'], expected) def test_cleanup_requires_bound_marker_and_preserves_evidence_and_other_roots(self): code,manifest=self.case.trial()