From ca4bce1e4266fc9991fb6e8a12246ae720768af9 Mon Sep 17 00:00:00 2001 From: Paul Brackin Date: Fri, 30 Oct 2026 05:48:54 -0700 Subject: [PATCH] Gate self-hosted integration jobs on test failures; keep setup failures advisory --- .github/workflows/build-test.yml | 113 ++++++++++++++++++------------- 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 785a962..87564f6 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -364,11 +364,8 @@ jobs: Windows, X64 ] - # Advisory, same contract as macos-arm-integration: self-hosted failures - # here are more often environmental than defects in the wheel under test, - # so a red must not gate the matrix. The last step surfaces the failure - # as a warning annotation and a job-summary entry. - continue-on-error: true + # Same contract as macos-arm-integration: runner and server bring-up is + # advisory, the tests gate. timeout-minutes: 60 defaults: run: @@ -383,6 +380,8 @@ jobs: - uses: actions/checkout@v4 - name: Resolve the remote Docker host + id: docker_host + continue-on-error: true run: | case "$DOCKER_HOST" in tcp://*) ;; @@ -402,6 +401,8 @@ jobs: path: aerospike_native - uses: actions/setup-python@v6 + id: python + continue-on-error: true with: python-version: ${{ matrix.python }} allow-prereleases: true @@ -409,6 +410,7 @@ jobs: - name: Run Aerospike server id: aerospike + continue-on-error: true uses: aerospike/shared-workflows/.github/actions/setup-aerospike-server@49381314f7afd64afb580b77bab8ac263040c27e # PyPI metadata SIGPIPE fix (post-v5.0.0) with: server-tag: ${{ env.AEROSPIKE_SERVER_TAG }} @@ -419,21 +421,29 @@ jobs: oidc-audience: ${{ vars.OIDC_AUDIENCE || 'aerospike' }} - name: Assert the server is the one we asked for + id: assert_server + continue-on-error: true # SERVER_HOST from the resolve step points it at the remote VM. env: EXPECTED_BUILD: ${{ env.AEROSPIKE_SERVER_TAG }} run: python .github/assert-server.py - name: Install wheel + id: install_wheel + continue-on-error: true # This job runs on a persistent runner and the version string is # stable across merges, so a plain install is a no-op against # whatever build a previous run left behind. run: python -m pip install --force-reinstall *.whl working-directory: aerospike_native - name: Install pytest + id: install_pytest + continue-on-error: true run: python -m pip install pytest pytest-asyncio - name: Assert the installed package matches the wheel under test + id: assert_install + continue-on-error: true run: | python - <<'EOF' import hashlib, pathlib, zipfile @@ -451,14 +461,28 @@ jobs: print(f"aerospike_native at {installed_init} matches {wheel.name} ({expected[:12]})") EOF + # A failed setup step leaves the job green (GitHub has no "warn" + # conclusion), so say so where a reviewer will see it. + - name: Surface an advisory setup failure + if: contains(steps.*.outcome, 'failure') + run: | + echo "::warning title=Windows integration not run (advisory)::python ${{ matrix.python }}: runner or server setup failed on the self-hosted Windows runner, so the tests were skipped" + { + echo "### :warning: Windows integration not run - advisory" + echo "" + echo "Python ${{ matrix.python }} on the self-hosted GCP Windows runner." + echo "Setup failed before the tests, so this leg has no coverage. Check" + echo "the runner and its paired Docker VM, then re-run the job." + } >> "$GITHUB_STEP_SUMMARY" + - name: Debug module contents + if: ${{ !contains(steps.*.outcome, 'failure') }} run: | python -c "import aerospike_native; print('Module imported successfully')" python -c "import aerospike_native; print('Available functions:', [x for x in dir(aerospike_native) if not x.startswith('_')])" - name: Run tests - id: itest - continue-on-error: true + if: ${{ !contains(steps.*.outcome, 'failure') }} env: # The server is on the paired Linux VM, not localhost. Single node, # so the client keeps talking to this seed address throughout; the @@ -477,20 +501,6 @@ jobs: python .github/scripts/server_log_window.py "${{ steps.aerospike.outputs.container-names }}" "$PNC_FAILURE_TIMES" exit 1 fi - # GitHub has no "warn" job conclusion, so a soft-failed job would - # otherwise report as green with nothing to notice. - - name: Surface an advisory failure - if: steps.itest.outcome == 'failure' - run: | - echo "::warning title=Windows integration failed (advisory)::python ${{ matrix.python }} failed on the self-hosted Windows runner; not gating the matrix" - { - echo "### :warning: Windows integration failed - advisory" - echo "" - echo "Python ${{ matrix.python }} on the self-hosted GCP Windows runner." - echo "This does not gate the matrix. Check the runner and its paired" - echo "Docker VM before concluding the code under test is at fault, and" - echo "do not read a later green here as proof it was fixed." - } >> "$GITHUB_STEP_SUMMARY" macos-x86: runs-on: macos-latest-large @@ -626,15 +636,13 @@ jobs: macOS, ARM64 ] - # Advisory only: this is the one self-hosted bare-metal runner, and its - # failures have been environmental (a same-version pip install silently - # no-oping, framework python shadowing the venv) rather than defects in - # the code under test, so a red here must not gate the matrix. Every - # other target -- including the GitHub-hosted macOS jobs -- still fails - # hard. The failure is surfaced as a warning annotation and in the job - # summary by the last step: GitHub has no "warn" conclusion, so without - # that continue-on-error would make the failure invisible instead. - continue-on-error: true + # This is the one self-hosted bare-metal runner, and its environment can + # break independently of the code under test (a same-version pip install + # silently no-oping, framework python shadowing the venv). Runner and + # server bring-up, through the check that the wheel under test is the one + # installed, is advisory: a failure there skips the tests with a warning. + # Once setup succeeds the tests run against a verified environment, so a + # test failure fails the job like any other target. strategy: matrix: python: [ @@ -655,12 +663,15 @@ jobs: path: aerospike_native - uses: actions/setup-python@v6 + id: python + continue-on-error: true with: python-version: ${{ matrix.python }} allow-prereleases: true - name: Run Aerospike server id: aerospike + continue-on-error: true uses: aerospike/shared-workflows/.github/actions/setup-aerospike-server@49381314f7afd64afb580b77bab8ac263040c27e # PyPI metadata SIGPIPE fix (post-v5.0.0) with: server-tag: ${{ env.AEROSPIKE_SERVER_TAG }} @@ -671,20 +682,28 @@ jobs: oidc-audience: ${{ vars.OIDC_AUDIENCE || 'aerospike' }} - name: Assert the server is the one we asked for + id: assert_server + continue-on-error: true env: EXPECTED_BUILD: ${{ env.AEROSPIKE_SERVER_TAG }} run: python3 .github/assert-server.py - name: Install wheel + id: install_wheel + continue-on-error: true # This job runs on a persistent bare-metal runner and the version # string is stable across merges, so a plain install is a no-op # against whatever build a previous run left behind. run: python3 -m pip install --force-reinstall *.whl working-directory: aerospike_native - name: Install pytest + id: install_pytest + continue-on-error: true run: python3 -m pip install pytest pytest-asyncio - name: Assert the installed package matches the wheel under test + id: assert_install + continue-on-error: true run: | python3 - <<'EOF' import hashlib, pathlib, zipfile @@ -702,14 +721,28 @@ jobs: print(f"aerospike_native at {installed_init} matches {wheel.name} ({expected[:12]})") EOF + # A failed setup step leaves the job green (GitHub has no "warn" + # conclusion), so say so where a reviewer will see it. + - name: Surface an advisory setup failure + if: contains(steps.*.outcome, 'failure') + run: | + echo "::warning title=macOS ARM integration not run (advisory)::python ${{ matrix.python }}: runner or server setup failed on the self-hosted runner, so the tests were skipped" + { + echo "### :warning: macOS ARM integration not run - advisory" + echo "" + echo "Python ${{ matrix.python }} on the self-hosted bare-metal runner." + echo "Setup failed before the tests, so this leg has no coverage. Check" + echo "the runner environment, then re-run the job." + } >> "$GITHUB_STEP_SUMMARY" + - name: Debug module contents + if: ${{ !contains(steps.*.outcome, 'failure') }} run: | python3 -c "import aerospike_native; print('Module imported successfully')" python3 -c "import aerospike_native; print('Available functions:', [x for x in dir(aerospike_native) if not x.startswith('_')])" - name: Run tests - id: itest - continue-on-error: true + if: ${{ !contains(steps.*.outcome, 'failure') }} env: AEROSPIKE_HOST: localhost:3000 AEROSPIKE_USE_SERVICES_ALTERNATE: "true" @@ -725,22 +758,6 @@ jobs: python3 .github/scripts/server_log_window.py "${{ steps.aerospike.outputs.container-names }}" "$PNC_FAILURE_TIMES" exit 1 fi - # GitHub has no "warn" job conclusion, so a soft-failed job would - # otherwise report as green with nothing to notice. Emit an annotation - # and a summary line, and keep the log as an artifact so an advisory - # failure is still diagnosable. - - name: Surface an advisory failure - if: steps.itest.outcome == 'failure' - run: | - echo "::warning title=macOS ARM integration failed (advisory)::python ${{ matrix.python }} failed on the self-hosted runner; not gating the matrix" - { - echo "### :warning: macOS ARM integration failed - advisory" - echo "" - echo "Python ${{ matrix.python }} on the self-hosted bare-metal runner." - echo "This does not gate the matrix. Check the runner environment" - echo "before concluding the code under test is at fault, and do not" - echo "read a later green here as proof it was fixed." - } >> "$GITHUB_STEP_SUMMARY" macos-arm-freethreaded: