diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 420c59ea..5dd66245 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -124,6 +124,30 @@ runs: TRUNK_API_TOKEN: ${{ inputs.trunk-token }} TRUNK_VARIANT: smoke-test-variant + - name: Run quarantine lookup failure abort test (should fail fast, skip slow_test) + id: run-quarantine-abort-test + shell: bash + working-directory: ${{ github.action_path }} + run: | + set +e + timeout 25s bundle exec rspec spec/test_spec.rb spec/slow_test.rb --format documentation + EXIT_CODE=$? + if [ $EXIT_CODE -eq 124 ]; then + echo "ERROR: Test run exceeded 25s timeout — slow_test was not skipped (abort regression)" + exit 1 + elif [ $EXIT_CODE -eq 0 ]; then + echo "ERROR: Expected non-zero exit (fail_test should fail)" + exit 1 + else + echo "SUCCESS: Quarantine lookup failure aborted run before slow_test (exit ${EXIT_CODE})" + fi + env: + TRUNK_QUARANTINE_QUERY_FAILURE_EXIT: "true" + TRUNK_ORG_URL_SLUG: fake-org + TRUNK_API_TOKEN: fake-token + TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS: "0" + TRUNK_API_CLIENT_RETRY_COUNT: "0" + - name: Run knapsack_pro queue and verify a single local bundle shell: bash if: inputs.knapsack-pro-test-suite-token-rspec != '' diff --git a/.github/actions/test_ruby_gem_uploads/spec/slow_test.rb b/.github/actions/test_ruby_gem_uploads/spec/slow_test.rb new file mode 100644 index 00000000..db688946 --- /dev/null +++ b/.github/actions/test_ruby_gem_uploads/spec/slow_test.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +describe 'slow_test' do + it 'passes after a 30 second delay' do + sleep 30 + expect(true).to be true + end +end diff --git a/constants/src/lib.rs b/constants/src/lib.rs index 40b81d82..a83c54bc 100644 --- a/constants/src/lib.rs +++ b/constants/src/lib.rs @@ -50,6 +50,9 @@ pub const TRUNK_TEST_PROCESS_EXIT_CODE_ENV: &str = "TRUNK_TEST_PROCESS_EXIT_CODE pub const TRUNK_VALIDATION_REPORT_ENV: &str = "TRUNK_VALIDATION_REPORT"; pub const TRUNK_SHOW_FAILURE_MESSAGES_ENV: &str = "TRUNK_SHOW_FAILURE_MESSAGES"; pub const TRUNK_DEBUG_ENV: &str = "TRUNK_DEBUG"; +// RSpec-only: when set to "true", aborts the RSpec run if quarantine lookup fails. +// Handled in rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb, not the CLI. +pub const TRUNK_QUARANTINE_QUERY_FAILURE_EXIT_ENV: &str = "TRUNK_QUARANTINE_QUERY_FAILURE_EXIT"; // TRUNK_* environment variables to capture in bundle metadata for debugging. // TRUNK_API_TOKEN_ENV is intentionally omitted. @@ -73,6 +76,7 @@ pub const TRUNK_ENVS_TO_CAPTURE: &[&str] = &[ TRUNK_VARIANT_ENV, TRUNK_USE_UNCLONED_REPO_ENV, TRUNK_DISABLE_QUARANTINING_ENV, + TRUNK_QUARANTINE_QUERY_FAILURE_EXIT_ENV, TRUNK_ALLOW_EMPTY_TEST_RESULTS_ENV, TRUNK_DRY_RUN_ENV, TRUNK_TEST_PROCESS_EXIT_CODE_ENV, diff --git a/rspec-trunk-flaky-tests/README.md b/rspec-trunk-flaky-tests/README.md index 7ec5153c..9b54cbb8 100644 --- a/rspec-trunk-flaky-tests/README.md +++ b/rspec-trunk-flaky-tests/README.md @@ -106,6 +106,16 @@ require 'trunk_spec_helper' For a complete list of environment variables that the gem accepts, see [`lib/trunk_spec_helper.rb`](lib/trunk_spec_helper.rb). The gem uses the same environment variables as the Trunk Analytics CLI for configuration overrides. `TRUNK_ORG_URL_SLUG` and `TRUNK_API_TOKEN` must be set to activate the plugin. +#### `TRUNK_QUARANTINE_QUERY_FAILURE_EXIT` + +When set to `true`, the RSpec run aborts on the first quarantine lookup failure. The current failing example is still marked as failed, but remaining examples are skipped and not executed. Partial test results are still uploaded via the normal `close` hook. + +By default (when unset or not `true`), quarantine lookup failures print a warning and the run continues as normal. + +```bash +TRUNK_QUARANTINE_QUERY_FAILURE_EXIT=true bundle exec rspec +``` + #### `TRUNK_LOCAL_UPLOAD_DIR` (Experimental) > **⚠️ Experimental Feature**: This feature is experimental. Please reach out to support@trunk.io before attempting to use it. diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 68578f81..1b6f5a89 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -27,6 +27,8 @@ # TRUNK_USE_UNCLONED_REPO - Set to 'true' for uncloned repo mode # TRUNK_LOCAL_UPLOAD_DIR - Directory to save test results locally (disables upload) # TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds) +# TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine +# lookup fails (remaining examples are skipped) # DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk # require 'rspec/core' @@ -84,6 +86,10 @@ def trunk_disabled ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil? end +def quarantine_query_failure_exit? + ENV['TRUNK_QUARANTINE_QUERY_FAILURE_EXIT'] == 'true' +end + # we want to cache the test report in memory so we can add to it as we go and reduce the number of API calls $test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}", nil) $failure_encountered_and_quarantining_disabled = false @@ -126,6 +132,10 @@ def handle_quarantine_check(exception) puts 'Failed to check quarantining status, no failures will be quarantined'.yellow $failure_encountered_and_quarantine_lookup_failed = true end + if quarantine_query_failure_exit? + puts 'Quarantine lookup failed, exiting early'.red + RSpec.world.wants_to_quit = true + end set_exception_core(exception) elsif is_quarantined_result.quarantining_disabled_for_repo unless $failure_encountered_and_quarantining_disabled @@ -365,6 +375,12 @@ def add_test_case(example) RSpec.configure do |c| next if trunk_disabled + c.before(:example) do + if $failure_encountered_and_quarantine_lookup_failed && quarantine_query_failure_exit? + skip('Quarantine lookup failed, skipping test run') + end + end + c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close end