From dabeaf7ef5a18c663298c0afd028b027032e3d53 Mon Sep 17 00:00:00 2001 From: Calvin Bayer <50014594+calvinbayer@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:16:16 +0100 Subject: [PATCH 1/2] [SDTEST-4060] Add Go test automation exclusions and preserve active protected failures --- ddto/README.md | 43 + ddto/integration/compatibility_test.go | 2 +- ddto/integration/mini_crash_session_test.go | 2 +- .../mini_exclusions_acceptance_test.go | 1376 +++++++++++++++++ ddto/integration/mini_exclusions_api_test.go | 172 +++ ddto/integration/mini_exclusions_atf_test.go | 137 ++ .../mini_exclusions_callback_test.go | 87 ++ .../mini_exclusions_cleanup_goexit_test.go | 116 ++ .../mini_exclusions_declarations_test.go | 430 ++++++ .../mini_exclusions_omitted_test.go | 104 ++ .../integration/mini_exclusions_owner_test.go | 35 + .../mini_exclusions_parallel_cleanup_test.go | 199 +++ .../mini_exclusions_process_test.go | 125 ++ .../mini_exclusions_promoted_cache_test.go | 144 ++ .../mini_exclusions_retry_edges_test.go | 460 ++++++ ...ini_exclusions_terminal_acceptance_test.go | 276 ++++ ddto/integration/mini_exclusions_test.go | 481 ++++++ ddto/integration/mini_exclusions_tia_test.go | 203 +++ ddto/integration/mini_git_state_test.go | 2 +- ddto/integration/mini_test.go | 32 +- ddto/integration/parity_testify_test.go | 2 +- ddto/internal/runner/exclusions.go | 354 +++++ ddto/internal/runner/exclusions_test.go | 170 ++ ddto/internal/runner/run.go | 34 +- .../thirdparty/dd-trace-go/ADAPTATIONS.md | 199 +++ .../thirdparty/dd-trace-go/SOURCE.json | 59 +- .../integrations/civisibility_crash_test.go | 266 ++++ .../integrations/gotesting/exclusions.go | 541 +++++++ .../exclusions_serialization_test.go | 110 ++ .../integrations/gotesting/exclusions_test.go | 318 ++++ .../gotesting/fuzz_events_test.go | 2 +- .../integrations/gotesting/instrumentation.go | 384 ++++- .../gotesting/instrumentation_orchestrion.go | 46 +- .../gotesting/quarantine_process.go | 217 ++- .../integrations/gotesting/reflections.go | 23 + .../gotesting/retry_attempt_integration.go | 3 + .../integrations/gotesting/retry_process.go | 298 +++- .../gotesting/retry_process_control.go | 56 +- .../gotesting/retry_process_deferred.go | 115 +- .../gotesting/retry_process_selection.go | 42 + .../gotesting/retry_process_selection_test.go | 53 + .../gotesting/test_execution_finalizer.go | 16 +- .../integrations/gotesting/testify.go | 106 +- .../gotesting/testify_lookup_test.go | 27 + .../integrations/gotesting/testing.go | 38 +- .../integrations/gotesting/testingF.go | 2 +- .../integrations/manual_api_ddtest.go | 50 +- ddto/scripts/ci_shards.json | 14 +- ddto/scripts/test_ci_shards.py | 6 +- ddto/testopt/exclusions.go | 22 + internal/testdrive/probe_ownership_test.go | 99 +- main_test.go | 51 +- 52 files changed, 7839 insertions(+), 310 deletions(-) create mode 100644 ddto/integration/mini_exclusions_acceptance_test.go create mode 100644 ddto/integration/mini_exclusions_api_test.go create mode 100644 ddto/integration/mini_exclusions_atf_test.go create mode 100644 ddto/integration/mini_exclusions_callback_test.go create mode 100644 ddto/integration/mini_exclusions_cleanup_goexit_test.go create mode 100644 ddto/integration/mini_exclusions_declarations_test.go create mode 100644 ddto/integration/mini_exclusions_omitted_test.go create mode 100644 ddto/integration/mini_exclusions_owner_test.go create mode 100644 ddto/integration/mini_exclusions_parallel_cleanup_test.go create mode 100644 ddto/integration/mini_exclusions_process_test.go create mode 100644 ddto/integration/mini_exclusions_promoted_cache_test.go create mode 100644 ddto/integration/mini_exclusions_retry_edges_test.go create mode 100644 ddto/integration/mini_exclusions_terminal_acceptance_test.go create mode 100644 ddto/integration/mini_exclusions_test.go create mode 100644 ddto/integration/mini_exclusions_tia_test.go create mode 100644 ddto/internal/runner/exclusions.go create mode 100644 ddto/internal/runner/exclusions_test.go create mode 100644 ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions.go create mode 100644 ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_serialization_test.go create mode 100644 ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_test.go create mode 100644 ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection.go create mode 100644 ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection_test.go create mode 100644 ddto/testopt/exclusions.go diff --git a/ddto/README.md b/ddto/README.md index 5f91a1829..3ed585c27 100644 --- a/ddto/README.md +++ b/ddto/README.md @@ -275,6 +275,49 @@ runs native Go without instrumentation and prints a warning. The [validation contract](docs/validation.md) distinguishes loopback protocol checks from live intake, UI and downstream-project validation. +## Test automation exclusions + +With the default Mini runtime, add either of these comments directly above an +individual Go test function or Testify test method: + +```go +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestCritical(t *testing.T) { + // A terminal failure must fail the command, without Datadog retries. +} +``` + +`prevent_failure_suppression` prevents Datadog quarantine and disabling from +hiding this test's failure. `no_auto_retry` excludes Datadog automatic retries, +early flake detection and attempt-to-fix. Each can be used independently. Neither +changes native skips, name filters, Test Impact Analysis selection or native +repetitions such as `-count`. An excluded Testify method retains its first result +when Datadog retries its parent suite; ordinary sibling methods can still retry. + +The comments are compiled into the binary before execution policy is selected, +including separate test packages. Source files need not be present at runtime. +Native `t.Run` cases inherit their executing parent declaration, including +shared callbacks and parallel rows. The same callback used by an ordinary parent +keeps ordinary policy. Individual Testify methods use their own declarations. A method annotation applies +when its parent is selected and runs; it does not schedule a method whose whole +parent test is skipped or disabled before the method is discovered. A selected +protected method can override its own disabled assignment, and a quarantined +parent cannot hide its protected terminal failure. Annotations on a suite type or +`TestMain` are not supported. The reference `--runtime=sdk` is unchanged. + +For manual Mini instrumentation, register declarations during package +initialization with `testopt.RegisterTestExclusions(TestCritical, +testopt.TestExclusions{PreventFailureSuppression: true, NoAutoRetry: true})`. +For a Testify method, pass its method expression, such as `(*MySuite).TestCritical`. +Do not register or change declarations while tests are running. + +Emitted test events include `test.test_management.prevent_failure_suppression` +and/or `test.test_management.no_auto_retry` set to `true`, including known skipped +executions. Absent annotations omit these tags. Backend-assigned quarantine, +disabled and attempt-to-fix properties remain reported even when annotations +change execution policy. Setting a span tag alone does not activate an exclusion. + ## Environment variables Set these variables before invoking `ddtest instrument` or a compiled test diff --git a/ddto/integration/compatibility_test.go b/ddto/integration/compatibility_test.go index b9831e17d..5454ef9e8 100644 --- a/ddto/integration/compatibility_test.go +++ b/ddto/integration/compatibility_test.go @@ -114,7 +114,7 @@ func (c *capture) fail(err error) { func testEnv(extra ...string) []string { // Deliberately avoid inheriting API keys or CI configuration into fixtures. // GORACE reaches race-enabled fixtures; CI uses it to skip their exit delay. - names := []string{"PATH", "HOME", "USER", "GOCACHE", "GOMODCACHE", "GOTMPDIR", "TMPDIR", "GOTOOLCHAIN", "CGO_ENABLED", "GOOS", "GOARCH", "GOMAXPROCS", "GORACE", "GOPROXY", "GOSUMDB", "SYSTEMROOT", "SystemRoot", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "TEMP", "TMP", "COMSPEC", "PATHEXT"} + names := []string{"PATH", "HOME", "USER", "GOCACHE", "GOMODCACHE", "GOTMPDIR", "TMPDIR", "GOTOOLCHAIN", "SDKROOT", "CC", "CXX", "CGO_CFLAGS", "CGO_CPPFLAGS", "CGO_CXXFLAGS", "CGO_LDFLAGS", "MACOSX_DEPLOYMENT_TARGET", "CGO_ENABLED", "GOOS", "GOARCH", "GOMAXPROCS", "GORACE", "GOPROXY", "GOSUMDB", "SYSTEMROOT", "SystemRoot", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "TEMP", "TMP", "COMSPEC", "PATHEXT"} var env []string for _, name := range names { if v, ok := os.LookupEnv(name); ok { diff --git a/ddto/integration/mini_crash_session_test.go b/ddto/integration/mini_crash_session_test.go index 1e2ca3a39..338459347 100644 --- a/ddto/integration/mini_crash_session_test.go +++ b/ddto/integration/mini_crash_session_test.go @@ -322,7 +322,7 @@ func TestMiniTestSessionSpansTheCommand(t *testing.T) { // ddto runs in another repository and moves go test with -C: the // session's Git metadata is the test processes'. other := t.TempDir() - for _, args := range [][]string{{"init", "-q"}, {"remote", "add", "origin", "https://example.com/other.git"}, {"-c", "user.name=fixture", "-c", "user.email=fixture@example.com", "commit", "-q", "--allow-empty", "-m", "other"}} { + for _, args := range [][]string{{"init", "-q"}, {"remote", "add", "origin", "https://example.com/other.git"}, {"-c", "user.name=fixture", "-c", "user.email=fixture@example.com", "-c", "commit.gpgsign=false", "commit", "-q", "--allow-empty", "-m", "other"}} { if out, stderr, code := command(t, other, testEnv(), "git", args...); code != 0 { t.Fatalf("git %v: %s %s", args, out, stderr) } diff --git a/ddto/integration/mini_exclusions_acceptance_test.go b/ddto/integration/mini_exclusions_acceptance_test.go new file mode 100644 index 000000000..f4ee6b00d --- /dev/null +++ b/ddto/integration/mini_exclusions_acceptance_test.go @@ -0,0 +1,1376 @@ +package integration + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniReviewNestedRows(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"review_row_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestCriticalRows(t *testing.T){rowPath(t,0)} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestDeepCriticalRows(t *testing.T){rowPath(t,3)} +//ddtest:no_auto_retry +func TestNoRetryRows(t *testing.T){rowPath(t,0)} +//ddtest:prevent_failure_suppression +func TestProtectedRecovery(t *testing.T){recoverRow(t)} +func recoverRow(t *testing.T){f,e:=os.OpenFile(os.Getenv("ROW_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};f.WriteString("x");f.Close();d,e:=os.ReadFile(os.Getenv("ROW_COUNTER"));if e!=nil{t.Fatal(e)};if len(d)==1{t.Error("first recoverable failure")}} +func rowPath(t *testing.T,depth int){if depth>0{t.Run("level",func(t *testing.T){rowPath(t,depth-1)});return};t.Run("row",func(t *testing.T){f,e:=os.OpenFile(os.Getenv("ROW_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("ran");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)};t.Error("protected row")})} +type InnerRowSuite struct{suite.Suite} +func(s *InnerRowSuite) TestMethod(){if e:=os.WriteFile(os.Getenv("ROW_COUNTER"),[]byte("boundary"),0600);e!=nil{s.T().Fatal(e)}} + +type RowSuite struct{suite.Suite} +func(s *RowSuite) TestMethod(){s.T().Run("callback",TestCriticalRows)} +func(s *RowSuite) TestSharedPaths(){s.T().Run("protected",TestCriticalRows);s.T().Run("unprotected",TestNoRetryRows)} +func(s *RowSuite) TestDeep(){s.T().Run("callback",TestDeepCriticalRows)} +func(s *RowSuite) TestNoRetry(){s.T().Run("callback",TestNoRetryRows)} +func(s *RowSuite) TestProtectedRecovery(){s.T().Run("callback",TestProtectedRecovery)} +func(s *RowSuite) TestOrdinaryRecovery(){recoverRow(s.T())} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func(s *RowSuite) TestBoundary(){suite.Run(s.T(),new(InnerRowSuite))} +func TestRowSuite(t *testing.T){suite.Run(t,new(RowSuite))} +`}) + bin := filepath.Join(t.TempDir(), executableName("review-row.test")) + o, e, c := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if c != 0 { + t.Fatalf("compile%d %s %s", c, o, e) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "row") + r, g := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestRowSuite$/^TestMethod$"}, Policy: policySettings{Disabled: true, AttemptToFix: true, ManagementTarget: "TestRowSuite/TestMethod/callback/row", ManagementSuite: "review_row_test.go/RowSuite"}, Env: []string{"ROW_COUNTER=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false"}}) + data, err := os.ReadFile(counter) + if g.code != 1 || err != nil || string(data) != "ran" { + t.Fatalf("protected row exit%d calls%q err%v output%s %s", g.code, data, err, g.out, g.stderr) + } + count := 0 + for _, ev := range r.events { + if ev["type"] != "test" { + continue + } + m := ev["content"].(map[string]any)["meta"].(map[string]any) + if m["test.name"] != "TestRowSuite/TestMethod/callback/row" { + continue + } + count++ + requireReviewNoRetryOwnership(t, m) + if m["test.status"] != "fail" || m["test.test_management.prevent_failure_suppression"] != "true" || m["test.test_management.no_auto_retry"] != "true" || m["test.test_management.is_test_disabled"] != "true" || m["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("row metadata%v", m) + } + } + if count != 1 { + t.Fatalf("row events%d", count) + } + }) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode+"-shared-owner-cache", func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestRowSuite$/^TestSharedPaths$"}, Policy: policySettings{}, Env: []string{"ROW_COUNTER=" + calls, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false"}}) + raw, err := os.ReadFile(calls) + if got.code != 1 || err != nil || string(raw) != "ranran" { + t.Fatalf("shared declaration execution exit%d calls%q err%v %s %s", got.code, raw, err, got.out, got.stderr) + } + for _, entry := range []struct { + path, status string + protected bool + }{{"protected", "fail", true}, {"unprotected", "fail", false}} { + name := "TestRowSuite/TestSharedPaths/" + entry.path + "/row" + events := reviewTestEvents(receiver, name) + if len(events) != 1 { + t.Fatalf("shared path %s events%d: %v", name, len(events), receiver.events) + } + meta := events[0] + requireReviewNoRetryOwnership(t, meta) + if meta["test.status"] != entry.status || meta["test.test_management.no_auto_retry"] != "true" || (meta["test.test_management.prevent_failure_suppression"] == "true") != entry.protected || meta["test.test_management.is_test_disabled"] != nil || meta["test.test_management.is_attempt_to_fix"] != nil { + t.Fatalf("shared owner cache leaked policy: %v", meta) + } + } + }) + } + + for _, mode := range []string{"in_process", "process"} { + for _, tc := range []struct { + name, method, path string + prevent, disabled, atf, recovery bool + }{ + {"deep", "TestDeep", "/callback/level/level/level/row", true, true, true, false}, + {"shared-no-retry", "TestNoRetry", "/callback/row", false, true, true, false}, + {"pfs-recovery", "TestProtectedRecovery", "/callback", true, false, false, true}, + {"ordinary-recovery", "TestOrdinaryRecovery", "", false, false, false, true}, + {"actual-method-boundary", "TestBoundary", "/TestMethod", false, false, false, false}, + } { + t.Run(mode+"-"+tc.name, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "row") + name := "TestRowSuite/" + tc.method + tc.path + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestRowSuite$/^" + tc.method + "$"}, Policy: policySettings{Retry: tc.recovery, Disabled: tc.disabled, AttemptToFix: tc.atf, ManagementTarget: name, ManagementSuite: "review_row_test.go/RowSuite"}, Env: []string{"ROW_COUNTER=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=" + fmt.Sprint(tc.recovery), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + wantCode := 0 + if tc.disabled && tc.prevent { + wantCode = 1 + } + if got.code != wantCode { + t.Fatalf("exit%d want%d %s %s", got.code, wantCode, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if tc.disabled && !tc.prevent { + if !os.IsNotExist(err) { + t.Fatalf("disabled shared callback executed %q %v", data, err) + } + } else { + want := "ran" + if tc.name == "actual-method-boundary" { + want = "boundary" + } + if tc.recovery { + want = "xx" + } + if err != nil || string(data) != want { + t.Fatalf("callback count%q want%q err%v", data, want, err) + } + } + events := reviewTestEvents(receiver, name) + wantCount := 1 + if tc.recovery && mode == "in_process" { + wantCount = 2 + } + if len(events) != wantCount { + t.Fatalf("%s events%d want%d: %v", name, len(events), wantCount, receiver.events) + } + for _, meta := range events { + if (meta["test.test_management.prevent_failure_suppression"] == "true") != tc.prevent || (meta["test.test_management.no_auto_retry"] == "true") != (tc.disabled) { + t.Fatalf("declaration boundary policy: %v", meta) + } + if tc.disabled { + requireReviewNoRetryOwnership(t, meta) + if meta["test.test_management.is_test_disabled"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("lost assignments: %v", meta) + } + } + } + }) + } + } + +} + +func checkMiniReviewWorkerManagement(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"review_worker_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type WorkerLeaf struct{suite.Suite} +//ddtest:no_auto_retry +func(s *WorkerLeaf) TestLeaf(){workerLeafMark(s.T())} +func workerLeafMark(t *testing.T){f,e:=os.OpenFile(os.Getenv("LEAF_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("ran");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)}} +type ProtectedWorkerLeaf struct{suite.Suite} +//ddtest:no_auto_retry +//ddtest:prevent_failure_suppression +func(s *ProtectedWorkerLeaf) TestLeaf(){workerLeafMark(s.T());s.T().Error("protected leaf failure")} +func workerCarriers(t *testing.T,n int){if n>0{t.Run("wrapper",func(t *testing.T){workerCarriers(t,n-1)});return};if os.Getenv("PROTECTED_LEAF")=="true"{suite.Run(t,new(ProtectedWorkerLeaf))}else{suite.Run(t,new(WorkerLeaf))}} +func TestWorkerRoot(t *testing.T){f,e:=os.OpenFile(os.Getenv("ROOT_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("first attempt before discovery");return};if !os.IsExist(e){t.Fatal(e)};depth:=1;if os.Getenv("DEEP_WORKER")=="true"{depth=3};workerCarriers(t,depth)} +`}) + bin := filepath.Join(t.TempDir(), executableName("review-worker.test")) + o, e, c := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if c != 0 { + t.Fatalf("compile%d %s %s", c, o, e) + } + for _, mode := range []string{"in_process", "process"} { + for _, deep := range []bool{false, true} { + for _, protected := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/deep=%t/protected=%t", mode, deep, protected), func(t *testing.T) { + name := "TestWorkerRoot/wrapper/TestLeaf" + if deep { + name = "TestWorkerRoot/wrapper/wrapper/wrapper/TestLeaf" + } + counter := filepath.Join(t.TempDir(), "leaf") + r, g := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestWorkerRoot$"}, Policy: policySettings{Disabled: true, AttemptToFix: true, ManagementTarget: "TestWorkerRoot", ManagementSuite: "review_worker_test.go"}, Env: []string{"DEEP_WORKER=" + fmt.Sprint(deep), "PROTECTED_LEAF=" + fmt.Sprint(protected), "LEAF_COUNTER=" + counter, "ROOT_COUNTER=" + filepath.Join(t.TempDir(), "root"), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false"}}) + data, err := os.ReadFile(counter) + wantCode := 0 + if protected { + wantCode = 1 + } + if g.code != wantCode || (!protected && !os.IsNotExist(err)) || (protected && (err != nil || string(data) != "ran")) { + t.Fatalf("disabled leaf ran exit%d err%v output%s %s", g.code, err, g.out, g.stderr) + } + count := 0 + for _, ev := range r.events { + if ev["type"] != "test" { + continue + } + m := ev["content"].(map[string]any)["meta"].(map[string]any) + if m["test.name"] != name { + continue + } + count++ + requireReviewNoRetryOwnership(t, m) + wantStatus := "skip" + if protected { + wantStatus = "fail" + } + if (m["test.test_management.prevent_failure_suppression"] == "true") != protected || m["test.status"] != wantStatus || m["test.test_management.no_auto_retry"] != "true" || m["test.test_management.is_test_disabled"] != "true" || m["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("leaf metadata%v", m) + } + } + if count != 1 { + t.Fatalf("leaf events%d", count) + } + }) + } + } + } +} + +// The nearest exact ATF=false assignment must survive multiple ordinary +// carriers, including the process retry that discovers them after attempt one. +func checkMiniReviewWorkerOverride(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"worker_override_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type OverrideWorkerLeaf struct{suite.Suite} +//ddtest:no_auto_retry +func(s *OverrideWorkerLeaf) TestLeaf(){f,e:=os.OpenFile(os.Getenv("OVERRIDE_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{s.T().Fatal(e)};f.WriteString("x");f.Close()} +func TestWorkerOverride(t *testing.T){f,e:=os.OpenFile(os.Getenv("OVERRIDE_ROOT"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("first ATF attempt failed");return};if !os.IsExist(e){t.Fatal(e)};t.Run("override",func(t *testing.T){t.Run("ordinary",func(t *testing.T){suite.Run(t,new(OverrideWorkerLeaf))})})} +`}) + bin := filepath.Join(t.TempDir(), executableName("worker-override.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + receiver := &parityReceiver{policy: policySettings{AttemptToFix: true}, side: map[string][][]byte{}, requests: map[string]int{}} + server := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/test-management/tests") { + tests := map[string]any{} + for name, atf := range map[string]bool{"TestWorkerOverride": true, "TestWorkerOverride/override": false} { + tests[name] = map[string]any{"properties": map[string]bool{"attempt_to_fix": atf}} + } + writePolicyJSON(w, map[string]any{"data": map[string]any{"id": "override", "type": "ci_app_libraries_tests_request", "attributes": map[string]any{"modules": map[string]any{"example.com/dd-ci-testing-fixture_test": map[string]any{"suites": map[string]any{"worker_override_test.go": map[string]any{"tests": tests}}}}}}}) + return + } + receiver.handler(w, r) + })) + defer server.Close() + calls, root := filepath.Join(t.TempDir(), "calls"), filepath.Join(t.TempDir(), "root") + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_URL="+server.URL, "DD_TRACE_AGENT_URL="+server.URL, "DD_API_KEY=fixture", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE="+mode, "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES=2", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "OVERRIDE_CALLS="+calls, "OVERRIDE_ROOT="+root), bin, "-test.run=^TestWorkerOverride$") + if code != 1 { + t.Fatalf("first failed ATF attempt must still block: exit%d %s %s", code, out, stderr) + } + data, err := os.ReadFile(calls) + if err != nil || string(data) != "x" { + t.Fatalf("leaf calls%q err%v %s %s", data, err, out, stderr) + } + events := reviewTestEvents(receiver, "TestWorkerOverride/override/ordinary/TestLeaf") + if len(events) != 1 { + t.Fatalf("leaf events%d: %v", len(events), receiver.events) + } + meta := events[0] + requireReviewNoRetryOwnership(t, meta) + if meta["test.status"] != "pass" || meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.is_attempt_to_fix"] != nil || meta["test.test_management.is_test_disabled"] != nil || meta["test.test_management.is_quarantined"] != nil { + t.Fatalf("nearest false override lost: %v", meta) + } + }) + } +} + +// Crash finalization uses effective execution policy while preserving backend +// assignment tags. NoRetry alone must retain its preexisting crash behavior. +func checkMiniExcludedCrashFinalStatus(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("os";"testing";"time") +ANNOTATIONS +func TestCrashPolicy(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("CRASH_POLICY_ENTERED"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)} + if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)} + if os.Getenv("CRASH_POLICY_MODE")=="background"{go func(){panic("protected background panic sentinel")}()} + time.Sleep(time.Minute) +} +` + for _, flags := range []struct { + name, comments string + prevent, noRetry bool + }{ + {"pfs", "//ddtest:prevent_failure_suppression", true, false}, + {"both", "//ddtest:prevent_failure_suppression\n//ddtest:no_auto_retry", true, true}, + {"no-retry", "//ddtest:no_auto_retry", false, true}, + } { + t.Run(flags.name, func(t *testing.T) { + writeBuildFixture(t, dir, map[string]string{"crash_policy_test.go": strings.Replace(source, "ANNOTATIONS", flags.comments, 1)}) + for _, mode := range []string{"timeout", "background"} { + t.Run(mode, func(t *testing.T) { + bin := filepath.Join(t.TempDir(), executableName("crash-policy.test")) + args := exclusionBuildArgs(bin, false) + if mode == "background" { + args = append([]string{"test", "--panic-hook"}, args[1:]...) + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, args...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, assignment := range []string{"quarantined", "disabled-atf", "unmanaged"} { + t.Run(assignment, func(t *testing.T) { + marker := filepath.Join(t.TempDir(), "entered") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestCrashPolicy$", "-test.timeout=250ms"}, Policy: policySettings{Quarantined: assignment == "quarantined", Disabled: assignment == "disabled-atf", AttemptToFix: assignment == "disabled-atf", ManagementTarget: "TestCrashPolicy", ManagementSuite: "crash_policy_test.go"}, Env: []string{"DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=in_process", "CRASH_POLICY_MODE=" + mode, "CRASH_POLICY_ENTERED=" + marker}}) + // Exact published-head baseline: NoRetry+quarantine still executes and + // crashes; NoRetry+disabled excludes ATF and skips the body entirely. + skipped := !flags.prevent && assignment == "disabled-atf" + wantCode, wantStatus, wantFinal := 2, "fail", "fail" + if skipped { + wantCode, wantStatus, wantFinal = 0, "skip", "skip" + } else if !flags.prevent && assignment == "quarantined" { + wantFinal = "skip" + } + data, err := os.ReadFile(marker) + if skipped { + if !os.IsNotExist(err) { + t.Fatalf("disabled NoRetry body executed %q err%v", data, err) + } + } else if err != nil || string(data) != "x" { + t.Fatalf("crash body calls%q want x err%v exit%d %s %s", data, err, got.code, got.out, got.stderr) + } + if got.code != wantCode { + t.Fatalf("command exit%d want%d %s %s", got.code, wantCode, got.out, got.stderr) + } + events := reviewTestEvents(receiver, "TestCrashPolicy") + if len(events) != 1 { + t.Fatalf("crash events%d want1: %v", len(events), receiver.events) + } + meta := events[0] + if meta["test.status"] != wantStatus || meta["test.final_status"] != wantFinal { + t.Fatalf("crash outcome status=%v final=%v want%s/%s: %v", meta["test.status"], meta["test.final_status"], wantStatus, wantFinal, meta) + } + for key, want := range map[string]bool{"prevent_failure_suppression": flags.prevent, "no_auto_retry": flags.noRetry, "is_quarantined": assignment == "quarantined", "is_test_disabled": assignment == "disabled-atf", "is_attempt_to_fix": assignment == "disabled-atf"} { + tag := "test.test_management." + key + if want && meta[tag] != "true" || !want && meta[tag] != nil { + t.Fatalf("crash assignment/annotation %s=%v wanttrue=%t: %v", tag, meta[tag], want, meta) + } + } + requireReviewNoRetryOwnership(t, meta) + if skipped { + if meta["error.type"] != nil || meta["error.message"] != nil { + t.Fatalf("skipped body gained crash diagnostics: %v", meta) + } + return + } + errorType, sentinel := "timeout", "test timed out after 250ms" + if mode == "background" { + errorType, sentinel = "panic", "protected background panic sentinel" + } + if meta["error.type"] != errorType || !strings.Contains(fmt.Sprint(meta["error.message"]), sentinel) || !strings.Contains(got.out+got.stderr, sentinel) { + t.Fatalf("lost crash type/value %s/%s: %v\n%s\n%s", errorType, sentinel, meta, got.out, got.stderr) + } + }) + } + }) + } + }) + } +} + +func checkMiniEarlyWorkerTerminalState(t *testing.T) { + t.Run("completed", checkMiniEarlyCompletedWorkerState) + t.Run("active", checkMiniEarlyActiveWorkerState) +} + +const earlyCompletedWorkerSource = `package fixture_test +import("testing";"os";"runtime";"github.com/stretchr/testify/suite") +func appendEarly(t *testing.T,key string)int{path:=os.Getenv(key);f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)};d,e:=os.ReadFile(path);if e!=nil{t.Fatal(e)};return len(d)} +type EarlyWorkerSuite struct{suite.Suite} +//ddtest:no_auto_retry +EXTRA +func(s *EarlyWorkerSuite) TestOnce(){n:=appendEarly(s.T(),"EARLY_NR_CALLS");if n==1&&os.Getenv("EARLY_PROTECTED_FAIL")=="true"{s.T().Error("first protected declaration failure")}} +func TestEarlyWorkerRoot(t *testing.T){ + appendEarly(t,"EARLY_ROOT_CALLS") + f,e:=os.OpenFile(os.Getenv("EARLY_ROOT_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("first root failure before discovery");return};if !os.IsExist(e){t.Fatal(e)} + if os.Getenv("EARLY_ORDINARY_ONLY")!="true"{suite.Run(t,new(EarlyWorkerSuite))} + t.Run("ordinary",func(t *testing.T){ + appendEarly(t,"EARLY_CHILD_CALLS") + f,e:=os.OpenFile(os.Getenv("EARLY_CHILD_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();switch os.Getenv("EARLY_TERMINAL"){case "panic":panic("controlled ordinary worker panic sentinel");case "goexit":runtime.Goexit()}}else if !os.IsExist(e){t.Fatal(e)} + }) +} +` + +// These markers distinguish native invocations from report-event ownership. +func earlyWorkerEnv(t *testing.T, terminal string, failed bool) ([]string, string, string, string) { + t.Helper() + scratch := t.TempDir() + nr, roots, children := filepath.Join(scratch, "nr"), filepath.Join(scratch, "roots"), filepath.Join(scratch, "children") + env := []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "EARLY_NR_CALLS=" + nr, "EARLY_ROOT_CALLS=" + roots, "EARLY_CHILD_CALLS=" + children, "EARLY_ROOT_FIRST=" + filepath.Join(scratch, "root-first"), "EARLY_CHILD_FIRST=" + filepath.Join(scratch, "child-first"), "EARLY_TERMINAL=" + terminal, "EARLY_PROTECTED_FAIL=" + fmt.Sprint(failed)} + return env, nr, roots, children +} + +func requireEarlyMarker(t *testing.T, path, want string) { + t.Helper() + data, err := os.ReadFile(path) + if want == "" { + if !os.IsNotExist(err) { + t.Fatalf("unexpected callback marker%q data%q err%v", path, data, err) + } + return + } + if err != nil || string(data) != want { + t.Fatalf("callback marker%q=%q want%q err%v", path, data, want, err) + } +} + +func checkMiniEarlyCompletedWorkerState(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + // Native Go provides the terminal outcome independently of Mini. Its two + // invocations perform the first root failure, then one actual declaration. + writeBuildFixture(t, dir, map[string]string{"early_worker_test.go": strings.Replace(earlyCompletedWorkerSource, "EXTRA\n", "", 1)}) + native := filepath.Join(t.TempDir(), executableName("early-native.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), "go", exclusionBuildArgs(native, true)...) + if code != 0 { + t.Fatalf("native compile%d %s %s", code, out, stderr) + } + for _, terminal := range []string{"none", "panic", "goexit"} { + for _, failed := range []bool{false, true} { + t.Run(fmt.Sprintf("native/%s/failed=%t", terminal, failed), func(t *testing.T) { + env, nr, roots, children := earlyWorkerEnv(t, terminal, failed) + out, stderr, code := command(t, dir, testEnv(append(env, "DD_CIVISIBILITY_ENABLED=false")...), native, "-test.run=^TestEarlyWorkerRoot$", "-test.count=2") + wantCode := 2 + if terminal == "none" { + wantCode = 1 + } + if code != wantCode { + t.Fatalf("native exit%d want%d %s %s", code, wantCode, out, stderr) + } + requireEarlyMarker(t, nr, "x") + requireEarlyMarker(t, roots, "xx") + requireEarlyMarker(t, children, "x") + }) + } + } + for _, protected := range []bool{false, true} { + extra := "" + if protected { + extra = "//ddtest:prevent_failure_suppression\n" + } + writeBuildFixture(t, dir, map[string]string{"early_worker_test.go": strings.Replace(earlyCompletedWorkerSource, "EXTRA\n", extra, 1)}) + bin := filepath.Join(t.TempDir(), executableName(fmt.Sprintf("early-completed-%t.test", protected))) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, terminal := range []string{"none", "panic", "goexit"} { + for _, failed := range []bool{false, true} { + if failed && !protected { + continue + } + t.Run(fmt.Sprintf("protected=%t/failed=%t/%s", protected, failed, terminal), func(t *testing.T) { + env, nr, roots, children := earlyWorkerEnv(t, terminal, failed) + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestEarlyWorkerRoot$"}, Policy: policySettings{Retry: true}, Env: env}) + wantCode, wantRoots := 0, 3 + if terminal == "none" && !failed { + wantRoots = 2 + } + if failed { + wantCode = 1 + if terminal != "none" { + wantCode = 2 + } + } + if got.code != wantCode { + t.Fatalf("completed declaration terminal exit%d want%d %s %s", got.code, wantCode, got.out, got.stderr) + } + requireEarlyMarker(t, nr, "x") + requireEarlyMarker(t, roots, strings.Repeat("x", wantRoots)) + requireEarlyMarker(t, children, strings.Repeat("x", wantRoots-1)) + events := reviewTestEvents(receiver, "TestEarlyWorkerRoot/TestOnce") + if len(events) != 1 { + t.Fatalf("completed NoRetry events%d want1: %v", len(events), receiver.events) + } + meta := events[0] + wantStatus := "pass" + if failed { + wantStatus = "fail" + } + if meta["test.status"] != wantStatus || meta["test.final_status"] != wantStatus || meta["test.test_management.no_auto_retry"] != "true" || (meta["test.test_management.prevent_failure_suppression"] == "true") != protected { + t.Fatalf("first completed declaration event lost/replaced: %v", meta) + } + requireReviewNoRetryOwnership(t, meta) + if failed && !strings.Contains(fmt.Sprint(meta["error.message"]), "first protected declaration failure") { + t.Fatalf("original completed failure lost: %v", meta) + } + rootEvents := reviewTestEvents(receiver, "TestEarlyWorkerRoot") + if len(rootEvents) != wantRoots { + t.Fatalf("root event ownership count%d want%d: %v", len(rootEvents), wantRoots, receiver.events) + } + }) + } + } + if !protected { + for _, terminal := range []string{"panic", "goexit"} { + t.Run("ordinary-only/"+terminal, func(t *testing.T) { + env, nr, roots, children := earlyWorkerEnv(t, terminal, false) + env = append(env, "EARLY_ORDINARY_ONLY=true") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestEarlyWorkerRoot$"}, Policy: policySettings{Retry: true}, Env: env}) + if got.code != 0 { + t.Fatalf("ordinary terminal must retain retry recovery: exit%d %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, nr, "") + requireEarlyMarker(t, roots, "xxx") + requireEarlyMarker(t, children, "xx") + if len(reviewTestEvents(receiver, "TestEarlyWorkerRoot/TestOnce")) != 0 || len(reviewTestEvents(receiver, "TestEarlyWorkerRoot")) != 3 { + t.Fatalf("ordinary retry event ownership changed: %v", receiver.events) + } + }) + } + } + } +} + +const earlyActiveWorkerSource = `package fixture_test +import("testing";"os";"runtime") +func pendingAppend(t *testing.T,key string){f,e:=os.OpenFile(os.Getenv(key),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)}} +//ddtest:no_auto_retry +func TestEarlierCompleted(t *testing.T){pendingAppend(t,"PENDING_EARLIER")} +//ddtest:no_auto_retry +EXTRA +func TestPendingExcluded(t *testing.T){pendingAppend(t,"PENDING_ENTERED");t.Parallel();pendingAppend(t,"PENDING_COMPLETED")} +func TestPendingRoot(t *testing.T){ + pendingAppend(t,"PENDING_ROOT_CALLS") + f,e:=os.OpenFile(os.Getenv("PENDING_ROOT_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("first root failure");return};if !os.IsExist(e){t.Fatal(e)} + if os.Getenv("PENDING_WITH_EARLIER")=="true"{t.Run("earlier",TestEarlierCompleted)} + t.Run("pending",TestPendingExcluded) + t.Run("ordinary",func(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("PENDING_CHILD_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e!=nil{if !os.IsExist(e){t.Fatal(e)};return};f.Close() + entered,e:=os.ReadFile(os.Getenv("PENDING_ENTERED"));if e!=nil||string(entered)!="x"{t.Fatalf("writer precondition entered%q err%v",entered,e)} + if _,e=os.Stat(os.Getenv("PENDING_COMPLETED"));!os.IsNotExist(e){t.Fatalf("writer precondition: pending child already completed %v",e)} + pendingAppend(t,"PENDING_OBSERVED_INFLIGHT") + if os.Getenv("PENDING_TERMINAL")=="goexit"{runtime.Goexit()};panic("pending ordinary worker panic sentinel") + }) +} +` + +// Native t.Parallel is the barrier: the parent cannot execute the terminal +// child until the annotated callback entered and suspended before completion. +func checkMiniEarlyActiveWorkerState(t *testing.T) { + dir, driver := prepareMiniFixture(t) + for _, protected := range []bool{false, true} { + extra := "" + if protected { + extra = "//ddtest:prevent_failure_suppression\n" + } + writeBuildFixture(t, dir, map[string]string{"pending_worker_test.go": strings.Replace(earlyActiveWorkerSource, "EXTRA\n", extra, 1)}) + bins := map[string]string{} + for _, kind := range []string{"native", "Mini"} { + bin := filepath.Join(t.TempDir(), executableName(fmt.Sprintf("pending-%s-%t.test", kind, protected))) + build := driver + if kind == "native" { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + bins[kind] = bin + } + for _, terminal := range []string{"panic", "goexit"} { + for _, earlier := range []bool{false, true} { + if earlier && !protected { + continue + } + for _, kind := range []string{"native", "Mini"} { + if kind == "native" && earlier { + continue + } + t.Run(fmt.Sprintf("protected=%t/earlier=%t/%s/%s", protected, earlier, terminal, kind), func(t *testing.T) { + scratch := t.TempDir() + entered, completed, roots, observed, earlierCalls := filepath.Join(scratch, "entered"), filepath.Join(scratch, "completed"), filepath.Join(scratch, "roots"), filepath.Join(scratch, "observed"), filepath.Join(scratch, "earlier") + args := []string{"-test.run=^TestPendingRoot$"} + if kind == "native" { + args = append(args, "-test.count=2") + } + receiver, got := runParityCase(t, dir, bins[kind], parityCase{Args: args, Policy: policySettings{Retry: true}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "PENDING_ENTERED=" + entered, "PENDING_COMPLETED=" + completed, "PENDING_ROOT_CALLS=" + roots, "PENDING_OBSERVED_INFLIGHT=" + observed, "PENDING_EARLIER=" + earlierCalls, "PENDING_WITH_EARLIER=" + fmt.Sprint(earlier), "PENDING_ROOT_FIRST=" + filepath.Join(scratch, "root-first"), "PENDING_CHILD_FIRST=" + filepath.Join(scratch, "child-first"), "PENDING_TERMINAL=" + terminal}}) + if got.code != 2 { + t.Fatalf("active NoRetry must preserve native terminal exit2; got%d %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, entered, "x") + requireEarlyMarker(t, completed, "") + requireEarlyMarker(t, roots, "xx") + requireEarlyMarker(t, observed, "x") + sentinel := "pending ordinary worker panic sentinel" + if terminal == "goexit" { + sentinel = "runtime.Goexit" + } + if !strings.Contains(got.out+got.stderr, sentinel) { + t.Fatalf("lost native terminal value%s %s %s", sentinel, got.out, got.stderr) + } + if kind == "native" { + return + } + if len(reviewTestEvents(receiver, "TestPendingRoot/pending")) != 0 { + t.Fatalf("unfinished declaration synthesized a completed event: %v", receiver.events) + } + rootEvents := reviewTestEvents(receiver, "TestPendingRoot") + if len(rootEvents) != 2 { + t.Fatalf("active terminal root event count%d want2: %v", len(rootEvents), receiver.events) + } + for _, meta := range rootEvents { + if meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("active child's annotation activated on ordinary owner: %v", meta) + } + } + earlierEvents := reviewTestEvents(receiver, "TestPendingRoot/earlier") + if !earlier { + requireEarlyMarker(t, earlierCalls, "") + if len(earlierEvents) != 0 { + t.Fatalf("invented earlier event: %v", earlierEvents) + } + return + } + requireEarlyMarker(t, earlierCalls, "x") + if len(earlierEvents) != 1 { + t.Fatalf("completed earlier event lost: %v", receiver.events) + } + meta := earlierEvents[0] + requireReviewNoRetryOwnership(t, meta) + if meta["test.status"] != "pass" || meta["test.final_status"] != "pass" || meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("completed earlier event changed: %v", meta) + } + }) + } + } + } + } +} + +const pfsWorkerTerminalSource = `package fixture_test +import("os";"runtime";"testing") +func pfsWorkerRecord(t *testing.T,key,value string)int{path:=os.Getenv(key);f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString(value);e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)};d,e:=os.ReadFile(path);if e!=nil{t.Fatal(e)};return len(d)} +ANNOTATIONS +func TestPFSWorkerCallback(t *testing.T){ + if os.Getenv("PFS_WORKER_SELF_PARALLEL")=="true"{t.Parallel()} + pfsWorkerRecord(t,"PFS_WORKER_CHILD","x") + if os.Getenv("PFS_WORKER_QUEUED")=="true"{t.Run("queued",func(t *testing.T){t.Parallel();pfsWorkerRecord(t,"PFS_WORKER_ORDER","q")})} + cleanup:=func(){pfsWorkerRecord(t,"PFS_WORKER_ORDER","c");panic("protected worker cleanup sentinel")} + switch os.Getenv("PFS_WORKER_TERMINAL"){ + case "cleanup-panic":t.Cleanup(cleanup);return + case "body-cleanup-panic":t.Cleanup(cleanup) + case "body-cleanup-skip":t.Cleanup(func(){pfsWorkerRecord(t,"PFS_WORKER_ORDER","c");t.SkipNow()}) + case "body-cleanup-fail":t.Cleanup(func(){pfsWorkerRecord(t,"PFS_WORKER_ORDER","c");t.FailNow()}) + case "body-cleanup-goexit":t.Cleanup(func(){pfsWorkerRecord(t,"PFS_WORKER_ORDER","c");runtime.Goexit()}) + case "goexit":runtime.Goexit() + } + panic("protected worker body sentinel") +} +func TestPFSWorkerRoot(t *testing.T){ + pfsWorkerRecord(t,"PFS_WORKER_ROOT","x") + if os.Getenv("PFS_WORKER_NATIVE")!="true"{f,e:=os.OpenFile(os.Getenv("PFS_WORKER_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("ordinary first failure before protected discovery");return};if !os.IsExist(e){t.Fatal(e)}} + t.Run("critical",TestPFSWorkerCallback) +} +` + +// PFS alone keeps native terminal semantics in ordinary worker callbacks; +// Both retains its established managed-event protocol. No child event is +// invented for a PFS-only ordinary callback whose owner is the retry family. +func checkMiniPFSWorkerTerminalPolicy(t *testing.T) { + dir, driver := prepareMiniFixture(t) + cases := []struct { + mode string + queued bool + nativeExit int + order string + selfParallel bool + }{ + {"panic", false, 2, "", false}, {"goexit", false, 2, "", false}, {"cleanup-panic", false, 2, "c", false}, {"body-cleanup-panic", false, 2, "c", false}, {"body-cleanup-skip", false, 0, "c", false}, {"body-cleanup-fail", false, 1, "c", false}, {"body-cleanup-goexit", false, 2, "c", false}, + {"cleanup-panic", true, 2, "qc", false}, {"body-cleanup-panic", true, 2, "c", false}, + {"cleanup-panic", true, 2, "qc", true}, + } + for _, kind := range []string{"native", "pfs", "both"} { + t.Run(kind, func(t *testing.T) { + annotations := "" + if kind != "native" { + annotations = "//ddtest:prevent_failure_suppression" + } + if kind == "both" { + annotations += "\n//ddtest:no_auto_retry" + } + writeBuildFixture(t, dir, map[string]string{"pfs_worker_terminal_test.go": strings.Replace(pfsWorkerTerminalSource, "ANNOTATIONS", annotations, 1)}) + bin := filepath.Join(t.TempDir(), executableName("pfs-worker-"+kind+".test")) + build := driver + if kind == "native" { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, tc := range cases { + t.Run(fmt.Sprintf("%s/queued=%t/self-parallel=%t", tc.mode, tc.queued, tc.selfParallel), func(t *testing.T) { + scratch := t.TempDir() + roots, child, order := filepath.Join(scratch, "roots"), filepath.Join(scratch, "child"), filepath.Join(scratch, "order") + env := []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "PFS_WORKER_ROOT=" + roots, "PFS_WORKER_CHILD=" + child, "PFS_WORKER_ORDER=" + order, "PFS_WORKER_FIRST=" + filepath.Join(scratch, "first"), "PFS_WORKER_TERMINAL=" + tc.mode, "PFS_WORKER_QUEUED=" + fmt.Sprint(tc.queued), "PFS_WORKER_SELF_PARALLEL=" + fmt.Sprint(tc.selfParallel), "PFS_WORKER_NATIVE=" + fmt.Sprint(kind == "native")} + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestPFSWorkerRoot$", "-test.parallel=2", "-test.timeout=20s"}, Policy: policySettings{Disabled: kind != "native", AttemptToFix: kind != "native", ManagementTarget: "TestPFSWorkerRoot", ManagementSuite: "pfs_worker_terminal_test.go"}, Env: env}) + wantCode := tc.nativeExit + if kind == "both" && wantCode != 0 { + wantCode = 1 + } + if got.code != wantCode { + t.Fatalf("%s terminal exit%d want%d %s %s", kind, got.code, wantCode, got.out, got.stderr) + } + wantRoots := "xx" + if kind == "native" { + wantRoots = "x" + } + requireEarlyMarker(t, roots, wantRoots) + requireEarlyMarker(t, child, "x") + requireEarlyMarker(t, order, tc.order) + if kind == "native" { + return + } + rootEvents := reviewTestEvents(receiver, "TestPFSWorkerRoot") + if len(rootEvents) != 2 { + t.Fatalf("family event count%d want2: %v", len(rootEvents), receiver.events) + } + var retry map[string]any + for _, meta := range rootEvents { + if meta["test.test_management.is_test_disabled"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" || meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("root assignment/declaration identity changed: %v", meta) + } + if meta["test.is_retry"] == "true" { + if retry != nil { + t.Fatal("multiple retry-family events") + } + retry = meta + } else if meta["test.status"] != "fail" || !strings.Contains(fmt.Sprint(meta["error.message"]), "ordinary first failure before protected discovery") { + t.Fatalf("lost initial ordinary failure: %v", meta) + } + } + if retry == nil || retry["test.retry_reason"] != "attempt_to_fix" || retry["test.test_management.attempt_to_fix_passed"] != "false" { + t.Fatalf("missing original ATF attempt metadata: %v", retry) + } + wantStatus, wantFinal := "fail", "fail" + if tc.nativeExit == 0 { + wantStatus, wantFinal = "pass", "skip" + } + if retry["test.status"] != wantStatus || retry["test.final_status"] != wantFinal { + t.Fatalf("protected/cancelled family outcome %v want%s/%s", retry, wantStatus, wantFinal) + } + if tc.mode == "body-cleanup-skip" { + for _, key := range []string{"error.type", "error.message", "error.stack"} { + if retry[key] != nil { + t.Fatalf("native cleanup SkipNow cancellation revived %s: %v", key, retry) + } + } + } + if kind == "pfs" { + if len(reviewTestEvents(receiver, "TestPFSWorkerRoot/critical")) != 0 { + t.Fatalf("ordinary PFS callback gained an unsupported child event: %v", receiver.events) + } + testCount := 0 + for _, event := range receiver.events { + if event["type"] == "test" { + testCount++ + } + } + if testCount != 2 { + t.Fatalf("PFS family-only event count%d want2: %v", testCount, receiver.events) + } + if tc.mode == "panic" || tc.mode == "goexit" || tc.mode == "cleanup-panic" || tc.mode == "body-cleanup-panic" || tc.mode == "body-cleanup-goexit" { + if retry["error.type"] != "panic" { + t.Fatalf("surviving native terminal lost panic type: %v", retry) + } + } + } else { + events := reviewTestEvents(receiver, "TestPFSWorkerRoot/critical") + if len(events) != 1 { + t.Fatalf("managed callback events%d want1: %v", len(events), receiver.events) + } + meta := events[0] + requireReviewNoRetryOwnership(t, meta) + status := "fail" + if tc.nativeExit == 0 { + status = "skip" + } + if meta["test.status"] != status || meta["test.final_status"] != status || meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.prevent_failure_suppression"] != "true" || meta["test.test_management.is_test_disabled"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("managed callback outcome/assignment changed: %v", meta) + } + } + sessions := 0 + for _, event := range receiver.events { + if event["type"] != "test_session_end" { + continue + } + sessions++ + meta := event["content"].(map[string]any)["meta"].(map[string]any) + status := "fail" + if wantCode == 0 { + status = "pass" + } + if meta["test.status"] != status { + t.Fatalf("session failed to preserve command outcome: %v", meta) + } + } + if sessions != 1 { + t.Fatalf("session count%d want1", sessions) + } + }) + } + }) + } + t.Run("ordinary-retry-recovery", checkMiniPFSWorkerRetryRecovery) + t.Run("unmanaged-queued-terminal", checkMiniPFSUnmanagedQueuedTerminal) + t.Run("concurrent-irreversible-failure", checkMiniPFSConcurrentIrreversibleWorkerFailure) +} + +// Native callbacks, Testify rows, and actual methods retain ordinary ATR +// recovery when PFS is present without NoRetry. Declaration and reporting +// ownership can differ; positive family events and body counters prove recovery. +func checkMiniPFSWorkerRetryRecovery(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"pfs_recovery_test.go": `package fixture_test +import("os";"testing";"github.com/stretchr/testify/suite") +func pfsRecoveryCall(t *testing.T)int{path:=os.Getenv("PFS_RECOVERY_CHILD");f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};f.Close();d,e:=os.ReadFile(path);if e!=nil{t.Fatal(e)};return len(d)} +//ddtest:prevent_failure_suppression +func TestPFSRecoverCallback(t *testing.T){if pfsRecoveryCall(t)==1{panic("recoverable protected callback sentinel")}} +type PFSRecoverRow struct{suite.Suite} +func(s *PFSRecoverRow) TestMethod(){s.T().Run("critical",TestPFSRecoverCallback)} +type PFSRecoverMethod struct{suite.Suite} +//ddtest:prevent_failure_suppression +func(s *PFSRecoverMethod) TestMethod(){if pfsRecoveryCall(s.T())==1{panic("recoverable protected method sentinel")}} +func TestPFSRecoveryRoot(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("PFS_RECOVERY_ROOT"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};f.Close() + f,e=os.OpenFile(os.Getenv("PFS_RECOVERY_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("first ordinary failure");return};if !os.IsExist(e){t.Fatal(e)} + switch os.Getenv("PFS_RECOVERY_KIND"){case "callback":t.Run("critical",TestPFSRecoverCallback);case "row":suite.Run(t,new(PFSRecoverRow));case "method":suite.Run(t,new(PFSRecoverMethod));default:t.Fatal("invalid recovery kind")} +} +`}) + bin := filepath.Join(t.TempDir(), executableName("pfs-recovery.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, kind := range []string{"callback", "row", "method"} { + t.Run(kind, func(t *testing.T) { + scratch := t.TempDir() + roots, child := filepath.Join(scratch, "roots"), filepath.Join(scratch, "child") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestPFSRecoveryRoot$"}, Policy: policySettings{Retry: true}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "PFS_RECOVERY_KIND=" + kind, "PFS_RECOVERY_ROOT=" + roots, "PFS_RECOVERY_CHILD=" + child, "PFS_RECOVERY_FIRST=" + filepath.Join(scratch, "first")}}) + if got.code != 0 { + t.Fatalf("PFS-only ordinary retry recovery failed: exit%d %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, roots, "xxx") + requireEarlyMarker(t, child, "xx") + rootEvents := reviewTestEvents(receiver, "TestPFSRecoveryRoot") + if len(rootEvents) != 3 { + t.Fatalf("recovered family events%d want3: %v", len(rootEvents), receiver.events) + } + for _, meta := range rootEvents { + if meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("child policy leaked into ordinary retry owner: %v", meta) + } + } + }) + } +} + +// The same queued-child barrier is respected when the ordinary process owner +// has ATR without management metadata: surviving PFS terminals still block, +// while the otherwise permitted retry invokes the callback again. +func checkMiniPFSUnmanagedQueuedTerminal(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"pfs_worker_terminal_test.go": strings.Replace(pfsWorkerTerminalSource, "ANNOTATIONS", "//ddtest:prevent_failure_suppression", 1)}) + bin := filepath.Join(t.TempDir(), executableName("pfs-unmanaged-queued.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, tc := range []struct{ mode, order string }{{"body-cleanup-panic", "cc"}, {"cleanup-panic", "qcqc"}} { + t.Run(tc.mode, func(t *testing.T) { + scratch := t.TempDir() + roots, child, order := filepath.Join(scratch, "roots"), filepath.Join(scratch, "child"), filepath.Join(scratch, "order") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestPFSWorkerRoot$", "-test.parallel=2", "-test.timeout=20s"}, Policy: policySettings{Retry: true}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "PFS_WORKER_ROOT=" + roots, "PFS_WORKER_CHILD=" + child, "PFS_WORKER_ORDER=" + order, "PFS_WORKER_FIRST=" + filepath.Join(scratch, "first"), "PFS_WORKER_TERMINAL=" + tc.mode, "PFS_WORKER_QUEUED=true"}}) + if got.code != 2 { + t.Fatalf("unmanaged PFS terminal exit%d want2 %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, roots, "xxx") + requireEarlyMarker(t, child, "xx") + requireEarlyMarker(t, order, tc.order) + events := reviewTestEvents(receiver, "TestPFSWorkerRoot") + if len(events) != 3 { + t.Fatalf("unmanaged ATR root event count%d want3: %v", len(events), receiver.events) + } + for _, meta := range events { + for _, key := range []string{"test.test_management.no_auto_retry", "test.test_management.prevent_failure_suppression", "test.test_management.is_test_disabled", "test.test_management.is_attempt_to_fix", "test.test_management.is_quarantined"} { + if meta[key] != nil { + t.Fatalf("unmanaged owner gained %s: %v", key, meta) + } + } + } + }) + } +} + +// An ordinary sibling can terminate a worker while a protected callback is +// still blocked. Its already-observed native failure must survive even though +// the protected callback has not returned or run cleanup. +func checkMiniPFSConcurrentIrreversibleWorkerFailure(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("os";"testing") +var irreversibleReady chan struct{} +func irreversibleRecord(t *testing.T,key string){f,e:=os.OpenFile(os.Getenv(key),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)}} +ANNOTATIONS +func TestBlockedProtected(t *testing.T){t.Parallel();irreversibleRecord(t,"IRREVERSIBLE_CHILD");t.Error("irreversible protected parallel failure");close(irreversibleReady);select{}} +func TestConcurrentPFSRoot(t *testing.T){ + irreversibleRecord(t,"IRREVERSIBLE_ROOT") + if os.Getenv("IRREVERSIBLE_NATIVE")!="true"{f,e:=os.OpenFile(os.Getenv("IRREVERSIBLE_FIRST"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if e==nil{f.Close();t.Error("ordinary first failure");return};if !os.IsExist(e){t.Fatal(e)}} + irreversibleReady=make(chan struct{}) + t.Run("protected",TestBlockedProtected) + t.Run("ordinary",func(t *testing.T){t.Parallel();<-irreversibleReady;irreversibleRecord(t,"IRREVERSIBLE_TERMINAL");panic("ordinary sibling panic after irreversible PFS failure")}) +} +` + for _, kind := range []string{"native", "pfs", "ordinary"} { + t.Run(kind, func(t *testing.T) { + annotations := "" + if kind == "pfs" { + annotations = "//ddtest:prevent_failure_suppression" + } + writeBuildFixture(t, dir, map[string]string{"irreversible_pfs_test.go": strings.Replace(source, "ANNOTATIONS", annotations, 1)}) + bin := filepath.Join(t.TempDir(), executableName("irreversible-"+kind+".test")) + build := driver + + if kind == "native" { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + scratch := t.TempDir() + roots, child, terminal := filepath.Join(scratch, "roots"), filepath.Join(scratch, "child"), filepath.Join(scratch, "terminal") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestConcurrentPFSRoot$", "-test.parallel=2", "-test.timeout=10s"}, Policy: policySettings{Disabled: kind != "native", AttemptToFix: kind != "native", ManagementTarget: "TestConcurrentPFSRoot", ManagementSuite: "irreversible_pfs_test.go"}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "IRREVERSIBLE_ROOT=" + roots, "IRREVERSIBLE_CHILD=" + child, "IRREVERSIBLE_TERMINAL=" + terminal, "IRREVERSIBLE_FIRST=" + filepath.Join(scratch, "first"), "IRREVERSIBLE_NATIVE=" + fmt.Sprint(kind == "native")}}) + r, re := os.ReadFile(roots) + c, ce := os.ReadFile(child) + p, pe := os.ReadFile(terminal) + rows := []map[string]any{} + for _, event := range receiver.events { + if event["type"] == "test" { + rows = append(rows, event["content"].(map[string]any)["meta"].(map[string]any)) + } + } + t.Logf("IRREVERSIBLE kind=%s cmd%d roots%q child%q terminal%q rows%d", kind, got.code, r, c, p, len(rows)) + for _, meta := range rows { + t.Logf("ROW name=%v status=%v final=%v PFS=%v disabled=%v ATF=%v error=%v", meta["test.name"], meta["test.status"], meta["test.final_status"], meta["test.test_management.prevent_failure_suppression"], meta["test.test_management.is_test_disabled"], meta["test.test_management.is_attempt_to_fix"], meta["error.message"]) + } + if re != nil || ce != nil || pe != nil || len(r) == 0 || len(c) == 0 || string(c) != string(p) { + t.Fatalf("causal markersmissing root%q/%v child%q/%v terminal%q/%v %s %s", r, re, c, ce, p, pe, got.out, got.stderr) + } + + wantCode, wantRoots := 2, "xx" + if kind == "ordinary" { + wantCode = 0 + } + if kind == "native" { + wantRoots = "x" + } + if got.code != wantCode { + t.Fatalf("irreversible protected failure terminal exit%d want%d %s %s", got.code, wantCode, got.out, got.stderr) + } + requireEarlyMarker(t, roots, wantRoots) + requireEarlyMarker(t, child, "x") + requireEarlyMarker(t, terminal, "x") + if kind == "native" { + return + } + if len(rows) != 2 || len(reviewTestEvents(receiver, "TestConcurrentPFSRoot")) != 2 { + t.Fatalf("concurrent family-only event count%d want2: %v", len(rows), receiver.events) + } + for _, meta := range rows { + if meta["test.test_management.is_test_disabled"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" || meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("concurrent family assignment/annotation ownership changed: %v", meta) + } + if meta["test.is_retry"] != "true" { + continue + } + final := "fail" + if kind == "ordinary" { + final = "skip" + } + if meta["test.status"] != "fail" || meta["test.final_status"] != final || meta["error.type"] != "panic" || meta["error.message"] != "ordinary sibling panic after irreversible PFS failure" { + t.Fatalf("irreversible failure lost from concurrent terminal outcome: %v", meta) + } + } + + }) + } +} + +// A completed child's trusted protected failure survives a later crash of its +// still-open assigned parent. A passing child or a separate fresh event does +// not create that fact, and public annotation tags remain declaration-local. +func checkMiniProtectedDescendantCrashPolicy(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("os";"testing";"time") +func descendantRecord(t *testing.T,key string){f,e:=os.OpenFile(os.Getenv(key),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)}} +CHILD_ANNOTATION +func TestDescendantCrashChild(t *testing.T){descendantRecord(t,"DESCENDANT_CHILD");if os.Getenv("DESCENDANT_PASS")!="true"{t.Error("protected descendant assertion sentinel")}} +ROOT_ANNOTATION +func TestDescendantCrashParent(t *testing.T){ + descendantRecord(t,"DESCENDANT_PARENT") + if os.Getenv("DESCENDANT_NO_CHILD")!="true"{t.Run("critical",TestDescendantCrashChild)} + descendantRecord(t,"DESCENDANT_RETURNED") + if os.Getenv("DESCENDANT_MODE")=="background"{go func(){panic("parent background crash after protected child")}()} + time.Sleep(time.Minute) +} +` + for _, kind := range []string{"protected-child", "ordinary-child", "direct-pfs-root", "no-child", "passing-child", "native"} { + t.Run(kind, func(t *testing.T) { + childAnnotation, rootAnnotation := "", "" + if kind == "protected-child" || kind == "passing-child" || kind == "native" { + childAnnotation = "//ddtest:prevent_failure_suppression" + } + if kind == "direct-pfs-root" { + rootAnnotation = "//ddtest:prevent_failure_suppression" + } + fixture := strings.ReplaceAll(strings.ReplaceAll(source, "CHILD_ANNOTATION", childAnnotation), "ROOT_ANNOTATION", rootAnnotation) + writeBuildFixture(t, dir, map[string]string{"descendant_crash_test.go": fixture}) + for _, mode := range []string{"timeout", "background"} { + t.Run(mode, func(t *testing.T) { + bin := filepath.Join(t.TempDir(), executableName("descendant-crash.test")) + args := exclusionBuildArgs(bin, false) + build := driver + if kind == "native" { + build = "go" + } else if mode == "background" { + args = append([]string{"test", "--panic-hook"}, args[1:]...) + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, args...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + scratch := t.TempDir() + parent, child, returned := filepath.Join(scratch, "parent"), filepath.Join(scratch, "child"), filepath.Join(scratch, "returned") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestDescendantCrashParent$", "-test.timeout=250ms"}, Policy: policySettings{Quarantined: kind != "native", ManagementTarget: "TestDescendantCrashParent", ManagementSuite: "descendant_crash_test.go"}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=in_process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "DESCENDANT_PARENT=" + parent, "DESCENDANT_CHILD=" + child, "DESCENDANT_RETURNED=" + returned, "DESCENDANT_MODE=" + mode, "DESCENDANT_NO_CHILD=" + fmt.Sprint(kind == "no-child"), "DESCENDANT_PASS=" + fmt.Sprint(kind == "passing-child")}}) + if got.code != 2 { + t.Fatalf("native process crash exit%d want2 %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, parent, "x") + requireEarlyMarker(t, returned, "x") + wantChild := "x" + if kind == "no-child" { + wantChild = "" + } + requireEarlyMarker(t, child, wantChild) + if kind == "native" { + if len(receiver.events) != 0 { + t.Fatal("native control emitted telemetry") + } + return + } + parents := reviewTestEvents(receiver, "TestDescendantCrashParent") + children := reviewTestEvents(receiver, "TestDescendantCrashParent/critical") + wantChildren := 1 + if kind == "no-child" { + wantChildren = 0 + } + if len(parents) != 1 || len(children) != wantChildren { + t.Fatalf("actual descendant crash event counts parent%d child%d want1/%d: %v", len(parents), len(children), wantChildren, receiver.events) + } + testCount := 0 + for _, event := range receiver.events { + if event["type"] == "test" { + testCount++ + } + } + if testCount != 1+wantChildren { + t.Fatalf("unexpected extra test events%d want%d", testCount, 1+wantChildren) + } + meta := parents[0] + wantFinal := "skip" + if kind == "protected-child" || kind == "direct-pfs-root" { + wantFinal = "fail" + } + if meta["test.status"] != "fail" || meta["test.final_status"] != wantFinal || meta["test.test_management.is_quarantined"] != "true" || (meta["test.test_management.prevent_failure_suppression"] == "true") != (kind == "direct-pfs-root") || meta["test.test_management.no_auto_retry"] != nil { + t.Fatalf("descendant crash trusted fact/assignment/declaration changed: %v", meta) + } + requireReviewNoRetryOwnership(t, meta) + errorType, sentinel := "timeout", "test timed out after 250ms" + if mode == "background" { + errorType, sentinel = "panic", "parent background crash after protected child" + } + if meta["error.type"] != errorType || !strings.Contains(fmt.Sprint(meta["error.message"]), sentinel) { + t.Fatalf("parent crash cause lost: %v", meta) + } + if wantChildren == 0 { + return + } + cm := children[0] + childFinal := "skip" + if kind == "protected-child" || kind == "direct-pfs-root" || kind == "passing-child" { + childFinal = "fail" + } + childStatus := "fail" + if kind == "passing-child" { + childStatus, childFinal = "pass", "pass" + } + protected := kind == "protected-child" || kind == "direct-pfs-root" || kind == "passing-child" + if cm["test.status"] != childStatus || cm["test.final_status"] != childFinal || cm["test.test_management.is_quarantined"] != "true" || (cm["test.test_management.prevent_failure_suppression"] == "true") != protected || cm["test.test_management.no_auto_retry"] != nil { + t.Fatalf("completed child outcome/assignment changed: %v", cm) + } + requireReviewNoRetryOwnership(t, cm) + if kind == "passing-child" { + if cm["error.type"] != nil || cm["error.message"] != nil { + t.Fatalf("fresh passing child acquired a protected failure: %v", cm) + } + } else if cm["error.message"] != "protected descendant assertion sentinel" { + t.Fatalf("original completed child assertion lost: %v", cm) + } + }) + } + }) + } + t.Run("active-child", checkMiniActiveProtectedDescendantCrashPolicy) + t.Run("retained-atf", checkMiniRetainedDescendantCrashPolicy) +} + +// ATF deliberately retains earlier protected failures across all permitted +// attempts. NoRetry cache omission and ordinary child's later pass both leave +// that trusted family truth intact when the second root crashes before return. +func checkMiniRetainedDescendantCrashPolicy(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("os";"testing";"time") +func retainedRecord(t *testing.T,key string)int{path:=os.Getenv(key);f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)};d,e:=os.ReadFile(path);if e!=nil{t.Fatal(e)};return len(d)} +//ddtest:prevent_failure_suppression +EXTRA +func TestRetainedProtectedChild(t *testing.T){if retainedRecord(t,"RETAINED_CHILD")==1{t.Error("first protected child failure before parent retry")}} +func TestRetainedProtectedParent(t *testing.T){ + n:=retainedRecord(t,"RETAINED_PARENT") + if n==1{t.Run("critical",TestRetainedProtectedChild);retainedRecord(t,"RETAINED_RETURNED");return} + if os.Getenv("RETAINED_NO_RETRY")=="true"{retainedRecord(t,"RETAINED_OMITTED")}else{t.Run("critical",TestRetainedProtectedChild);retainedRecord(t,"RETAINED_HEALTHY")} + if os.Getenv("RETAINED_MODE")=="background"{go func(){panic("parent crash after retained or recovered protected child")}()} + time.Sleep(time.Minute) +} +` + for _, noRetry := range []bool{true, false} { + extra := "" + if noRetry { + extra = "//ddtest:no_auto_retry\n" + } + writeBuildFixture(t, dir, map[string]string{"retained_descendant_crash_test.go": strings.Replace(source, "EXTRA\n", extra, 1)}) + for _, mode := range []string{"timeout", "background"} { + t.Run(fmt.Sprintf("no-retry=%t/%s", noRetry, mode), func(t *testing.T) { + bin := filepath.Join(t.TempDir(), executableName("retained-descendant.test")) + args := exclusionBuildArgs(bin, false) + if mode == "background" { + args = append([]string{"test", "--panic-hook"}, args[1:]...) + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, args...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + scratch := t.TempDir() + paths := map[string]string{} + for _, key := range []string{"PARENT", "CHILD", "RETURNED", "OMITTED", "HEALTHY"} { + paths[key] = filepath.Join(scratch, strings.ToLower(key)) + } + env := []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=in_process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "RETAINED_MODE=" + mode, "RETAINED_NO_RETRY=" + fmt.Sprint(noRetry)} + for key, path := range paths { + env = append(env, "RETAINED_"+key+"="+path) + } + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestRetainedProtectedParent$", "-test.timeout=250ms"}, Policy: policySettings{Retry: true, Quarantined: true, AttemptToFix: true, ManagementTarget: "TestRetainedProtectedParent", ManagementSuite: "retained_descendant_crash_test.go"}, Env: env}) + if got.code != 2 { + t.Fatalf("retained protected descendant parent crash exit%d want2 %s %s", got.code, got.out, got.stderr) + } + requireEarlyMarker(t, paths["PARENT"], "xx") + requireEarlyMarker(t, paths["RETURNED"], "x") + childCalls, wantChildren := "xx", 2 + if noRetry { + childCalls, wantChildren = "x", 1 + } + requireEarlyMarker(t, paths["CHILD"], childCalls) + if noRetry { + requireEarlyMarker(t, paths["OMITTED"], "x") + requireEarlyMarker(t, paths["HEALTHY"], "") + } else { + requireEarlyMarker(t, paths["OMITTED"], "") + requireEarlyMarker(t, paths["HEALTHY"], "x") + } + parents := reviewTestEvents(receiver, "TestRetainedProtectedParent") + children := reviewTestEvents(receiver, "TestRetainedProtectedParent/critical") + if len(parents) != 2 || len(children) != wantChildren { + t.Fatalf("retained attempt event counts parent%d child%d want2/%d: %v", len(parents), len(children), wantChildren, receiver.events) + } + var retry map[string]any + for _, meta := range parents { + if meta["test.test_management.is_quarantined"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" || meta["test.test_management.prevent_failure_suppression"] != nil || meta["test.test_management.no_auto_retry"] != nil { + t.Fatalf("retained parent assignment/declaration changed: %v", meta) + } + if meta["test.is_retry"] == "true" { + if retry != nil { + t.Fatal("duplicate retry parent event") + } + retry = meta + } + } + if retry == nil || retry["test.status"] != "fail" || retry["test.final_status"] != "fail" || retry["test.retry_reason"] != "attempt_to_fix" { + t.Fatalf("known retained ATF protection lost before crash: %v", retry) + } + errorType := "timeout" + if mode == "background" { + errorType = "panic" + } + if retry["error.type"] != errorType { + t.Fatalf("retained parent crash cause lost: %v", retry) + } + failed, passed := 0, 0 + for _, meta := range children { + if meta["test.test_management.prevent_failure_suppression"] != "true" || (meta["test.test_management.no_auto_retry"] == "true") != noRetry || meta["test.test_management.is_quarantined"] != "true" || meta["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("retained child annotation/assignment lost: %v", meta) + } + switch meta["test.status"] { + case "fail": + failed++ + if meta["test.final_status"] != "fail" || meta["error.message"] != "first protected child failure before parent retry" { + t.Fatalf("original child failure lost: %v", meta) + } + case "pass": + passed++ + if meta["test.final_status"] != "pass" || meta["error.type"] != nil { + t.Fatalf("later child pass corrupted: %v", meta) + } + default: + t.Fatalf("unexpected child outcome: %v", meta) + } + if noRetry { + requireReviewNoRetryOwnership(t, meta) + } + } + wantPassed := 1 + if noRetry { + wantPassed = 0 + } + if failed != 1 || passed != wantPassed { + t.Fatalf("original/later child outcomes failed%d passed%d want1/%d", failed, passed, wantPassed) + } + }) + } + } +} + +// A failure that returned to an active child is already irreversible even when +// the child body and its cleanup never finish before the process crashes. +func checkMiniActiveProtectedDescendantCrashPolicy(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("os";"testing") +var activeCrashReady=make(chan struct{}) +func activeCrashMark(t *testing.T,key string){if e:=os.WriteFile(os.Getenv(key),[]byte("x"),0600);e!=nil{t.Fatal(e)}} +CHILD_ANNOTATION +func TestActiveCrashChild(t *testing.T){ + activeCrashMark(t,"ACTIVE_CHILD") + switch os.Getenv("ACTIVE_FAILURE"){case "Fail":t.Fail();case "Error":t.Error("active child assertion before crash");case "Errorf":t.Errorf("active child %s before crash","assertion");default:t.Fatal("unknown failure operation")} + activeCrashMark(t,"ACTIVE_AFTER_ERROR");close(activeCrashReady);select{} +} +ROOT_ANNOTATION +func TestActiveCrashParent(t *testing.T){activeCrashMark(t,"ACTIVE_PARENT");if os.Getenv("ACTIVE_MODE")=="background"{go func(){<-activeCrashReady;panic("parent crash while protected child is active")}()};t.Run("critical",TestActiveCrashChild);activeCrashMark(t,"ACTIVE_RETURNED")} +` + for _, kind := range []string{"protected-child", "ordinary-child", "direct-pfs-root", "native-protected", "native-ordinary"} { + t.Run(kind, func(t *testing.T) { + childAnnotation, rootAnnotation := "", "" + native := strings.HasPrefix(kind, "native-") + if kind == "protected-child" || kind == "native-protected" { + childAnnotation = "//ddtest:prevent_failure_suppression" + } + if kind == "direct-pfs-root" { + rootAnnotation = "//ddtest:prevent_failure_suppression" + } + fixture := strings.ReplaceAll(strings.ReplaceAll(source, "CHILD_ANNOTATION", childAnnotation), "ROOT_ANNOTATION", rootAnnotation) + writeBuildFixture(t, dir, map[string]string{"active_crash_test.go": fixture}) + failures := []string{"Error"} + if kind == "protected-child" || kind == "ordinary-child" { + failures = []string{"Fail", "Error", "Errorf"} + } + for _, mode := range []string{"timeout", "background"} { + t.Run(mode, func(t *testing.T) { + bin := filepath.Join(t.TempDir(), executableName("active-crash.test")) + args := exclusionBuildArgs(bin, false) + build := driver + if native { + build = "go" + } else if mode == "background" { + args = append([]string{"test", "--panic-hook"}, args[1:]...) + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, args...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + for _, failure := range failures { + t.Run(failure, func(t *testing.T) { + scratch := t.TempDir() + paths := map[string]string{} + env := []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=in_process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "ACTIVE_MODE=" + mode, "ACTIVE_FAILURE=" + failure} + for _, key := range []string{"PARENT", "CHILD", "AFTER_ERROR", "RETURNED"} { + paths[key] = filepath.Join(scratch, key) + env = append(env, "ACTIVE_"+key+"="+paths[key]) + } + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestActiveCrashParent$", "-test.timeout=250ms"}, Policy: policySettings{Quarantined: !native, ManagementTarget: "TestActiveCrashParent", ManagementSuite: "active_crash_test.go"}, Env: env}) + if got.code != 2 { + t.Fatalf("active child native terminal exit%d want2 %s %s", got.code, got.out, got.stderr) + } + for _, key := range []string{"PARENT", "CHILD", "AFTER_ERROR"} { + requireEarlyMarker(t, paths[key], "x") + } + requireEarlyMarker(t, paths["RETURNED"], "") + if native { + if len(receiver.events) != 0 { + t.Fatal("native active-child control emitted telemetry") + } + return + } + parents := reviewTestEvents(receiver, "TestActiveCrashParent") + children := reviewTestEvents(receiver, "TestActiveCrashParent/critical") + count := 0 + for _, event := range receiver.events { + if event["type"] == "test" { + count++ + } + } + if len(parents) != 1 || len(children) != 1 || count != 2 { + t.Fatalf("active crash exact counts parent%d child%d total%d want1/1/2: %v", len(parents), len(children), count, receiver.events) + } + final := "fail" + if kind == "ordinary-child" { + final = "skip" + } + errorType, sentinel := "timeout", "test timed out after 250ms" + if mode == "background" { + errorType, sentinel = "panic", "parent crash while protected child is active" + } + for _, row := range []struct { + meta map[string]any + protected bool + }{{parents[0], kind == "direct-pfs-root"}, {children[0], kind != "ordinary-child"}} { + m := row.meta + if m["test.status"] != "fail" || m["test.final_status"] != final || m["test.test_management.is_quarantined"] != "true" || (m["test.test_management.prevent_failure_suppression"] == "true") != row.protected || m["test.test_management.no_auto_retry"] != nil { + t.Fatalf("active irreversible failure or assignment/declaration lost: %v", m) + } + requireReviewNoRetryOwnership(t, m) + if m["error.type"] != errorType || !strings.Contains(fmt.Sprint(m["error.message"]), sentinel) { + t.Fatalf("active event crash cause lost: %v", m) + } + } + }) + } + }) + } + }) + } +} diff --git a/ddto/integration/mini_exclusions_api_test.go b/ddto/integration/mini_exclusions_api_test.go new file mode 100644 index 000000000..607d20d07 --- /dev/null +++ b/ddto/integration/mini_exclusions_api_test.go @@ -0,0 +1,172 @@ +package integration + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniPublicMethodExclusions(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"api_helpers.go": `package fixture +import("os";"github.com/stretchr/testify/suite";"github.com/DataDog/ddtest/ddto/testopt") +type OrdinaryAPI struct{suite.Suite} +func(s OrdinaryAPI) TestCritical(){ + f,err:=os.OpenFile(os.Getenv("API_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + s.T().Error("ordinary method must keep normal policy") +} +// This package function deliberately has a method's name and first parameter. +// Its registration must not register the unrelated method expression above. +func TestCritical(OrdinaryAPI){} +func init(){if os.Getenv("SKIP_PACKAGE_FUNCTION_REGISTRATION")!="true"{testopt.RegisterTestExclusions(TestCritical,testopt.TestExclusions{NoAutoRetry:true,PreventFailureSuppression:true})}} +`, "public_api_test.go": `package fixture_test +import("os";"testing";fixture "example.com/dd-ci-testing-fixture";"github.com/stretchr/testify/suite";"github.com/DataDog/ddtest/ddto/testopt") +func recordAPI(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("API_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} + t.Error("API critical failure") +} +// Keep the ordinary control in the same module as the other suites while its +// declaring method and same-signature package function stay in the helper. +type OrdinaryAPI struct{fixture.OrdinaryAPI} +type ValueAPI struct{suite.Suite} +func(s ValueAPI) TestCritical(){recordAPI(s.T())} +type PointerAPI struct{suite.Suite} +func(s *PointerAPI) TestCritical(){recordAPI(s.T())} +type PromotedBaseAPI struct{suite.Suite} +func(s PromotedBaseAPI) TestCritical(){recordAPI(s.T())} +type PromotedAPI struct{PromotedBaseAPI} +type GenericAPI[T any] struct{suite.Suite} +func(s GenericAPI[T]) TestCritical(){recordAPI(s.T())} +func init(){ + flags:=testopt.TestExclusions{NoAutoRetry:true,PreventFailureSuppression:true} + testopt.RegisterTestExclusions(ValueAPI.TestCritical,flags) + testopt.RegisterTestExclusions((*PointerAPI).TestCritical,flags) + testopt.RegisterTestExclusions(PromotedAPI.TestCritical,flags) + testopt.RegisterTestExclusions(GenericAPI[int].TestCritical,flags) +} +func TestPublicAPI(t *testing.T){ + switch os.Getenv("API_KIND"){ + case "value":suite.Run(t,new(ValueAPI)) + case "pointer":suite.Run(t,new(PointerAPI)) + case "promoted":suite.Run(t,new(PromotedAPI)) + case "generic-int":suite.Run(t,new(GenericAPI[int])) + case "generic-string":suite.Run(t,new(GenericAPI[string])) + case "ordinary":suite.Run(t,new(OrdinaryAPI)) + default:t.Fatal("unknown API suite") + } +} +`}) + bin := filepath.Join(t.TempDir(), executableName("public-api.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile API fixture %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, kind := range []string{"value", "pointer", "promoted", "generic-int", "generic-string", "ordinary"} { + t.Run(mode+"-"+kind, func(t *testing.T) { + suiteName := "" + for _, quarantined := range []bool{false, true} { + name := "normal" + if quarantined { + name = "quarantined" + } + t.Run(name, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestPublicAPI$"}, Policy: policySettings{Retry: true, Quarantined: quarantined, ManagementTarget: "TestPublicAPI/TestCritical", ManagementSuite: suiteName}, Env: []string{"API_KIND=" + kind, "API_COUNTER=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + ordinary := kind == "ordinary" + wantExit := 1 + if ordinary && quarantined { + wantExit = 0 + } + if got.code != wantExit { + t.Fatalf("exit%d want%d: %s %s", got.code, wantExit, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil && !(ordinary && quarantined && os.IsNotExist(err)) { + t.Fatal(err) + } + if ordinary && quarantined { + // Ordinary quarantine can suppress the body at the + // selected management boundary. Compare the same + // case without the package-function registration. + baselineCounter := filepath.Join(t.TempDir(), "baseline-calls") + baselineReceiver, baseline := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestPublicAPI$"}, Policy: policySettings{Retry: true, Quarantined: true, ManagementTarget: "TestPublicAPI/TestCritical", ManagementSuite: suiteName}, Env: []string{"API_KIND=" + kind, "API_COUNTER=" + baselineCounter, "SKIP_PACKAGE_FUNCTION_REGISTRATION=true", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + baselineData, baselineErr := os.ReadFile(baselineCounter) + if baselineErr != nil && !os.IsNotExist(baselineErr) { + t.Fatal(baselineErr) + } + if baseline.code != got.code || string(baselineData) != string(data) { + t.Fatalf("package-function registration changed ordinary quarantine: callbacks=%q baseline=%q exit=%d baseline=%d", data, baselineData, got.code, baseline.code) + } + baselineEvents := 0 + for _, event := range baselineReceiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestPublicAPI/TestCritical" { + continue + } + baselineEvents++ + if meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil || meta["test.test_management.is_quarantined"] != "true" { + t.Fatalf("ordinary quarantine baseline metadata: %v", meta) + } + } + if baselineEvents == 0 { + t.Fatal("missing ordinary quarantine baseline event") + } + } + if !ordinary && string(data) != "x" { + t.Fatalf("excluded method calls=%q want=x", data) + } + if ordinary && !quarantined && string(data) != "xxx" { + t.Fatalf("ordinary method calls=%q want=xxx", data) + } + events := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestPublicAPI/TestCritical" { + continue + } + events++ + actualSuite, _ := meta["test.suite"].(string) + if !quarantined { + suiteName = actualSuite + } + if ordinary { + if meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("package function registration leaked to ordinary method: %v", meta) + } + } else { + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.prevent_failure_suppression"] != "true" || meta["test.status"] != "fail" { + t.Fatalf("API exclusion lost: %v", meta) + } + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode"} { + if meta[key] != nil { + t.Fatalf("excluded API method inherited retry: %v", meta) + } + } + } + if quarantined && meta["test.test_management.is_quarantined"] != "true" { + t.Fatalf("raw backend assignment lost: %v", meta) + } + } + if !ordinary && events != 1 { + t.Fatalf("excluded API events=%d want=1", events) + } + if events == 0 || strings.TrimSpace(suiteName) == "" { + t.Fatalf("missing actual method metadata: events=%d suite=%q", events, suiteName) + } + }) + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_atf_test.go b/ddto/integration/mini_exclusions_atf_test.go new file mode 100644 index 000000000..eb940adf7 --- /dev/null +++ b/ddto/integration/mini_exclusions_atf_test.go @@ -0,0 +1,137 @@ +package integration + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniProtectedAggregateFailure(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"protected_aggregate_test.go": `package fixture_test +import("os";"testing";"github.com/stretchr/testify/suite") +type AggregateInner struct{suite.Suite} +//ddtest:prevent_failure_suppression +func(s *AggregateInner) TestCritical(){ + path:=os.Getenv("AGGREGATE_COUNTER") + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + data,err:=os.ReadFile(path);if err!=nil{s.T().Fatal(err)} + if len(data)==1{s.T().Error("early protected failure followed by successful attempts")} +} +func TestProtectedAggregate(t *testing.T){suite.Run(t,new(AggregateInner))} +`}) + bin := filepath.Join(t.TempDir(), executableName("protected-aggregate.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, profile := range []struct{ mode, family string }{ + {"in_process", "atf"}, {"process", "atf"}, {"in_process", "atr"}, {"in_process", "efd"}, + } { + mode, family := profile.mode, profile.family + t.Run(mode+"-"+family, func(t *testing.T) { + atf, efd := family == "atf", family == "efd" + receiver := &parityReceiver{policy: policySettings{AttemptToFix: atf, Quarantined: true, Retry: !efd, EFD: efd}, side: map[string][][]byte{}, requests: map[string]int{}} + server := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/test-management/tests") { + suites := map[string]any{ + "protected_aggregate_test.go": map[string]any{"tests": map[string]any{"TestProtectedAggregate": map[string]any{"properties": map[string]bool{"quarantined": atf, "attempt_to_fix": false}}}}, + "protected_aggregate_test.go/AggregateInner": map[string]any{"tests": map[string]any{"TestProtectedAggregate/TestCritical": map[string]any{"properties": map[string]bool{"attempt_to_fix": atf}}}}, + } + writePolicyJSON(w, map[string]any{"data": map[string]any{"id": "aggregate", "type": "ci_app_libraries_tests_request", "attributes": map[string]any{"modules": map[string]any{"example.com/dd-ci-testing-fixture_test": map[string]any{"suites": suites}}}}}) + return + } + if strings.HasSuffix(r.URL.Path, "/ci/libraries/tests") { + rootKnown := "TestProtectedAggregate" + if efd { + rootKnown = "OtherKnownRoot" + } + tests := map[string]any{"protected_aggregate_test.go": []string{rootKnown}, "protected_aggregate_test.go/AggregateInner": []string{"OtherKnownMethod"}} + writePolicyJSON(w, map[string]any{"data": map[string]any{"id": "aggregate", "type": "ci_app_libraries_tests", "attributes": map[string]any{"tests": map[string]any{"example.com/dd-ci-testing-fixture_test": tests}}}}) + return + } + receiver.handler(w, r) + })) + defer server.Close() + counter := filepath.Join(t.TempDir(), "calls") + env := testEnv("DD_CIVISIBILITY_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_URL="+server.URL, "DD_TRACE_AGENT_URL="+server.URL, "DD_API_KEY=fixture", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE="+mode, "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES=2", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", fmt.Sprintf("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=%t", !efd), fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", efd), "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_MAX_RETRIES=2", "AGGREGATE_COUNTER="+counter) + out, stderr, code := command(t, dir, env, bin, "-test.run=^TestProtectedAggregate$") + want := 0 + if atf { + want = 1 + } + if code != want { + t.Errorf("aggregate protection exit=%d want=%d: %s %s", code, want, out, stderr) + } + data, err := os.ReadFile(counter) + if err != nil { + t.Fatal(err) + } + // Selected Testify methods deliberately fall back to one native + // execution when fresh subtest retry layouts are unavailable. The + // configured ATF assignment must not hide that terminal failure. + atfCalls := "x" + // Only race binaries enable quarantine subtree workers. ATF's + // configured count is total executions, so that path runs twice. + if mode == "process" && os.Getenv("PARITY_TEST_MODE") == "race" { + atfCalls = "xx" + } + if atf && string(data) != atfCalls || family == "atr" && string(data) != "xx" || efd && len(data) < 2 { + t.Fatalf("%s callbacks=%q", family, data) + } + childEvents, failedEvents, passedEvents := 0, 0, 0 + fallbackSeen := false + parentFinal, childFinal := "", "" + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestProtectedAggregate/TestCritical" { + childEvents++ + stack, _ := meta["error.stack"].(string) + fallbackSeen = fallbackSeen || strings.Contains(stack, "runRetryAttemptCapabilityFallback") + if meta["test.status"] == "fail" { + failedEvents++ + } + if meta["test.status"] == "pass" { + passedEvents++ + } + if value, ok := meta["test.final_status"].(string); ok { + childFinal = value + } + if meta["test.test_management.prevent_failure_suppression"] != "true" || atf && meta["test.test_management.is_quarantined"] != "true" { + t.Fatalf("protected child assignment/annotation lost: %v", meta) + } + if atf && meta["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("ATF owner lost: %v", meta) + } + } + if meta["test.name"] == "TestProtectedAggregate" { + parentFinal, _ = meta["test.final_status"].(string) + } + } + if childEvents != len(data) || failedEvents != 1 || !atf && passedEvents < 1 || atf && passedEvents != len(atfCalls)-1 { + t.Fatalf("callbacks=%q events=%d failed=%d passed=%d", data, childEvents, failedEvents, passedEvents) + } + if atf { + if len(atfCalls) == 1 && !fallbackSeen { + t.Fatal("single ATF execution lacked actual native fallback stack evidence") + } + if len(atfCalls) > 1 && fallbackSeen { + t.Fatal("worker ATF continuations unexpectedly took native fallback") + } + if childFinal != "fail" || parentFinal != "fail" { + t.Fatalf("aggregate failure masked: childFinal=%q parentFinal=%q", childFinal, parentFinal) + } + } else if childFinal != "pass" || parentFinal == "fail" { + t.Fatalf("legitimate %s recovery stayed protected: child=%q parent=%q", family, childFinal, parentFinal) + } + }) + } +} diff --git a/ddto/integration/mini_exclusions_callback_test.go b/ddto/integration/mini_exclusions_callback_test.go new file mode 100644 index 000000000..5ca36e21d --- /dev/null +++ b/ddto/integration/mini_exclusions_callback_test.go @@ -0,0 +1,87 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +func checkMiniTestifyNativeCallback(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"callback_test.go": `package fixture_test +import("os";"testing";"github.com/stretchr/testify/suite") +//ddtest:no_auto_retry +//ddtest:prevent_failure_suppression +func TestCriticalCallback(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("CALLBACK_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};f.WriteString("x");f.Close();t.Error("protected callback") +} +//ddtest:prevent_failure_suppression +func TestProtectedCallback(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("CALLBACK_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};f.WriteString("x");f.Close();t.Error("protected callback without NoRetry") +} +type CallbackSuite struct{suite.Suite} +func(s *CallbackSuite) TestMethod(){if os.Getenv("PROTECTED_ONLY")=="true"{s.T().Run("critical",TestProtectedCallback)}else{s.T().Run("critical",TestCriticalCallback)}} +func TestCallbackSuite(t *testing.T){suite.Run(t,new(CallbackSuite))} +type CallbackFirst struct{suite.Suite} +//ddtest:no_auto_retry +func(s *CallbackFirst) TestMethod(){s.T().Run("critical",TestCriticalCallback)} +type CallbackSecond struct{suite.Suite} +//ddtest:no_auto_retry +func(s *CallbackSecond) TestMethod(){s.T().Run("critical",TestCriticalCallback)} +func TestSwitchCallback(t *testing.T){ + f,e:=os.OpenFile(os.Getenv("PARENT_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if e==nil{f.Close();suite.Run(t,new(CallbackFirst))}else if os.IsExist(e){suite.Run(t,new(CallbackSecond))}else{t.Fatal(e)} +} +`}) + bin := filepath.Join(t.TempDir(), executableName("callback.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, tc := range []struct { + name, root string + managed bool + }{ + {"ordinary", "TestCallbackSuite", false}, + {"quarantined", "TestCallbackSuite", true}, + {"switch-declarations", "TestSwitchCallback", false}, + } { + // Only race-enabled process isolation executes quarantined selected + // methods; the existing capability fallback does not discover callbacks. + if tc.managed && (mode != "process" || os.Getenv("PARITY_TEST_MODE") != "race") { + continue + } + t.Run(mode+"-"+tc.name, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "counter") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^" + tc.root + "$"}, Policy: policySettings{Retry: !tc.managed, Quarantined: tc.managed, ManagementTarget: tc.root + "/TestMethod", ManagementSuite: "callback_test.go/CallbackSuite"}, Env: []string{"PROTECTED_ONLY=" + fmt.Sprint(tc.managed), "CALLBACK_COUNTER=" + counter, "PARENT_COUNTER=" + filepath.Join(t.TempDir(), "parent"), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=" + fmt.Sprint(!tc.managed), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("exit%d %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "x" { + t.Fatalf("callback count=%q err=%v", data, err) + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != tc.root+"/TestMethod/critical" { + continue + } + count++ + if (meta["test.test_management.no_auto_retry"] == "true") != !tc.managed || meta["test.test_management.prevent_failure_suppression"] != "true" { + t.Fatalf("lost callback policy: %v", meta) + } + } + if count != 1 { + t.Fatalf("callback events=%d", count) + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_cleanup_goexit_test.go b/ddto/integration/mini_exclusions_cleanup_goexit_test.go new file mode 100644 index 000000000..c2675ce1c --- /dev/null +++ b/ddto/integration/mini_exclusions_cleanup_goexit_test.go @@ -0,0 +1,116 @@ +package integration + +import ( + "os" + "path/filepath" + "testing" +) + +func checkMiniExcludedCleanupGoexit(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"cleanup_goexit_test.go": `package fixture_test +import("testing";"os";"runtime") +func terminalCleanupBody(t *testing.T){ + if os.Getenv("TERMINAL_PARALLEL_CHILD")=="true"{ + t.Run("queued",func(t *testing.T){t.Parallel();if err:=os.WriteFile(os.Getenv("TERMINAL_CHILD_ADMITTED"),[]byte("ran"),0600);err!=nil{t.Fatal(err)}}) + } + t.Cleanup(func(){ + switch os.Getenv("TERMINAL_CLEANUP"){ + case "failnow":t.FailNow() + case "skipnow":t.SkipNow() + case "goexit":runtime.Goexit() + } + }) + panic("body terminal sentinel") +} +//ddtest:no_auto_retry +func TestNoRetryTerminal(t *testing.T){terminalCleanupBody(t)} +//ddtest:prevent_failure_suppression +func TestPreventTerminal(t *testing.T){terminalCleanupBody(t)} +func TestTerminalCarrier(t *testing.T){ + if os.Getenv("TERMINAL_DECLARATION")=="TestNoRetryTerminal"{t.Run("selected",TestNoRetryTerminal)}else{t.Run("selected",TestPreventTerminal)} +} +func TestTerminalLater(t *testing.T){if err:=os.WriteFile(os.Getenv("TERMINAL_LATER"),[]byte("ran"),0600);err!=nil{t.Fatal(err)}} +`}) + native := filepath.Join(t.TempDir(), executableName("native-goexit.test")) + out, stderr, code := command(t, dir, testEnv(), "go", exclusionBuildArgs(native, false)...) + if code != 0 { + t.Fatalf("native compile: %d %s %s", code, out, stderr) + } + mini := filepath.Join(t.TempDir(), executableName("mini-goexit.test")) + out, stderr, code = command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(mini, false)...) + if code != 0 { + t.Fatalf("Mini compile: %d %s %s", code, out, stderr) + } + for _, decl := range []struct{ name, annotation string }{{"TestNoRetryTerminal", "no_auto_retry"}, {"TestPreventTerminal", "prevent_failure_suppression"}} { + for _, scope := range []string{"top", "subtest", "queued-child"} { + for _, cleanup := range []string{"failnow", "skipnow", "goexit"} { + for _, target := range []struct{ name, bin string }{{"native", native}, {"Mini", mini}} { + t.Run(decl.name+"/"+scope+"/"+cleanup+"/"+target.name, func(t *testing.T) { + selected := decl.name + if scope == "subtest" { + selected = "TestTerminalCarrier" + } + later := filepath.Join(t.TempDir(), "later") + admitted := filepath.Join(t.TempDir(), "admitted") + parallel := "false" + if scope == "queued-child" { + parallel = "true" + } + receiver, got := runParityCase(t, dir, target.bin, parityCase{ + Args: []string{"-test.run=^(" + selected + "|TestTerminalLater)$"}, + Env: []string{"TERMINAL_CLEANUP=" + cleanup, "TERMINAL_DECLARATION=" + decl.name, "TERMINAL_LATER=" + later, "TERMINAL_CHILD_ADMITTED=" + admitted, "TERMINAL_PARALLEL_CHILD=" + parallel}, + }) + want := 2 + status := "fail" + if scope != "queued-child" { + if cleanup == "failnow" { + want = 1 + } else if cleanup == "skipnow" { + want = 0 + status = "skip" + } + } + if got.code != want { + t.Fatalf("terminal cleanup exit=%d want%d: %s %s", got.code, want, got.out, got.stderr) + } + _, err := os.Stat(later) + if want < 2 { + if err != nil { + t.Fatalf("next selected test did not run: %v", err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("next selected test ran after terminal: %v", err) + } + if _, err := os.Stat(admitted); !os.IsNotExist(err) { + t.Fatalf("queued parallel child ran after terminal: %v", err) + } + if target.name == "Mini" { + name := decl.name + if scope == "subtest" { + name = "TestTerminalCarrier/selected" + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != name { + continue + } + count++ + if meta["test.status"] != status || meta["test.test_management."+decl.annotation] != "true" { + t.Fatalf("terminal cleanup event=%v", meta) + } + } + if count != 1 { + t.Fatalf("terminal cleanup events=%d want1", count) + } + } + }) + } + } + } + } +} diff --git a/ddto/integration/mini_exclusions_declarations_test.go b/ddto/integration/mini_exclusions_declarations_test.go new file mode 100644 index 000000000..169485c66 --- /dev/null +++ b/ddto/integration/mini_exclusions_declarations_test.go @@ -0,0 +1,430 @@ +package integration + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// An imported declaring receiver and its promoted wrappers keep one policy. +// Explicit overrides and unrelated declarations with the same name do not. +func checkMiniImportedPromotedExclusions(t *testing.T) { + dir, driver := prepareMiniFixture(t) + helper := installExternalTestifyHelper(t, dir) + // Dots in the last path component are escaped in compiler runtime names. + for _, modDir := range []string{dir, helper} { + modPath := filepath.Join(modDir, "go.mod") + data, err := os.ReadFile(modPath) + if err != nil { + t.Fatal(err) + } + data = []byte(strings.ReplaceAll(string(data), "example.com/external-testkit", "example.com/external-testkit.v1")) + if err := os.WriteFile(modPath, data, 0600); err != nil { + t.Fatal(err) + } + } + writeBuildFixture(t, helper, map[string]string{"base.go": `package testkit +import("os";"path/filepath";"strings";"testing";"github.com/stretchr/testify/suite") +type Base struct{suite.Suite} +func record(t *testing.T){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(t.Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} +} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func(s (Alias)) TestCritical(){record(s.T());s.T().Error("protected imported declaration")} +//ddtest:no_auto_retry +func(s (Base)) TestValue(){record(s.T())} +type hiddenSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *hiddenSuite) TestHidden(){record(s.T());s.T().Error("private imported receiver")} +func NewHidden() suite.TestingSuite{return new(hiddenSuite)} +`, "aliases.go": `package testkit +type Alias = Intermediate +type Intermediate = *Base +`}) + writeBuildFixture(t, dir, map[string]string{"declarations_test.go": `package fixture_test +import("os";"path/filepath";"strings";"testing";"example.com/external-testkit.v1";"github.com/stretchr/testify/suite") +type Derived struct{testkit.Base} +type Override struct{testkit.Base} +type Ordinary struct{suite.Suite} +type LocalPointer struct{suite.Suite} +type LocalAlias = LocalPointerAlias +type LocalPointerAlias = *LocalPointer +//ddtest:no_auto_retry +func(s (LocalAlias)) TestLocalPointer(){recordDeclaration(s.T());s.T().Error("local pointer alias")} + +func recordDeclaration(t *testing.T){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(t.Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} +} +func(s *Override) TestCritical(){recordDeclaration(s.T());s.T().Error("ordinary override")} +func(s *Ordinary) TestCritical(){recordDeclaration(s.T());s.T().Error("ordinary independent declaration")} +func(s *Ordinary) TestValue(){recordDeclaration(s.T())} +func TestLocalPointer(t *testing.T){suite.Run(t,new(LocalPointer))} +func TestDerived(t *testing.T){suite.Run(t,new(Derived))} +func TestOverride(t *testing.T){suite.Run(t,new(Override))} +func TestOrdinaryDeclaration(t *testing.T){suite.Run(t,new(Ordinary))} +func TestHiddenDeclaration(t *testing.T){suite.Run(t,testkit.NewHidden())} +`}) + bin := filepath.Join(t.TempDir(), executableName("declarations.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + // Declaration registrations must stay in the consuming binary, including an + // imported package which supplies no tests and whose source is now absent. + if err := os.Rename(filepath.Join(helper, "base.go"), filepath.Join(helper, "base.go.saved")); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counters := t.TempDir() + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^Test(Derived|Override|OrdinaryDeclaration|HiddenDeclaration|LocalPointer)$"}, Policy: policySettings{Retry: true, Quarantined: true, ManagementTarget: "TestDerived/TestCritical", ManagementSuite: "/Derived"}, Env: []string{"DECLARATION_COUNTER=" + counters, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + counts := map[string]string{ + "TestDerived/TestCritical": "x", "TestDerived/TestValue": "x", + "TestHiddenDeclaration/TestHidden": "x", + "TestLocalPointer/TestLocalPointer": "x", + "TestOverride/TestCritical": "xxx", "TestOverride/TestValue": "x", + "TestOrdinaryDeclaration/TestCritical": "xxx", "TestOrdinaryDeclaration/TestValue": "xxx", + } + for name, want := range counts { + data, err := os.ReadFile(filepath.Join(counters, strings.ReplaceAll(name, "/", "_"))) + if err != nil || string(data) != want { + t.Fatalf("%s calls=%q want=%q err=%v", name, data, want, err) + } + } + protected, value, hidden := 0, 0, 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if name == "TestDerived/TestCritical" { + protected++ + for _, key := range []string{"prevent_failure_suppression", "no_auto_retry", "is_quarantined"} { + if meta["test.test_management."+key] != "true" { + t.Fatalf("imported promoted declaration lost %s: %v", key, meta) + } + } + } else if name == "TestHiddenDeclaration/TestHidden" || name == "TestLocalPointer/TestLocalPointer" { + hidden++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("unexported imported receiver lost declaration: %v", meta) + } + } else if name == "TestDerived/TestValue" || name == "TestOverride/TestValue" { + value++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("value receiver declaration lost: %v", meta) + } + if meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("independent method inherited sibling annotation: %v", meta) + } + } else if strings.HasPrefix(name, "TestOverride/") || strings.HasPrefix(name, "TestOrdinaryDeclaration/") { + for _, key := range []string{"prevent_failure_suppression", "no_auto_retry"} { + if meta["test.test_management."+key] != nil { + t.Fatalf("override or unrelated declaration leaked %s: %v", key, meta) + } + } + } + } + if protected != 1 || value != 2 || hidden != 2 { + t.Fatalf("protected events=%d value events=%d hidden events=%d", protected, value, hidden) + } + }) + } +} + +// Cgo's compiler-selected Go sources participate in the same declaration scan +// as ordinary files, for both the local package and an imported suite helper. +func checkMiniCgoImportedExclusions(t *testing.T) { + dir, driver := prepareMiniFixture(t) + out, stderr, code := command(t, dir, testEnv(), "go", "env", "CGO_ENABLED") + if code != 0 { + t.Fatalf("native cgo configuration: %s %s", out, stderr) + } + if strings.TrimSpace(out) != "1" { + t.Skip("native toolchain disables cgo") + } + helper := installExternalTestifyHelper(t, dir) + writeBuildFixture(t, helper, map[string]string{"cgo.go": `package testkit +import "C" +import("os";"path/filepath";"strings";"github.com/stretchr/testify/suite") +type CgoBase struct{suite.Suite} +type CgoAlias = CgoBase +//ddtest:no_auto_retry +func(s *CgoAlias) TestCgo(){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(s.T().Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + s.T().Error("imported cgo annotation") +} +`}) + writeBuildFixture(t, dir, map[string]string{"cgo.go": `package fixture +import "C" +import("os";"path/filepath";"strings";"github.com/stretchr/testify/suite") +type LocalCgo struct{suite.Suite} +type LocalAlias = LocalCgo +//ddtest:no_auto_retry +func(s *LocalAlias) TestCgo(){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(s.T().Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + s.T().Error("local cgo annotation") +} +`, "cgo_test.go": `package fixture +import("testing";"example.com/external-testkit";"github.com/stretchr/testify/suite") +func TestImportedCgo(t *testing.T){suite.Run(t,new(testkit.CgoBase))} +func TestLocalCgo(t *testing.T){suite.Run(t,new(LocalCgo))} +`}) + bin := filepath.Join(t.TempDir(), executableName("cgo-declarations.test")) + out, stderr, code = command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile native cgo fixture %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counters := t.TempDir() + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^Test(Imported|Local)Cgo$"}, Policy: policySettings{Retry: true}, Env: []string{"DECLARATION_COUNTER=" + counters, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + for _, name := range []string{"TestImportedCgo/TestCgo", "TestLocalCgo/TestCgo"} { + data, err := os.ReadFile(filepath.Join(counters, strings.ReplaceAll(name, "/", "_"))) + if err != nil || string(data) != "x" { + t.Fatalf("%s calls=%q want=x err=%v", name, data, err) + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != name { + continue + } + count++ + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.is_retry"] != nil || meta["test.retry_reason"] != nil { + t.Fatalf("cgo declaration tags: %v", meta) + } + } + if count != 1 { + t.Fatalf("%s emitted %d events, want one", name, count) + } + } + }) + } +} + +// A runtime dependency's internal test package cannot import testopt: doing so +// would create a cycle through the dependency under test. Its bootstrap and +// declaration registrations must both live in the external test package. +func checkMiniRuntimeDependencyExclusions(t *testing.T) { + _, driver := prepareMiniFixture(t) + root, err := filepath.Abs("..") + if err != nil { + t.Fatal(err) + } + sources := t.TempDir() + writeBuildFixture(t, sources, map[string]string{"annotation_test.go": `package minitracer +import("os";"testing") +//ddtest:no_auto_retry +func TestAnnotatedRuntimeDependency(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("DECLARATION_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} + t.Error("annotated internal runtime-dependency test") +} +`}) + encoded, err := json.Marshal(map[string]any{"Replace": map[string]string{filepath.Join(root, "internal", "minitracer", "annotation_test.go"): filepath.Join(sources, "annotation_test.go")}}) + if err != nil { + t.Fatal(err) + } + overlay := filepath.Join(sources, "overlay.json") + if err := os.WriteFile(overlay, encoded, 0600); err != nil { + t.Fatal(err) + } + bin := filepath.Join(t.TempDir(), executableName("runtime-dependency.test")) + args := exclusionBuildArgs(bin, true) + args = append(args[:len(args)-1], "-overlay="+overlay, "./internal/minitracer") + out, stderr, code := command(t, root, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, args...) + if code != 0 { + t.Fatalf("compile runtime-dependency internal annotation %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, root, bin, parityCase{Args: []string{"-test.run=^TestAnnotatedRuntimeDependency$"}, Policy: policySettings{Retry: true}, Env: []string{"DECLARATION_COUNTER=" + counter, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "x" { + t.Fatalf("calls=%q err=%v", data, err) + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestAnnotatedRuntimeDependency" { + continue + } + count++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("runtime-dependency declaration lost: %v", meta) + } + } + if count != 1 { + t.Fatalf("events=%d want=1", count) + } + }) + } +} + +func checkMiniGenericDeclarations(t *testing.T) { + dir, driver := prepareMiniFixture(t) + helper := installExternalTestifyHelper(t, dir) + writeBuildFixture(t, helper, map[string]string{"generic.go": `package testkit +import("os";"path/filepath";"strings";"testing";"github.com/stretchr/testify/suite") +type hiddenGeneric[T any] struct{suite.Suite} +func genericRecord(t *testing.T){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(t.Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} +} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func(s (*hiddenGeneric[T])) TestCritical(){genericRecord(s.T());s.T().Error("imported generic critical")} +//ddtest:no_auto_retry +func(s (hiddenGeneric[T])) TestValue(){genericRecord(s.T())} +func(s *hiddenGeneric[T]) TestOrdinary(){genericRecord(s.T())} +func NewGeneric[T any]() suite.TestingSuite{return new(hiddenGeneric[T])} +`}) + writeBuildFixture(t, dir, map[string]string{"generic_test.go": `package fixture_test +import("os";"path/filepath";"strings";"testing";"example.com/external-testkit";"github.com/stretchr/testify/suite") +type LocalGeneric[A,B any] struct{suite.Suite} +type UnrelatedGeneric[T any] struct{suite.Suite} +func(s *UnrelatedGeneric[T]) TestCritical(){recordGeneric(s.T());s.T().Error("ordinary unrelated generic")} +func(s UnrelatedGeneric[T]) TestValue(){recordGeneric(s.T())} + +func recordGeneric(t *testing.T){ + path:=filepath.Join(os.Getenv("DECLARATION_COUNTER"),strings.ReplaceAll(t.Name(),"/","_")) + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)} +} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func(s (*LocalGeneric[A,B])) TestCritical(){recordGeneric(s.T());s.T().Error("local generic critical")} +//ddtest:no_auto_retry +func(s (LocalGeneric[A,B])) TestValue(){recordGeneric(s.T())} +func(s *LocalGeneric[A,B]) TestOrdinary(){recordGeneric(s.T())} +func TestUnrelatedGenericInt(t *testing.T){recordGeneric(t);suite.Run(t,new(UnrelatedGeneric[int]))} +func TestLocalGenericInt(t *testing.T){recordGeneric(t);suite.Run(t,new(LocalGeneric[int,string]))} +func TestLocalGenericString(t *testing.T){recordGeneric(t);suite.Run(t,new(LocalGeneric[string,int]))} +func TestImportedGenericInt(t *testing.T){recordGeneric(t);suite.Run(t,testkit.NewGeneric[int]())} +func TestImportedGenericString(t *testing.T){recordGeneric(t);suite.Run(t,testkit.NewGeneric[string]())} +`}) + bin := filepath.Join(t.TempDir(), executableName("generic-declarations.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile generic receivers %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counters := t.TempDir() + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^Test(Local|Imported|Unrelated)Generic(Int|String)$"}, Policy: policySettings{Retry: true}, Env: []string{"DECLARATION_COUNTER=" + counters, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + // An unrelated generic receiver with identical method names keeps + // ordinary retry behavior and receives no declaration flags. + for _, suffix := range []string{"", "/TestCritical", "/TestValue"} { + name := "TestUnrelatedGenericInt" + suffix + data, err := os.ReadFile(filepath.Join(counters, strings.ReplaceAll(name, "/", "_"))) + if err != nil || string(data) != "xxx" { + t.Fatalf("%s callbacks=%q want=xxx err=%v", name, data, err) + } + } + unrelatedEvents := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if !strings.HasPrefix(name, "TestUnrelatedGenericInt/") { + continue + } + unrelatedEvents++ + if meta["test.test_management.no_auto_retry"] != nil || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("unrelated generic inherited annotation: %v", meta) + } + } + // Ordinary process-worker subtest spans retain the existing replay + // scope: callback counters above prove retries even when only the + // initial ordinary event is emitted. Excluded spans stay exact below. + ordinaryEvents := 3 + if mode == "process" { + ordinaryEvents = 1 + } + if unrelatedEvents != 2*ordinaryEvents { + t.Fatalf("unrelated generic events=%d want=%d", unrelatedEvents, 2*ordinaryEvents) + } + for _, root := range []string{"TestLocalGenericInt", "TestLocalGenericString", "TestImportedGenericInt", "TestImportedGenericString"} { + for suffix, want := range map[string]string{"": "xxx", "/TestCritical": "x", "/TestValue": "x", "/TestOrdinary": "xxx"} { + name := root + suffix + data, err := os.ReadFile(filepath.Join(counters, strings.ReplaceAll(name, "/", "_"))) + if err != nil || string(data) != want { + t.Fatalf("%s callbacks=%q want=%q err=%v", name, data, want, err) + } + } + counts := map[string]int{} + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if !strings.HasPrefix(name, root+"/") { + continue + } + method := strings.TrimPrefix(name, root+"/") + counts[method]++ + critical := method == "TestCritical" + excluded := critical || method == "TestValue" + if excluded { + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("generic annotation lost: %v", meta) + } + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode"} { + if meta[key] != nil { + t.Fatalf("generic excluded test retried: %v", meta) + } + } + } else if meta["test.test_management.no_auto_retry"] != nil { + t.Fatalf("generic sibling inherited exclusion: %v", meta) + } + if critical { + if meta["test.test_management.prevent_failure_suppression"] != "true" || meta["test.status"] != "fail" { + t.Fatalf("generic critical policy lost: %v", meta) + } + } else if meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("generic sibling inherited critical policy: %v", meta) + } + } + if counts["TestCritical"] != 1 || counts["TestValue"] != 1 || counts["TestOrdinary"] != ordinaryEvents { + t.Fatalf("%s event counts=%v", root, counts) + } + } + }) + } +} diff --git a/ddto/integration/mini_exclusions_omitted_test.go b/ddto/integration/mini_exclusions_omitted_test.go new file mode 100644 index 000000000..7e532dc39 --- /dev/null +++ b/ddto/integration/mini_exclusions_omitted_test.go @@ -0,0 +1,104 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniOmittedExcludedFailure(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"omitted_test.go": `package fixture_test +import("os";"testing";"flag";"strings";"github.com/stretchr/testify/suite";"github.com/DataDog/ddtest/ddto/testopt") +type OmittedSuite struct{suite.Suite} +func(s *OmittedSuite)TestCritical(){if err:=os.WriteFile(os.Getenv("OMITTED_CALLS"),[]byte("x"),0600);err!=nil{s.T().Fatal(err)};s.T().Error("sole excluded execution fails")} +func init(){testopt.RegisterTestExclusions((*OmittedSuite).TestCritical,testopt.TestExclusions{NoAutoRetry:os.Getenv("OMITTED_NO_RETRY")=="true",PreventFailureSuppression:os.Getenv("OMITTED_PROTECTED")=="true"})} +func TestOmittedRoot(t *testing.T){if strings.Contains(flag.Lookup("test.run").Value.String(),"TestCritical"){suite.Run(t,new(OmittedSuite));return};f,err:=os.OpenFile(os.Getenv("OMITTED_MARKER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.WriteString("initial");f.Close();if os.Getenv("OMITTED_LATE")=="true"{if os.Getenv("OMITTED_EARLY_PASS")=="true"{return};t.Error("retry before first discovery");return};suite.Run(t,new(OmittedSuite));t.Error("retry parent independently");return};if !os.IsExist(err){t.Fatal(err)};if os.Getenv("OMITTED_LATE")=="true"{data,err:=os.ReadFile(os.Getenv("OMITTED_MARKER"));if err!=nil{t.Fatal(err)};if string(data)=="initial"{if err:=os.WriteFile(os.Getenv("OMITTED_MARKER"),[]byte("discovered"),0600);err!=nil{t.Fatal(err)};suite.Run(t,new(OmittedSuite));t.Error("retry after first discovery");return}};if os.Getenv("OMITTED_REDISCOVER")=="true"{suite.Run(t,new(OmittedSuite))}} +`}) + bin := filepath.Join(t.TempDir(), executableName("omitted.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, family := range []string{"atr", "efd"} { + for _, flags := range []struct{ noRetry, protected, quarantined, childQuarantined, rediscover, late, earlyPass bool }{{true, false, false, false, false, false, false}, {true, true, false, false, false, false, false}, {true, true, true, false, false, false, false}, {false, true, false, false, false, false, false}, {true, false, false, true, false, false, false}, {true, false, false, true, true, false, false}, {true, false, false, true, false, true, false}, {true, false, false, false, false, true, true}, {true, true, false, false, false, true, true}, {false, true, false, false, false, true, true}, {true, false, false, true, false, true, true}} { + if flags.earlyPass && family != "efd" { + continue + } + t.Run(fmt.Sprintf("%s/%s/noRetry=%t/protected=%t/quarantined=%t/childQuarantined=%t/rediscover=%t/late=%t/earlyPass=%t", mode, family, flags.noRetry, flags.protected, flags.quarantined, flags.childQuarantined, flags.rediscover, flags.late, flags.earlyPass), func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + target, suiteName := "TestOmittedRoot", "omitted_test.go" + if flags.childQuarantined { + target = "TestOmittedRoot/TestCritical" + suiteName = "omitted_test.go/OmittedSuite" + } + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestOmittedRoot$"}, Policy: policySettings{Retry: family == "atr", EFD: family == "efd", Quarantined: flags.quarantined || flags.childQuarantined, ManagementTarget: target, ManagementSuite: suiteName}, Env: []string{"OMITTED_CALLS=" + calls, "OMITTED_MARKER=" + filepath.Join(t.TempDir(), "marker"), fmt.Sprintf("OMITTED_EARLY_PASS=%t", flags.earlyPass), fmt.Sprintf("OMITTED_LATE=%t", flags.late), fmt.Sprintf("OMITTED_REDISCOVER=%t", flags.rediscover), fmt.Sprintf("OMITTED_NO_RETRY=%t", flags.noRetry), fmt.Sprintf("OMITTED_PROTECTED=%t", flags.protected), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, fmt.Sprintf("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=%t", family == "atr"), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", family == "efd"), "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_MAX_RETRIES=2"}}) + want := 0 + if flags.noRetry && !flags.childQuarantined { + want = 1 + } + if got.code != want { + t.Errorf("retained sole failure exit=%d want%d: %s %s", got.code, want, got.out, got.stderr) + } + data, err := os.ReadFile(calls) + if err != nil && !(flags.childQuarantined && os.IsNotExist(err)) || !flags.childQuarantined && string(data) != "x" { + t.Fatalf("sole declaration execution=%q err=%v", data, err) + } + parentFinal := "" + childEvents := 0 + parentPasses := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestOmittedRoot/TestCritical" { + childEvents++ + if flags.childQuarantined { + if string(data) == "x" && !strings.Contains(fmt.Sprint(meta["error.message"]), "sole excluded execution fails") { + t.Fatalf("expected actual quarantined declaration failure, got %v", meta) + } + if meta["test.test_management.is_quarantined"] != "true" || meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("quarantine assignment changed: %v", meta) + } + } else if meta["test.status"] != "fail" { + t.Fatalf("excluded initial failure=%v", meta) + } + } + if meta["test.name"] == "TestOmittedRoot" { + if meta["test.status"] == "pass" { + parentPasses++ + } + if flags.earlyPass && meta["test.has_failed_all_retries"] == "true" { + t.Fatalf("earlier passing attempt was erased: %v", meta) + } + if status, ok := meta["test.final_status"].(string); ok { + parentFinal = status + } + } + } + final := "pass" + if flags.noRetry && !flags.childQuarantined { + final = "fail" + } + wantChildren := 1 + // Ordinary subtests discovered only in a process retry retain the existing + // excluded-only event transport boundary. + if mode == "process" && flags.earlyPass && !flags.noRetry { + wantChildren = 0 + } + if flags.earlyPass && parentPasses == 0 { + t.Fatal("initial passing parent execution was erased") + } + if parentFinal != final || childEvents != wantChildren { + t.Fatalf("retained verdict final=%q want%s childEvents=%d", parentFinal, final, childEvents) + } + }) + } + } + } +} diff --git a/ddto/integration/mini_exclusions_owner_test.go b/ddto/integration/mini_exclusions_owner_test.go new file mode 100644 index 000000000..b91d83a1b --- /dev/null +++ b/ddto/integration/mini_exclusions_owner_test.go @@ -0,0 +1,35 @@ +package integration + +import ( + "os" + "path/filepath" + "testing" +) + +func checkMiniInheritedOwnerCache(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"owner_test.go": `package fixture_test +import("os";"testing") +func sharedOwnerRow(t *testing.T){f,err:=os.OpenFile(os.Getenv("OWNER_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};f.WriteString("r");f.Close();if _,err:=os.Stat(os.Getenv("OWNER_SECOND"));err==nil{t.Error("second declaration row fails")}} +//ddtest:no_auto_retry +func TestFirstOwner(t *testing.T){t.Run("row",sharedOwnerRow)} +//ddtest:no_auto_retry +func TestSecondOwner(t *testing.T){t.Run("row",sharedOwnerRow)} +func TestOwnerSwitch(t *testing.T){f,err:=os.OpenFile(os.Getenv("OWNER_SECOND"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.Close();os.Remove(os.Getenv("OWNER_SECOND"));t.Run("case",TestFirstOwner);os.WriteFile(os.Getenv("OWNER_SECOND"),[]byte("second"),0600);t.Error("retry parent");return};if !os.IsExist(err){t.Fatal(err)};t.Run("case",TestSecondOwner)} +`}) + bin := filepath.Join(t.TempDir(), executableName("owner.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + _, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestOwnerSwitch$"}, Policy: policySettings{Retry: true}, Env: []string{"OWNER_CALLS=" + calls, "OWNER_SECOND=" + filepath.Join(t.TempDir(), "second"), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + data, err := os.ReadFile(calls) + if got.code != 1 || err != nil || string(data) != "rr" { + t.Fatalf("inherited declaration collision calls=%q err=%v exit=%d %s %s", data, err, got.code, got.out, got.stderr) + } + }) + } +} diff --git a/ddto/integration/mini_exclusions_parallel_cleanup_test.go b/ddto/integration/mini_exclusions_parallel_cleanup_test.go new file mode 100644 index 000000000..87cefd668 --- /dev/null +++ b/ddto/integration/mini_exclusions_parallel_cleanup_test.go @@ -0,0 +1,199 @@ +package integration + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniExcludedParallelCleanup(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"parallel_cleanup_test.go": `package fixture_test +import("testing";"os";"runtime";"sync") +func parallelExcludedCleanup(t *testing.T){ + t.Parallel() + t.Run("parallel-child",func(t *testing.T){t.Parallel()}) + t.Cleanup(func(){ + switch os.Getenv("CLEANUP_OPERATION"){ + case "setenv": t.Setenv("DDTEST_EXCLUDED_PARALLEL_CLEANUP","changed") + case "chdir": t.Chdir(os.Getenv("CLEANUP_TARGET_DIR")) + } + if err:=os.WriteFile(os.Getenv("CLEANUP_RETURNED"),[]byte("returned"),0600);err!=nil{t.Fatal(err)} + }) +} +//ddtest:no_auto_retry +func TestParallelNoRetry(t *testing.T){parallelExcludedCleanup(t)} +//ddtest:prevent_failure_suppression +func TestParallelPrevent(t *testing.T){parallelExcludedCleanup(t)} +var cleanupLeafDone=make(chan struct{}) +var cleanupBlockerActive=make(chan struct{}) +var cleanupParentDone=make(chan struct{}) +var cleanupRunReturned=make(chan struct{}) +var cleanupBlockerOnce sync.Once +//ddtest:no_auto_retry +func TestCleanupSchedulerA(t *testing.T){ + t.Parallel() + t.Run("sequential",func(t *testing.T){ + t.Run("leaf",func(t *testing.T){ + t.Parallel() + if os.Getenv("CLEANUP_SCHEDULER_PANIC")=="body-panic"{ + if err:=os.WriteFile(os.Getenv("CLEANUP_CHILD_ADMITTED"),[]byte("ran"),0600);err!=nil{t.Fatal(err)} + } + t.Cleanup(func(){ + close(cleanupLeafDone) + <-cleanupBlockerActive + for i:=0;i<10;i++{runtime.Gosched()} + }) + }) + t.Cleanup(func(){ + switch os.Getenv("CLEANUP_SCHEDULER_PANIC"){ + case "panic": panic("native sequential cleanup panic") + case "skip": t.SkipNow() + case "failnow": t.FailNow() + case "goexit": runtime.Goexit() + } + close(cleanupParentDone) + }) + if os.Getenv("CLEANUP_SCHEDULER_PANIC")=="body-panic"{panic("native sequential body panic")} + }) + close(cleanupRunReturned) +} +func TestCleanupSchedulerB(t *testing.T){ + t.Parallel() + <-cleanupLeafDone + for _,name:=range []string{"first","second"} { + t.Run(name,func(t *testing.T){ + t.Parallel() + cleanupBlockerOnce.Do(func(){close(cleanupBlockerActive)}) + mode:=os.Getenv("CLEANUP_SCHEDULER_PANIC") + if mode=="skip"||mode=="failnow"||mode=="goexit"{<-cleanupRunReturned}else{<-cleanupParentDone} + }) + } +} +`}) + native := filepath.Join(t.TempDir(), executableName("native-parallel-cleanup.test")) + out, stderr, code := command(t, dir, testEnv(), "go", exclusionBuildArgs(native, false)...) + if code != 0 { + t.Fatalf("native compile: %d %s %s", code, out, stderr) + } + mini := filepath.Join(t.TempDir(), executableName("mini-parallel-cleanup.test")) + out, stderr, code = command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(mini, false)...) + if code != 0 { + t.Fatalf("Mini compile: %d %s %s", code, out, stderr) + } + for _, tc := range []struct{ name, annotation string }{ + {"TestParallelNoRetry", "no_auto_retry"}, + {"TestParallelPrevent", "prevent_failure_suppression"}, + } { + for _, operation := range []string{"setenv", "chdir", "ok"} { + for _, target := range []struct{ name, bin string }{{"native", native}, {"Mini", mini}} { + t.Run(tc.name+"/"+operation+"/"+target.name, func(t *testing.T) { + returned := filepath.Join(t.TempDir(), "returned") + receiver, got := runParityCase(t, dir, target.bin, parityCase{ + Args: []string{"-test.run=^" + tc.name + "$", "-test.parallel=1"}, + Env: []string{"CLEANUP_OPERATION=" + operation, "CLEANUP_TARGET_DIR=" + t.TempDir(), "CLEANUP_RETURNED=" + returned}, + }) + want := 2 + if operation == "ok" { + want = 0 + } + if got.code != want { + t.Fatalf("cleanup exit=%d want%d: %s %s", got.code, want, got.out, got.stderr) + } + _, err := os.Stat(returned) + if operation == "ok" { + if err != nil { + t.Fatalf("ordinary parallel cleanup did not finish: %v", err) + } + } else { + if !os.IsNotExist(err) { + t.Fatalf("parallel cleanup escaped native guard: %v", err) + } + if !strings.Contains(got.stderr, "can not use t.Parallel") { + t.Fatalf("native parallel guard missing: %s", got.stderr) + } + } + if target.name == "Mini" { + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != tc.name { + continue + } + count++ + status := "fail" + if operation == "ok" { + status = "pass" + } + if meta["test.status"] != status || meta["test.test_management."+tc.annotation] != "true" { + t.Fatalf("parallel cleanup event=%v", meta) + } + } + if count != 1 { + t.Fatalf("parent cleanup events=%d want1", count) + } + } + }) + } + } + } + for _, mode := range []string{"pass", "panic", "body-panic", "skip", "failnow", "goexit"} { + for _, target := range []struct{ name, bin string }{{"native", native}, {"Mini", mini}} { + t.Run("sequential-cleanup-order/"+mode+"/"+target.name, func(t *testing.T) { + admitted := filepath.Join(t.TempDir(), "admitted") + receiver, got := runParityCase(t, dir, target.bin, parityCase{ + Args: []string{"-test.run=^TestCleanupScheduler(A|B)$", "-test.parallel=2", "-test.timeout=10s"}, + Env: []string{"GOMAXPROCS=1", "CLEANUP_SCHEDULER_PANIC=" + mode, "CLEANUP_CHILD_ADMITTED=" + admitted}, + }) + want := 0 + if mode == "panic" || mode == "body-panic" { + want = 2 + } else if mode == "failnow" { + want = 1 + } + if got.code != want { + t.Fatalf("native cleanup scheduling exit=%d want%d: %s %s", got.code, want, got.out, got.stderr) + } + if mode == "panic" && !strings.Contains(got.stderr, "native sequential cleanup panic") { + t.Fatalf("native terminal cleanup panic missing: %s", got.stderr) + } + if target.name == "Mini" && (mode == "skip" || mode == "failnow" || mode == "goexit") { + status := "pass" + if mode == "skip" { + status = "skip" + } else if mode == "failnow" { + status = "fail" + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestCleanupSchedulerA/sequential" { + count++ + if meta["test.status"] != status || meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("cleanup Goexit event=%v want%s", meta, status) + } + } + } + if count != 1 { + t.Fatalf("cleanup Goexit events=%d want1", count) + } + } + if mode == "body-panic" { + if !strings.Contains(got.stderr, "native sequential body panic") { + t.Fatalf("native terminal body panic missing: %s", got.stderr) + } + if _, err := os.Stat(admitted); !os.IsNotExist(err) { + t.Fatalf("parallel child ran after terminal parent body panic: %v", err) + } + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_process_test.go b/ddto/integration/mini_exclusions_process_test.go new file mode 100644 index 000000000..c9b0ab751 --- /dev/null +++ b/ddto/integration/mini_exclusions_process_test.go @@ -0,0 +1,125 @@ +package integration + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniExclusionProcessContinuations(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"continuation_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +func appendContinuation(t *testing.T,path string)int{f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)};data,err:=os.ReadFile(path);if err!=nil{t.Fatal(err)};return len(data)} +type ContinuationOuter struct{suite.Suite} +func(s *ContinuationOuter) TestQuarantine(){suite.Run(s.T(),new(ContinuationFix))} +type ContinuationFix struct{suite.Suite} +func(s *ContinuationFix) TestFix(){n:=appendContinuation(s.T(),os.Getenv("FIX_CALLS"));if os.Getenv("LATE_PROTECTED")!="true"||n>1{suite.Run(s.T(),new(ContinuationLeaf))}} +type ContinuationLeaf struct{suite.Suite} +//ddtest:no_auto_retry +//ddtest:prevent_failure_suppression +func(s *ContinuationLeaf) TestCritical(){appendContinuation(s.T(),os.Getenv("EXCLUDED_CALLS"));if os.Getenv("LATE_PROTECTED")=="true"{s.T().Error("protected continuation-only failure")}} +func TestContinuationRoot(t *testing.T){f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.Close();return};if !os.IsExist(err){t.Fatal(err)};suite.Run(t,new(ContinuationOuter))} +`}) + bin := filepath.Join(t.TempDir(), executableName("continuation.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, "test", "-c", "-race", "-mod=mod", "-o", bin, ".") + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, late := range []bool{false, true} { + t.Run(fmt.Sprintf("late-protected=%t", late), func(t *testing.T) { + receiver := &parityReceiver{policy: policySettings{AttemptToFix: true, Quarantined: true}, side: map[string][][]byte{}, requests: map[string]int{}} + server := newServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/test-management/tests") { + suites := map[string]any{} + for _, entry := range []struct { + suite, name string + properties map[string]bool + }{ + {"continuation_test.go", "TestContinuationRoot", map[string]bool{"attempt_to_fix": true}}, + {"continuation_test.go/ContinuationOuter", "TestContinuationRoot/TestQuarantine", map[string]bool{"quarantined": true, "attempt_to_fix": false}}, + {"continuation_test.go/ContinuationFix", "TestContinuationRoot/TestQuarantine/TestFix", map[string]bool{"attempt_to_fix": true}}, + } { + suites[entry.suite] = map[string]any{"tests": map[string]any{entry.name: map[string]any{"properties": entry.properties}}} + } + writePolicyJSON(w, map[string]any{"data": map[string]any{"id": "continuation", "type": "ci_app_libraries_tests_request", "attributes": map[string]any{"modules": map[string]any{"example.com/dd-ci-testing-fixture_test": map[string]any{"suites": suites}}}}}) + return + } + receiver.handler(w, r) + })) + defer server.Close() + calls, fix, root := filepath.Join(t.TempDir(), "excluded"), filepath.Join(t.TempDir(), "fix"), filepath.Join(t.TempDir(), "root") + env := testEnv("DD_CIVISIBILITY_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_ENABLED=true", "DD_CIVISIBILITY_AGENTLESS_URL="+server.URL, "DD_TRACE_AGENT_URL="+server.URL, "DD_API_KEY=fixture", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES=2", "POC_RETRY_COUNTER="+root, "EXCLUDED_CALLS="+calls, "FIX_CALLS="+fix, fmt.Sprintf("LATE_PROTECTED=%t", late)) + out, stderr, code := command(t, dir, env, bin, "-test.run=^TestContinuationRoot$") + data, err := os.ReadFile(calls) + want := 0 + if late { + want = 1 + } + if code != want || err != nil || string(data) != "x" { + t.Fatalf("continuation cache/outcome: exit=%d want%d calls=%q err=%v %s %s", code, want, data, err, out, stderr) + } + leaf, rootStatus := 0, "" + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if strings.HasSuffix(name, "/TestCritical") { + leaf++ + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.prevent_failure_suppression"] != "true" || meta["test.is_retry"] != nil { + t.Fatalf("excluded continuation event=%v", meta) + } + } + if name == "TestContinuationRoot" { + rootStatus, _ = meta["test.final_status"].(string) + } + } + if leaf != 1 || late && rootStatus != "fail" { + t.Fatalf("continuation events=%d rootfinal=%q", leaf, rootStatus) + } + }) + } +} + +func checkMiniExcludedIsolationRoot(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"isolated_exclusion_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type IsolatedExcludedSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *IsolatedExcludedSuite) TestOnly(){if err:=os.WriteFile(os.Getenv("ISOLATED_CALL"),[]byte("x"),0600);err!=nil{s.T().Fatal(err)}} +func TestIsolatedExcluded(t *testing.T){suite.Run(t,new(IsolatedExcludedSuite))} +`}) + bin := filepath.Join(t.TempDir(), executableName("isolated-exclusion.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, "test", "-c", "-race", "-mod=mod", "-o", bin, ".") + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + calls := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestIsolatedExcluded$/^TestOnly$"}, Policy: policySettings{Quarantined: true, ManagementTarget: "TestIsolatedExcluded/TestOnly", ManagementSuite: "isolated_exclusion_test.go/IsolatedExcludedSuite"}, Env: []string{"ISOLATED_CALL=" + calls, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process"}}) + data, err := os.ReadFile(calls) + if got.code != 0 || err != nil || string(data) != "x" { + t.Fatalf("excluded isolation root rejected: exit%d calls=%q err=%v %s %s", got.code, data, err, got.out, got.stderr) + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestIsolatedExcluded/TestOnly" { + count++ + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.is_quarantined"] != "true" || meta["test.is_retry"] != nil { + t.Fatalf("excluded isolation root event=%v", meta) + } + } + } + if count != 1 { + t.Fatalf("excluded isolation root events=%d", count) + } +} diff --git a/ddto/integration/mini_exclusions_promoted_cache_test.go b/ddto/integration/mini_exclusions_promoted_cache_test.go new file mode 100644 index 000000000..ab1538358 --- /dev/null +++ b/ddto/integration/mini_exclusions_promoted_cache_test.go @@ -0,0 +1,144 @@ +package integration + +import ( + "os" + "path/filepath" + "testing" +) + +// Promoted wrappers keep their reporting identities, while automatic retry +// replay is keyed by the declaring method and its native test path. +func checkMiniPromotedDeclarationCache(t *testing.T) { + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + writeBuildFixture(t, dir, map[string]string{"promoted_cache_test.go": `package fixture_test +import("os";"testing";"github.com/stretchr/testify/suite") +func recordPromoted(t *testing.T,path,letter string)bool{ + previous,err:=os.ReadFile(path);if err!=nil&&!os.IsNotExist(err){t.Fatal(err)} + f,err:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)} + _,err=f.WriteString(letter);f.Close();if err!=nil{t.Fatal(err)} + return len(previous)==0 +} +func runPromoted(t *testing.T){if recordPromoted(t,os.Getenv("METHOD_COUNTER"),"x"){t.Error("initial excluded failure must remain cached")}} +type PointerBase struct{suite.Suite} +//ddtest:no_auto_retry +func(s *PointerBase) TestOnce(){runPromoted(s.T())} +type PointerA struct{PointerBase} +type PointerB struct{PointerBase} +type ValueBase struct{suite.Suite} +//ddtest:no_auto_retry +func(s ValueBase) TestOnce(){runPromoted(s.T())} +type ValueA struct{ValueBase} +type ValueB struct{ValueBase} +type GenericBase[T any] struct{suite.Suite} +//ddtest:no_auto_retry +func(s *GenericBase[T]) TestOnce(){runPromoted(s.T())} +type GenericA struct{GenericBase[int]} +type GenericB struct{GenericBase[string]} +type Override struct{PointerBase} +//ddtest:no_auto_retry +func(s *Override) TestOnce(){recordPromoted(s.T(),os.Getenv("METHOD_COUNTER"),"o");s.T().Error("independent explicit override")} +type Unrelated struct{suite.Suite} +//ddtest:no_auto_retry +func(s *Unrelated) TestOnce(){recordPromoted(s.T(),os.Getenv("METHOD_COUNTER"),"o");s.T().Error("independent unrelated declaration")} +type Ordinary struct{PointerBase} +func(s *Ordinary) TestOnce(){recordPromoted(s.T(),os.Getenv("METHOD_COUNTER"),"o");s.T().Error("independent ordinary override")} +func TestSwitchPromoted(t *testing.T){ + first:=recordPromoted(t,os.Getenv("PARENT_COUNTER"),"p") + switch os.Getenv("PROMOTED_KIND"){ + case "pointer":if first{suite.Run(t,new(PointerA))}else{suite.Run(t,new(PointerB))} + case "value":if first{suite.Run(t,new(ValueA))}else{suite.Run(t,new(ValueB))} + case "generic":if first{suite.Run(t,new(GenericA))}else{suite.Run(t,new(GenericB))} + case "override":if first{suite.Run(t,new(PointerA))}else{suite.Run(t,new(Override))} + case "unrelated":if first{suite.Run(t,new(PointerA))}else{suite.Run(t,new(Unrelated))} + case "ordinary":if first{suite.Run(t,new(PointerA))}else{suite.Run(t,new(Ordinary))} + default:t.Fatal("unknown suite kind") + } +} +`}) + bin := filepath.Join(t.TempDir(), executableName("promoted-cache.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, kind := range []string{"pointer", "value", "generic", "override", "unrelated", "ordinary"} { + t.Run(mode+"-"+kind, func(t *testing.T) { + counters := t.TempDir() + methods := filepath.Join(counters, "methods") + parents := filepath.Join(counters, "parents") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestSwitchPromoted$"}, Policy: policySettings{Retry: true}, Env: []string{"METHOD_COUNTER=" + methods, "PARENT_COUNTER=" + parents, "PROMOTED_KIND=" + kind, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("cached declaration failure recovered: exit%d %s %s", got.code, got.out, got.stderr) + } + want := "x" + if kind == "override" || kind == "unrelated" { + want = "xo" + } + if kind == "ordinary" { + want = "xoo" + } + data, err := os.ReadFile(methods) + if err != nil || string(data) != want { + t.Fatalf("method calls=%q want=%q err=%v", data, want, err) + } + data, err = os.ReadFile(parents) + if err != nil || string(data) != "ppp" { + t.Fatalf("parent calls=%q want=ppp err=%v", data, err) + } + excluded, ordinary := 0, 0 + suites := map[string]int{} + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestSwitchPromoted/TestOnce" { + continue + } + if meta["test.test_management.no_auto_retry"] == "true" { + excluded++ + suiteName, _ := meta["test.suite"].(string) + suites[suiteName]++ + if meta["test.status"] != "fail" { + t.Fatalf("excluded result changed: %v", meta) + } + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode"} { + if meta[key] != nil { + t.Fatalf("excluded actual execution inherited retry: %v", meta) + } + } + } else { + ordinary++ + } + } + wantExcluded := 1 + if kind == "override" || kind == "unrelated" { + wantExcluded = 2 + } + firstSuite := "/PointerA" + if kind == "value" { + firstSuite = "/ValueA" + } + if kind == "generic" { + firstSuite = "/GenericA" + } + if suites[firstSuite] != 1 { + t.Fatalf("reflected wrapper reporting changed: suites=%v want %s once", suites, firstSuite) + } + if kind == "override" && suites["promoted_cache_test.go/Override"] != 1 { + t.Fatalf("override reporting changed: %v", suites) + } + if kind == "unrelated" && suites["promoted_cache_test.go/Unrelated"] != 1 { + t.Fatalf("unrelated reporting changed: %v", suites) + } + if excluded != wantExcluded { + t.Fatalf("excluded events=%d want=%d", excluded, wantExcluded) + } + if kind != "ordinary" && ordinary != 0 { + t.Fatalf("unexpected ordinary events=%d", ordinary) + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_retry_edges_test.go b/ddto/integration/mini_exclusions_retry_edges_test.go new file mode 100644 index 000000000..1f2f70a95 --- /dev/null +++ b/ddto/integration/mini_exclusions_retry_edges_test.go @@ -0,0 +1,460 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func checkMiniDeclarationCacheIdentity(t *testing.T) { + for _, excludedSecond := range []bool{false, true} { + t.Run(fmt.Sprint("second-excluded=", excludedSecond), func(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + annotation := "" + if excludedSecond { + annotation = "//ddtest:no_auto_retry" + } + source := strings.ReplaceAll(`package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +func recordCacheCall(t *testing.T,letter string){f,err:=os.OpenFile(os.Getenv("CACHE_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString(letter);f.Close();if err!=nil{t.Fatal(err)}} +type FirstCacheSuite struct{suite.Suite} +type SecondCacheSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *FirstCacheSuite) TestSame(){recordCacheCall(s.T(),"a")} +__SECOND_ANNOTATION__ +func(s *SecondCacheSuite) TestSame(){recordCacheCall(s.T(),"b");s.T().Error("different declaration must execute")} +func TestSwitch(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err==nil{f.Close();suite.Run(t,new(FirstCacheSuite));t.Error("retry parent");return};if !os.IsExist(err){t.Fatal(err)} + suite.Run(t,new(SecondCacheSuite)) +} +`, "__SECOND_ANNOTATION__", annotation) + writeBuildFixture(t, dir, map[string]string{"cache_suite_test.go": source}) + bin := filepath.Join(t.TempDir(), executableName("cache-identity.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + t.Run(mode, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "calls") + _, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestSwitch$"}, Policy: policySettings{Retry: true}, Env: []string{"CACHE_CALLS=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("different declaration's failure was hidden: %d %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + want := "abb" + if excludedSecond { + want = "ab" + } + if err != nil || string(data) != want { + t.Fatalf("declaration calls=%q want=%q err=%v", data, want, err) + } + }) + } + }) + } +} + +func checkMiniNestedWorkerDeclarations(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"nested_worker_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +func recordNestedCall(t *testing.T,letter string){f,err:=os.OpenFile(os.Getenv("NESTED_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString(letter);f.Close();if err!=nil{t.Fatal(err)}} +type OuterExcludedSuite struct{suite.Suite} +type InnerOrdinarySuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *OuterExcludedSuite) TestOuter(){recordNestedCall(s.T(),"o");suite.Run(s.T(),new(InnerOrdinarySuite))} +func(s *InnerOrdinarySuite) TestInner(){recordNestedCall(s.T(),"i")} +func TestNestedWorker(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err==nil{f.Close();t.Error("fail before discovery");return};if !os.IsExist(err){t.Fatal(err)} + suite.Run(t,new(OuterExcludedSuite)) +} +`}) + bin := filepath.Join(t.TempDir(), executableName("nested-worker.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, efd := range []bool{false, true} { + t.Run(fmt.Sprintf("%s-efd=%t", mode, efd), func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestNestedWorker$"}, Policy: policySettings{Retry: !efd, EFD: efd}, Env: []string{"NESTED_CALLS=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, fmt.Sprintf("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=%t", !efd), fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", efd), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 0 { + t.Fatalf("valid nested worker result failed: %d %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "oi" { + t.Fatalf("nested callback calls=%q: %v", data, err) + } + outer := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestNestedWorker/TestOuter" { + outer++ + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.is_retry"] != nil { + t.Fatalf("outer declaration metadata: %v", meta) + } + } + if meta["test.name"] == "TestNestedWorker/TestOuter/TestInner" && meta["test.test_management.no_auto_retry"] != nil { + t.Fatalf("ordinary nested method inherited exclusion: %v", meta) + } + } + if outer != 1 { + t.Fatalf("outer execution events=%d", outer) + } + }) + } + } +} + +func checkMiniLargeExclusionPayload(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"large_exclusion_test.go": `package fixture_test +import("testing";"os";"fmt";"strings";"github.com/stretchr/testify/suite") +type LargeExcludedSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *LargeExcludedSuite) TestExcluded(){for i:=0;i<4096;i++{s.T().Run(fmt.Sprintf("row-%03d-%s",i,strings.Repeat("x",110)),func(t *testing.T){f,err:=os.OpenFile(os.Getenv("LARGE_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)}})}} +func(s *LargeExcludedSuite) TestRecovery(){f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.Close();s.T().Error("recover on retry")}else if !os.IsExist(err){s.T().Fatal(err)}} +func TestLargeRoot(t *testing.T){if path:=os.Getenv("LARGE_DISCOVERY");path!=""{f,err:=os.OpenFile(path,os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.Close();t.Error("discover excluded rows during retry");return};if !os.IsExist(err){t.Fatal(err)}};suite.Run(t,new(LargeExcludedSuite))} +`}) + bin := filepath.Join(t.TempDir(), executableName("large-exclusion.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, late := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/late=%t", mode, late), func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "calls") + discovery := "" + if late { + discovery = filepath.Join(t.TempDir(), "discovery") + } + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestLargeRoot$"}, Policy: policySettings{Retry: true}, Env: []string{"LARGE_CALLS=" + counter, "LARGE_DISCOVERY=" + discovery, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 0 { + t.Fatalf("large excluded cache/events blocked ordinary retry: %d %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || len(data) != 4096 { + t.Fatalf("excluded row callbacks=%d want4096 err=%v", len(data), err) + } + seen := map[string]bool{} + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if name != "TestLargeRoot/TestExcluded" && !strings.HasPrefix(name, "TestLargeRoot/TestExcluded/") { + continue + } + if seen[name] || meta["test.test_management.no_auto_retry"] != "true" || meta["test.is_retry"] != nil { + t.Fatalf("excluded row duplicated or declaration lost: %v", meta) + } + seen[name] = true + } + if len(seen) != 4097 { + t.Fatalf("excluded execution events=%d want4097", len(seen)) + } + }) + } + } +} + +func checkMiniExcludedCleanupTerminal(t *testing.T) { + for _, directive := range []string{"no_auto_retry", "prevent_failure_suppression"} { + t.Run(directive, func(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"cleanup_terminal_test.go": `package fixture_test +import("testing";"os") +//ddtest:` + directive + ` +func TestCleanupTerminal(t *testing.T){t.Cleanup(func(){panic("excluded cleanup terminal")})} +func TestLaterThanCleanup(t *testing.T){if err:=os.WriteFile(os.Getenv("AFTER_CLEANUP"),[]byte("ran"),0600);err!=nil{t.Fatal(err)}} +`}) + native := filepath.Join(t.TempDir(), executableName("native.test")) + out, stderr, code := command(t, dir, testEnv(), "go", "test", "-c", "-o", native, ".") + if code != 0 { + t.Fatalf("native compile: %d %s %s", code, out, stderr) + } + bin := filepath.Join(t.TempDir(), executableName("cleanup.test")) + out, stderr, code = command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("Mini compile: %d %s %s", code, out, stderr) + } + for _, target := range []struct{ name, bin string }{{"native", native}, {"Mini", bin}} { + t.Run(target.name, func(t *testing.T) { + after := filepath.Join(t.TempDir(), "after") + receiver, got := runParityCase(t, dir, target.bin, parityCase{Args: []string{"-test.run=^Test(CleanupTerminal|LaterThanCleanup)$"}, Env: []string{"AFTER_CLEANUP=" + after}}) + if got.code != 2 || !strings.Contains(got.stderr, "excluded cleanup terminal") { + t.Fatalf("cleanup panic: exit=%d out=%s stderr=%s", got.code, got.out, got.stderr) + } + if _, err := os.Stat(after); !os.IsNotExist(err) { + t.Fatalf("later test ran after terminal cleanup: %v", err) + } + if target.name == "Mini" { + var tests []map[string]any + for _, event := range receiver.events { + if event["type"] == "test" { + tests = append(tests, event["content"].(map[string]any)) + } + } + if len(tests) != 1 { + t.Fatalf("cleanup events=%v", tests) + } + meta := tests[0]["meta"].(map[string]any) + if meta["test.status"] != "fail" || meta["test.test_management."+directive] != "true" { + t.Fatalf("cleanup final event=%v", meta) + } + } + }) + } + }) + } +} + +func checkMiniNestedManagementCache(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"nested_managed_test.go": `package fixture_test +import("testing";"os";"strings";"github.com/stretchr/testify/suite") +type NestedManagedSuite struct{suite.Suite} +type OuterManagedSuite struct{suite.Suite} +func(s *OuterManagedSuite) TestManaged(){if os.Getenv("NESTED_NO_RETRY_ANCESTOR")=="true"{suite.Run(s.T(),new(NoRetryAncestor))}else{suite.Run(s.T(),new(NestedManagedSuite))}} +type NoRetryAncestor struct{suite.Suite} +//ddtest:no_auto_retry +func(s *NoRetryAncestor) TestAncestor(){suite.Run(s.T(),new(NestedManagedSuite))} +func managedPath(t *testing.T,key string)string{p:=os.Getenv(key);mark:="/"+os.Getenv("MANAGED_METHOD");if !strings.HasSuffix(t.Name(),mark)&&!strings.Contains(t.Name(),mark+"/"){p+=".other"};return p} +func appendManagedCall(t *testing.T)int {p:=managedPath(t,"MANAGED_CALLS");f,err:=os.OpenFile(p,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString("x");f.Close();if err!=nil{t.Fatal(err)};data,err:=os.ReadFile(p);if err!=nil{t.Fatal(err)};return len(data)} +func runManagedRows(t *testing.T){callback:=func(t *testing.T){f,err:=os.OpenFile(managedPath(t,"MANAGED_ROWS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString(t.Name()+"\n");f.Close();if err!=nil{t.Fatal(err)}};t.Run("same",callback);t.Run("same",callback)} +//ddtest:no_auto_retry +func(s *NestedManagedSuite) TestOnce(){appendManagedCall(s.T());runManagedRows(s.T())} +//ddtest:no_auto_retry +//ddtest:prevent_failure_suppression +func(s *NestedManagedSuite) TestProtected(){n:=appendManagedCall(s.T());runManagedRows(s.T());if os.Getenv("FILTERED_PROTECTED_FAIL")=="true"||os.Getenv("MANAGED_METHOD")=="TestProtected"&&n==1{s.T().Error("first protected callback must remain failed")}} +func TestNestedManaged(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);first:=err==nil;if first{f.Close()}else if !os.IsExist(err){t.Fatal(err)} + suite.Run(t,new(OuterManagedSuite));if first&&os.Getenv("NESTED_NO_PARENT_FAILURE")!="true"{t.Error("retry the parent")} +} +`}) + bin := filepath.Join(t.TempDir(), executableName("nested-managed.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + t.Run("native-filter", func(t *testing.T) { + path := filepath.Join(dir, "nested_managed_test.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.WriteFile(path, source, 0600); err != nil { + t.Error(err) + } + }) + on := strings.Replace(string(source), "//ddtest:no_auto_retry\nfunc(s *NestedManagedSuite) TestOnce", "func(s *NestedManagedSuite) TestOnce", 1) + off := strings.Replace(on, "//ddtest:no_auto_retry\n//ddtest:prevent_failure_suppression\nfunc(s *NestedManagedSuite) TestProtected", "func(s *NestedManagedSuite) TestProtected", 1) + if off == on { + t.Fatal("filter control annotation missing") + } + for _, variant := range []struct { + name, source string + native bool + }{{"native", on, true}, {"annotation-off", off, false}, {"annotation-on", on, false}} { + t.Run(variant.name, func(t *testing.T) { + if err := os.WriteFile(path, []byte(variant.source), 0600); err != nil { + t.Fatal(err) + } + target := filepath.Join(t.TempDir(), executableName("filter.test")) + builder := driver + if variant.native { + builder = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), builder, exclusionBuildArgs(target, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + calls := filepath.Join(t.TempDir(), "calls") + rows := filepath.Join(t.TempDir(), "rows") + receiver, got := runParityCase(t, dir, target, parityCase{Args: []string{"-test.run=^TestNestedManaged$/^TestManaged$/^TestOnce$"}, Policy: policySettings{Quarantined: true, ManagementTarget: "TestNestedManaged/TestManaged", ManagementSuite: "nested_managed_test.go/OuterManagedSuite"}, Env: []string{"MANAGED_CALLS=" + calls, "MANAGED_ROWS=" + rows, "MANAGED_METHOD=TestOnce", "FILTERED_PROTECTED_FAIL=true", "NESTED_NO_PARENT_FAILURE=true", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process"}}) + _, otherErr := os.Stat(calls + ".other") + t.Logf("filter variant=%s exit=%d filteredCallbackAdmitted=%t", variant.name, got.code, otherErr == nil) + if got.code != 0 { + t.Fatalf("filtered protected method changed selected ordinary command: exit=%d %s %s events=%v", got.code, got.out, got.stderr, receiver.events) + } + if variant.native || variant.name == "annotation-on" { + if !os.IsNotExist(otherErr) { + t.Fatalf("filtered-out method ran: %v", otherErr) + } + } + }) + } + }) + t.Run("protected-excluded-ancestor", func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + rows := filepath.Join(t.TempDir(), "rows") + _, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestNestedManaged$/^TestManaged$/^TestAncestor$/^TestProtected$"}, Policy: policySettings{EFD: true, Quarantined: true, ManagementTarget: "TestNestedManaged/TestManaged", ManagementSuite: "nested_managed_test.go/OuterManagedSuite"}, Env: []string{"MANAGED_CALLS=" + calls, "MANAGED_ROWS=" + rows, "MANAGED_METHOD=TestProtected", "NESTED_NO_RETRY_ANCESTOR=true", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=true"}}) + data, err := os.ReadFile(calls) + if got.code != 1 || err != nil || string(data) != "x" { + t.Fatalf("excluded ancestor lost protected descendant failure: exit=%d calls=%q err=%v %s %s", got.code, data, err, got.out, got.stderr) + } + }) + qNativeInvocations := 2 + t.Run("ordinary-count-control", func(t *testing.T) { + path := filepath.Join(dir, "nested_managed_test.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + ordinary := strings.Replace(string(source), "//ddtest:no_auto_retry\nfunc(s *NestedManagedSuite) TestOnce", "func(s *NestedManagedSuite) TestOnce", 1) + if ordinary == string(source) { + t.Fatal("ordinary control annotation missing") + } + if err = os.WriteFile(path, []byte(ordinary), 0600); err != nil { + t.Fatal(err) + } + ordinaryBin := filepath.Join(t.TempDir(), executableName("ordinary-managed.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(ordinaryBin, true)...) + if err = os.WriteFile(path, source, 0600); err != nil { + t.Fatal(err) + } + if code != 0 { + t.Fatalf("ordinary compile %d: %s %s", code, out, stderr) + } + calls := filepath.Join(t.TempDir(), "calls") + rows := filepath.Join(t.TempDir(), "rows") + _, got := runParityCase(t, dir, ordinaryBin, parityCase{Args: []string{"-test.run=^TestNestedManaged$/^TestManaged$/^TestOnce$", "-test.count=2"}, Policy: policySettings{Retry: true, Quarantined: true, ManagementTarget: "TestNestedManaged/TestManaged", ManagementSuite: "nested_managed_test.go/OuterManagedSuite"}, Env: []string{"MANAGED_CALLS=" + calls, "MANAGED_ROWS=" + rows, "MANAGED_METHOD=TestOnce", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + data, err := os.ReadFile(calls) + t.Logf("ordinary nested quarantine calls=%q exit=%d", data, got.code) + if got.code != 0 || err != nil || len(data) < 1 { + t.Fatalf("ordinary nested isolation baseline changed: calls=%q err=%v exit=%d %s %s", data, err, got.code, got.out, got.stderr) + } + qNativeInvocations = min(len(data), 2) + }) + for _, mode := range []string{"in_process", "process"} { + for _, atf := range []bool{false, true} { + for _, efd := range []bool{false, true} { + for _, protected := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/atf=%t/efd=%t/protected=%t", mode, atf, efd, protected), func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + rows := filepath.Join(t.TempDir(), "rows") + wantCalls := 2 + method := "TestOnce" + want := 0 + if protected { + method = "TestProtected" + want = 1 + } + // The existing native masking fallback does not admit this nested + // quarantine declaration in in-process mode; retain its boundary. + if !atf && mode == "in_process" { + want, wantCalls = 0, 0 + } else if !atf && !efd { + // The ordinary control above has one admitted isolated + // invocation for this native-count/parent-retry combination. + wantCalls = qNativeInvocations + } + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestNestedManaged$/^TestManaged$/^" + method + "$", "-test.count=2"}, Policy: policySettings{Retry: !efd, EFD: efd, Quarantined: !atf, AttemptToFix: atf, ManagementTarget: "TestNestedManaged/TestManaged", ManagementSuite: "nested_managed_test.go/OuterManagedSuite"}, Env: []string{"MANAGED_CALLS=" + calls, "MANAGED_ROWS=" + rows, "MANAGED_METHOD=" + method, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, fmt.Sprintf("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=%t", !efd), fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", efd), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != want { + t.Fatalf("nested policy outcome=%d want%d: %s %s", got.code, want, got.out, got.stderr) + } + data, err := os.ReadFile(calls) + if len(data) != wantCalls || got.code != want { + for _, event := range receiver.events { + if event["type"] == "test" { + meta := event["content"].(map[string]any)["meta"].(map[string]any) + t.Logf("event name=%v status=%v noRetry=%v protected=%v retry=%v reason=%v", meta["test.name"], meta["test.status"], meta["test.test_management.no_auto_retry"], meta["test.test_management.prevent_failure_suppression"], meta["test.is_retry"], meta["test.retry_reason"]) + } + } + } + if wantCalls == 0 { + if !os.IsNotExist(err) { + t.Fatalf("unadmitted declaration ran: %q %v", data, err) + } + } else if err != nil || len(data) != wantCalls { + t.Fatalf("excluded callback calls=%q want%d err=%v", data, wantCalls, err) + } + rowData, rowErr := os.ReadFile(rows) + if wantCalls > 0 { + lines := strings.Fields(string(rowData)) + counts := map[string]int{} + for _, line := range lines { + counts[line]++ + } + if rowErr != nil || len(lines) != wantCalls*2 || len(counts) != 2 { + t.Fatalf("shared duplicate row callbacks=%q err=%v", rowData, rowErr) + } + for _, count := range counts { + if count != wantCalls { + t.Fatalf("native count/duplicate row callbacks=%v", counts) + } + } + } else if !os.IsNotExist(rowErr) { + t.Fatalf("unadmitted rows ran: %q %v", rowData, rowErr) + } + count := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestNestedManaged/TestManaged/"+method { + count++ + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.is_retry"] != nil || meta["test.retry_reason"] != nil { + t.Fatalf("excluded event=%v", meta) + } + } + } + if count != wantCalls { + t.Fatalf("excluded events=%d want%d", count, wantCalls) + } + }) + } + } + } + } +} + +func checkMiniAncestorDeclarationCache(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"ancestor_cache_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +func ancestorCall(t *testing.T,s string){f,err:=os.OpenFile(os.Getenv("ANCESTOR_CALLS"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{t.Fatal(err)};_,err=f.WriteString(s);f.Close();if err!=nil{t.Fatal(err)}} +type IndependentLeaf struct{suite.Suite} +//ddtest:no_auto_retry +func(s *IndependentLeaf) TestLeaf(){ancestorCall(s.T(),"d");s.T().Run("generated",func(t *testing.T){ancestorCall(t,"r")})} +type FirstAncestor struct{suite.Suite} +type SecondAncestor struct{suite.Suite} +//ddtest:no_auto_retry +func(s *FirstAncestor) TestSame(){ancestorCall(s.T(),"a");suite.Run(s.T(),new(IndependentLeaf))} +//ddtest:no_auto_retry +func(s *SecondAncestor) TestSame(){ancestorCall(s.T(),"b");suite.Run(s.T(),new(IndependentLeaf))} +type OrdinaryAncestorOuter struct{suite.Suite} +func(s *OrdinaryAncestorOuter) TestOuter(){f,err:=os.OpenFile(os.Getenv("ANCESTOR_SWITCH"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if err==nil{f.Close();suite.Run(s.T(),new(FirstAncestor));s.T().Error("retry ordinary ancestor");return};if !os.IsExist(err){s.T().Fatal(err)};suite.Run(s.T(),new(SecondAncestor))} +func TestSwitchedAncestor(t *testing.T){f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600);if os.Getenv("ANCESTOR_ORDINARY")=="true"{if err==nil{f.Close();t.Error("discover ordinary wrapper on retry");return};if !os.IsExist(err){t.Fatal(err)};suite.Run(t,new(OrdinaryAncestorOuter));return};if err==nil{f.Close();suite.Run(t,new(FirstAncestor));t.Error("retry parent");return};if !os.IsExist(err){t.Fatal(err)};suite.Run(t,new(SecondAncestor))} +`}) + bin := filepath.Join(t.TempDir(), executableName("ancestor-cache.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, ordinary := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/ordinary=%t", mode, ordinary), func(t *testing.T) { + calls := filepath.Join(t.TempDir(), "calls") + _, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestSwitchedAncestor$"}, Policy: policySettings{Retry: true}, Env: []string{"ANCESTOR_CALLS=" + calls, "ANCESTOR_SWITCH=" + filepath.Join(t.TempDir(), "switch"), fmt.Sprintf("ANCESTOR_ORDINARY=%t", ordinary), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + data, err := os.ReadFile(calls) + if got.code != 0 || err != nil || string(data) != "adrb" { + t.Fatalf("changed ancestor lost independent descendant history: calls=%q err=%v exit=%d %s %s", data, err, got.code, got.out, got.stderr) + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_terminal_acceptance_test.go b/ddto/integration/mini_exclusions_terminal_acceptance_test.go new file mode 100644 index 000000000..0f975955e --- /dev/null +++ b/ddto/integration/mini_exclusions_terminal_acceptance_test.go @@ -0,0 +1,276 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func requireReviewNoRetryOwnership(t *testing.T, meta map[string]any) { + t.Helper() + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode", "test.test_management.attempt_to_fix_passed"} { + if meta[key] != nil { + t.Fatalf("excluded declaration inherited %s: %v", key, meta) + } + } +} + +func reviewTestEvents(receiver *parityReceiver, name string) []map[string]any { + var result []map[string]any + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == name { + result = append(result, meta) + } + } + return result +} + +// Native is the oracle for panic cancellation and queued-parallel precedence. +// All variants compile the same source; only declaration comments differ. File +// markers prove actual callbacks/admission independently of reports and logs. +func checkMiniTerminalProvenance(t *testing.T) { + dir, driver := prepareMiniFixture(t) + source := `package fixture_test +import("testing";"os";"runtime";"strings") +func terminalMark(t *testing.T, key string){f,e:=os.OpenFile(os.Getenv("TERMINAL_ORDER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};defer f.Close();if _,e=f.WriteString(key+"\n");e!=nil{t.Fatal(e)}} +func terminalExit(t *testing.T,mode string){switch {case strings.Contains(mode,"goexit"):runtime.Goexit();case strings.Contains(mode,"skip"):t.SkipNow();case strings.Contains(mode,"fail"):t.FailNow()}} +//ANNOTATIONS +func TestTerminalProvenance(t *testing.T){ + if os.Getenv("TERMINAL_QUEUED")=="true"{t.Run("queued",func(t *testing.T){t.Parallel();if e:=os.WriteFile(os.Getenv("TERMINAL_ADMITTED"),[]byte("admitted"),0600);e!=nil{t.Fatal(e)}})} + mode:=os.Getenv("TERMINAL_MODE") + t.Cleanup(func(){terminalMark(t,"first");if strings.HasSuffix(mode,"-final"){terminalExit(t,mode)};panic("cleanup first sentinel")}) + if mode!="one"{t.Cleanup(func(){terminalMark(t,"second");if strings.HasSuffix(mode,"-early"){terminalExit(t,mode)};if strings.HasPrefix(mode,"dynamic"){t.Cleanup(func(){terminalMark(t,"dynamic");panic("cleanup dynamic sentinel")})};panic("cleanup second sentinel")})} + if os.Getenv("TERMINAL_BODY")=="true"{panic("body sentinel")} +} +func TestTerminalCarrier(t *testing.T){t.Run("row",TestTerminalProvenance)} +` + bins := map[string]string{} + for _, variant := range []string{"native", "off", "on"} { + annotations := "" + if variant == "on" { + annotations = "//ddtest:no_auto_retry\n//ddtest:prevent_failure_suppression" + } + writeBuildFixture(t, dir, map[string]string{"terminal_provenance_test.go": strings.Replace(source, "//ANNOTATIONS", annotations, 1)}) + bin := filepath.Join(t.TempDir(), executableName("terminal-"+variant+".test")) + build := driver + if variant == "native" { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile %s: %d %s %s", variant, code, out, stderr) + } + bins[variant] = bin + } + sentinels := []string{"body sentinel", "cleanup first sentinel", "cleanup second sentinel", "cleanup dynamic sentinel"} + modes := []string{"one", "multiple", "dynamic", "goexit-final", "skip-final", "fail-final", "goexit-early", "skip-early", "fail-early", "dynamic-goexit-final", "dynamic-skip-final", "dynamic-fail-final"} + for _, name := range []string{"TestTerminalProvenance", "TestTerminalCarrier"} { + for _, mode := range modes { + for _, body := range []bool{false, true} { + for _, queued := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/body=%t/queued=%t", name, mode, body, queued), func(t *testing.T) { + var native execution + var nativeOrder string + var nativeAdmitted bool + for _, variant := range []string{"native", "off", "on"} { + t.Run(variant, func(t *testing.T) { + order, admitted := filepath.Join(t.TempDir(), "order"), filepath.Join(t.TempDir(), "admitted") + receiver, got := runParityCase(t, dir, bins[variant], parityCase{Args: []string{"-test.run=^" + name + "$", "-test.v"}, Env: []string{"DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false", "TERMINAL_MODE=" + mode, "TERMINAL_BODY=" + fmt.Sprint(body), "TERMINAL_QUEUED=" + fmt.Sprint(queued), "TERMINAL_ORDER=" + order, "TERMINAL_ADMITTED=" + admitted}}) + raw, err := os.ReadFile(order) + if err != nil { + t.Fatalf("cleanup callbacks did not execute: %v, exit%d %s %s", err, got.code, got.out, got.stderr) + } + _, err = os.Stat(admitted) + wasAdmitted := err == nil + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + if queued && body && wasAdmitted { + t.Fatal("terminal body admitted queued child") + } + if queued && !body && !wasAdmitted { + t.Fatal("normally returned body did not admit queued child") + } + if variant == "native" { + native = got + nativeOrder = string(raw) + nativeAdmitted = wasAdmitted + return + } + // Ordinary SDK events may pre-finalize the body failure before + // cleanup SkipNow cancels its panic; preserve that existing lifetime. + wantCode := native.code + if variant == "off" && body && !queued && (mode == "skip-final" || mode == "dynamic-skip-final") { + wantCode = 1 + } + if got.code != wantCode || string(raw) != nativeOrder || wasAdmitted != nativeAdmitted { + t.Fatalf("%s differs from native: exit%d/%d order%q/%q admitted%t/%t\n%s\n%s", variant, got.code, native.code, raw, nativeOrder, wasAdmitted, nativeAdmitted, got.out, got.stderr) + } + output, nativeOutput := got.out+got.stderr, native.out+native.stderr + for _, sentinel := range sentinels { + if strings.Contains(output, sentinel) != strings.Contains(nativeOutput, sentinel) { + t.Fatalf("%s command diagnostic %q differs from native\nMini:%s\nNative:%s", variant, sentinel, output, nativeOutput) + } + } + eventName := name + if name == "TestTerminalCarrier" { + eventName += "/row" + } + events := reviewTestEvents(receiver, eventName) + wantEvents := 1 + // A returned ordinary body closes before native cleanup; a later + // fatal cleanup can terminate the process before its buffered flush. + if variant == "off" && !body && native.code == 2 { + wantEvents = 0 + } + if len(events) != wantEvents { + t.Fatalf("%s events=%d want%d: %v", eventName, len(events), wantEvents, receiver.events) + } + if variant == "off" { + return + } // Pinned ordinary event lifetime ends before cleanup. + meta := events[0] + requireReviewNoRetryOwnership(t, meta) + if meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management.prevent_failure_suppression"] != "true" { + t.Fatalf("missing eager policy: %v", meta) + } + diagnostics := fmt.Sprint(meta["error.message"]) + fmt.Sprint(meta["error.stack"]) + for _, sentinel := range sentinels { + if strings.Contains(diagnostics, sentinel) != strings.Contains(nativeOutput, sentinel) { + t.Fatalf("eager event diagnostic %q differs from native\nmeta:%v\nNative:%s", sentinel, meta, nativeOutput) + } + } + wantStatus := "fail" + if native.code == 0 { + wantStatus = "pass" + if strings.Contains(mode, "skip") { + wantStatus = "skip" + } + } + if meta["test.status"] != wantStatus { + t.Fatalf("eager status=%v want%s: %v", meta["test.status"], wantStatus, meta) + } + // Original source identity must survive in the combined stack. + if strings.Contains(diagnostics, "sentinel") && !strings.Contains(fmt.Sprint(meta["error.stack"]), "terminal_provenance_test.go:") { + t.Fatalf("lost original panic source: %v", meta) + } + }) + } + }) + } + } + } + } +} + +// Selecting a quarantined Testify method uses the existing actual race worker. +// PFS on the native callback must preserve both panics and block suppression. +func checkMiniTerminalWorkerProvenance(t *testing.T) { + if os.Getenv("PARITY_TEST_MODE") != "race" { + t.Skip("selected quarantine executes callbacks only in actual race/process isolation") + } + dir, driver := prepareMiniFixture(t) + installExternalTestifyHelper(t, dir) + + source := `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +//ddtest:prevent_failure_suppression +//EXTRA +func TestWorkerPanicCallback(t *testing.T){ + if os.Getenv("WORKER_QUEUED")=="true"{t.Run("queued",func(t *testing.T){t.Parallel();if e:=os.WriteFile(os.Getenv("WORKER_ADMITTED"),[]byte("admitted"),0600);e!=nil{t.Fatal(e)}})} + t.Cleanup(func(){workerPanicMark(t,"first");panic("worker cleanup first sentinel")}) + if os.Getenv("WORKER_MULTI")=="true"{t.Cleanup(func(){workerPanicMark(t,"second");t.Cleanup(func(){workerPanicMark(t,"dynamic");panic("worker cleanup dynamic sentinel")});panic("worker cleanup second sentinel")})} + if os.Getenv("WORKER_BODY")=="true"{panic("worker body sentinel")} +} +func workerPanicMark(t *testing.T,key string){f,e:=os.OpenFile(os.Getenv("WORKER_ORDER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};defer f.Close();if _,e=f.WriteString(key+"\n");e!=nil{t.Fatal(e)}} +type WorkerPanicSuite struct{suite.Suite} +func(s *WorkerPanicSuite) TestMethod(){s.T().Run("critical",TestWorkerPanicCallback)} +func TestWorkerPanicSuite(t *testing.T){suite.Run(t,new(WorkerPanicSuite))} +` + for _, noRetry := range []bool{false, true} { + t.Run(fmt.Sprintf("no-retry=%t", noRetry), func(t *testing.T) { + extra := "" + if noRetry { + extra = "//ddtest:no_auto_retry\n" + } + writeBuildFixture(t, dir, map[string]string{"worker_panic_test.go": strings.Replace(source, "//EXTRA\n", extra, 1)}) + bins := map[string]string{} + for _, kind := range []string{"native", "Mini"} { + bin := filepath.Join(t.TempDir(), executableName("worker-panic-"+kind+".test")) + build := driver + if kind == "native" { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile%d %s %s", code, out, stderr) + } + bins[kind] = bin + } + for _, multi := range []bool{false, true} { + for _, body := range []bool{false, true} { + for _, queued := range []bool{false, true} { + t.Run(fmt.Sprintf("multi=%t/body=%t/queued=%t", multi, body, queued), func(t *testing.T) { + var native execution + var nativeOrder string + for _, kind := range []string{"native", "Mini"} { + order := filepath.Join(t.TempDir(), "order") + admitted := filepath.Join(t.TempDir(), "admitted") + receiver, got := runParityCase(t, dir, bins[kind], parityCase{Args: []string{"-test.run=^TestWorkerPanicSuite$/^TestMethod$"}, Policy: policySettings{Quarantined: kind == "Mini", ManagementTarget: "TestWorkerPanicSuite/TestMethod", ManagementSuite: "worker_panic_test.go/WorkerPanicSuite"}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=false", "WORKER_MULTI=" + fmt.Sprint(multi), "WORKER_BODY=" + fmt.Sprint(body), "WORKER_ORDER=" + order, "WORKER_QUEUED=" + fmt.Sprint(queued), "WORKER_ADMITTED=" + admitted}}) + _, admissionErr := os.Stat(admitted) + if admissionErr != nil && !os.IsNotExist(admissionErr) { + t.Fatal(admissionErr) + } + if queued && ((admissionErr == nil) == body) { + t.Errorf("%s queued admission=%t body panic=%t %s %s", kind, admissionErr == nil, body, got.out, got.stderr) + } + raw, err := os.ReadFile(order) + if err != nil { + t.Fatalf("cleanup missing: %v %s %s", err, got.out, got.stderr) + } + if kind == "native" { + native = got + nativeOrder = string(raw) + if got.code != 2 { + t.Fatalf("native panic exit%d %s %s", got.code, got.out, got.stderr) + } + continue + } + if got.code != 1 || string(raw) != nativeOrder { + t.Errorf("worker panic failed to block/preserve callback order exit%d order%q native%q %s %s", got.code, raw, nativeOrder, got.out, got.stderr) + } + events := reviewTestEvents(receiver, "TestWorkerPanicSuite/TestMethod/critical") + if len(events) != 1 { + t.Fatalf("worker callback events%d %v", len(events), receiver.events) + } + meta := events[0] + diag := fmt.Sprint(meta["error.message"]) + fmt.Sprint(meta["error.stack"]) + if meta["test.status"] != "fail" || meta["test.test_management.prevent_failure_suppression"] != "true" || (meta["test.test_management.no_auto_retry"] == "true") != noRetry { + t.Fatalf("worker callback policy/status %v", meta) + } + for _, sentinel := range []string{"worker body sentinel", "worker cleanup first sentinel", "worker cleanup second sentinel", "worker cleanup dynamic sentinel"} { + want := strings.Contains(native.out+native.stderr, sentinel) + // The established process protocol sends nested panic text in + // bounded event fields, without forwarding it to the parent console. + // Verified against the published runtime before setting this baseline. + if strings.Contains(diag, sentinel) != want || strings.Contains(got.out+got.stderr, sentinel) { + t.Errorf("worker diagnostic %q event=%t command=%t native=%t: %v\n%s\n%s\nNative:%s %s", sentinel, strings.Contains(diag, sentinel), strings.Contains(got.out+got.stderr, sentinel), want, meta, got.out, got.stderr, native.out, native.stderr) + } + } + if !strings.Contains(fmt.Sprint(meta["error.stack"]), "worker_panic_test.go:") { + t.Fatalf("worker original source lost: %v", meta) + } + } + }) + } + } + } + }) + } +} diff --git a/ddto/integration/mini_exclusions_test.go b/ddto/integration/mini_exclusions_test.go new file mode 100644 index 000000000..c4381de8d --- /dev/null +++ b/ddto/integration/mini_exclusions_test.go @@ -0,0 +1,481 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestMiniDeclarationExclusions checks command outcome and actual CI events, +// including process retry children, with no dependency on runtime source reads. +func TestMiniDeclarationExclusions(t *testing.T) { + cases := []struct { + name, comments string + prevent, noRetry bool + }{ + {"both", "//ddtest:prevent_failure_suppression\n//ddtest:no_auto_retry\n", true, true}, + {"no-retry", "//ddtest:no_auto_retry\n", false, true}, + {"prevent-suppression", "//ddtest:prevent_failure_suppression\n", true, false}, + {"ordinary", "", false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir, driver := prepareMiniFixture(t) + path := filepath.Join(dir, "sample_test.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + data = []byte(strings.Replace(string(data), "func TestManaged(", tc.comments+"func TestManaged(", 1)) + if err = os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + bin := filepath.Join(t.TempDir(), executableName("exclusions.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile: %d %s %s", code, out, stderr) + } + // Compiled annotation metadata survives source removal and process retries. + if err = os.Rename(path, path+".saved"); err != nil { + t.Fatal(err) + } + for _, profile := range []string{"disabled", "quarantined", "attempt_to_fix", "efd", ""} { + t.Run("profile-"+profile, func(t *testing.T) { + got := executeProfile(t, dir, bin, []string{"-test.run=^TestManaged$", "-mode=managed"}, true, profile == "", profile) + wantCode := 1 + if !tc.prevent && (profile == "disabled" || profile == "quarantined") { + wantCode = 0 + } + if got.code != wantCode { + t.Fatalf("exit=%d want=%d\n%s\n%v", got.code, wantCode, got.stderr, got.events) + } + count := 0 + for _, event := range got.events { + if !strings.Contains(event, `"type":"test"`) { + continue + } + count++ + for key, want := range map[string]bool{"prevent_failure_suppression": tc.prevent, "no_auto_retry": tc.noRetry} { + tag := `"test.test_management.` + key + `":"true"` + if strings.Contains(event, tag) != want { + t.Fatalf("annotation %s expected=%t: %s", key, want, event) + } + } + assigned := map[string]string{"disabled": "is_test_disabled", "quarantined": "is_quarantined", "attempt_to_fix": "is_attempt_to_fix"}[profile] + if assigned != "" && !strings.Contains(event, `"test.test_management.`+assigned+`":"true"`) { + t.Fatalf("lost assigned property: %s", event) + } + if tc.noRetry && strings.Contains(event, `"test.is_retry":"true"`) { + t.Fatalf("excluded test retried: %s", event) + } + } + if count == 0 || tc.noRetry && count != 1 { + t.Fatalf("test events=%d: %v", count, got.events) + } + if !tc.noRetry && (profile == "efd" || profile == "") && count < 2 { + t.Fatalf("unexcluded test did not retry: %v", got.events) + } + }) + } + }) + } + t.Run("table-declarations", checkMiniTableExclusions) + t.Run("top-level-tia", checkMiniTopLevelExcludedTIA) + t.Run("cleanup-terminal", checkMiniExcludedCleanupTerminal) + t.Run("terminal-provenance", checkMiniTerminalProvenance) + t.Run("crash-final-status", checkMiniExcludedCrashFinalStatus) + t.Run("descendant-crash-policy", checkMiniProtectedDescendantCrashPolicy) + t.Run("terminal-worker-provenance", checkMiniTerminalWorkerProvenance) + t.Run("parallel-cleanup", checkMiniExcludedParallelCleanup) + t.Run("cleanup-goexit", checkMiniExcludedCleanupGoexit) +} + +// An excluded Testify method must not rerun with its parent suite, and an +// ordinary sibling must still receive the parent's automatic retries. +func TestMiniTestifyExclusionsKeepSiblingRetries(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"exclusion_suite_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type ExclusionSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *ExclusionSuite) TestOnce(){ + f,err:=os.OpenFile(os.Getenv("EXCLUSION_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + if os.Getenv("EXCLUSION_FAIL")=="true" {tt:=s.T();tt.Cleanup(func(){tt.Error("excluded cleanup failure")})} +} +//ddtest:no_auto_retry +func(s ExclusionSuite) TestValue(){ + f,err:=os.OpenFile(os.Getenv("VALUE_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} +} +func(s *ExclusionSuite) TestSibling(){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err==nil{f.Close();s.T().Error("recoverable sibling failure")}else if !os.IsExist(err){s.T().Fatal(err)} +} +func TestExclusionSuite(t *testing.T){suite.Run(t,new(ExclusionSuite))} +`}) + bin := filepath.Join(t.TempDir(), executableName("testify-exclusions.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile: %d %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, failed := range []bool{false, true} { + for _, efd := range []bool{false, true} { + for _, managed := range []bool{false, true} { + t.Run(fmt.Sprint("managed=", managed, "-efd=", efd, "-")+mode+"-failed="+fmt.Sprint(failed), func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "once") + valueCounter := filepath.Join(t.TempDir(), "value") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestExclusionSuite$", "-test.count=2"}, Policy: policySettings{Retry: true, EFD: efd, Quarantined: managed, AttemptToFix: managed, ManagementTarget: "TestExclusionSuite", ManagementSuite: "exclusion_suite_test.go"}, Env: []string{"DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "EXCLUSION_COUNTER=" + counter, "VALUE_COUNTER=" + valueCounter, fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", efd), fmt.Sprintf("EXCLUSION_FAIL=%t", failed)}}) + want := 0 + if failed && !managed { + want = 1 + } + if got.code != want { + t.Fatalf("exit %d want %d: %s %s", got.code, want, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "xx" { + t.Fatalf("excluded method executed %q: %v", data, err) + } + valueData, err := os.ReadFile(valueCounter) + if err != nil || string(valueData) != "xx" { + t.Fatalf("value receiver executed %q: %v", valueData, err) + } + tests := 0 + valueTests := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.test_management.no_auto_retry"] == "true" { + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode", "test.test_management.attempt_to_fix_passed"} { + if meta[key] != nil { + t.Fatalf("excluded method inherited %s: %v", key, meta) + } + } + if managed && meta["test.test_management.is_attempt_to_fix"] != "true" { + t.Fatalf("lost ATF assignment: %v", meta) + } + } + if meta["test.name"] == "TestExclusionSuite/TestValue" { + valueTests++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("missing value receiver tag: %v", meta) + } + } + if meta["test.name"] == "TestExclusionSuite/TestOnce" { + tests++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("missing method declaration tag: %v", meta) + } + } + } + if tests != 2 || valueTests != 2 { + t.Fatalf("excluded method emitted %d events", tests) + } + }) + } + } + } + } + t.Run("subtest-gate", func(t *testing.T) { checkMiniSubtestGate(t, dir, bin) }) + t.Run("imported-promoted-declarations", checkMiniImportedPromotedExclusions) + t.Run("generic-declarations", checkMiniGenericDeclarations) + t.Run("cgo-declarations", checkMiniCgoImportedExclusions) + t.Run("runtime-dependency-declarations", checkMiniRuntimeDependencyExclusions) + t.Run("declaration-cache", checkMiniDeclarationCacheIdentity) + t.Run("promoted-declaration-cache", checkMiniPromotedDeclarationCache) + t.Run("public-method-exclusions", checkMiniPublicMethodExclusions) + t.Run("testify-native-callback", checkMiniTestifyNativeCallback) + t.Run("nested-callback-policy", checkMiniReviewNestedRows) + t.Run("early-worker-terminal-state", checkMiniEarlyWorkerTerminalState) + t.Run("pfs-worker-terminal-policy", checkMiniPFSWorkerTerminalPolicy) + t.Run("ordinary-worker-management", checkMiniReviewWorkerManagement) + t.Run("ordinary-worker-atf-override", checkMiniReviewWorkerOverride) + t.Run("inherited-owner-cache", checkMiniInheritedOwnerCache) + t.Run("protected-aggregate-failure", checkMiniProtectedAggregateFailure) + t.Run("omitted-excluded-failure", checkMiniOmittedExcludedFailure) + t.Run("ancestor-declaration-cache", checkMiniAncestorDeclarationCache) + t.Run("nested-worker-declarations", checkMiniNestedWorkerDeclarations) + t.Run("late-worker-tia", checkMiniLateExcludedTIA) + t.Run("process-continuations", checkMiniExclusionProcessContinuations) + t.Run("isolated-declaration-root", checkMiniExcludedIsolationRoot) + t.Run("nested-management-cache", checkMiniNestedManagementCache) + t.Run("large-exclusion-payload", checkMiniLargeExclusionPayload) + t.Run("protected-suppression", checkMiniTestifyProtection) + t.Run("late-declaration", checkMiniLateExclusionDeclaration) +} + +func checkMiniTestifyProtection(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"protected_suite_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type ProtectedSuite struct{suite.Suite} +//ddtest:prevent_failure_suppression +func(s *ProtectedSuite) TestCritical(){ + if os.Getenv("PROTECTED_RECOVER")=="true" || os.Getenv("PROTECTED_LATE")=="true" { + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err!=nil && os.IsExist(err){if os.Getenv("PROTECTED_RECOVER")=="true" {return}}else{if err!=nil{s.T().Fatal(err)};f.Close();if os.Getenv("PROTECTED_LATE")=="true"{return}} + } + s.T().Error("protected failure") +} +type ProtectedCleanupSuite struct{suite.Suite} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func(s *ProtectedCleanupSuite) TestCritical(){tt:=s.T();tt.Cleanup(func(){tt.Error("protected cleanup failure")})} +func TestProtectedSuite(t *testing.T){ + if os.Getenv("PROTECTED_SKIP")=="true"{t.Skip("native parent skip")} + if os.Getenv("PROTECTED_CLEANUP")=="true"{suite.Run(t,new(ProtectedCleanupSuite))}else{suite.Run(t,new(ProtectedSuite))} +} +`}) + bin := filepath.Join(t.TempDir(), executableName("protected.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, tc := range []struct { + name, target, suite string + disabled, quarantined, recover, skip, attempt, late, cleanup, efd bool + want int + }{ + {name: "parent-disabled", target: "TestProtectedSuite", suite: "protected_suite_test.go", disabled: true, want: 0}, + {name: "native-parent-skip", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, skip: true, want: 0}, + {name: "parent-quarantine-atf-earlier-failure", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, attempt: true, recover: true, want: 1}, + {name: "efd-retry-recovers", target: "TestProtectedSuite/TestCritical", suite: "protected_suite_test.go/ProtectedSuite", quarantined: true, recover: true, efd: true, want: 0}, + {name: "parent-quarantine-atf-late-failure", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, attempt: true, late: true, want: 1}, + {name: "parent-quarantined-cleanup", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, cleanup: true, want: 1}, + {name: "parent-quarantine-atf-cleanup", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, attempt: true, cleanup: true, want: 1}, + {name: "parent-quarantined", target: "TestProtectedSuite", suite: "protected_suite_test.go", quarantined: true, want: 1}, + {name: "selected-method-disabled", target: "TestProtectedSuite/TestCritical", suite: "protected_suite_test.go/ProtectedSuite", disabled: true, want: 1}, + {name: "retry-recovers", target: "TestProtectedSuite/TestCritical", suite: "protected_suite_test.go/ProtectedSuite", quarantined: true, recover: true, want: 0}, + } { + t.Run(mode+"-"+tc.name, func(t *testing.T) { + method := "TestCritical" + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestProtectedSuite$/^" + method + "$"}, Policy: policySettings{Retry: true, EFD: tc.efd, Disabled: tc.disabled, Quarantined: tc.quarantined, AttemptToFix: tc.attempt, ManagementTarget: tc.target, ManagementSuite: tc.suite}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", fmt.Sprintf("PROTECTED_RECOVER=%t", tc.recover), fmt.Sprintf("PROTECTED_SKIP=%t", tc.skip), fmt.Sprintf("PROTECTED_LATE=%t", tc.late), fmt.Sprintf("PROTECTED_CLEANUP=%t", tc.cleanup), fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", tc.efd)}}) + if got.code != tc.want { + t.Fatalf("exit %d want %d: %s %s", got.code, tc.want, got.out, got.stderr) + } + seen := false + lastRootStatus := "" + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] == "TestProtectedSuite/"+method { + seen = true + if meta["test.test_management.prevent_failure_suppression"] != "true" { + t.Fatalf("missing declaration: %v", meta) + } + if tc.cleanup && (meta["test.status"] != "fail" || meta["test.test_management.no_auto_retry"] != "true" || meta["test.is_retry"] != nil) { + t.Fatalf("protected cleanup outcome lost: %v", meta) + } + assignment := "test.test_management.is_quarantined" + if tc.disabled { + assignment = "test.test_management.is_test_disabled" + } + if meta[assignment] != "true" { + t.Fatalf("lost assignment: %v", meta) + } + } else if meta["test.test_management.prevent_failure_suppression"] != nil { + t.Fatalf("annotation leaked into ancestor: %v", meta) + } else if meta["test.name"] == "TestProtectedSuite" { + lastRootStatus, _ = meta["test.final_status"].(string) + } + } + if tc.want == 1 && lastRootStatus != "fail" { + t.Fatalf("protected failure root final_status=%q", lastRootStatus) + } + if tc.recover && tc.want == 0 && lastRootStatus != "pass" { + t.Fatalf("recovered root final_status=%q", lastRootStatus) + } + if !seen && tc.want != 0 { + t.Fatal("protected method not reported") + } + }) + } + } +} + +func checkMiniTableExclusions(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"rows_test.go": `package fixture_test +import "testing" +func sharedRow(t *testing.T){t.Parallel();t.Run("nested",func(t *testing.T){t.Error("row failed")})} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestCriticalRows(t *testing.T){for _,name:=range []string{"a","b"}{t.Run(name,sharedRow)}} +func TestOrdinaryRows(t *testing.T){for _,name:=range []string{"a","b"}{t.Run(name,sharedRow)}} +`}) + bin := filepath.Join(t.TempDir(), executableName("table.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, profile := range []string{"disabled", "quarantined", "attempt_to_fix"} { + t.Run(mode+"-"+profile, func(t *testing.T) { + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^Test(CriticalRows|OrdinaryRows)$", "-test.parallel=4"}, Policy: policySettings{Retry: true, Disabled: profile == "disabled", Quarantined: profile == "quarantined", AttemptToFix: profile == "attempt_to_fix", ManagementTarget: "TestCriticalRows", ManagementSuite: "rows_test.go"}, Env: []string{"DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + critical, ordinary := 0, 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + name, _ := meta["test.name"].(string) + if strings.HasPrefix(name, "TestCriticalRows") { + critical++ + for _, key := range []string{"prevent_failure_suppression", "no_auto_retry"} { + if meta["test.test_management."+key] != "true" { + t.Fatalf("missing generated case annotation: %v", meta) + } + } + if meta["test.is_retry"] != nil { + t.Fatalf("annotated generated case retried: %v", meta) + } + } else if strings.HasPrefix(name, "TestOrdinaryRows") { + ordinary++ + for _, key := range []string{"prevent_failure_suppression", "no_auto_retry"} { + if meta["test.test_management."+key] != nil { + t.Fatalf("shared callback leaked policy: %v", meta) + } + } + } + } + if critical != 5 || ordinary < 7 { + t.Fatalf("events critical=%d ordinary=%d", critical, ordinary) + } + }) + } + } +} + +func exclusionBuildArgs(bin string, modules bool) []string { + args := []string{"test", "-c", "-o", bin} + if modules { + args = append(args, "-mod=mod") + } + if os.Getenv("PARITY_TEST_MODE") == "race" { + args = append(args, "-race") + } + return append(args, ".") +} + +func checkMiniLateExclusionDeclaration(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"late_suite_test.go": `package fixture_test +import("testing";"os";"github.com/stretchr/testify/suite") +type LateSuite struct{suite.Suite} +//ddtest:no_auto_retry +func(s *LateSuite) TestOnce(){ + f,err:=os.OpenFile(os.Getenv("EXCLUSION_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} + if os.Getenv("LATE_FAIL")=="true"{s.T().Error("excluded failure")} +} +func(s *LateSuite) TestSibling(){if os.Getenv("LATE_SIBLING_PASS")!="true" {s.T().Error("parent must retry")}} +func TestLateSuite(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err==nil{f.Close();if os.Getenv("LATE_INITIAL_PASS")!="true"{t.Error("fail before discovering the suite")};return};if !os.IsExist(err){t.Fatal(err)} + suite.Run(t,new(LateSuite)) +} +`}) + bin := filepath.Join(t.TempDir(), executableName("late.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + for _, mode := range []string{"in_process", "process"} { + for _, family := range []string{"atr", "efd", "atf-parent", "atr-method-atf"} { + efd := family == "efd" + atf := family == "atf-parent" || family == "atr-method-atf" + target, suiteName := "TestLateSuite", "late_suite_test.go" + if family == "atr-method-atf" { + target = "TestLateSuite/TestOnce" + suiteName += "/LateSuite" + } + t.Run(mode+"-"+family, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "once") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestLateSuite$"}, Policy: policySettings{Retry: !efd, EFD: efd, AttemptToFix: atf, ManagementTarget: target, ManagementSuite: suiteName}, Env: []string{"EXCLUSION_COUNTER=" + counter, fmt.Sprintf("LATE_INITIAL_PASS=%t", family == "atf-parent"), fmt.Sprintf("LATE_FAIL=%t", atf), fmt.Sprintf("LATE_SIBLING_PASS=%t", family == "atr-method-atf"), "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, fmt.Sprintf("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=%t", !efd), fmt.Sprintf("DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=%t", efd), "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_INTERNAL_PARALLEL_EARLY_FLAKE_DETECTION_ENABLED=true", "DD_CIVISIBILITY_RETRY_PROCESS_MAX_CONCURRENCY=2"}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "x" { + t.Fatalf("late-discovered excluded callback executed %q: %v", data, err) + } + events := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestLateSuite/TestOnce" { + continue + } + events++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("missing late declaration: %v", meta) + } + if atf && (meta["test.test_management.is_attempt_to_fix"] != "true" || meta["test.status"] != "fail") { + t.Fatalf("excluded ATF assignment/result lost: %v", meta) + } + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode", "test.test_management.attempt_to_fix_passed"} { + if meta[key] != nil { + t.Fatalf("first excluded execution inherited retry marker %s: %v", key, meta) + } + } + } + if events != 1 { + t.Fatalf("late-discovered excluded execution events=%d", events) + } + }) + } + } +} + +func checkMiniSubtestGate(t *testing.T, dir, bin string) { + for _, mode := range []string{"in_process", "process"} { + for _, profile := range []string{"disabled", "quarantined", "attempt_to_fix"} { + t.Run(mode+"-"+profile, func(t *testing.T) { + counter := filepath.Join(t.TempDir(), "once") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestExclusionSuite$/^TestOnce$"}, Policy: policySettings{Retry: true, Disabled: profile == "disabled", Quarantined: profile == "quarantined", AttemptToFix: profile == "attempt_to_fix", ManagementTarget: "TestExclusionSuite/TestOnce", ManagementSuite: "exclusion_suite_test.go/ExclusionSuite"}, Env: []string{"DD_CIVISIBILITY_SUBTEST_FEATURES_ENABLED=false", "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "EXCLUSION_COUNTER=" + counter, "EXCLUSION_FAIL=true"}}) + if got.code != 1 { + t.Fatalf("gated backend action controlled command: %d %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if err != nil || string(data) != "x" { + t.Fatalf("excluded callback count %q: %v", data, err) + } + events := 0 + for _, event := range receiver.events { + if event["type"] != "test" { + continue + } + meta := event["content"].(map[string]any)["meta"].(map[string]any) + if meta["test.name"] != "TestExclusionSuite/TestOnce" { + continue + } + events++ + assigned := map[string]string{"disabled": "is_test_disabled", "quarantined": "is_quarantined", "attempt_to_fix": "is_attempt_to_fix"}[profile] + if meta["test.status"] != "fail" || meta["test.test_management.no_auto_retry"] != "true" || meta["test.test_management."+assigned] != "true" { + t.Fatalf("gated execution/assignment mismatch: %v", meta) + } + if meta["test.is_retry"] != nil || meta["test.test_management.attempt_to_fix_passed"] != nil { + t.Fatalf("excluded method claimed retry: %v", meta) + } + } + if events != 1 { + t.Fatalf("gated excluded event count=%d", events) + } + }) + } + } +} diff --git a/ddto/integration/mini_exclusions_tia_test.go b/ddto/integration/mini_exclusions_tia_test.go new file mode 100644 index 000000000..d69ccc56b --- /dev/null +++ b/ddto/integration/mini_exclusions_tia_test.go @@ -0,0 +1,203 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// A method first discovered by a retry worker emits its only event through +// excluded-event replay; that event must retain the worker's TIA decision. +func checkMiniLateExcludedTIA(t *testing.T) { + dir, driver := prepareTestifyFixture(t, false) + writeBuildFixture(t, dir, map[string]string{"late_tia_test.go": `package fixture_test +import("os";"testing";"github.com/stretchr/testify/suite") +type LateTIA struct{suite.Suite} +func(s *LateTIA) record(){ + f,err:=os.OpenFile(os.Getenv("EXCLUSION_COUNTER"),os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if err!=nil{s.T().Fatal(err)} + _,err=f.WriteString("x");f.Close();if err!=nil{s.T().Fatal(err)} +} +//ddtest:no_auto_retry +func(s *LateTIA) TestSkipped(){s.record()} +//dd:test.unskippable +//ddtest:no_auto_retry +func(s *LateTIA) TestForced(){s.record()} +func(s *LateTIA) TestSibling(){s.T().Error("parent must retry again after discovery")} +func TestLateTIA(t *testing.T){ + f,err:=os.OpenFile(os.Getenv("POC_RETRY_COUNTER"),os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err==nil{f.Close();t.Error("first invocation does not discover the suite");return};if !os.IsExist(err){t.Fatal(err)} + suite.Run(t,new(LateTIA)) +} +`}) + bin := filepath.Join(t.TempDir(), executableName("late-tia.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, true)...) + if code != 0 { + t.Fatalf("compile %d: %s %s", code, out, stderr) + } + // Native Testify method instrumentation retains the existing top-level-only + // TIA scope (see the itr-method-sdk-limit parity case). Compare that native + // baseline with TIA off/on; worker subtree policy supports method decisions. + for _, profile := range []struct { + name, mode string + tia bool + }{ + {"native-baseline", "in_process", false}, + {"native-tia", "in_process", true}, + {"worker-tia", "process", true}, + } { + mode := profile.mode + for _, forced := range []bool{false, true} { + t.Run(fmt.Sprintf("%s-forced-%t", profile.name, forced), func(t *testing.T) { + method := "TestSkipped" + if forced { + method = "TestForced" + } + name := "TestLateTIA/" + method + counter := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestLateTIA$/^(" + method + "|TestSibling)$"}, Policy: policySettings{ITR: profile.tia, Retry: true, SkippableTarget: name, SkippableSuite: "late_tia_test.go/LateTIA"}, Env: []string{"EXCLUSION_COUNTER=" + counter, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=" + mode, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2"}}) + if got.code != 1 { + t.Fatalf("exit %d: %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if forced || mode == "in_process" { + if err != nil || string(data) != "x" { + t.Fatalf("forced callback=%q err=%v", data, err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("skipped callback ran: %q err=%v", data, err) + } + events, sessions := 0, 0 + for _, event := range receiver.events { + content := event["content"].(map[string]any) + meta, _ := content["meta"].(map[string]any) + if event["type"] == "test" && meta["test.name"] == name { + events++ + if meta["test.test_management.no_auto_retry"] != "true" { + t.Fatalf("declaration lost: %v", meta) + } + if mode == "in_process" { + if meta["test.status"] != "pass" || meta["test.itr.forced_run"] != nil || meta["test.skipped_by_itr"] != nil { + t.Fatalf("native method TIA baseline changed: %v", meta) + } + } else if forced { + if meta["test.itr.forced_run"] != "true" || meta["test.status"] != "pass" || meta["test.skipped_by_itr"] != nil { + t.Fatalf("forced TIA result lost: %v", meta) + } + } else if meta["test.skipped_by_itr"] != "true" || meta["test.status"] != "skip" { + t.Fatalf("skipped TIA result lost: %v", meta) + } + for _, key := range []string{"test.is_retry", "test.retry_reason", "test.retry.execution_mode"} { + if meta[key] != nil { + t.Fatalf("first execution inherited %s: %v", key, meta) + } + } + } + if event["type"] == "test_session_end" { + sessions++ + metrics, _ := content["metrics"].(map[string]any) + count := "0" + if value, ok := metrics["test.itr.tests_skipping.count"]; ok { + count = fmt.Sprint(value) + } + want := "1" + if forced || mode == "in_process" { + want = "0" + } + if count != want { + t.Fatalf("actual TIA skipped count=%v want=%v session=%v", count, want, content) + } + if !forced && mode == "process" && meta["_dd.ci.itr.tests_skipped"] != "true" { + t.Fatalf("actual TIA skip not reflected in session: %v", content) + } + } + } + if events != 1 || sessions != 1 { + t.Fatalf("events=%d sessions=%d want one each", events, sessions) + } + }) + } + } +} + +// Both declaration policies preserve native-supported top-level TIA decisions. +func checkMiniTopLevelExcludedTIA(t *testing.T) { + for _, annotation := range []string{"//ddtest:no_auto_retry", "//ddtest:prevent_failure_suppression", "//ddtest:no_auto_retry\n//ddtest:prevent_failure_suppression"} { + t.Run(annotation, func(t *testing.T) { + dir, driver := prepareMiniFixture(t) + writeBuildFixture(t, dir, map[string]string{"top_tia_test.go": `package fixture_test +import("testing";"os") +func writeTopTIA(t *testing.T){if err:=os.WriteFile(os.Getenv("TOP_TIA_CALL"),[]byte("x"),0600);err!=nil{t.Fatal(err)}} +` + annotation + ` +func TestTopSkipped(t *testing.T){writeTopTIA(t)} +` + annotation + ` +//dd:test.unskippable +func TestTopForced(t *testing.T){writeTopTIA(t)} +`}) + bin := filepath.Join(t.TempDir(), executableName("top-tia.test")) + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), driver, exclusionBuildArgs(bin, false)...) + if code != 0 { + t.Fatalf("compile%d: %s %s", code, out, stderr) + } + for _, forced := range []bool{false, true} { + t.Run(fmt.Sprint("forced=", forced), func(t *testing.T) { + name := "TestTopSkipped" + if forced { + name = "TestTopForced" + } + counter := filepath.Join(t.TempDir(), "calls") + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^" + name + "$"}, Policy: policySettings{ITR: true, Retry: true, SkippableTarget: name, SkippableSuite: "top_tia_test.go"}, Env: []string{"TOP_TIA_CALL=" + counter, "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true"}}) + if got.code != 0 { + t.Fatalf("TIA policy changed command exit%d: %s %s", got.code, got.out, got.stderr) + } + data, err := os.ReadFile(counter) + if forced { + if err != nil || string(data) != "x" { + t.Fatalf("forced callback=%q %v", data, err) + } + } else if !os.IsNotExist(err) { + t.Fatalf("skipped callback ran: %q %v", data, err) + } + events := 0 + for _, event := range receiver.events { + content := event["content"].(map[string]any) + meta, _ := content["meta"].(map[string]any) + if event["type"] == "test" && meta["test.name"] == name { + events++ + for _, flag := range []string{"no_auto_retry", "prevent_failure_suppression"} { + if (meta["test.test_management."+flag] == "true") != strings.Contains(annotation, flag) { + t.Fatalf("TIA event lost declaration %s: %v", flag, meta) + } + } + if forced { + if meta["test.itr.forced_run"] != "true" || meta["test.status"] != "pass" { + t.Fatalf("forced event=%v", meta) + } + } else if meta["test.skipped_by_itr"] != "true" || meta["test.status"] != "skip" { + t.Fatalf("skipped event=%v", meta) + } + } + if event["type"] == "test_session_end" { + metrics, _ := content["metrics"].(map[string]any) + count := int64(0) + if value, ok := metrics["test.itr.tests_skipping.count"]; ok { + count = toInt64(value) + } + want := int64(1) + if forced { + want = 0 + } + if count != want { + t.Fatalf("native skipped count=%d want%d", count, want) + } + } + } + if events != 1 { + t.Fatalf("native TIA execution events=%d", events) + } + }) + } + }) + } +} diff --git a/ddto/integration/mini_git_state_test.go b/ddto/integration/mini_git_state_test.go index ffe8745b6..7ed96921d 100644 --- a/ddto/integration/mini_git_state_test.go +++ b/ddto/integration/mini_git_state_test.go @@ -156,7 +156,7 @@ func TestMiniGitStateHandoff(t *testing.T) { // A state from another checkout is ignored. other := t.TempDir() - for _, gitArgs := range [][]string{{"init", "--initial-branch=main"}, {"-c", "user.name=Other", "-c", "user.email=other@example.invalid", "commit", "--allow-empty", "-m", "Other"}} { + for _, gitArgs := range [][]string{{"init", "--initial-branch=main"}, {"-c", "user.name=Other", "-c", "user.email=other@example.invalid", "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-m", "Other"}} { if out, stderr, code := command(t, other, testEnv(), "git", gitArgs...); code != 0 { t.Fatalf("other repository: %s %s", out, stderr) } diff --git a/ddto/integration/mini_test.go b/ddto/integration/mini_test.go index b0b5fcccb..93de9a7eb 100644 --- a/ddto/integration/mini_test.go +++ b/ddto/integration/mini_test.go @@ -165,15 +165,15 @@ func canonicalMiniStack(stack string) string { return strings.Join(lines, "\n") } -// These calls moved with cleanup-aware shutdown and the fuzz lifecycle port. Match +// These calls moved with cleanup-aware shutdown, fuzz, and test exclusions. Match // both the function and its exact location; other frames stay strict. var adaptedMiniCallSites = []struct{ function, mini, sdk string }{ - {"integrations/gotesting.applyAdditionalFeaturesToTestFunc.func2", "integrations/gotesting/instrumentation.go:775", "integrations/gotesting/instrumentation.go:775"}, - {"integrations/gotesting.runTestWithRetry", "integrations/gotesting/instrumentation.go:1011", "integrations/gotesting/instrumentation.go:1011"}, - {"integrations/gotesting.runRetryAttemptCapabilityFallback", "integrations/gotesting/instrumentation.go:1138", "integrations/gotesting/instrumentation.go:1138"}, - {"integrations/gotesting.(*M).executeInternalTest.func1", "integrations/gotesting/testing.go:882", "integrations/gotesting/testing.go:864"}, - {"integrations/gotesting.instrumentTestingTFuncWithSourceOptions.func1.1", "integrations/gotesting/instrumentation_orchestrion.go:427", "integrations/gotesting/instrumentation_orchestrion.go:424"}, - {"integrations/gotesting.instrumentTestingTFuncWithSourceOptions.func1", "integrations/gotesting/instrumentation_orchestrion.go:433", "integrations/gotesting/instrumentation_orchestrion.go:430"}, + {"integrations/gotesting.applyAdditionalFeaturesToTestFunc.func2", "integrations/gotesting/instrumentation.go:825", "integrations/gotesting/instrumentation.go:775"}, + {"integrations/gotesting.runTestWithRetry", "integrations/gotesting/instrumentation.go:1072", "integrations/gotesting/instrumentation.go:1011"}, + {"integrations/gotesting.runRetryAttemptCapabilityFallback", "integrations/gotesting/instrumentation.go:1199", "integrations/gotesting/instrumentation.go:1138"}, + {"integrations/gotesting.(*M).executeInternalTest.func1", "integrations/gotesting/testing.go:895", "integrations/gotesting/testing.go:864"}, + {"integrations/gotesting.instrumentTestingTFuncWithSourceOptions.func1.1", "integrations/gotesting/instrumentation_orchestrion.go:451", "integrations/gotesting/instrumentation_orchestrion.go:424"}, + {"integrations/gotesting.instrumentTestingTFuncWithSourceOptions.func1", "integrations/gotesting/instrumentation_orchestrion.go:457", "integrations/gotesting/instrumentation_orchestrion.go:430"}, {"integrations/gotesting.runRetryAttemptBody", "integrations/gotesting/retry_attempt_runner.go:452", "integrations/gotesting/retry_attempt_runner.go:452"}, {"integrations/gotesting.runFreshRetryAttemptOwner", "integrations/gotesting/retry_attempt_runner.go:202", "integrations/gotesting/retry_attempt_runner.go:202"}, } @@ -244,12 +244,12 @@ func canonicalMiniAssertionLocation(location string) string { func TestCanonicalMiniStackMapsOnlyAdaptedCallSite(t *testing.T) { sdk := "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting.(*M).executeInternalTest.func1\n\t/sdk/internal/civisibility/integrations/gotesting/testing.go:864\nexample.com/app.TestFailure\n\t/work/app_test.go:17" - mini := "github.com/DataDog/ddtest/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting.(*M).executeInternalTest.func1\n\t/poc/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go:882\nexample.com/app.TestFailure\n\t/work/app_test.go:17" + mini := "github.com/DataDog/ddtest/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting.(*M).executeInternalTest.func1\n\t/poc/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go:895\nexample.com/app.TestFailure\n\t/work/app_test.go:17" want := canonicalMiniStack(sdk) if canonicalMiniStack(mini) != want { t.Fatal("adapted call site was not mapped to the pinned SDK") } - for _, changed := range []string{strings.ReplaceAll(mini, "testing.go:882", "testing.go:883"), strings.ReplaceAll(mini, "app_test.go:17", "app_test.go:18"), strings.ReplaceAll(mini, "executeInternalTest.func1", "executeInternalTest.func2")} { + for _, changed := range []string{strings.ReplaceAll(mini, "testing.go:895", "testing.go:896"), strings.ReplaceAll(mini, "app_test.go:17", "app_test.go:18"), strings.ReplaceAll(mini, "executeInternalTest.func1", "executeInternalTest.func2")} { if canonicalMiniStack(changed) == want { t.Fatal("canonicalization hid a different location or function") } @@ -444,8 +444,8 @@ func TestCanonicalMiniCallSitesRetainOtherLocations(t *testing.T) { function string mini, sdk int }{ - {"instrumentTestingTFuncWithSourceOptions.func1.1", 427, 424}, - {"instrumentTestingTFuncWithSourceOptions.func1", 433, 430}, + {"instrumentTestingTFuncWithSourceOptions.func1.1", 451, 424}, + {"instrumentTestingTFuncWithSourceOptions.func1", 457, 430}, } { function := "integrations/gotesting." + site.function location := fmt.Sprintf("integrations/gotesting/instrumentation_orchestrion.go:%d", site.mini) @@ -478,22 +478,22 @@ func TestTestifyMovedCallSitesInFormattedDiagnostics(t *testing.T) { file := "integrations/gotesting/instrumentation_orchestrion.go:" sdk := "test panicked: fixture\n goroutine 12 [running]:\n github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting.instrumentTestingTFuncWithSourceOptions.func1(0x123)\n \t" + sdkPrefix + file + "430 +0x123\n" mini := strings.ReplaceAll(sdk, "github.com/DataDog/dd-trace-go/v2/internal/civisibility/", "github.com/DataDog/ddtest/ddto/internal/thirdparty/dd-trace-go/civisibility/") - mini = strings.ReplaceAll(strings.ReplaceAll(mini, sdkPrefix, miniPrefix), file+"430", file+"433") + mini = strings.ReplaceAll(strings.ReplaceAll(mini, sdkPrefix, miniPrefix), file+"430", file+"457") if canonicalTestifyDiagnostic(sdk) != canonicalTestifyDiagnostic(mini) { t.Error("indented stack mapping lost") } sdk = "\n\tError Trace:\t/work/app_test.go:18\n\t \t\t\t\t" + sdkPrefix + file + "430\n\tError: \tfixture failed" - mini = strings.ReplaceAll(strings.ReplaceAll(sdk, sdkPrefix, miniPrefix), file+"430", file+"433") + mini = strings.ReplaceAll(strings.ReplaceAll(sdk, sdkPrefix, miniPrefix), file+"430", file+"457") if canonicalTestifyDiagnostic(sdk) != canonicalTestifyDiagnostic(mini) { t.Error("formatted assertion source mapping lost") } - for _, changed := range []string{strings.ReplaceAll(mini, file+"433", file+"434"), strings.ReplaceAll(mini, "app_test.go:18", "app_test.go:19"), strings.ReplaceAll(mini, "fixture failed", "different failure")} { + for _, changed := range []string{strings.ReplaceAll(mini, file+"457", file+"458"), strings.ReplaceAll(mini, "app_test.go:18", "app_test.go:19"), strings.ReplaceAll(mini, "fixture failed", "different failure")} { if canonicalTestifyDiagnostic(sdk) == canonicalTestifyDiagnostic(changed) { t.Error("formatted assertion erased a real difference") } } - text := "message references " + miniPrefix + file + "433" - if !strings.Contains(canonicalTestifyDiagnostic(text), file+"433") { + text := "message references " + miniPrefix + file + "457" + if !strings.Contains(canonicalTestifyDiagnostic(text), file+"457") { t.Error("rewrote arbitrary message text") } } diff --git a/ddto/integration/parity_testify_test.go b/ddto/integration/parity_testify_test.go index 4623e7d3e..9bc15f8e0 100644 --- a/ddto/integration/parity_testify_test.go +++ b/ddto/integration/parity_testify_test.go @@ -41,7 +41,7 @@ func installExternalTestifyHelperWithTempDir(t *testing.T, dir string, tempDir f t.Helper() kit := tempDir() for name, source := range map[string]string{ - "go.mod": "module example.com/external-testkit\n\ngo 1.26.0\nrequire github.com/stretchr/testify v1.11.1\n", + "go.mod": "module example.com/external-testkit\n\ngo 1.25.0\nrequire github.com/stretchr/testify v1.11.1\n", "run.go": "package testkit\nimport(\"testing\";\"github.com/stretchr/testify/suite\")\nfunc Run(t *testing.T,s suite.TestingSuite){suite.Run(t,s)}\n", } { if err := os.WriteFile(filepath.Join(kit, name), []byte(source), 0600); err != nil { diff --git a/ddto/internal/runner/exclusions.go b/ddto/internal/runner/exclusions.go new file mode 100644 index 000000000..3b20d3c57 --- /dev/null +++ b/ddto/internal/runner/exclusions.go @@ -0,0 +1,354 @@ +package runner + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +type exclusionDeclaration struct { + expression string + receiver string + valueReceiver bool + genericReceiver bool + name string + prevent, noRetry bool +} + +// exclusionDeclarations reads only the compiler-selected files, including the +// caller's overlay. The registration is compiled, so runtime source is optional. +func exclusionDeclarations(p goPackage, files []string, replacements map[string]string) (map[string][]exclusionDeclaration, error) { + declarations := map[string][]exclusionDeclaration{} + files = append([]string(nil), files...) + sort.Strings(files) + for _, name := range files { + logical := filepath.Join(p.Dir, name) + source := logical + if replacement, ok := replacements[logical]; ok { + source = replacement + } + if source == "" { + continue + } + data, err := os.ReadFile(source) + if err != nil { + return nil, err + } + if !bytes.Contains(data, []byte("//ddtest:")) { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), logical, data, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + return nil, err + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Doc == nil || fn.Body == nil { + continue + } + d := exclusionDeclaration{name: fn.Name.Name, expression: fn.Name.Name} + for _, comment := range fn.Doc.List { + switch strings.TrimSpace(comment.Text) { + case "//ddtest:prevent_failure_suppression": + d.prevent = true + case "//ddtest:no_auto_retry": + d.noRetry = true + } + } + if !d.prevent && !d.noRetry { + continue + } + if !strings.HasPrefix(fn.Name.Name, "Test") || fn.Name.Name == "TestMain" { + return nil, fmt.Errorf("%s: ddtest exclusions require an individual Test function or Testify Test method", logical) + } + if fn.Recv != nil { + if len(fn.Recv.List) != 1 { + return nil, fmt.Errorf("%s: unsupported annotated test receiver", logical) + } + receiver := ast.Unparen(fn.Recv.List[0].Type) + if star, ok := receiver.(*ast.StarExpr); ok { + receiver = ast.Unparen(star.X) + } else { + d.valueReceiver = true + } + switch indexed := receiver.(type) { + case *ast.IndexExpr: + receiver = ast.Unparen(indexed.X) + d.genericReceiver = true + case *ast.IndexListExpr: + receiver = ast.Unparen(indexed.X) + d.genericReceiver = true + } + id, ok := receiver.(*ast.Ident) + if !ok { + return nil, fmt.Errorf("%s: unsupported annotated test receiver", logical) + } + d.receiver = id.Name + d.expression = "(*" + id.Name + ")." + fn.Name.Name + } + declarations[file.Name.Name] = append(declarations[file.Name.Name], d) + } + } + return declarations, nil +} + +// go list separates active import-C sources from ordinary Go source files. +func exclusionPackageFiles(p goPackage) []string { + return append(append([]string(nil), p.GoFiles...), p.CgoFiles...) +} + +func exclusionRegistrationSources(p goPackage, replacements map[string]string, runtimePackage string) (map[string]string, error) { + return exclusionRegistrationSourcesForBoundary(p, replacements, runtimePackage, false) +} + +func exclusionRegistrationSourcesForBoundary(p goPackage, replacements map[string]string, runtimePackage string, external bool) (map[string]string, error) { + files := append(append(append([]string{}, exclusionPackageFiles(p)...), p.TestGoFiles...), p.XTestGoFiles...) + declarations, err := exclusionDeclarations(p, files, replacements) + if err != nil { + return nil, err + } + var aliases map[string]map[string]exclusionReceiverAlias + callsByPackage := map[string][]string{} + packages := make([]string, 0, len(declarations)) + for pkg := range declarations { + packages = append(packages, pkg) + } + sort.Strings(packages) + for _, pkg := range packages { + entries := declarations[pkg] + destination := pkg + if external && pkg == p.Name { + destination = p.Name + "_test" + } + for _, d := range entries { + if d.receiver != "" { + if aliases == nil { + aliases, err = exclusionReceiverAliases(p, files, replacements) + if err != nil { + return nil, err + } + } + d = resolveExclusionReceiver(d, aliases[pkg]) + d.expression = "(*" + d.receiver + ")." + d.name + } + if external && pkg == p.Name || d.genericReceiver { + declarationPath := p.ImportPath + if pkg != p.Name { + declarationPath += "_test" + } + for _, name := range exclusionRuntimeNames(declarationPath, d) { + callsByPackage[destination] = append(callsByPackage[destination], fmt.Sprintf("__dd_ci_runtime.RegisterTestExclusionsByName(%q, __dd_ci_runtime.TestExclusions{PreventFailureSuppression: %t, NoAutoRetry: %t})", name, d.prevent, d.noRetry)) + } + } else { + callsByPackage[destination] = append(callsByPackage[destination], fmt.Sprintf("__dd_ci_runtime.RegisterTestExclusions(%s, __dd_ci_runtime.TestExclusions{PreventFailureSuppression: %t, NoAutoRetry: %t})", d.expression, d.prevent, d.noRetry)) + } + } + } + sources := map[string]string{} + for pkg, calls := range callsByPackage { + sources[pkg] = "package " + pkg + "\nimport __dd_ci_runtime " + fmt.Sprintf("%q", runtimePackage) + "\nfunc init() {\n" + strings.Join(calls, "\n") + "\n}\n" + } + return sources, nil +} + +// The compiler escapes special bytes and dots in the final import-path segment +// before using an import path as a runtime function-name prefix. +func exclusionRuntimePrefix(path string) string { + lastSlash := strings.LastIndexByte(path, '/') + var prefix strings.Builder + for i := 0; i < len(path); i++ { + c := path[i] + if c <= ' ' || c == '.' && i > lastSlash || c == '%' || c == '"' || c >= 0x7f { + fmt.Fprintf(&prefix, "%%%02x", c) + } else { + prefix.WriteByte(c) + } + } + return prefix.String() +} + +func exclusionRuntimeNames(path string, d exclusionDeclaration) []string { + prefix := exclusionRuntimePrefix(path) + if d.genericReceiver { + d.receiver += "[...]" + } + if d.receiver == "" { + return []string{prefix + "." + d.name} + } + names := []string{prefix + ".(*" + d.receiver + ")." + d.name} + if d.valueReceiver { + names = append(names, prefix+"."+d.receiver+"."+d.name) + } + return names +} + +// Receiver aliases use the defined type's runtime symbol, even when a method +// spells its receiver through a package-local alias from another selected file. +// Defined types (without '=') retain their independent identities. +type exclusionReceiverAlias struct { + name string + pointer bool +} + +func resolveExclusionReceiver(d exclusionDeclaration, aliases map[string]exclusionReceiverAlias) exclusionDeclaration { + seen := map[string]bool{} + for aliases[d.receiver].name != "" && !seen[d.receiver] { + seen[d.receiver] = true + alias := aliases[d.receiver] + d.receiver = alias.name + if alias.pointer { + d.valueReceiver = false + } + } + return d +} + +func exclusionReceiverAliases(p goPackage, files []string, replacements map[string]string) (map[string]map[string]exclusionReceiverAlias, error) { + aliases := map[string]map[string]exclusionReceiverAlias{} + for _, name := range files { + logical := filepath.Join(p.Dir, name) + source := logical + if replacement, ok := replacements[logical]; ok { + source = replacement + } + if source == "" { + continue + } + data, err := os.ReadFile(source) + if err != nil { + return nil, err + } + file, err := parser.ParseFile(token.NewFileSet(), logical, data, parser.SkipObjectResolution) + if err != nil { + return nil, err + } + if aliases[file.Name.Name] == nil { + aliases[file.Name.Name] = map[string]exclusionReceiverAlias{} + } + for _, declaration := range file.Decls { + group, ok := declaration.(*ast.GenDecl) + if !ok || group.Tok != token.TYPE { + continue + } + for _, spec := range group.Specs { + typ := spec.(*ast.TypeSpec) + if !typ.Assign.IsValid() { + continue + } + targetType := ast.Unparen(typ.Type) + alias := exclusionReceiverAlias{} + if star, ok := targetType.(*ast.StarExpr); ok { + targetType = ast.Unparen(star.X) + alias.pointer = true + } + if target, ok := targetType.(*ast.Ident); ok { + alias.name = target.Name + aliases[file.Name.Name][typ.Name.Name] = alias + } + } + } + } + return aliases, nil +} + +// Imported suites are ordinary packages: _test.go registrations in the defining +// package would never join the consumer's binary. Register their compiled names +// in the consumer instead, without editing cached modules or importing Mini into +// dependencies that Mini already imports. +func exclusionDependencyRegistrations(root goPackage, graph packageGraph, replacements map[string]string, excluded map[string]bool) (string, error) { + pending := append(append(append([]string{}, root.Imports...), root.TestImports...), root.XTestImports...) + reached := map[string]bool{} + var paths []string + for len(pending) > 0 { + path := pending[len(pending)-1] + pending = pending[:len(pending)-1] + if reached[path] || excluded[path] { + continue + } + reached[path] = true + p := graph[path] + if p == nil || p.Standard || p.Module == nil || p.Error != nil || p.ownRuntime || p.ImportPath == miniPackage || strings.HasPrefix(p.ImportPath, miniModule+"/") { + continue + } + paths = append(paths, path) + pending = append(pending, p.Imports...) + } + sort.Strings(paths) + var calls []string + for _, path := range paths { + p := graph[path] + declarations, err := exclusionDeclarations(*p, exclusionPackageFiles(*p), replacements) + if err != nil { + return "", err + } + var aliases map[string]map[string]exclusionReceiverAlias + for _, d := range declarations[p.Name] { + if d.receiver != "" { + aliases, err = exclusionReceiverAliases(*p, exclusionPackageFiles(*p), replacements) + if err != nil { + return "", err + } + break + } + } + for _, d := range declarations[p.Name] { + d = resolveExclusionReceiver(d, aliases[p.Name]) + names := exclusionRuntimeNames(path, d) + for _, name := range names { + calls = append(calls, fmt.Sprintf("__dd_ci_runtime.RegisterTestExclusionsByName(%q, __dd_ci_runtime.TestExclusions{PreventFailureSuppression: %t, NoAutoRetry: %t})", name, d.prevent, d.noRetry)) + } + } + } + if len(calls) == 0 { + return "", nil + } + return "func init() {\n" + strings.Join(calls, "\n") + "\n}\n", nil +} + +// Extend the native package graph only for test imports not already listed. +// Dependency package tests are deliberately excluded; only active GoFiles and CgoFiles join +// an import, and this query preserves the caller's module/build/overlay flags. +func exclusionDependencyGraph(packages []goPackage, graph packageGraph, list func(...string) ([]goPackage, error)) (packageGraph, error) { + missing := map[string]bool{} + for _, p := range packages { + if p.ImportPath == miniPackage || p.ImportPath == sdkPackage || p.Standard || p.ownRuntime { + continue + } + for _, imports := range [][]string{p.TestImports, p.XTestImports} { + for _, path := range imports { + if graph[path] == nil { + missing[path] = true + } + } + } + } + if len(missing) == 0 { + return graph, nil + } + paths := make([]string, 0, len(missing)) + for path := range missing { + paths = append(paths, path) + } + sort.Strings(paths) + dependencies, err := list(append([]string{"-deps"}, paths...)...) + if err != nil { + return nil, err + } + extended := make(packageGraph, len(graph)+len(dependencies)) + for path, p := range graph { + extended[path] = p + } + for i := range dependencies { + p := &dependencies[i] + if extended[p.ImportPath] == nil { + extended[p.ImportPath] = p + } + } + return extended, nil +} diff --git a/ddto/internal/runner/exclusions_test.go b/ddto/internal/runner/exclusions_test.go new file mode 100644 index 000000000..151f5137c --- /dev/null +++ b/ddto/internal/runner/exclusions_test.go @@ -0,0 +1,170 @@ +package runner + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExclusionRegistrationsKeepPackagesAndDeclarationsSeparate(t *testing.T) { + dir := t.TempDir() + files := map[string]string{ + "internal_test.go": "package fixture\nimport \"testing\"\n//ddtest:no_auto_retry\nfunc TestInternal(t *testing.T) {}\ntype Suite struct{}\n//ddtest:prevent_failure_suppression\nfunc (s (*Suite)) TestMethod() {}\n//ddtest:no_auto_retry\nfunc (s (Suite)) TestValue() {}\n", + "external_test.go": "package fixture_test\nimport \"testing\"\n//ddtest:no_auto_retry\n//ddtest:prevent_failure_suppression\nfunc TestExternal(t *testing.T) {}\n", + } + for name, data := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + p := goPackage{Dir: dir, Name: "fixture", TestGoFiles: []string{"internal_test.go"}, XTestGoFiles: []string{"external_test.go"}} + got, err := exclusionRegistrationSources(p, nil, miniPackage) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !strings.Contains(got["fixture"], "RegisterTestExclusions((*Suite).TestValue") || !strings.Contains(got["fixture"], "RegisterTestExclusions((*Suite).TestMethod") || strings.Contains(got["fixture"], "TestExternal") || !strings.Contains(got["fixture_test"], "PreventFailureSuppression: true, NoAutoRetry: true") { + t.Fatalf("registrations: %v", got) + } + for _, src := range got { + if _, err := parser.ParseFile(token.NewFileSet(), "generated.go", src, 0); err != nil { + t.Fatal(err) + } + } + p.ImportPath = "example.com/runtime.v1" + boundary, err := exclusionRegistrationSourcesForBoundary(p, nil, miniPackage, true) + if err != nil || len(boundary) != 1 || !strings.Contains(boundary["fixture_test"], `RegisterTestExclusionsByName("example.com/runtime%2ev1.TestInternal"`) || !strings.Contains(boundary["fixture_test"], `RegisterTestExclusionsByName("example.com/runtime%2ev1.(*Suite).TestMethod"`) || !strings.Contains(boundary["fixture_test"], "RegisterTestExclusions(TestExternal") { + t.Fatalf("external runtime dependency registrations: %v %v", boundary, err) + } + t.Run("generic-receivers", func(t *testing.T) { + generic := filepath.Join(dir, "generic.go") + source := "package fixture\ntype Suite[T any] struct{}\n//ddtest:no_auto_retry\nfunc (s (*Suite[T])) TestPointer(){}\n//ddtest:no_auto_retry\nfunc (s (Suite[T])) TestValue(){}\n" + if err := os.WriteFile(generic, []byte(source), 0600); err != nil { + t.Fatal(err) + } + external := filepath.Join(dir, "generic_external_test.go") + if err := os.WriteFile(external, []byte("package fixture_test\ntype Suite[A,B any] struct{}\n//ddtest:no_auto_retry\nfunc (s (*Suite[A,B])) TestPointer(){}\n"), 0600); err != nil { + t.Fatal(err) + } + gp := goPackage{Dir: dir, Name: "fixture", ImportPath: "example.com/generic.v1", GoFiles: []string{"generic.go"}, XTestGoFiles: []string{"generic_external_test.go"}, TestImports: []string{"example.com/helper"}} + got, err := exclusionRegistrationSources(gp, nil, miniPackage) + if err != nil { + t.Fatal(err) + } + for pkg, names := range map[string][]string{ + "fixture": {"example.com/generic%2ev1.(*Suite[...]).TestPointer", "example.com/generic%2ev1.(*Suite[...]).TestValue", "example.com/generic%2ev1.Suite[...].TestValue"}, + "fixture_test": {"example.com/generic%2ev1_test.(*Suite[...]).TestPointer"}, + } { + for _, name := range names { + if !strings.Contains(got[pkg], `RegisterTestExclusionsByName("`+name+`"`) { + t.Fatalf("missing %s: %v", name, got) + } + } + } + // Importing the generic declaration requires no concrete type argument. + gp.Module = &struct { + Path, Version, Dir string + Main bool + Replace *struct{ Path, Dir, Version string } + }{Path: "example.com/generic.v1"} + imported, err := exclusionDependencyRegistrations(goPackage{Imports: []string{gp.ImportPath}}, packageGraph{gp.ImportPath: &gp}, nil, nil) + if err != nil || !strings.Contains(imported, `RegisterTestExclusionsByName("example.com/generic%2ev1.(*Suite[...]).TestPointer"`) { + t.Fatalf("generic imported declarations: %s %v", imported, err) + } + boundary, err := exclusionRegistrationSourcesForBoundary(gp, nil, miniPackage, true) + if err != nil || len(boundary) != 1 || !strings.Contains(boundary["fixture_test"], `"example.com/generic%2ev1.(*Suite[...]).TestPointer"`) { + t.Fatalf("generic runtime dependency boundary: %v %v", boundary, err) + } + }) + // An edited overlay and annotation removal replace rather than retain policy. + overlay := filepath.Join(dir, "replacement.go") + if err := os.WriteFile(overlay, []byte("package fixture\nfunc TestInternal() {}\n"), 0600); err != nil { + t.Fatal(err) + } + got, err = exclusionRegistrationSources(p, map[string]string{filepath.Join(dir, "internal_test.go"): overlay}, miniPackage) + if err != nil || got["fixture"] != "" { + t.Fatalf("removed annotation retained: %v %v", got, err) + } +} + +func TestExclusionRegistrationsRejectNonTestDeclarations(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "sample_test.go"), []byte("package fixture\n//ddtest:no_auto_retry\nfunc helper() {}\n"), 0600); err != nil { + t.Fatal(err) + } + _, err := exclusionRegistrationSources(goPackage{Dir: dir, TestGoFiles: []string{"sample_test.go"}}, nil, miniPackage) + if err == nil { + t.Fatal("accepted a non-test annotation") + } +} + +func TestImportedExclusionRegistrationsFollowNativeReachabilityAndOverlay(t *testing.T) { + dir := t.TempDir() + for name, data := range map[string]string{ + "suite.go": "package helper\ntype Base struct{}\n//ddtest:no_auto_retry\nfunc (*Base) TestCritical(){}\n//ddtest:no_auto_retry\nfunc (Base) TestValue(){}\n", + "suite_test.go": "package helper\n//ddtest:no_auto_retry\nfunc TestDependencyOnly(){}\n", + "overlay.go": "package helper\ntype Base struct{}\n//ddtest:prevent_failure_suppression\nfunc (Alias) TestCritical(){}\n//ddtest:no_auto_retry\nfunc (ValueAlias) TestValue(){}\n//ddtest:no_auto_retry\nfunc (*Defined) TestDistinct(){}\n", + "aliases.go": "package helper\ntype Alias = Wrong\ntype ValueAlias = Wrong\ntype Defined Base\n", + "aliases_overlay.go": "package helper\ntype Alias = Intermediate\ntype Intermediate = (*Base)\ntype ValueAlias = Base\ntype Defined Base\n", + "unselected.go": "package helper\ntype Intermediate = Wrong\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + root := goPackage{ImportPath: "example.com/root", TestImports: []string{"example.com/helper.v1"}} + helper := goPackage{Dir: dir, Name: "helper", ImportPath: "example.com/helper.v1", CgoFiles: []string{"suite.go", "aliases.go"}, TestGoFiles: []string{"suite_test.go"}, DepOnly: true} + helper.Module = &struct { + Path, Version, Dir string + Main bool + Replace *struct{ Path, Dir, Version string } + }{Path: "example.com/helper.v1"} + requested := []string{} + graph, err := exclusionDependencyGraph([]goPackage{root}, packageGraph{}, func(paths ...string) ([]goPackage, error) { + requested = append(requested, paths...) + return []goPackage{helper}, nil + }) + if err != nil || strings.Join(requested, " ") != "-deps example.com/helper.v1" { + t.Fatalf("query=%v err=%v", requested, err) + } + overlay := map[string]string{filepath.Join(dir, "suite.go"): filepath.Join(dir, "overlay.go"), filepath.Join(dir, "aliases.go"): filepath.Join(dir, "aliases_overlay.go")} + got, err := exclusionDependencyRegistrations(root, graph, overlay, nil) + if err != nil { + t.Fatal(err) + } + // cgo declarations also register inside a local test package, where the + // compiler itself resolves the receiver aliases in method expressions. + local, err := exclusionRegistrationSources(helper, overlay, miniPackage) + if err != nil || !strings.Contains(local["helper"], "RegisterTestExclusions((*Base).TestCritical") || !strings.Contains(local["helper"], "RegisterTestExclusions((*Base).TestValue") { + t.Fatalf("local cgo declarations: %v %v", local, err) + } + for _, wanted := range []string{ + `RegisterTestExclusionsByName("example.com/helper%2ev1.(*Base).TestCritical", __dd_ci_runtime.TestExclusions{PreventFailureSuppression: true, NoAutoRetry: false})`, + `RegisterTestExclusionsByName("example.com/helper%2ev1.(*Base).TestValue"`, + `RegisterTestExclusionsByName("example.com/helper%2ev1.Base.TestValue"`, + `RegisterTestExclusionsByName("example.com/helper%2ev1.(*Defined).TestDistinct"`, + } { + if !strings.Contains(got, wanted) { + t.Fatalf("missing %s: %s", wanted, got) + } + } + if strings.Contains(got, ".(*Alias).") || strings.Contains(got, ".ValueAlias.") || strings.Contains(got, "Wrong") || strings.Contains(got, ".(*Base).TestDistinct") || strings.Contains(got, "helper%2ev1.Base.TestCritical") { + t.Fatalf("receiver alias identity or defined type was changed: %s", got) + } + if strings.Contains(got, "TestDependencyOnly") { + t.Fatalf("dependency's tests joined consumer: %s", got) + } + if _, err := parser.ParseFile(token.NewFileSet(), "generated.go", "package fixture\n"+got, 0); err != nil { + t.Fatal(err) + } + got, err = exclusionDependencyRegistrations(root, graph, overlay, map[string]bool{"example.com/helper.v1": true}) + if err != nil || got != "" { + t.Fatalf("runtime dependency registered: %s %v", got, err) + } + got, err = exclusionDependencyRegistrations(goPackage{ImportPath: "example.com/unrelated"}, graph, overlay, nil) + if err != nil || got != "" { + t.Fatalf("unreachable helper registered: %s %v", got, err) + } +} diff --git a/ddto/internal/runner/run.go b/ddto/internal/runner/run.go index 9067307f8..53f1d3afa 100644 --- a/ddto/internal/runner/run.go +++ b/ddto/internal/runner/run.go @@ -35,7 +35,8 @@ type goPackage struct { DepOnly bool // Listed by -deps only as a dependency of the named packages. GoFiles, TestGoFiles, XTestGoFiles []string Imports, TestImports, XTestImports []string - // ForTest and the other files are listed only for file coverage. + // CgoFiles also participates in declaration discovery; the remaining fields + // below are listed only for file coverage. ForTest string CgoFiles, EmbedFiles, TestEmbedFiles, XTestEmbedFiles []string EmbedPatterns, TestEmbedPatterns, XTestEmbedPatterns []string @@ -427,7 +428,7 @@ func prepare(ctx context.Context, dir string, opts options, runtime Runtime, pro return packages, goRejected(err) } list := func(patterns ...string) ([]goPackage, error) { - return listWith("Dir,Name,ImportPath,Standard,DepOnly,GoFiles,TestGoFiles,XTestGoFiles,Imports,TestImports,XTestImports,Module,Error", patterns...) + return listWith("Dir,Name,ImportPath,Standard,DepOnly,GoFiles,CgoFiles,TestGoFiles,XTestGoFiles,Imports,TestImports,XTestImports,Module,Error", patterns...) } if runtime == Mini && opts.mod != "mod" && plan.Workfile == "" { plan.Modfile, e = preprovideMini(ctx, dir, opts, temp, replacements, progress) @@ -637,6 +638,13 @@ func prepare(ctx context.Context, dir string, opts options, runtime Runtime, pro // Packages that the runtime imports keep it in an external test package, // the only test package that may import them back. runtimeImports := graph.imported(selectedRuntime) + exclusionGraph := graph + if runtime == Mini { + exclusionGraph, e = exclusionDependencyGraph(packages, graph, list) + if e != nil { + return plan, e + } + } internalImports := 0 for _, p := range packages { // ddto lists testing itself; only the caller's patterns select its tests. @@ -670,6 +678,28 @@ func prepare(ctx context.Context, dir string, opts options, runtime Runtime, pro // its directory into the source. Identical packages still share files. content = "package " + testPackage + "\nimport __dd_ci_runtime " + fmt.Sprintf("%q", runtimePackage) + "\nfunc init() { __dd_ci_runtime.RegisterTestPackage() }\n" + callerEnvironment } + if runtime == Mini { + importedRegistrations, err := exclusionDependencyRegistrations(p, exclusionGraph, replacements, runtimeImports) + if err != nil { + return plan, err + } + content += importedRegistrations + } + if runtime == Mini { + registrations, err := exclusionRegistrationSourcesForBoundary(p, replacements, runtimePackage, runtimeImports[p.ImportPath]) + if err != nil { + return plan, err + } + for pkg, source := range registrations { + filename := "zz_dd_ci_exclusions_test.go" + if pkg != p.Name { + filename = "zz_dd_ci_exclusions_external_test.go" + } + if e = add(filepath.Join(p.Dir, filename), source); e != nil { + return plan, e + } + } + } if e = add(filepath.Join(p.Dir, "zz_dd_ci_visibility_test.go"), content); e != nil { return plan, e } diff --git a/ddto/internal/thirdparty/dd-trace-go/ADAPTATIONS.md b/ddto/internal/thirdparty/dd-trace-go/ADAPTATIONS.md index cd0b76302..8b91e99ac 100644 --- a/ddto/internal/thirdparty/dd-trace-go/ADAPTATIONS.md +++ b/ddto/internal/thirdparty/dd-trace-go/ADAPTATIONS.md @@ -3049,3 +3049,202 @@ best-effort behavior. OS-release parsing skips malformed lines without a value. `TestContainerReadersHandleReadErrors`, `TestContainerReaderStopsAtFirstID` and `TestOSReleaseHandlesMalformedLinesAndReadErrors` cover these paths; tip's scanner analyzer checks the error handling with `go vet`. + +## Test automation exclusion declarations + +Mini's `gotesting/exclusions.go` owns compiled per-function declarations and +per-native-parent-invocation result caches. `testopt.RegisterTestExclusions` +registers the declaration API. The CLI reads only Go-selected sources and embeds +comment declarations in package initialization, preserving overlays, build tags, +external test packages and method identity without runtime source reads. + +Feature metadata projects quarantine/disabled and retry policy after retaining +backend assignments separately. Execution metadata reapplies exclusions after +parent management propagation; span setters do not provide declaration policy. +Process-retry snapshots carry both declarations and assignments. Excluded Testify +methods replay their native result without executing their callback or emitting +an additional event when their parent retries. The cache is scoped to one native +parent invocation, is synchronized for parallel descendants, and its bounded +identity map crosses the existing private process-control transport. Ordinary +siblings and native repetitions keep their normal execution. + +Checks: `TestMiniDeclarationExclusions` verifies independent/together/absent +annotations, original assignments, disabling, quarantine, EFD, ATR, ATF, native +command outcomes and annotation survival without sources. The Testify regression +`TestMiniTestifyExclusionsKeepSiblingRetries` checks both retry modes and retains +an excluded failure while a sibling recovers. The runner tests cover distinct +internal/external packages, method declarations, overlays and annotation removal. + +- Plain Go table rows inherit the resolved parent declaration's exclusion policy, + including shared callbacks and parallel rows, while Testify methods retain + their own declarations. Value-receiver methods register the pointer method + expression Testify actually invokes. Protected terminal descendant failures + remain blocking through a quarantined ancestor; the marker is per attempt and + does not retain a failure after an otherwise permitted retry recovers it. + +- Isolated retries carry resolved declaration/assignment metadata and protected + descendant outcomes in their bounded control/result protocol. Newly discovered + excluded declarations return their first native result and one test event to + the parent; later retries replay the cached native result without executing or + reporting the declaration again. Their first execution never inherits the + parent's retry marker. EFD remains serial when a package has no-auto-retry + declarations, so a newly discovered declaration publishes its result before + another attempt is scheduled. Protected outcome markers bypass ancestor + masking without changing ATR/EFD recovery or ATF aggregation. + +- Excluded-result keys include both the native test path and compiled declaring + function identity, so a parent switching suites cannot replay another method's + outcome. In-process ATF keeps protected failure history for its strict family; + ATR/EFD still use their existing recovery rules. Ordinary nested declarations + stay outside the excluded-only worker event payload. Exclusion handoffs use + the existing bounded extended wire budget in both directions; ordinary + payloads keep their smaller limit. + +- Managed subtree launches retain the caller's native descendant run filters. + `retry_process_selection.go` reuses the existing testing matcher grammar to + narrow only the managed root prefix, including native alternative patterns. + Descendant ATF continuations exchange the same exclusion history on each + launch; their protected terminal failures participate in strict ATF package + and final-status aggregation without changing ordinary ATR/EFD recovery. + Selected isolation-root cache identities use the same native-path validation + as descendant identities. Replayed excluded worker events retain actual TIA + skip/forced-run flags and session skip accounting. + +- A promoted Testify method keeps its reflected source/suite identity while a + separate declaring-function identity keys exclusion replay across derived + wrappers. Late-discovered excluded execution events use the existing 16 MiB + byte-bounded result envelope without inheriting the unrelated subtree-node + count limit; all execution identities/statuses and cache limits stay checked. +- Native lifecycle draining preserves the parallel flag until user cleanup has + finished, then neutralizes only a slot already released. Sequential parents + reacquire their scheduler slot after cleanup, matching native ordering. + Direct unsuppressed body terminals run cleanup before admitting queued + parallel children; managed retry/quarantine attempts retain their lifecycle. + +- Public method-expression registration resolves exact reflected value/pointer + counterparts and their canonical declaring function; ordinary package + functions with matching signatures cannot register unrelated methods. + Ordinary process-worker wrappers share the invocation exclusion cache, so + independently declared descendants retain their first result across a later + ancestor switch. + Inherited native-row cache keys include their owning declaration as well as + the row callback, preventing a later declaration from reusing another owner's + passing row at the same native path. + +- Native cleanup Goexit retains its scheduler unwind and body-terminal + replacement behavior. Fuzz callbacks retain the fuzz lifecycle rather than + entering direct testing.T panic handling. + +- Parent retry verdicts retain excluded descendants' first failed native result + when a later attempt omits their declaration. Effective quarantined/disabled + outcomes remain suppressed without failure protection, including replay and + transport; native invocations keep separate caches. + Earlier passing EFD parent samples cannot redeem a newly discovered excluded + failure; aggregate verdicts and final status retain that failure without + rewriting the actual sample history. +- Extended result envelopes budget optional diagnostics across both subtree + and excluded execution records. Trimming preserves every required record and + the existing wire limits; mandatory overflow fails explicitly. + +- Testify reporting attribution remains attached to native descendants, but a + separate method-boundary check resolves exclusion ownership. Explicit native + callbacks keep their own policy, and their unannotated native rows inherit + that callback's policy rather than the surrounding method's. Actual nested + Testify methods remain independent. Cache declaration keys use the same + distinction, retaining canonical promoted-method identities and explicit + callback history when a parent switches suite declarations. + Promoted-method lookup stops at an interface shadowing boundary instead of + borrowing a deeper concrete declaration's annotations. Explicit callback + entries remain independent during cache compaction, including callbacks + beneath excluded methods. + +- Ordinary process-worker carriers now retain their native identity and resolved + cached management context. Descendants inherit the immediate carrier's + effective policy, original assignments and attempt ownership, including an + exact intermediate Attempt-to-Fix override. NoRetry can therefore remove + Attempt to Fix while retaining an inherited disabled assignment and skip. + This adds neither an ordinary carrier event nor a backend request or retry. + +- Eager annotation cleanup observes native cleanup terminals without replacing + `testing.common.runCleanup`. Each callback is observed once, including + callbacks registered by a cleanup, while native cancellation, LIFO order, + recursive drain and Goexit retain control. Original queued-child shape selects + native cleanup panic handling before the scheduler drain clears that shape. + Body and cleanup panic values and their origin stacks remain separate until + native precedence determines which diagnostics survive; cleanup Goexit does + not blindly resurrect or erase a prior panic. Top-level, native-child and + isolated-worker finalization retain the surviving secondary diagnostics in + existing bounded output/error fields. Worker updates replace only their own + body-panic record, preserving independently owned descendant failures. + Only the observer's own added frames are omitted from captured cleanup + stacks; source-frame comparisons and unrelated function frames stay strict. + Ordinary annotation-off events retain the pinned SDK's body-return lifetime, + and fuzz retains its existing terminal precedence. + +- In an isolated worker, an annotated unsuppressed body terminal leaves queued + parallel descendants behind their unopened native barrier. The worker detaches + only that terminal queue so consuming the panic for subtree-result publication + cannot admit children that native `doPanic` would never run. Original queued + shape still determines cleanup precedence; managed quarantine and nonterminal + child draining retain their existing behavior. + +- Crash and timeout finalization reads an immutable per-test suppression policy + registered from effective Go execution metadata before assignment tags are set. + Compiled failure-suppression exclusions therefore remain effective when + quarantined/disabled assignment tags are retained for attribution. No-retry + alone keeps effective suppression. Registration and crash/normal close share + the existing test mutex; only the first registration on an open test is kept, + and finished or delayed-close events are never reopened. Manual API tests + without a registered policy retain legacy quarantined/disabled tag behavior; + mutable annotation tags cannot override registered execution policy. + +- Early controlled worker-panic results carry the same recorded no-retry cache, + completed excluded execution rows, and protected-descendant failure flag as + normal worker results. A shared snapshot helper reads the resolved root owner + without draining parallel work, finalizing coverage, or treating in-flight + tests as completed. Existing writer arbitration, control acknowledgement, + payload budgets, and validation limits remain unchanged. + +- No-retry callback admission and terminal publication share the completed-cache + mutex. Publication freezes new admission without waiting; an entered callback + remains active until its existing deferred completion record is stored. Both + controlled-terminal publishers retain completed facts but withhold recovery + acknowledgement when callbacks are still active, using the existing + uncommitted-terminal failure path instead of retrying unfinished declarations. + A callback rejected after the freeze has no execution or fabricated completion + record. No-retry alone does not acquire failure-suppression protection. + +- Ordinary process-worker children with compiled failure-suppression protection + resolve body and cleanup outcomes before publishing terminal results. Existing + cleanup helpers preserve native queued-child ordering and cancellation by + cleanup SkipNow/FailNow; surviving terminal or failed state marks ancestors + before publication. The child preserves original panic diagnostics and the + existing bounded owner record, without creating an additional child event. + Unprotected ordinary-worker lifecycle and terminal admission/history protocols + retain their existing behavior. + +- An original native Fail on a compiled protected worker child marks its resolved + result owner immediately, before another concurrent child can publish a + terminal envelope. Recursive native failure propagation cannot infer protection + from a protected ancestor; existing managed/unmasked-failure handling remains + unchanged. Resolved terminal finalization also marks its captured owner directly + before ancestry propagation, without eagerly failing a cancellable body panic. + +- Crash finalization preserves trusted protected-descendant failures separately + from the immutable base suppression policy. Each concrete open event keeps a + durable protected-failure fact under its existing close mutex; mutable tags + cannot establish or clear it. Execution metadata forwards trusted marks through + an atomic event binding, including marks received before binding, without + racing ordinary event-interface assignments. Fresh-attempt aliases copy only + the current binding, not historical failure state, and fresh false snapshot + resets remain intact. Closed, finished, and delayed-close events keep their + existing outcome; manual API suppression behavior remains unchanged when no + trusted protected failure was recorded. + +- Normal native Fail hooks immediately forward compiled protected-failure facts + to the active event and its native ancestors, before a blocked child can miss + crash finalization. Only an original native Fail qualifies; recursive parent + propagation and logging helpers cannot infer protection from ancestor tags. + The native `*testing.common` receiver uses a validated common-parent accessor, + while typed tests retain the existing accessor. Worker failure handling, native + cleanup cancellation, and fresh-attempt retry state remain unchanged. diff --git a/ddto/internal/thirdparty/dd-trace-go/SOURCE.json b/ddto/internal/thirdparty/dd-trace-go/SOURCE.json index e111de40f..bf0126677 100644 --- a/ddto/internal/thirdparty/dd-trace-go/SOURCE.json +++ b/ddto/internal/thirdparty/dd-trace-go/SOURCE.json @@ -11,7 +11,7 @@ "files": [ { "path": "ADAPTATIONS.md", - "sha256": "9caddd5071da1f0571e72916e0be10bf93f935794cbbb734b24b8bfc5f972fd5", + "sha256": "3c49447a7c4aaba1aad107ac63c6e60858a4a6ab8b521662283053857153658b", "role": "local" }, { @@ -170,7 +170,7 @@ }, { "path": "civisibility/integrations/civisibility_crash_test.go", - "sha256": "f18cc9c317ed74ee8dc03dc26b24293fc43d5039e9fb7d5b55268aac275de2d6", + "sha256": "adf775e1210b1638c74acfda7aad670929b9798a6c66c96d5ff03b693b8ee1f3", "role": "local" }, { @@ -491,6 +491,21 @@ "sha256": "aae2fdd85a264a16dcd368e48d0de8d795265b66bee97b24b107aa25919f39c4", "role": "local" }, + { + "path": "civisibility/integrations/gotesting/exclusions.go", + "sha256": "2cae972c76e6947a7bb049eb4eb312224b1889b27ca2ab40cb91a9294130aa66", + "role": "local" + }, + { + "path": "civisibility/integrations/gotesting/exclusions_serialization_test.go", + "sha256": "9b7739216d38cddc31a07e649838f5eaeec3a6a7863cb37e23b721a94889b5ca", + "role": "local" + }, + { + "path": "civisibility/integrations/gotesting/exclusions_test.go", + "sha256": "60d8e4b41364453934b2022f200818717866d584675ab3f2b7290740831c3912", + "role": "local" + }, { "path": "civisibility/integrations/gotesting/final_status.go", "sha256": "70bd07ef012e1f9b522773697052da49fa4123e7f2158569a5ccdd8c8c338405", @@ -512,7 +527,7 @@ }, { "path": "civisibility/integrations/gotesting/fuzz_events_test.go", - "sha256": "7900882ff78cdec52996df39b519611c682539a8358261d603f470d8b576d4ab", + "sha256": "97d0c9a147e31abb94201f93c8ecc793bef25969f2cd8b8eb170353c1511adc6", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/fuzz_events_test.go", "source_sha256": "dce83df9f524068447d65afea0be2fff7ad494b1520d266e8b4f139038e6d624" @@ -530,14 +545,14 @@ }, { "path": "civisibility/integrations/gotesting/instrumentation.go", - "sha256": "952c52a00444fa78ee091afb3ec50b74443cfff613a3b7f04c53a65a1cbec92a", + "sha256": "7b895439ae6658e2f6f22adb421061c39a3a550a27ec4667073394fc3e657500", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/instrumentation.go", "source_sha256": "19d482de88ae510e34db412273a5cf940eacc1192da0b26947622df76b52cdc6" }, { "path": "civisibility/integrations/gotesting/instrumentation_orchestrion.go", - "sha256": "212a133449ea14564b3396dc80e196cb2b14530c5c71efa08d57179179c21afc", + "sha256": "ebce207fe4803130ca75878659f4a1cfdd2979395fe5d9bdee223bb61e92b11c", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go", "source_sha256": "e07e0e5d272cdf6148fea865f53b1f1f40dfe29b60b4c461bfe29a5dd52784b3" @@ -582,7 +597,7 @@ }, { "path": "civisibility/integrations/gotesting/quarantine_process.go", - "sha256": "7b177dc04251df3be578603d780c69d07b107a8bb5e137aba7a7b8f39b9d5302", + "sha256": "af144e0fbeea68934e63b439f874a42755615b8971b6930179917eac475fcf89", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/quarantine_process.go", "source_sha256": "127b790b31e406e5b1d4702ef09e0a7b37cfff06488f0f97b20fa987d67cd8e5" @@ -603,7 +618,7 @@ }, { "path": "civisibility/integrations/gotesting/reflections.go", - "sha256": "171f8bf4e45f4f072ecbc9afc45036fb0c010a7ad920238473533e5f2e722db0", + "sha256": "022e9559545bd573e2f76a88223f1664f5ece7ce75d9350a6788d654a05aa637", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/reflections.go", "source_sha256": "171f8bf4e45f4f072ecbc9afc45036fb0c010a7ad920238473533e5f2e722db0" @@ -624,7 +639,7 @@ }, { "path": "civisibility/integrations/gotesting/retry_attempt_integration.go", - "sha256": "d367782ce485ededfd1cba4b43c185e281c12fffb81d4cfc76045321c03e4179", + "sha256": "b38bf832832d13248fd1611adfa97350bf718df8190618dadd680a24430397b4", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/retry_attempt_integration.go", "source_sha256": "1bd6b3bda3bee19f6615784d506a15d47d5d0f981b7f71bc1cca4ae235cdd25a" @@ -716,21 +731,21 @@ }, { "path": "civisibility/integrations/gotesting/retry_process.go", - "sha256": "b24765cf84558616b6d872d76a75c855fde87a41db993d51a8ec065b73068410", + "sha256": "f8dff80a4810f2bee39a4bf7300771b13c213d4f80a0349cc39121f0f06349eb", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/retry_process.go", "source_sha256": "e1bdfd9174b86b856a05dbeb4278d386e8126cbb272248b9f7ab00bf8ee8feca" }, { "path": "civisibility/integrations/gotesting/retry_process_control.go", - "sha256": "3f99f1d424457953e22466f26c78a2be0ff318354b25a8c7974f167d338292ab", + "sha256": "4ffbac8e6a72b026c817ab0adb1bc4acfa173ee545812e2982dc9b7c06ee045a", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/retry_process_control.go", "source_sha256": "7671d328d8c28bd29eb6c81e15dddc8623feb776a4e4dfb2e0211a81dd95932a" }, { "path": "civisibility/integrations/gotesting/retry_process_deferred.go", - "sha256": "bdec3ffa9e6653cb0e6c670fe113e74cb2be78e2fda8ebbddcdfbeb908a09d41", + "sha256": "91aee63cbad14a19603ea488426d12d207cf0496601c62661c3a1a24cd578f12", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/retry_process_deferred.go", "source_sha256": "f5efa31791a06e6b2481092015eb5346a4ac4e91302c605cf9b976a5e5cc404e" @@ -749,6 +764,16 @@ "source_path": "internal/civisibility/integrations/gotesting/retry_process_other.go", "source_sha256": "2a2f6325be7112cbdbf0a6aa42c3ed1bf1d3a7bf2c3427cd35196052371700b9" }, + { + "path": "civisibility/integrations/gotesting/retry_process_selection.go", + "sha256": "c3b724f1054890f28d12b5904fc915f718dbe4455bd80b69da226e24eb251990", + "role": "local" + }, + { + "path": "civisibility/integrations/gotesting/retry_process_selection_test.go", + "sha256": "1ff5d0863f9f9f04f4820c874d1a3eecad80dc9ee0f93cf49815577ce36aae1d", + "role": "local" + }, { "path": "civisibility/integrations/gotesting/retry_process_unix.go", "sha256": "a28364b631d7c607cd9d6bafc3a6397feea8163a9ea4b3fa2344332b56be875e", @@ -782,7 +807,7 @@ }, { "path": "civisibility/integrations/gotesting/test_execution_finalizer.go", - "sha256": "6e2610895036b3ef313309bca94523768ff5f6081f5b9e6fd5028d8594013ce3", + "sha256": "fb4c0297516eccc1a9d2e8e51554a79a913a0950e8a43f11ca4d550de843ea3a", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/test_execution_finalizer.go", "source_sha256": "76a569dea1c97bee062050d21e9c3a8c1984420a6069ff9987e779cf3a7c8dc7" @@ -804,19 +829,19 @@ }, { "path": "civisibility/integrations/gotesting/testify.go", - "sha256": "51edc0cf0a37d55c9e69d26fff77c7a5e8d51db628b424d0775a385c9eca5fc8", + "sha256": "bff4861a8d5ada4535e136e073ebd6d63461a794fd18869c9affc07e5ab30d54", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/testify.go", "source_sha256": "25105ea7f1a1d46c78c639de90bf61a925c3942adb45dc73dbbdc6fe7e0248c1" }, { "path": "civisibility/integrations/gotesting/testify_lookup_test.go", - "sha256": "e4fe0e6f735329dc8309f7bbb3ccb70b0b962f7a94347dc932427caaa6cceb12", + "sha256": "7a07b21361d9f230f9a5d33169b6946a25d662f2700293cc555bbc8f675a3596", "role": "local" }, { "path": "civisibility/integrations/gotesting/testing.go", - "sha256": "c644e4ad28faecf891358d178b788e3d7856c6881751941112b5d37bc17d5d5f", + "sha256": "4d79d49dfecb5017583778252a79f3cbc4653b88358b06890d08cc1970574b5b", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/testing.go", "source_sha256": "73324a56f1b1c581df8620a4f67a51743b198b2470d43f9b645425fff6a63646" @@ -844,7 +869,7 @@ }, { "path": "civisibility/integrations/gotesting/testingF.go", - "sha256": "900326ae2ae5743993ae4a769eef8187d4c2f3713d1fa4cdb4dd556bee2cb96d", + "sha256": "bf4f9990c8727cea07dcb8824caa78769925f939e38d9f525448d2202f46fa3e", "role": "adapted", "source_path": "internal/civisibility/integrations/gotesting/testingF.go", "source_sha256": "7a62a0af9a53bd8fa7abccf320cf47ed62acc559268973c08c937a4e5bd31ab8" @@ -972,7 +997,7 @@ }, { "path": "civisibility/integrations/manual_api_ddtest.go", - "sha256": "d9cbc667b426af61658ea0f349d2b7cd1d9b661da8fb26f6e59e68453d047a3e", + "sha256": "bbb3b9244a3c53639618d87d791695670e10e4cf610a5ad0486f12b37eadcd27", "role": "adapted", "source_path": "internal/civisibility/integrations/manual_api_ddtest.go", "source_sha256": "7380f18e54348da3d6c9515d1f6e31f8c9fb33e8423cd90791c305cf3d51afc2" diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/civisibility_crash_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/civisibility_crash_test.go index d0fb730fc..f77863601 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/civisibility_crash_test.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/civisibility_crash_test.go @@ -161,3 +161,269 @@ func TestFailForCrashIsAtomicWithClose(t *testing.T) { } } } + +// Assignment and user tags describe a test; only a configured execution +// policy can override the legacy manual-API crash suppression fallback. +func TestFailForCrashUsesEffectiveFailureSuppression(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("crash-policy") + pfs := "test.test_management.prevent_failure_suppression" + noRetry := "test.test_management.no_auto_retry" + for _, tc := range []struct { + name string + tags []string + policy []bool + want string + }{ + {"legacy-unmanaged", nil, nil, constants.TestStatusFail}, + {"legacy-manual-quarantine", []string{constants.TestIsQuarantined}, nil, constants.TestStatusSkip}, + {"legacy-manual-disabled", []string{constants.TestIsDisabled}, nil, constants.TestStatusSkip}, + {"forged-pfs-only", []string{pfs}, nil, constants.TestStatusFail}, + {"forged-pfs-manual-quarantine", []string{pfs, constants.TestIsQuarantined}, nil, constants.TestStatusSkip}, + {"forged-pfs-manual-disabled", []string{pfs, constants.TestIsDisabled}, nil, constants.TestStatusSkip}, + {"effective-false-quarantine", []string{constants.TestIsQuarantined}, []bool{false}, constants.TestStatusFail}, + {"effective-false-disabled-atf", []string{constants.TestIsDisabled, constants.TestIsAttempToFix}, []bool{false}, constants.TestStatusFail}, + {"effective-true-no-retry-quarantine", []string{noRetry, constants.TestIsQuarantined}, []bool{true}, constants.TestStatusSkip}, + {"effective-true-no-retry-disabled-atf", []string{noRetry, constants.TestIsDisabled, constants.TestIsAttempToFix}, []bool{true}, constants.TestStatusSkip}, + {"effective-true-without-assignment-tags", nil, []bool{true}, constants.TestStatusSkip}, + {"forged-pfs-cannot-change-configured-policy", []string{pfs, constants.TestIsQuarantined}, []bool{true}, constants.TestStatusSkip}, + {"first-false-is-immutable", []string{constants.TestIsQuarantined}, []bool{false, true}, constants.TestStatusFail}, + {"first-true-is-immutable", nil, []bool{true, false}, constants.TestStatusSkip}, + } { + t.Run(tc.name, func(t *testing.T) { + event := suite.CreateTest(tc.name).(*tslvTest) + for _, enabled := range tc.policy { + SetTestCrashFailureSuppression(event, enabled) + } + for _, tag := range tc.tags { + event.SetTag(tag, "true") + } + require.True(t, event.failForCrash("timeout", "crash policy sentinel", "original.go:42")) + status, _ := event.span.Meta(constants.TestStatus) + final, _ := event.span.Meta(constants.TestFinalStatus) + message, _ := event.span.Meta("error.message") + stack, _ := event.span.Meta("error.stack") + require.Equal(t, constants.TestStatusFail, status) + require.Equal(t, tc.want, final) + require.Equal(t, "crash policy sentinel", message) + require.Equal(t, "original.go:42", stack) + for _, tag := range tc.tags { + value, _ := event.span.Meta(tag) + require.Equal(t, "true", value, tag) + } + _, open := openTests.Load(event) + require.False(t, open) + require.False(t, event.failForCrash("panic", "late crash must not replace outcome", "later.go:99")) + SetTestCrashFailureSuppression(event, tc.want != constants.TestStatusSkip) + final, _ = event.span.Meta(constants.TestFinalStatus) + require.Equal(t, tc.want, final) + }) + } +} + +// Finished fuzz events can remain buffered until the pre-close barrier. A +// subsequent setter or process crash must not turn their completed pass into +// a crash result; ordinary already-closed events have the same guarantee. +func TestCrashFailureSuppressionKeepsClosedAndFinishedEvents(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + resetCrashForTesting(t) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("finished-policy") + closed := suite.CreateTest("Closed").(*tslvTest) + closed.Close(ResultStatusPass) + SetTestCrashFailureSuppression(closed, true) + require.False(t, closed.failForCrash("panic", "closed event", "")) + finished := suite.CreateTest("FuzzCompleted").(*tslvTest) + finished.SetTag(constants.TestIsQuarantined, "true") + MarkTestFinished(finished) + SetTestCrashFailureSuppression(finished, true) + require.False(t, finished.failForCrash("panic", "finished seed", "")) + require.True(t, TryPushCiVisibilityPreCloseAction(func() { finished.Close(ResultStatusPass) })) + running := suite.CreateTest("RunningProtected").(*tslvTest) + SetTestCrashFailureSuppression(running, false) + running.SetTag(constants.TestIsQuarantined, "true") + CloseForCrash("timeout", "active crash sentinel", "") + for _, event := range []*tslvTest{closed, finished} { + status, _ := event.span.Meta(constants.TestStatus) + final, _ := event.span.Meta(constants.TestFinalStatus) + _, hasError := event.span.Meta("error.message") + require.Equal(t, constants.TestStatusPass, status, event.name) + require.Equal(t, "", final, event.name) + require.False(t, hasError, event.name) + } + status, _ := running.span.Meta(constants.TestStatus) + final, _ := running.span.Meta(constants.TestFinalStatus) + require.Equal(t, constants.TestStatusFail, status) + require.Equal(t, constants.TestStatusFail, final) +} + +// Once an effective policy is configured, a concurrent opposite setter may +// neither revive suppression nor mix a normal close with a crash's outcome. +func TestCrashFailureSuppressionIsAtomicWithClose(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("concurrent-policy") + for i := range 200 { + event := suite.CreateTest("Policy" + strconv.Itoa(i)).(*tslvTest) + SetTestCrashFailureSuppression(event, false) + event.SetTag(constants.TestIsQuarantined, "true") + var wg sync.WaitGroup + wg.Add(3) + go func() { defer wg.Done(); SetTestCrashFailureSuppression(event, true) }() + go func() { defer wg.Done(); event.failForCrash("timeout", "immutable crash sentinel", "original.go:42") }() + go func() { + defer wg.Done() + event.SetTag(constants.TestFinalStatus, constants.TestStatusPass) + event.Close(ResultStatusPass) + }() + wg.Wait() + status, _ := event.span.Meta(constants.TestStatus) + final, _ := event.span.Meta(constants.TestFinalStatus) + message, failed := event.span.Meta("error.message") + switch { + case status == constants.TestStatusPass && !failed && final == constants.TestStatusPass: + case status == constants.TestStatusFail && message == "immutable crash sentinel" && final == constants.TestStatusFail: + default: + t.Fatalf("iteration%d mixed crash policy status%q final%q error%q", i, status, final, message) + } + } +} + +// Trusted execution facts override the immutable base management policy; raw +// tags remain descriptive and cannot manufacture a protected descendant. +func TestTrustedProtectedFailureOverridesCrashSuppression(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("trusted-descendant") + MarkTestProtectedFailure(nil) + for _, tc := range []struct { + name string + configured, mark, markBeforePolicy, forgePFS bool + want string + }{ + {"configured-quarantine-trusted", true, true, false, false, constants.TestStatusFail}, + {"legacy-manual-quarantine-trusted", false, true, false, false, constants.TestStatusFail}, + {"trusted-before-base-policy", true, true, true, false, constants.TestStatusFail}, + {"configured-quarantine-without-fact", true, false, false, false, constants.TestStatusSkip}, + {"legacy-manual-quarantine-without-fact", false, false, false, false, constants.TestStatusSkip}, + {"forged-pfs-is-not-trusted-fact", true, false, false, true, constants.TestStatusSkip}, + } { + t.Run(tc.name, func(t *testing.T) { + event := suite.CreateTest(tc.name).(*tslvTest) + if tc.markBeforePolicy { + MarkTestProtectedFailure(event) + } + if tc.configured { + SetTestCrashFailureSuppression(event, true) + } + event.SetTag(constants.TestIsQuarantined, "true") + if tc.forgePFS { + event.SetTag("test.test_management.prevent_failure_suppression", "true") + } + if tc.mark && !tc.markBeforePolicy { + MarkTestProtectedFailure(event) + MarkTestProtectedFailure(event) + } + require.True(t, event.failForCrash("timeout", "trusted descendant crash sentinel", "original.go:42")) + status, _ := event.span.Meta(constants.TestStatus) + final, _ := event.span.Meta(constants.TestFinalStatus) + message, _ := event.span.Meta("error.message") + require.Equal(t, constants.TestStatusFail, status) + require.Equal(t, tc.want, final) + require.Equal(t, "trusted descendant crash sentinel", message) + q, _ := event.span.Meta(constants.TestIsQuarantined) + require.Equal(t, "true", q) + if !tc.forgePFS { + _, synthetic := event.span.Meta("test.test_management.prevent_failure_suppression") + require.False(t, synthetic, "trusted descendant fact is not a declaration annotation") + } + }) + } + // A prior event's true fact must not be imported into a fresh manual event. + first := suite.CreateTest("FirstProtected").(*tslvTest) + SetTestCrashFailureSuppression(first, true) + first.SetTag(constants.TestIsQuarantined, "true") + MarkTestProtectedFailure(first) + first.Close(ResultStatusPass) + fresh := suite.CreateTest("FreshWithoutProtectedFailure").(*tslvTest) + SetTestCrashFailureSuppression(fresh, true) + fresh.SetTag(constants.TestIsQuarantined, "true") + require.True(t, fresh.failForCrash("timeout", "fresh event crash", "")) + final, _ := fresh.span.Meta(constants.TestFinalStatus) + require.Equal(t, constants.TestStatusSkip, final) +} + +func TestTrustedProtectedFailureKeepsClosedAndFinishedEvents(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + resetCrashForTesting(t) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("closed-trusted-descendant") + closed := suite.CreateTest("Closed").(*tslvTest) + closed.Close(ResultStatusPass) + MarkTestProtectedFailure(closed) + finished := suite.CreateTest("FinishedFuzzSeed").(*tslvTest) + MarkTestFinished(finished) + MarkTestProtectedFailure(finished) + require.True(t, TryPushCiVisibilityPreCloseAction(func() { finished.Close(ResultStatusPass) })) + running := suite.CreateTest("OpenQuarantined").(*tslvTest) + SetTestCrashFailureSuppression(running, true) + running.SetTag(constants.TestIsQuarantined, "true") + MarkTestProtectedFailure(running) + CloseForCrash("timeout", "open protected descendant crash", "") + for _, event := range []*tslvTest{closed, finished} { + status, _ := event.span.Meta(constants.TestStatus) + _, hasError := event.span.Meta("error.message") + require.Equal(t, constants.TestStatusPass, status, event.name) + require.False(t, hasError, event.name) + require.False(t, event.failForCrash("panic", "late crash", "")) + } + final, _ := running.span.Meta(constants.TestFinalStatus) + require.Equal(t, constants.TestStatusFail, final) +} + +func TestTrustedProtectedFailureIsAtomicWithCrashAndClose(t *testing.T) { + resetCIVisibilityBootstrapStateForTesting() + disableAdditionalFeaturesForBootstrapTest() + t.Cleanup(restoreCIVisibilityBootstrapForTesting) + initializeCIVisibilityLifecycleForTesting() + suite := CreateTestSession().GetOrCreateModule("m").GetOrCreateSuite("racing-trusted-descendant") + for i := range 200 { + event := suite.CreateTest("Trusted" + strconv.Itoa(i)).(*tslvTest) + SetTestCrashFailureSuppression(event, true) + event.SetTag(constants.TestIsQuarantined, "true") + MarkTestProtectedFailure(event) + var wg sync.WaitGroup + wg.Add(3) + go func() { defer wg.Done(); MarkTestProtectedFailure(event) }() + go func() { + defer wg.Done() + event.failForCrash("timeout", "already-observed protected failure", "original.go:42") + }() + go func() { + defer wg.Done() + event.SetTag(constants.TestFinalStatus, constants.TestStatusPass) + event.Close(ResultStatusPass) + }() + wg.Wait() + status, _ := event.span.Meta(constants.TestStatus) + final, _ := event.span.Meta(constants.TestFinalStatus) + message, failed := event.span.Meta("error.message") + switch { + case status == constants.TestStatusPass && !failed && final == constants.TestStatusPass: + case status == constants.TestStatusFail && message == "already-observed protected failure" && final == constants.TestStatusFail: + default: + t.Fatalf("iteration%d lost trusted fact or mixed outcomes status%q final%q message%q", i, status, final, message) + } + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions.go new file mode 100644 index 000000000..95cddf20c --- /dev/null +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions.go @@ -0,0 +1,541 @@ +package gotesting + +import ( + "reflect" + "runtime" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/DataDog/ddtest/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations" +) + +// TestExclusions describes per-declaration execution exclusions. Generated +// package initialization registers these before testing chooses execution policy. +type TestExclusions struct { + PreventFailureSuppression bool `json:"prevent_failure_suppression,omitempty"` + NoAutoRetry bool `json:"no_auto_retry,omitempty"` +} + +type testManagementAssignment struct{ quarantined, disabled, attemptToFix bool } + +var hasNoAutoRetryDeclarations atomic.Bool + +var testExclusionDeclarations sync.Map // runtime function name -> TestExclusions + +// RegisterTestExclusions records a compiled declaration, not mutable span tags. +func RegisterTestExclusions(fn any, exclusions TestExclusions) { + v := reflect.ValueOf(fn) + if !v.IsValid() || v.Kind() != reflect.Func || v.IsNil() { + return + } + f := runtime.FuncForPC(v.Pointer()) + if f == nil { + return + } + RegisterTestExclusionsByName(f.Name(), exclusions) + if v.Type().NumIn() == 0 { + return + } + receiver := v.Type().In(0) + methodName := f.Name()[strings.LastIndexByte(f.Name(), '.')+1:] + method, ok := receiver.MethodByName(methodName) + // A package function can have the same name/signature as a method expression. + // Only the exact reflected method may register its receiver counterparts. + if !ok || !method.Func.IsValid() || method.Func.Pointer() != v.Pointer() { + return + } + registerMethod := func(typ reflect.Type) { + m, exists := typ.MethodByName(methodName) + if !exists || !m.Func.IsValid() { + return + } + if compiled := runtime.FuncForPC(m.Func.Pointer()); compiled != nil { + RegisterTestExclusionsByName(compiled.Name(), exclusions) + } + if declaration := testifyMethodDeclaration(typ, m); declaration != nil { + RegisterTestExclusionsByName(declaration.Name(), exclusions) + } + } + registerMethod(receiver) + if receiver.Kind() == reflect.Pointer { + registerMethod(receiver.Elem()) + } else { + registerMethod(reflect.PointerTo(receiver)) + } +} + +func exclusionsForFunc(fn *runtime.Func) TestExclusions { + if fn != nil { + if value, ok := testExclusionDeclarations.Load(fn.Name()); ok { + return value.(TestExclusions) + } + } + return TestExclusions{} +} + +func (e TestExclusions) any() bool { return e.PreventFailureSuppression || e.NoAutoRetry } + +func applyFeatureExclusions(meta *additionalFeatureMetadata) { + if meta.exclusions.PreventFailureSuppression { + meta.isDisabled, meta.isQuarantined = false, false + } + if meta.exclusions.NoAutoRetry { + meta.isEarlyFlakeDetectionEnabled, meta.isFlakyTestRetriesEnabled = false, false + meta.isAttemptToFix, meta.shouldOrchestrateAttemptToFix = false, false + } +} + +func applyExecutionExclusions(meta *testExecutionMetadata) { + if meta == nil { + return + } + if meta.exclusions.PreventFailureSuppression { + meta.isDisabled, meta.isQuarantined = false, false + } + if meta.exclusions.NoAutoRetry { + meta.isEarlyFlakeDetectionEnabled, meta.isFlakyTestRetriesEnabled = false, false + meta.isAttemptToFix, meta.shouldOrchestrateAttemptToFix = false, false + meta.isARetry, meta.isEfdInParallel = false, false + meta.suppressParentRetryMetadata = true + } +} + +// Excluded descendants retain their native result across retries of their parent. +// Each native parent invocation owns a fresh cache; a caller's -count and ordinary +// repeated suite invocation therefore keep their native execution behavior. +type excludedTestResult struct { + Failed bool `json:"failed,omitempty"` + Skipped bool `json:"skipped,omitempty"` + Protected bool `json:"protected,omitempty"` + Inherited bool `json:"inherited,omitempty"` +} + +type excludedTestResults struct { + mu sync.Mutex + items map[string]excludedTestResult + active int + terminal bool +} + +// begin records an entered no-retry callback, unless terminal publication has +// already frozen admission. A rejected callback has no completed result. +func (results *excludedTestResults) begin() bool { + if results == nil { + return true + } + results.mu.Lock() + defer results.mu.Unlock() + if results.terminal { + return false + } + results.active++ + return true +} + +// end runs after the existing deferred record has stored the completed result. +func (results *excludedTestResults) end() { + if results == nil { + return + } + results.mu.Lock() + defer results.mu.Unlock() + results.active-- +} + +// freezeForTerminal closes admission and permits recovery only when every +// entered callback has completed. It never waits for in-flight callbacks. +func (results *excludedTestResults) freezeForTerminal() bool { + if results == nil { + return true + } + results.mu.Lock() + defer results.mu.Unlock() + results.terminal = true + return results.active == 0 +} + +func (results *excludedTestResults) record(t *testing.T, protected bool, declaration string) { + if results == nil { + return + } + results.mu.Lock() + defer results.mu.Unlock() + if results.items == nil { + results.items = map[string]excludedTestResult{} + } + direct, _ := testExclusionDeclarations.Load(declaration) + inherited := direct == nil || !direct.(TestExclusions).NoAutoRetry + if data := getTestifyTest(t); data != nil && declaration == exclusionDeclarationName(t, nil) { + if fields := getTestPrivateFields(t); fields != nil && fields.parent != nil && getBoundTestifyTest(*fields.parent) == data { + inherited = true + } + } + results.items[t.Name()+"\x00"+declaration] = excludedTestResult{Failed: t.Failed(), Skipped: t.Skipped(), Protected: protected, Inherited: inherited} +} + +func (results *excludedTestResults) snapshot() map[string]excludedTestResult { + if results == nil { + return nil + } + results.mu.Lock() + defer results.mu.Unlock() + if len(results.items) == 0 { + return nil + } + out := make(map[string]excludedTestResult, len(results.items)) + paths := make(map[string][]string, len(results.items)) + keys := make([]string, 0, len(results.items)) + for name, result := range results.items { + out[name] = result + path, _, _ := strings.Cut(name, "\x00") + paths[path] = append(paths[path], name) + keys = append(keys, name) + } + // Completed excluded ancestors replay their whole native subtree. Compact + // only inherited generated rows in the transported copy. Independently + // excluded declarations and local declaration-specific history remain intact. + sort.Slice(keys, func(i, j int) bool { + a, _, _ := strings.Cut(keys[i], "\x00") + b, _, _ := strings.Cut(keys[j], "\x00") + return len(a) > len(b) + }) + for _, key := range keys { + if !out[key].Inherited { + continue + } + path, _, _ := strings.Cut(key, "\x00") + for slash := strings.LastIndexByte(path, '/'); slash >= 0; slash = strings.LastIndexByte(path, '/') { + path = path[:slash] + ancestors := paths[path] + if len(ancestors) > 1 { + // Distinct declarations at one native path retain separate history. + break + } + if len(ancestors) == 1 { + ancestor := ancestors[0] + parent := out[ancestor] + child := out[key] + if child.Failed && !parent.Failed { + // A slash in a native label can share a lexical prefix + // without denoting its actual native parent. + break + } + if parent.Failed && child.Failed && child.Protected { + parent.Protected = true + out[ancestor] = parent + } + delete(out, key) + break + } + } + } + return out +} + +func (results *excludedTestResults) replay(t *testing.T, declaration string) bool { + if results == nil { + return false + } + results.mu.Lock() + result, ok := results.items[t.Name()+"\x00"+declaration] + results.mu.Unlock() + if !ok { + return false + } + if result.Failed { + if result.Protected { + markProtectedFailureAncestors(t) + } + t.Fail() + } + if result.Skipped { + t.SkipNow() + } + return true +} + +// Automatic retries of a parent retain each excluded descendant's first native +// result even when a later parent attempt no longer discovers that descendant. +func (results *excludedTestResults) failedDescendants(root string) (failed, protected bool) { + if results == nil { + return false, false + } + results.mu.Lock() + defer results.mu.Unlock() + for key, result := range results.items { + path, _, _ := strings.Cut(key, "\x00") + if result.Failed && strings.HasPrefix(path, root+"/") { + failed = true + protected = protected || result.Protected + } + } + return failed, protected +} + +func applyRetainedExcludedFailure(t *testing.T, meta *testExecutionMetadata) bool { + // In-process automatic families currently own only fresh top-level roots. + // Independent declarations at the same subtest path keep their own outcomes. + if meta == nil || !meta.usesFreshRetryAttemptRuntime || meta.identity == nil || len(meta.identity.Segments) != 1 { + return false + } + failed, protected := meta.exclusionResults.failedDescendants(t.Name()) + if failed { + if fields := getTestPrivateFields(t); fields != nil { + fields.SetFailed(true) + } + if protected { + markProtectedFailure(meta) + } + } + return failed +} + +// A selected protected descendant must remain blocking when a surrounding suite +// is quarantined. The marker belongs to one concrete attempt, so a legitimate +// retry which recovers the descendant does not retain an earlier failure. +func markProtectedFailureAncestors(t *testing.T) { + markProtectedFailureAncestorsFromFields(getTestPrivateFields(t)) +} + +func markProtectedFailureAncestorsFromFields(fields *commonPrivateFields) { + for fields != nil && fields.parent != nil && *fields.parent != nil { + if meta := getTestMetadataFromPointer(*fields.parent); meta != nil { + markProtectedFailure(meta) + } + fields = getCommonParentPrivateFields(fields) + } +} + +func applyProcessDirectiveExclusions(d *processRetrySubtreeDirective, e TestExclusions) { + d.AssignedDisabled = d.AssignedDisabled || d.Disabled + d.AssignedQuarantined = d.AssignedQuarantined || d.Quarantined + d.AssignedAttemptToFix = d.AssignedAttemptToFix || d.AttemptToFix + d.Exclusions = e + if e.PreventFailureSuppression { + d.Disabled, d.Quarantined = false, false + } + if e.NoAutoRetry { + d.AttemptToFix = false + } +} + +// exclusionPolicySource keeps source attribution separate from declaration +// ownership. Only an actual Testify method boundary substitutes a reflected +// method for the framework callback; native rows keep their callback identity. +func exclusionPolicySource(t *testing.T, source *runtime.Func, data *TestifyTest) *runtime.Func { + if isTestifyMethodBoundary(t, data) && data.methodFunc != nil && !exclusionsForFunc(source).any() { + return data.methodFunc + } + return source +} + +func exclusionsForChild(t *testing.T, source *runtime.Func, parent *testExecutionMetadata) TestExclusions { + data := getTestifyTest(t) + boundary := isTestifyMethodBoundary(t, data) + source = exclusionPolicySource(t, source, data) + e := exclusionsForFunc(source) + if parent != nil && (data == nil || !boundary && !e.any()) { + // An unannotated native row belongs to its immediate callback, even + // when Testify supplies the row's reporting attribution. Explicit + // callbacks and distinct Testify methods keep their own policy. + e.PreventFailureSuppression = e.PreventFailureSuppression || parent.exclusions.PreventFailureSuppression + e.NoAutoRetry = e.NoAutoRetry || parent.exclusions.NoAutoRetry + } + return e +} + +func (results *excludedTestResults) merge(incoming map[string]excludedTestResult) { + if results == nil || len(incoming) == 0 { + return + } + results.mu.Lock() + defer results.mu.Unlock() + if results.items == nil { + results.items = map[string]excludedTestResult{} + } + for name, result := range incoming { + if _, known := results.items[name]; !known { + results.items[name] = result + } + } +} + +// Send only cached management/selection data already owned by the parent. A +// retry child performs no new feature requests and reports no ordinary subtests. +func processRetryExclusionPolicy(meta *testExecutionMetadata) *processRetrySubtreeConfig { + if !hasNoAutoRetryDeclarations.Load() || meta == nil || meta.identity == nil { + return nil + } + cfg, err := buildProcessRetrySubtreeConfig(&quarantinedRaceProcessContext{}, &commonInfo{identity: meta.identity}, meta, nil) + if err != nil { + return nil + } + return cfg +} + +func validateProcessRetryExclusionPolicy(cfg *processRetrySubtreeConfig, root string) error { + if cfg == nil { + return nil + } + if cfg.Version != processRetrySubtreeVersion || cfg.SelectedRoot != root || cfg.Root.TestName != root || len(cfg.Directives) > processRetrySubtreeMaxDirectives || len(cfg.ITR) > processRetrySubtreeMaxDirectives { + return errProcessRetryControlInvalid + } + for _, directive := range cfg.Directives { + if !processRetryNameWithinRoot(directive.TestName, root) { + return errProcessRetryControlInvalid + } + } + for _, entry := range cfg.ITR { + if !processRetryNameWithinRoot(entry.TestName, root) { + return errProcessRetryControlInvalid + } + } + return nil +} + +func replayNewExcludedProcessTests(parent *testExecutionMetadata, attempt processRetryAttemptResult) { + parent.exclusionResults.merge(attempt.Result.ExclusionResults) + for _, result := range attempt.Result.ExcludedTests { + identity := newTestIdentity(result.ModuleName, result.SuiteName, result.TestName) + meta := &testExecutionMetadata{identity: identity, exclusions: result.Exclusions, assignment: testManagementAssignment{disabled: result.AssignedDisabled, quarantined: result.AssignedQuarantined, attemptToFix: result.AssignedAttemptToFix}, isDisabled: result.Disabled, isQuarantined: result.Quarantined, isItrSkipped: result.SkippedByITR, isItrForcedRun: result.ITRForcedRun, retryContinuationDecided: true} + info := &commonInfo{moduleName: result.ModuleName, suiteName: result.SuiteName, testName: result.TestName, identity: identity} + if settings := integrations.GetSettings(); settings != nil { + if settings.KnownTestsEnabled { + known, hasData := isKnownTest(info) + meta.isANewTest = hasData && !known + } + raw := &additionalFeatureMetadata{identity: identity, isTestManagementEnabled: settings.TestManagement.Enabled} + populateTestManagementMetadata(raw) + meta.assignment.disabled = meta.assignment.disabled || raw.isDisabled + meta.assignment.quarantined = meta.assignment.quarantined || raw.isQuarantined + if raw.managementMatchKind == testManagementMatchExact { + meta.assignment.attemptToFix = raw.isAttemptToFix + } + } + finishProcessRetryTestEvent(info, meta, processRetryAttemptFromSubtreeResult(result), nil, nil) + } +} + +func (results *excludedTestResults) snapshotForRoot(root string) map[string]excludedTestResult { + snapshot := results.snapshot() + for name := range snapshot { + if !excludedResultWithinRoot(name, root) { + delete(snapshot, name) + } + } + return snapshot +} + +// RegisterTestExclusionsByName is the generated-bootstrap bridge for reachable +// helper declarations. Canonical names avoid importing the helper back into its +// own runtime dependency. Call only during initialization, before tests execute. +func RegisterTestExclusionsByName(name string, exclusions TestExclusions) { + if strings.TrimSpace(name) == "" { + return + } + testExclusionDeclarations.Store(name, exclusions) + if exclusions.NoAutoRetry { + hasNoAutoRetryDeclarations.Store(true) + } +} + +func registerExcludedResultCleanup(t *testing.T, parent *testExecutionMetadata, e TestExclusions, declaration string) { + t.Cleanup(func() { + if t.Failed() && e.PreventFailureSuppression { + markProtectedFailureAncestors(t) + } + parent.exclusionResults.record(t, e.PreventFailureSuppression, declaration) + }) +} + +func registerProtectedChildCleanup(t *testing.T, meta *testExecutionMetadata) { + if !meta.exclusions.PreventFailureSuppression { + return + } + t.Cleanup(func() { + if t.Failed() { + markProtectedFailureAncestors(t) + } + }) +} + +func exclusionDeclarationName(t *testing.T, source *runtime.Func) string { + if data := getTestifyTest(t); isTestifyMethodBoundary(t, data) && !(source != nil && (data.methodFunc == nil || source.Name() != data.methodFunc.Name()) && exclusionsForFunc(source).any()) { + if data.declarationFunc != nil { + source = data.declarationFunc + } else if data.methodFunc != nil { + source = data.methodFunc + } + } + if source != nil { + return source.Name() + } + return "" +} + +// Generated native rows inherit their owning declaration's exclusion. Keep that +// owner in the cache identity when another declaration later uses the same row +// callback and native path. Explicit declarations retain their independent key. +func exclusionResultDeclaration(t *testing.T, source *runtime.Func, parent *testExecutionMetadata) string { + declaration := exclusionDeclarationName(t, source) + direct, _ := testExclusionDeclarations.Load(declaration) + if parent != nil && parent.exclusions.NoAutoRetry && parent.exclusionDeclaration != "" && + (direct == nil || !direct.(TestExclusions).NoAutoRetry) { + return parent.exclusionDeclaration + "\x01" + declaration + } + return declaration +} + +func setExecutionExclusions(t *testing.T, meta *testExecutionMetadata, source *runtime.Func, parent *testExecutionMetadata, exclusions TestExclusions) { + meta.exclusions = exclusions + meta.exclusionDeclaration = exclusionResultDeclaration(t, source, parent) +} + +// Cache identities append a compiled declaration after the native test path. +// A selected isolation root may itself be the excluded declaration. +func excludedResultWithinRoot(key, root string) bool { + path, declaration, declared := strings.Cut(key, "\x00") + if declared && declaration == "" { + return false + } + return declared && path == root || strings.HasPrefix(path, root+"/") +} + +// markProtectedFailure bridges a trusted execution fact to its bound event +// without racing the execution metadata's ordinary test interface assignment. +func markProtectedFailure(meta *testExecutionMetadata) { + if meta == nil { + return + } + meta.protectedDescendantFailure.Store(true) + if event := meta.protectedFailureEvent.Load(); event != nil { + integrations.MarkTestProtectedFailure(*event) + } +} + +// bindProtectedFailureEvent publishes the event before rechecking the fact, so +// concurrent marking cannot be lost in either binding order. Bind before the +// base crash policy is registered, which may otherwise suppress the event. +func bindProtectedFailureEvent(meta *testExecutionMetadata, test integrations.Test) { + if meta == nil { + return + } + meta.protectedFailureEvent.Store(&test) + if meta.protectedDescendantFailure.Load() { + integrations.MarkTestProtectedFailure(test) + } +} + +// markNativeProtectedFailure observes an original native Fail, before recursive +// parent propagation or finalization can obscure the protected callback owner. +func markNativeProtectedFailure(tb testing.TB) { + meta := getTestMetadata(tb) + if meta == nil || !meta.exclusions.PreventFailureSuppression || processRetryChildFailureIsPropagated() { + return + } + markProtectedFailure(meta) + markProtectedFailureAncestorsFromFields(getFailureHookPrivateFields(tb)) +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_serialization_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_serialization_test.go new file mode 100644 index 000000000..d8ea52afd --- /dev/null +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_serialization_test.go @@ -0,0 +1,110 @@ +package gotesting + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func TestExcludedProcessDiagnosticsFitWireBudget(t *testing.T) { + cfg := processRetryChildConfig{TestName: "Root", Attempt: 1, RetryReason: "retry", ExclusionPolicy: &processRetrySubtreeConfig{Version: processRetrySubtreeVersion, SelectedRoot: "Root", Root: processRetrySubtreeDirective{TestName: "Root", ModuleName: "fixture", SuiteName: "fixture_test.go"}}} + result := processRetryResult{Version: 1, TestName: cfg.TestName, Attempt: cfg.Attempt, RetryReason: cfg.RetryReason, Status: processRetryStatusPass, StartUnixNano: 1, FinishUnixNano: 2, DurationNanos: 1} + output := strings.Repeat("l", 4*1024) + for i := 0; i < 4096; i++ { + result.ExcludedTests = append(result.ExcludedTests, processRetrySubtreeResult{TestName: fmt.Sprintf("Root/row-%04d", i), ModuleName: "fixture", SuiteName: "fixture_test.go", Exclusions: TestExclusions{NoAutoRetry: true}, Status: processRetryStatusPass, StartUnixNano: 1, FinishUnixNano: 2, DurationNanos: 1, OutputTail: output}) + } + if err := validateProcessRetryResult(result, cfg); err != nil { + t.Fatalf("invalid source fixture: %v", err) + } + raw, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if len(raw) <= processRetryWireMaxBytes(true) { + t.Fatalf("fixture did not exceed wire budget: %d", len(raw)) + } + t.Logf("excluded records=%d unboundedbytes=%d limit=%d", len(result.ExcludedTests), len(raw), processRetryWireMaxBytes(true)) + payload, err := marshalProcessRetryResult(result) + if err != nil { + t.Fatalf("valid excluded executions must trim optional diagnostics: %v", err) + } + if len(payload) > processRetryWireMaxBytes(true) { + t.Fatalf("wire budget exceeded: %d", len(payload)) + } + var bounded processRetryResult + if err := json.Unmarshal(payload, &bounded); err != nil { + t.Fatal(err) + } + if err := validateProcessRetryResult(bounded, cfg); err != nil { + t.Fatalf("bounded result invalid: %v", err) + } + if bounded.Status != processRetryStatusPass || bounded.ResultError != "" || len(bounded.ExcludedTests) != 4096 { + t.Fatalf("required results lost: status=%s error=%s records=%d", bounded.Status, bounded.ResultError, len(bounded.ExcludedTests)) + } + for i, row := range bounded.ExcludedTests { + original := result.ExcludedTests[i] + if row.TestName != original.TestName || row.Status != original.Status || row.Exclusions != original.Exclusions { + t.Fatalf("required record%d changed: %v", i, row) + } + if !row.OutputTruncated || len(row.OutputTail) >= len(output) { + t.Fatalf("optional row%d diagnostics were not bounded", i) + } + if original.OutputTail != output || original.OutputTruncated { + t.Fatalf("caller row%d mutated", i) + } + } + mixedCfg := cfg + mixedCfg.Subtree = cfg.ExclusionPolicy + withSubtree := func(value processRetryResult) processRetryResult { + value.ModuleName, value.SuiteName = "fixture", "fixture_test.go" + value.Subtests = []processRetrySubtreeResult{{TestName: "Root/ordinary", ModuleName: "fixture", SuiteName: "fixture_test.go", Status: processRetryStatusPass, StartUnixNano: 1, FinishUnixNano: 2, DurationNanos: 1}} + return value + } + mixed := withSubtree(result) + if err := validateProcessRetryResult(mixed, mixedCfg); err != nil { + t.Fatalf("invalid mixed fixture: %v", err) + } + mixedPayload, err := marshalProcessRetryResult(mixed) + if err != nil { + t.Fatalf("mixed diagnostics must fit without dropping records: %v", err) + } + var mixedBounded processRetryResult + if err := json.Unmarshal(mixedPayload, &mixedBounded); err != nil { + t.Fatal(err) + } + if err := validateProcessRetryResult(mixedBounded, mixedCfg); err != nil { + t.Fatalf("invalid bounded mixed result: %v", err) + } + if len(mixedPayload) > processRetryWireMaxBytes(true) || len(mixedBounded.ExcludedTests) != 4096 || len(mixedBounded.Subtests) != 1 || mixedBounded.Status != processRetryStatusPass || mixedBounded.ResultError != "" { + t.Fatalf("mixed records changed: excluded=%d subtree=%d status=%s error=%s", len(mixedBounded.ExcludedTests), len(mixedBounded.Subtests), mixedBounded.Status, mixedBounded.ResultError) + } + for i, row := range mixedBounded.ExcludedTests { + if row.TestName != result.ExcludedTests[i].TestName || !row.OutputTruncated || row.Status != processRetryStatusPass || result.ExcludedTests[i].OutputTail != output { + t.Fatalf("mixed row%d changed required data or caller diagnostics", i) + } + } + // Required identity data can exceed the budget even while every field fits + // its individual bound. Never drop execution records to manufacture success. + for i := range result.ExcludedTests { + result.ExcludedTests[i].ModuleName = strings.Repeat("m", processRetryErrorMessageMaxBytes-2) + result.ExcludedTests[i].OutputTail = "" + } + if err := validateProcessRetryResult(result, cfg); err != nil { + t.Fatalf("invalid required-data fixture: %v", err) + } + if payload, err := marshalProcessRetryResult(result); err == nil { + t.Fatalf("oversized required records were not rejected: returnedbytes=%d", len(payload)) + } + requiredMixed := withSubtree(result) + if err := validateProcessRetryResult(requiredMixed, mixedCfg); err != nil { + t.Fatalf("invalid required mixed fixture: %v", err) + } + if payload, err := marshalProcessRetryResult(requiredMixed); err == nil { + t.Fatalf("oversized mixed required records were dropped: returnedbytes=%d", len(payload)) + } + ordinary := processRetryResult{Version: 1, TestName: strings.Repeat("r", processRetryResultMaxBytes), Status: processRetryStatusPass} + if payload, err := marshalProcessRetryResult(ordinary); err == nil { + t.Fatalf("ordinary64KiB budget expanded: returnedbytes=%d", len(payload)) + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_test.go new file mode 100644 index 000000000..430432902 --- /dev/null +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/exclusions_test.go @@ -0,0 +1,318 @@ +package gotesting + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "testing" +) + +// A protected descendant changes masking, never the retry family's recovery +// rule. An earlier failed attempt must not turn an ATR/EFD recovery into failure. +func TestProtectedProcessDescendantKeepsRetryFamilyAggregation(t *testing.T) { + t.Run("protected-only-in-descendant-atf-continuation", func(t *testing.T) { + group := &deferredProcessRetryGroup{protectedDescendantFailed: true, metadata: processRetryMetadataSnapshot{isAttemptToFix: true}} + group.outcomes.observe(false, false) + group.outcomes.observe(false, false) + if !group.packageFailed() { + t.Fatal("passing enclosing executions hid protected descendant continuation failure") + } + }) + for _, family := range []string{"atr", "efd", "attempt_to_fix"} { + for _, quarantined := range []bool{false, true} { + t.Run(family+map[bool]string{false: "", true: "-quarantined"}[quarantined], func(t *testing.T) { + group := &deferredProcessRetryGroup{protectedDescendantFailed: true, metadata: processRetryMetadataSnapshot{protectedDescendantFailure: true, isQuarantined: quarantined, isFlakyTestRetriesEnabled: family == "atr", isEarlyFlakeDetectionEnabled: family == "efd", isANewTest: family == "efd", isAttemptToFix: family == "attempt_to_fix"}} + group.outcomes.observe(true, false) + group.outcomes.observe(false, false) + if got, want := group.packageFailed(), family == "attempt_to_fix"; got != want { + t.Fatalf("recovered %s package failure=%t want=%t", family, got, want) + } + }) + } + } +} + +func TestExactAttemptToFixAssignmentClearsInheritedProperty(t *testing.T) { + parent := &testExecutionMetadata{isAttemptToFix: true, assignment: testManagementAssignment{attemptToFix: true}} + child := &testExecutionMetadata{hasExplicitAttemptToFix: true} + propagateTestExecutionMetadataFlags(child, parent) + if child.isAttemptToFix || child.assignment.attemptToFix { + t.Fatal("explicit false ATF assignment inherited parent's property") + } + child.assignment.attemptToFix = true + applyAdditionalFeatureMetadataToExecution(child, &additionalFeatureMetadata{hasExplicitAttemptToFix: true}) + if child.assignment.attemptToFix { + t.Fatal("explicit false ATF assignment retained stale inherited property") + } +} + +func TestExcludedResultsKeepDeclarationIdentity(t *testing.T) { + t.Run("excluded-execution-byte-budget", func(t *testing.T) { + cfg := processRetryChildConfig{TestName: "Root", Attempt: 1, RetryReason: "retry", ExclusionPolicy: &processRetrySubtreeConfig{Version: processRetrySubtreeVersion, SelectedRoot: "Root", Root: processRetrySubtreeDirective{TestName: "Root", ModuleName: "fixture", SuiteName: "fixture_test.go"}}} + result := processRetryResult{Version: 1, TestName: cfg.TestName, Attempt: cfg.Attempt, RetryReason: cfg.RetryReason, Status: processRetryStatusPass, StartUnixNano: 1, FinishUnixNano: 2, DurationNanos: 1} + for i := 0; i < 4097; i++ { + result.ExcludedTests = append(result.ExcludedTests, processRetrySubtreeResult{TestName: fmt.Sprintf("Root/Excluded/row-%04d", i), ModuleName: "example.com/fixture_test", SuiteName: "fixture_test.go/Suite", Exclusions: TestExclusions{NoAutoRetry: true}, Status: processRetryStatusPass, StartUnixNano: 1, FinishUnixNano: 2, DurationNanos: 1}) + } + payload, err := marshalProcessRetryResult(result) + if err != nil { + t.Fatal(err) + } + t.Logf("excluded execution records=%d bytes=%d wire-limit=%d", len(result.ExcludedTests), len(payload), processRetryWireMaxBytes(true)) + path := filepath.Join(t.TempDir(), "result.json") + if err = os.WriteFile(path, payload, 0600); err != nil { + t.Fatal(err) + } + read, _, err := readProcessRetryResult(path, cfg) + if err != nil || len(read.ExcludedTests) != 4097 { + t.Fatalf("valid execution records rejected/lost: count=%d err=%v", len(read.ExcludedTests), err) + } + result.ExcludedTests[len(result.ExcludedTests)-1].TestName = "Other/row" + if validateProcessRetryResult(result, cfg) == nil { + t.Fatal("unrelated execution identity accepted") + } + result.ExcludedTests[len(result.ExcludedTests)-1].TestName = result.ExcludedTests[0].TestName + if validateProcessRetryResult(result, cfg) == nil { + t.Fatal("duplicate execution identity accepted") + } + if err = os.WriteFile(path, bytes.Repeat([]byte{' '}, processRetryWireMaxBytes(true)+1), 0600); err != nil { + t.Fatal(err) + } + if _, _, err = readProcessRetryResult(path, cfg); err == nil { + t.Fatal("oversized excluded execution frame accepted") + } + result.ExcludedTests[0].TestName = "Root/" + string(bytes.Repeat([]byte{'x'}, processRetryWireMaxBytes(true))) + if _, err = marshalProcessRetryResult(result); err == nil { + t.Fatal("oversized execution writer dropped events or exceeded budget") + } + }) + t.Run("isolation-root-identity", func(t *testing.T) { + for _, tc := range []struct { + key string + want bool + }{{"Root\x00declaration", true}, {"Root/child\x00declaration", true}, {"Root/child", true}, {"Root", false}, {"Root\x00", false}, {"RootSibling\x00declaration", false}, {"Other/Root\x00declaration", false}} { + if got := excludedResultWithinRoot(tc.key, "Root"); got != tc.want { + t.Fatalf("cache key%q withinroot=%t want%t", tc.key, got, tc.want) + } + } + cache := &excludedTestResults{items: map[string]excludedTestResult{"Root\x00declaration": {}, "Other\x00declaration": {}}} + if snapshot := cache.snapshotForRoot("Root"); len(snapshot) != 1 { + t.Fatalf("selected-root cache lost: %v", snapshot) + } + }) + t.Run("compact-transported-subtree", func(t *testing.T) { + cache := &excludedTestResults{items: map[string]excludedTestResult{ + "Root/Excluded\x00parent": {Failed: true}, + "Root/Excluded/row\x00callback": {Failed: true, Protected: true, Inherited: true}, + "Root/Excluded/row/deep\x00callback": {Failed: true, Protected: true, Inherited: true}, + "Root/Other\x00first": {}, + "Root/Other\x00second": {}, + "Root/Other/row\x00callback": {}, + "Root/Prefix\x00passed": {}, + "Root/Prefix/failed\x00generated": {Failed: true, Protected: true, Inherited: true}, + }} + snapshot := cache.snapshot() + if len(snapshot) != 6 || !snapshot["Root/Excluded\x00parent"].Protected || len(cache.items) != 8 { + t.Fatalf("compaction lost protected outcome or declaration history: %v local=%v", snapshot, cache.items) + } + if !snapshot["Root/Prefix/failed\x00generated"].Protected { + t.Fatal("native slash-label failure was lost") + } + if _, ok := snapshot["Root/Other/row\x00callback"]; !ok { + t.Fatal("ambiguous ancestor discarded declaration-specific result") + } + }) + results := &excludedTestResults{} + t.Run("same-native-name", func(t *testing.T) { + results.record(t, false, "first-declaration") + if results.replay(t, "ordinary-other-declaration") { + t.Fatal("cache crossed declarations") + } + results.record(t, true, "second-declaration") + if len(results.snapshot()) != 2 { + t.Fatal("one declaration overwrote another") + } + if !results.replay(t, "first-declaration") || !results.replay(t, "second-declaration") { + t.Fatal("declared native result missing") + } + }) +} + +func TestRetainedExcludedFailuresKeepFamilyScope(t *testing.T) { + for _, tc := range []struct { + name, key string + failed, protected, wantFailed, wantProtected bool + }{ + {"descendant", "Root/child\x00declaration", true, true, true, true}, + {"inherited-owner", "Root/row\x00owner\x01callback", true, false, true, false}, + {"independent-at-root", "Root\x00another-declaration", true, true, false, false}, + {"other-native-root", "RootOther/child\x00declaration", true, true, false, false}, + {"passing-protected", "Root/child\x00declaration", false, true, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + cache := &excludedTestResults{items: map[string]excludedTestResult{tc.key: {Failed: tc.failed, Protected: tc.protected}}} + failed, protected := cache.failedDescendants("Root") + if failed != tc.wantFailed || protected != tc.wantProtected { + t.Fatalf("retained outcome=(%t,%t) want(%t,%t)", failed, protected, tc.wantFailed, tc.wantProtected) + } + }) + } + for _, quarantined := range []bool{false, true} { + for _, protected := range []bool{false, true} { + t.Run(fmt.Sprintf("deferred/quarantined=%t/protected=%t", quarantined, protected), func(t *testing.T) { + group := &deferredProcessRetryGroup{identity: *newTestIdentity("fixture", "fixture_test.go", "Root"), metadata: processRetryMetadataSnapshot{isQuarantined: quarantined, isFlakyTestRetriesEnabled: true, exclusionResults: &excludedTestResults{items: map[string]excludedTestResult{"Root/child\x00declaration": {Failed: true, Protected: protected}}}}} + group.outcomes.observe(true, false) + group.outcomes.observe(false, false) + if got, want := group.packageFailed(), !quarantined || protected; got != want { + t.Fatalf("retained group failure=%t want%t", got, want) + } + }) + } + } +} + +func TestExcludedResultsTerminalAdmission(t *testing.T) { + t.Run("nil-keeps-legacy-admission", func(t *testing.T) { + var results *excludedTestResults + if !results.begin() { + t.Fatal("nil result owner blocked legacy callback") + } + results.end() + if !results.freezeForTerminal() { + t.Fatal("nil result owner blocked legacy terminal recovery") + } + }) + t.Run("quiescent-freeze-closes-admission", func(t *testing.T) { + results := &excludedTestResults{} + if !results.freezeForTerminal() { + t.Fatal("quiescent terminal snapshot was not eligible") + } + calls := 0 + for range 10 { + if results.begin() { + calls++ + results.end() + } + } + if calls != 0 { + t.Fatalf("terminal snapshot admitted%d new callback bodies", calls) + } + if len(results.snapshot()) != 0 { + t.Fatal("terminal freeze synthesized a completed declaration") + } + }) + t.Run("active-callback-prevents-ready", func(t *testing.T) { + results := &excludedTestResults{} + if !results.begin() { + t.Fatal("initial callback was not admitted") + } + if results.freezeForTerminal() { + t.Fatal("terminal snapshot was eligible while admitted callback remained active") + } + if results.begin() { + results.end() + t.Fatal("terminal freeze admitted another callback") + } + if len(results.snapshot()) != 0 { + t.Fatal("active callback synthesized a completed declaration") + } + results.end() + if results.begin() { + results.end() + t.Fatal("ending active callback reopened terminal admission") + } + }) + t.Run("completed-record-precedes-end", func(t *testing.T) { + results := &excludedTestResults{} + if !results.begin() { + t.Fatal("callback was not admitted") + } + func() { + defer results.end() + defer results.record(t, true, "completed-before-end") + }() + if !results.freezeForTerminal() { + t.Fatal("completed callback left terminal snapshot ineligible") + } + snapshot := results.snapshot() + if len(snapshot) != 1 { + t.Fatalf("completed callback record lost: %v", snapshot) + } + for _, result := range snapshot { + if !result.Protected { + t.Fatalf("completed callback protection lost: %v", snapshot) + } + } + if results.begin() { + results.end() + t.Fatal("completed record reopened terminal admission") + } + }) + t.Run("recorded-but-still-active-remains-conservative", func(t *testing.T) { + results := &excludedTestResults{} + if !results.begin() { + t.Fatal("callback was not admitted") + } + results.record(t, true, "record-before-end") + if results.freezeForTerminal() { + t.Fatal("terminal snapshot ignored an admitted callback before its end") + } + if len(results.snapshot()) != 1 { + t.Fatal("terminal snapshot lost a completed record while another callback boundary remained active") + } + results.end() + if len(results.snapshot()) != 1 { + t.Fatal("ending callback discarded completed history") + } + }) +} + +// The accepted callback stays active until both competing operations return. +// A Ready-eligible freeze and an accepted active callback must never coexist, +// regardless of which operation wins admission's serialization boundary. +func TestExcludedResultsAdmissionCompetesWithTerminalFreeze(t *testing.T) { + for i := range 200 { + results := &excludedTestResults{} + start, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + admitted, ready := make(chan bool, 1), make(chan bool, 1) + go func() { + <-start + accepted := results.begin() + admitted <- accepted + if accepted { + <-release + results.record(t, false, "racing-completed-declaration") + results.end() + } + close(done) + }() + go func() { <-start; ready <- results.freezeForTerminal() }() + close(start) + accepted, eligible := <-admitted, <-ready + if accepted == eligible { + close(release) + <-done + t.Fatalf("iteration%d accepted-active=%t terminal-Ready=%t", i, accepted, eligible) + } + if len(results.snapshot()) != 0 { + close(release) + <-done + t.Fatalf("iteration%d terminal synthesized an unfinished declaration", i) + } + if results.begin() { + results.end() + close(release) + <-done + t.Fatalf("iteration%d terminal admitted a later callback", i) + } + close(release) + <-done + wantRecords := 0 + if accepted { + wantRecords = 1 + } + if got := len(results.snapshot()); got != wantRecords { + t.Fatalf("iteration%d completed history count%d want%d", i, got, wantRecords) + } + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/fuzz_events_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/fuzz_events_test.go index 9489424c0..0a82ba639 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/fuzz_events_test.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/fuzz_events_test.go @@ -173,7 +173,7 @@ func TestFuzzEventsPreserveCleanupPanicDetails(t *testing.T) { native.Cleanup(func() { remainingCleanupRan = true }) native.Cleanup(func() { panic("cleanup panic sentinel") }) meta := &testExecutionMetadata{cleanupResult: &testCleanupResult{}, isQuarantined: quarantined} - runAndApplyTestCleanupWithDurationOptions(native, meta, time.Millisecond, true) + runAndApplyTestCleanupWithDurationOptions(native, meta, time.Millisecond, true, false) require.Equal(t, "cleanup panic sentinel", meta.panicData) require.True(t, remainingCleanupRan) require.False(t, meta.cleanupResult.goexit, "panic is not Goexit") diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation.go index 942ef9c29..778c92cc0 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation.go @@ -33,6 +33,13 @@ type ( // testExecutionMetadata contains metadata regarding an unique *testing.T or *testing.B execution testExecutionMetadata struct { + protectedDescendantFailure atomic.Bool + protectedFailureEvent atomic.Pointer[integrations.Test] + exclusionResults *excludedTestResults + exclusionDeclaration string + exclusionPolicy *quarantinedRaceChildState + exclusions TestExclusions + assignment testManagementAssignment test integrations.Test // internal CI Visibility test event originalTest *testing.T // original test that was executed error atomic.Int32 // flag to check if the test event has error data already @@ -182,10 +189,19 @@ type ( // testCleanupResult captures how testing cleanup execution completed for a retry attempt. testCleanupResult struct { - panicData any // panic value returned by testing.common.runCleanup. - panicStacktrace string // stacktrace captured when cleanup returned a panic value. - goexit bool // true when cleanup called runtime.Goexit before runCleanup returned. - ran bool // true after this attempt has executed its testing cleanups. + panicData any // panic value returned by testing.common.runCleanup. + panicStacktrace string // stacktrace captured when cleanup returned a panic value. + goexit bool // true when cleanup called runtime.Goexit before runCleanup returned. + ran bool // true after this attempt has executed its testing cleanups. + observePanics bool + shapeCaptured bool + hadQueuedSubtests bool + terminals []testCleanupTerminal + } + testCleanupTerminal struct { + value any + stack string + goexit bool } retryGroupPolicyResult struct { @@ -196,6 +212,8 @@ type ( // additionalFeatureMetadata is the effective per-test state used to select and apply CI Visibility additional features. additionalFeatureMetadata struct { + exclusions TestExclusions + assignment testManagementAssignment identity *testIdentity // fully-qualified test or subtest identity isTestManagementEnabled bool // whether Test Management is enabled for the session isEarlyFlakeDetectionEnabled bool // whether EFD remains effective for this test after suppression @@ -528,6 +546,14 @@ func applyAdditionalFeatureMetadataToExecution(execMeta *testExecutionMetadata, if execMeta == nil || meta == nil { return } + execMeta.exclusions = meta.exclusions + execMeta.assignment.quarantined = execMeta.assignment.quarantined || meta.assignment.quarantined + execMeta.assignment.disabled = execMeta.assignment.disabled || meta.assignment.disabled + if meta.hasExplicitAttemptToFix { + execMeta.assignment.attemptToFix = meta.assignment.attemptToFix + } else { + execMeta.assignment.attemptToFix = execMeta.assignment.attemptToFix || meta.assignment.attemptToFix + } execMeta.identity = meta.identity if meta.hasExplicitQuarantined { // Exact Test Management data is applied before parent propagation, which still OR-inherits quarantine today. @@ -559,6 +585,7 @@ func applyAdditionalFeatureMetadataToExecution(execMeta *testExecutionMetadata, execMeta.isAModifiedTest = execMeta.isAModifiedTest || meta.isModified execMeta.efdFellBackToFlakyRetries = meta.efdFellBackToFlakyRetries execMeta.shouldOrchestrateAttemptToFix = meta.shouldOrchestrateAttemptToFix + applyExecutionExclusions(execMeta) } // syncFeatureMetadataFromExecution keeps wrapper-level metadata aligned with the latest concrete execution. @@ -587,6 +614,7 @@ func syncFeatureMetadataFromExecution(meta *additionalFeatureMetadata, execMeta meta.isNew = execMeta.isANewTest meta.isModified = execMeta.isAModifiedTest meta.efdFellBackToFlakyRetries = execMeta.efdFellBackToFlakyRetries + applyFeatureExclusions(meta) } // wrapWithAdditionalFeatureMetadata preloads metadata without entering retry isolation. @@ -645,7 +673,11 @@ func applyAdditionalFeaturesToTestFunc( _ = integrations.GetKnownTests() // If none of the additional features are enabled, return the original function. - if !settings.TestManagement.Enabled && !settings.EarlyFlakeDetection.Enabled && !settings.FlakyTestRetriesEnabled { + exclusions := testInfo.exclusions + if !exclusions.any() { + exclusions = exclusionsForFunc(testInfo.sourceFunc) + } + if !exclusions.any() && !settings.TestManagement.Enabled && !settings.EarlyFlakeDetection.Enabled && !settings.FlakyTestRetriesEnabled { return f } @@ -664,8 +696,18 @@ func applyAdditionalFeaturesToTestFunc( managementMatchKind: testManagementMatchNone, } - // Test Management feature + // Keep the backend assignment separate from effective annotation policy. populateTestManagementMetadata(&meta) + meta.exclusions = exclusions + meta.assignment.quarantined, meta.assignment.disabled, meta.assignment.attemptToFix = meta.isQuarantined, meta.isDisabled, meta.isAttemptToFix + if isSubtest && !settings.SubtestFeaturesEnabled { + // Retain known assignments as descriptive tags without activating + // backend subtest actions while their feature gate is disabled. + meta.isDisabled, meta.isQuarantined, meta.isAttemptToFix = false, false, false + meta.hasExplicitDisabled, meta.hasExplicitQuarantined, meta.hasExplicitAttemptToFix = false, false, false + meta.managementMatchKind = testManagementMatchNone + } + applyFeatureExclusions(&meta) // determine whether attempt-to-fix retries should be orchestrated at this level meta.shouldOrchestrateAttemptToFix = meta.isAttemptToFix @@ -682,12 +724,12 @@ func applyAdditionalFeaturesToTestFunc( } if isSubtest { - if !settings.SubtestFeaturesEnabled { + if !settings.SubtestFeaturesEnabled && !exclusions.any() { // Feature gate keeps legacy behaviour when subtests support is disabled. return f } // Require an exact match before applying subtest-specific directives; fallbacks remain parent-scoped. - if meta.managementMatchKind != testManagementMatchExact { + if meta.managementMatchKind != testManagementMatchExact && !exclusions.any() { return f } // Subtests currently inherit parent EFD/flaky retry behaviour; disable here to avoid double wrapping. @@ -754,6 +796,9 @@ func applyAdditionalFeaturesToTestFunc( switch selection.path { case additionalFeaturePathNone: + if exclusions.any() { + return wrapWithAdditionalFeatureMetadata(f, ptrMeta, false, false) + } return f case additionalFeaturePathMetadataOnly: return wrapWithAdditionalFeatureMetadata(f, ptrMeta, needsMetadataOnly, false) @@ -771,6 +816,11 @@ func applyAdditionalFeaturesToTestFunc( } var outcomes retryOutcomeAccumulator + var lastProtectedFailure, protectedFailureSeen bool + exclusionResults := &excludedTestResults{} + if isSubtest && parentExecMeta != nil && parentExecMeta.exclusionResults != nil { + exclusionResults = parentExecMeta.exclusionResults + } runTestWithRetry(&runTestWithRetryOptions{ targetFunc: f, @@ -789,6 +839,10 @@ func applyAdditionalFeaturesToTestFunc( retryAttemptMaskingFallback: ptrMeta.isDisabled || ptrMeta.isQuarantined, preExecMetaAdjust: func(execMeta *testExecutionMetadata, _ int) { // Synchronize the test execution metadata with the original test execution metadata. + execMeta.exclusionResults = exclusionResults + if ptrMeta.isAttemptToFix && protectedFailureSeen { + markProtectedFailure(execMeta) + } applyAdditionalFeatureMetadataToExecution(execMeta, ptrMeta) // An exact subtest wrapper owns its retry tags independently from the @@ -830,6 +884,9 @@ func applyAdditionalFeaturesToTestFunc( return retryCount }, postPerExecution: func(ptrToLocalT *testing.T, execMeta *testExecutionMetadata, executionIndex int, _ time.Duration) { + applyRetainedExcludedFailure(ptrToLocalT, execMeta) + lastProtectedFailure = execMeta.protectedDescendantFailure.Load() + protectedFailureSeen = protectedFailureSeen || lastProtectedFailure failed := ptrToLocalT.Failed() skipped := ptrToLocalT.Skipped() log.Debug("applyAdditionalFeaturesToTestFunc: postPerExecution called for execution %d, failed: %t, skipped: %t", executionIndex, failed, skipped) @@ -897,8 +954,12 @@ func applyAdditionalFeaturesToTestFunc( return false }, postOnRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T, result retryGroupPolicyResult) { + retainedFailure, retainedProtected := false, false + if !isSubtest { + retainedFailure, retainedProtected = exclusionResults.failedDescendants(t.Name()) + } // if the test is disabled or quarantined, skip the test result to the testing framework - if ptrMeta.isDisabled || ptrMeta.isQuarantined { + if (ptrMeta.isDisabled || ptrMeta.isQuarantined) && !(retainedProtected || lastProtectedFailure || ptrMeta.isAttemptToFix && protectedFailureSeen) { log.Debug("applyAdditionalFeaturesToTestFunc: Skipping test result for disabled or quarantined test") t.SkipNow() return @@ -913,7 +974,7 @@ func applyAdditionalFeaturesToTestFunc( // Attempt-to-fix owns result propagation when it is active, even if EFD or FTR // metadata is also present for tag compatibility. attemptToFixActive := ptrMeta.isAttemptToFix - if result.failfastRawFailure || result.lateFailure || attemptToFixActive && outcomes.anyFailed() { + if retainedFailure || result.failfastRawFailure || result.lateFailure || attemptToFixActive && outcomes.anyFailed() { tCommonPrivates.SetFailed(true) tParentCommonPrivates := getTestParentPrivateFields(t) if tParentCommonPrivates == nil { @@ -1169,7 +1230,10 @@ func shouldUseParallelEFD(options *runTestWithRetryOptions, execMeta *testExecut if options == nil || execMeta == nil { return false } - if !options.parallelEFDAllowed { + // A declaration first discovered during a retry must publish its one-shot + // result before another attempt can reuse it. Keep EFD retries serial in + // packages that contain no-auto-retry declarations. + if hasNoAutoRetryDeclarations.Load() || !options.parallelEFDAllowed { return false } if remainingAttempts <= 1 || maxConcurrency <= 1 { @@ -1340,43 +1404,251 @@ func runTestCleanup(t *testing.T, result *testCleanupResult) { } func runTestCleanupWithOptions(t *testing.T, result *testCleanupResult, neutralizeNativeParallelRelease bool) { - completeParallelSubtests(t, getTestPrivateFields(t), neutralizeNativeParallelRelease) + result.captureShape(t) + finishParallel := completeParallelSubtests(t, getTestPrivateFields(t), neutralizeNativeParallelRelease) + runTestCleanupCallbacksWithParallelRelease(t, result, finishParallel) +} + +// Preserve native parallel guards until all user cleanups have finished. +func runTestCleanupCallbacksWithParallelRelease(t *testing.T, result *testCleanupResult, finishParallel func(*testCleanupResult)) { + if finishParallel != nil { + defer finishParallel(result) + } runTestCleanupCallbacks(t, result) } +func (result *testCleanupResult) captureShape(t *testing.T) { + if !result.shapeCaptured { + result.hadQueuedSubtests = hasPendingParallelSubtests(t) + result.shapeCaptured = true + } +} + +// A factory with one stable code pointer lets an observer wrap only newly +// appended callbacks. Each callback is wrapped once, including one registered +// by an earlier cleanup, without rescanning the already wrapped prefix. +// +//go:noinline +func observeTestCleanupCallback(t *testing.T, result *testCleanupResult, callback func()) func() { + return func() { + returned := false + defer func() { + value := recover() + if value != nil { + result.terminals = append(result.terminals, testCleanupTerminal{value: value, stack: observedTestCleanupStack()}) + } else if !returned { + result.terminals = append(result.terminals, testCleanupTerminal{goexit: true}) + } + // Native runCleanup recursively drains remaining callbacks while + // unwinding panic/Goexit, so newly registered callbacks must be + // observed before control returns to that native defer. + observePendingTestCleanups(t, result) + if value != nil { + panic(value) + } + }() + callback() + returned = true + } +} + +var testCleanupObserverPC uintptr +var testCleanupObserverName string + +func init() { + testCleanupObserverPC = reflect.ValueOf(observeTestCleanupCallback(nil, nil, nil)).Pointer() + testCleanupObserverName = runtime.FuncForPC(testCleanupObserverPC).Name() +} + +func observePendingTestCleanups(t *testing.T, result *testCleanupResult) { + layout := getTestingInternalsLayout() + if layout == nil || layout.disabled || !layout.common.cleanups.available { + return + } + base := commonBaseForTest(t, layout) + callbacks := fieldPtr[[]func()](base, layout.common.cleanups) + mu := fieldPtr[sync.RWMutex](base, layout.common.mu) + if callbacks == nil || mu == nil { + return + } + mu.Lock() + defer mu.Unlock() + for i := len(*callbacks) - 1; i >= 0; i-- { + callback := (*callbacks)[i] + if callback == nil { + continue + } + if reflect.ValueOf(callback).Pointer() == testCleanupObserverPC { + break + } + (*callbacks)[i] = observeTestCleanupCallback(t, result, callback) + } +} + +func observedTestCleanupStack() string { + // Exclude only the observer's own added frames. The original cleanup, + // native runCleanup and caller frames keep their original identities. + lines := strings.SplitAfter(utils.GetStacktrace(2), "\n") + var stack strings.Builder + for i := 0; i+1 < len(lines); i += 2 { + name := strings.TrimSuffix(lines[i], "\n") + if name == testCleanupObserverName || strings.HasPrefix(name, testCleanupObserverName+".") { + continue + } + stack.WriteString(lines[i]) + stack.WriteString(lines[i+1]) + } + return stack.String() +} + func runTestCleanupCallbacks(t *testing.T, result *testCleanupResult) { + result.captureShape(t) result.ran = true done := make(chan struct{}) go func() { completed := false defer func() { - result.panicData = recover() - if result.panicData != nil { + if value := recover(); value != nil { + result.panicData = value result.panicStacktrace = utils.GetStacktrace(1) } else if !completed { result.goexit = true } + if result.observePanics && result.panicData != nil { + for i := len(result.terminals) - 1; i >= 0; i-- { + if !result.terminals[i].goexit { + result.panicStacktrace = result.terminals[i].stack + break + } + } + } close(done) }() - // Recover here rather than inside testing so the panic's cleanup frames - // are still available. Native runCleanup still drains remaining callbacks. - testingTRunCleanup(t, 0) + if result.observePanics { + observePendingTestCleanups(t, result) + } + if result.observePanics && result.hadQueuedSubtests { + // Native tRunner recovers cleanup panics for the queued-child + // shape. In particular, Goexit resumes after a later cleanup + // panic is recovered; normalPanic would spuriously fail it. + result.panicData = testingTRunCleanup(t, 1) + } else { + testingTRunCleanup(t, 0) + } completed = true }() <-done } +// resolveTestCleanupDiagnostics retains only the terminals native Go exposes. +// The original body value is separate from the effective body terminal because +// cleanup Goexit can cancel it or synthesize an unexpected-termination result. +func resolveTestCleanupDiagnostics(t *testing.T, meta *testExecutionMetadata, body any, bodyStack string, terminal any, terminalStack string) (any, string) { + if meta == nil || meta.cleanupResult == nil || !meta.cleanupResult.observePanics { + return terminal, terminalStack + } + result := meta.cleanupResult + panics := make([]testCleanupTerminal, 0, len(result.terminals)) + lastGoexit := false + for _, observed := range result.terminals { + lastGoexit = observed.goexit + if !observed.goexit { + panics = append(panics, observed) + } + } + if len(panics) == 0 { + return terminal, terminalStack + } + visibleBody := body != nil + primaryBody := body != nil + primaryCleanup := -1 + if result.hadQueuedSubtests { + if result.goexit { + // The outer tRunner defer rethrows a recovered body terminal. + // A returning cleanup panic exposes only its final value; a + // Goexit unwind instead retains its active preceding panic chain. + if terminal == nil || !lastGoexit { + panics = nil + } + if terminal == nil { + visibleBody = false + primaryBody = false + } + } else { + panics = panics[len(panics)-1:] + if terminal == nil { + terminal, terminalStack = panics[0].value, panics[0].stack + primaryBody = false + primaryCleanup = 0 + } + } + } else if result.goexit { + if terminal == nil { + panics = nil + visibleBody = false + primaryBody = false + } else { + primaryBody = false + } + } else { + primaryCleanup = len(panics) - 1 + terminal, terminalStack = panics[primaryCleanup].value, panics[primaryCleanup].stack + primaryBody = false + } + // Keep the primary panic's origin first, then retain every secondary + // diagnostic that native testing exposes. Logs can be disabled, so the + // existing error stack must carry these values and their original stacks + // through both local finalization and bounded worker serialization. + appendDiagnostic := func(message, stack string) { + t.Logf("%s\n%s", message, stack) + terminalStack += "\n" + message + "\n" + stack + } + if visibleBody && !primaryBody { + appendDiagnostic(fmt.Sprintf("panic: %v", body), bodyStack) + } + for i, observed := range panics { + if i != primaryCleanup { + appendDiagnostic(fmt.Sprintf("cleanup panicked with %v", observed.value), observed.stack) + } + } + return terminal, terminalStack +} + // completeParallelSubtests releases and waits for parallel subtests owned by a // Datadog-managed clone. It mirrors testing.tRunner's scheduler accounting: -// release the parent slot before unblocking children, then reacquire it for -// sequential parents before running cleanup. -func completeParallelSubtests(t *testing.T, localTPrivateFields *commonPrivateFields, neutralizeNativeParallelRelease bool) { - completeParallelSubtestsWithState( - localTPrivateFields, - getTestState(t), - !isParallelTest(t, localTPrivateFields), - neutralizeNativeParallelRelease, - ) +// release the parent slot before unblocking children. The returned function +// runs after user cleanups, then reacquires the slot for sequential parents +// or neutralizes the already-released slot for native parallel parents. +func completeParallelSubtests(t *testing.T, localTPrivateFields *commonPrivateFields, neutralizeNativeParallelRelease bool) func(*testCleanupResult) { + hasSubtests := localTPrivateFields != nil && localTPrivateFields.sub != nil && len(*localTPrivateFields.sub) > 0 + reacquireParent := !isParallelTest(t, localTPrivateFields) + testState := getTestState(t) + completeParallelSubtestsWithState(localTPrivateFields, testState) + if !hasSubtests { + return nil + } + if reacquireParent && testState != nil { + return func(cleanup *testCleanupResult) { + // Goexit unwinds native tRunner before its slot reacquire, including + // cleanup SkipNow/FailNow. Preserve that return path for the caller. + if cleanup != nil && cleanup.goexit { + return + } + // Native tRunner propagates terminal cleanup panics before reacquiring. + if meta := getTestMetadata(t); meta != nil && !meta.hasAdditionalFeatureWrapper && + !meta.isDisabled && !meta.isQuarantined && + meta.cleanupResult != nil && meta.cleanupResult.panicData != nil { + return + } + testingTestStateWaitParallel(testState) + } + } + if neutralizeNativeParallelRelease && !reacquireParent && localTPrivateFields.isParallel != nil { + // Draining t.sub already released this slot. After user cleanup, keep + // native tRunner from releasing it a second time when t.sub is empty. + return func(_ *testCleanupResult) { *localTPrivateFields.isParallel = false } + } + return nil } // startParallelWaitClock measures the wait for parallel subtests; tests @@ -1386,8 +1658,6 @@ var startParallelWaitClock = startNativeClock func completeParallelSubtestsWithState( localTPrivateFields *commonPrivateFields, testState *testingTestState, - reacquireParent bool, - neutralizeNativeParallelRelease bool, ) time.Duration { if localTPrivateFields == nil || localTPrivateFields.sub == nil || len(*localTPrivateFields.sub) == 0 { return 0 @@ -1410,16 +1680,6 @@ func completeParallelSubtestsWithState( <-*pvSub.signal } } - if testState != nil && reacquireParent { - testingTestStateWaitParallel(testState) - } - if neutralizeNativeParallelRelease && !reacquireParent && localTPrivateFields.isParallel != nil { - // A process-retry child drains native tRunner subtests before writing - // JSON. After we clear t.sub, Go's native tRunner would otherwise take - // its len(t.sub)==0 && t.isParallel release path and release the same - // scheduler slot twice. - *localTPrivateFields.isParallel = false - } return elapsed() } @@ -1451,7 +1711,7 @@ func runAndApplyTestCleanup(t *testing.T, execMeta *testExecutionMetadata) { // runAndApplyTestCleanupWithDuration adds user cleanup time to the test body // duration without counting CI Visibility work performed between them. func runAndApplyTestCleanupWithDuration(t *testing.T, execMeta *testExecutionMetadata, bodyDuration time.Duration) time.Duration { - return runAndApplyTestCleanupWithDurationOptions(t, execMeta, bodyDuration, false) + return runAndApplyTestCleanupWithDurationOptions(t, execMeta, bodyDuration, false, false) } func runAndApplyTestCleanupWithDurationOptions( @@ -1459,15 +1719,50 @@ func runAndApplyTestCleanupWithDurationOptions( execMeta *testExecutionMetadata, bodyDuration time.Duration, neutralizeNativeParallelRelease bool, + terminalBody bool, ) time.Duration { cleanupStart := time.Now() if execMeta != nil && execMeta.cleanupResult != nil && !execMeta.cleanupResult.ran { - runTestCleanupWithOptions(t, execMeta.cleanupResult, neutralizeNativeParallelRelease) + execMeta.cleanupResult.observePanics = execMeta.exclusions.any() && execMeta.fuzzEvents == nil + if terminalBody && execMeta.fuzzEvents == nil && !execMeta.hasAdditionalFeatureWrapper && !execMeta.isDisabled && !execMeta.isQuarantined { + // With queued children, native doPanic fails before cleanup and does + // not admit them. Without children, cleanup precedes panic recovery + // and its Goexit may replace the body terminal. Fuzz owns its lifecycle. + if hasPendingParallelSubtests(t) { + t.Fail() + } + runTestCleanupCallbacks(t, execMeta.cleanupResult) + } else { + runTestCleanupWithOptions(t, execMeta.cleanupResult, neutralizeNativeParallelRelease) + } applyTestCleanupResult(t, execMeta, execMeta.cleanupResult) } return bodyDuration + time.Since(cleanupStart) } +func hasPendingParallelSubtests(t *testing.T) bool { + fields := getTestPrivateFields(t) + return fields != nil && fields.sub != nil && len(*fields.sub) > 0 +} + +// Native cleanup runs before panic recovery when no parallel children are queued. +// Its Goexit can replace the body panic; a bare Goexit still needs native failure. +func resolveTestBodyTerminalAfterCleanup(t *testing.T, meta *testExecutionMetadata, bodyTerminal any, unexpectedTermination bool) any { + if (bodyTerminal != nil || unexpectedTermination) && meta != nil && meta.fuzzEvents == nil && !meta.hasAdditionalFeatureWrapper && + !meta.isDisabled && !meta.isQuarantined && meta.cleanupResult != nil && + meta.cleanupResult.goexit && !(meta.cleanupResult.shapeCaptured && meta.cleanupResult.hadQueuedSubtests) && !hasPendingParallelSubtests(t) { + fields := getTestPrivateFields(t) + if t.Skipped() || fields != nil && fields.GetFinished() { + return nil + } + return unexpectedTestTerminationMessage + } + if unexpectedTermination { + return unexpectedTestTerminationMessage + } + return bodyTerminal +} + // applyTestCleanupResult applies cleanup panics to the test attempt so retry // orchestration can make the normal retry decision. A cleanup FailNow or // SkipNow has already changed the testing.T state before Goexit. A bare cleanup @@ -1488,6 +1783,9 @@ func propagateTestExecutionMetadataFlags(execMeta *testExecutionMetadata, origin } // Propagate the test execution metadata + if execMeta.exclusionResults == nil { + execMeta.exclusionResults = originalExecMeta.exclusionResults + } execMeta.isANewTest = execMeta.isANewTest || originalExecMeta.isANewTest execMeta.isAModifiedTest = execMeta.isAModifiedTest || originalExecMeta.isAModifiedTest execMeta.isEarlyFlakeDetectionEnabled = execMeta.isEarlyFlakeDetectionEnabled || originalExecMeta.isEarlyFlakeDetectionEnabled @@ -1498,6 +1796,11 @@ func propagateTestExecutionMetadataFlags(execMeta *testExecutionMetadata, origin if execMeta.quarantinedRaceProcess == nil { execMeta.quarantinedRaceProcess = originalExecMeta.quarantinedRaceProcess } + execMeta.assignment.quarantined = execMeta.assignment.quarantined || originalExecMeta.assignment.quarantined || originalExecMeta.isQuarantined + execMeta.assignment.disabled = execMeta.assignment.disabled || originalExecMeta.assignment.disabled || originalExecMeta.isDisabled + if !execMeta.hasExplicitAttemptToFix { + execMeta.assignment.attemptToFix = execMeta.assignment.attemptToFix || originalExecMeta.assignment.attemptToFix || originalExecMeta.isAttemptToFix + } execMeta.isQuarantined = execMeta.isQuarantined || originalExecMeta.isQuarantined execMeta.isDisabled = execMeta.isDisabled || originalExecMeta.isDisabled if !execMeta.suppressParentRetryMetadata { @@ -1509,6 +1812,7 @@ func propagateTestExecutionMetadataFlags(execMeta *testExecutionMetadata, origin // Preserve attempt-to-fix inheritance only when the child didn't explicitly override it. execMeta.isAttemptToFix = true } + applyExecutionExclusions(execMeta) } // isAnEfdExecution checks if the current test execution is an Early Flake Detection execution. diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation_orchestrion.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation_orchestrion.go index 9520bafbf..57a50fb5b 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation_orchestrion.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/instrumentation_orchestrion.go @@ -254,7 +254,19 @@ func instrumentTestingTFuncWithSourceOptions( } } + declaration := exclusionPolicySource(t, originalFunc, testifyData) + exclusions := exclusionsForChild(t, originalFunc, parentExecMeta) + if parentExecMeta != nil && parentExecMeta.exclusionResults != nil { + if exclusions.NoAutoRetry && parentExecMeta.exclusionResults.replay(t, exclusionResultDeclaration(t, declaration, parentExecMeta)) { + return + } + if exclusions.NoAutoRetry { + registerExcludedResultCleanup(t, parentExecMeta, exclusions, exclusionResultDeclaration(t, declaration, parentExecMeta)) + } + } + subtestInfo := &commonInfo{ + exclusions: exclusions, moduleName: subtestIdentity.ModuleName, suiteName: suiteName, testName: subtestIdentity.FullName, @@ -262,6 +274,10 @@ func instrumentTestingTFuncWithSourceOptions( sourceFunc: originalFunc, } + if testifyData != nil { + subtestInfo.sourceFunc = testifyData.methodFunc + } + runSubtest := func(currentT *testing.T) { localIdentity := subtestIdentity if currentT.Name() != subtestIdentity.FullName { @@ -293,11 +309,12 @@ func instrumentTestingTFuncWithSourceOptions( execMeta = createTestMetadata(currentT, nil) defer deleteTestMetadata(currentT) } + setExecutionExclusions(currentT, execMeta, declaration, parentExecMeta, exclusions) execMeta.identity = localIdentity if drainNativeLifecycle && parentExecMeta != nil { execMeta.fuzzEvents = parentExecMeta.fuzzEvents } - if drainNativeLifecycle && execMeta.cleanupResult == nil { + if (drainNativeLifecycle || exclusions.any()) && execMeta.cleanupResult == nil { execMeta.cleanupResult = &testCleanupResult{} } @@ -365,11 +382,14 @@ func instrumentTestingTFuncWithSourceOptions( return } - unexpectedTermination := r == nil && processRetryUnexpectedTestTermination(currentT, bodyReturned) - duration := runAndApplyTestCleanupWithDurationOptions(currentT, execMeta, bodyDuration, drainNativeLifecycle) - if unexpectedTermination { - r = unexpectedTestTerminationMessage + bodyTerminal := r + bodyStack := terminalStack + if r != nil && bodyStack == "" && execMeta.cleanupResult != nil && execMeta.exclusions.any() { + bodyStack = utils.GetStacktrace(1) } + unexpectedTermination := r == nil && processRetryUnexpectedTestTermination(currentT, bodyReturned) + duration := runAndApplyTestCleanupWithDurationOptions(currentT, execMeta, bodyDuration, drainNativeLifecycle || exclusions.any(), r != nil || unexpectedTermination) + r = resolveTestBodyTerminalAfterCleanup(currentT, execMeta, r, unexpectedTermination) nativeTerminal := r if execMeta.fuzzEvents != nil && execMeta.cleanupResult != nil && execMeta.cleanupResult.goexit { // Cleanup Goexit replaces the body panic. Fatal/SkipNow keep @@ -385,6 +405,10 @@ func instrumentTestingTFuncWithSourceOptions( if r != nil && terminalStack == "" { terminalStack = utils.GetStacktrace(1) } + if execMeta.fuzzEvents == nil { + r, terminalStack = resolveTestCleanupDiagnostics(currentT, execMeta, bodyTerminal, bodyStack, r, terminalStack) + nativeTerminal = r + } if execMeta.fuzzEvents != nil { collectAndWriteLogs(currentT, test, nil) if r != nil { @@ -457,6 +481,10 @@ func instrumentSetErrorInfo(tb testing.TB, errType string, errMessage string, sk return } + if errType == "Fail" { + markNativeProtectedFailure(tb) + } + // Get the CI Visibility span and check if we can set the error type, message and stack ciTestItem := getTestMetadata(tb) if ciTestItem != nil && ciTestItem.test != nil && ciTestItem.error.CompareAndSwap(0, 1) { @@ -755,11 +783,11 @@ func instrumentTestifySuiteRun(t *testing.T, suite any) { defer release() if isProcessRetryChild() { execMeta := getTestMetadata(t) - if execMeta == nil || execMeta.quarantinedRaceChild == nil { + if execMeta == nil { return } - // The isolated subtree still needs the real Testify method for source, - // CODEOWNERS, impacted-test, and ITR metadata; it creates no child span. + // Retry children need the real Testify declaration for exclusions as well + // as subtree source, CODEOWNERS, impacted-test and TIA metadata. } log.Debug("instrumentTestifySuiteRun: instrumenting testify suite run") @@ -778,7 +806,7 @@ func instrumentTestifySuiteRunScoped(t *testing.T, suite any) func() { defer release() if isProcessRetryChild() { metadata := getTestMetadata(t) - if metadata == nil || metadata.quarantinedRaceChild == nil { + if metadata == nil { return func() {} } } diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/quarantine_process.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/quarantine_process.go index 68a859352..21c247ee2 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/quarantine_process.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/quarantine_process.go @@ -78,6 +78,11 @@ type processRetrySubtreeConfig struct { } type processRetrySubtreeDirective struct { + Exclusions TestExclusions `json:"exclusions,omitempty"` + AssignedDisabled bool `json:"assigned_disabled,omitempty"` + AssignedQuarantined bool `json:"assigned_quarantined,omitempty"` + AssignedAttemptToFix bool `json:"assigned_attempt_to_fix,omitempty"` + TestName string `json:"test_name"` ModuleName string `json:"module_name,omitempty"` SuiteName string `json:"suite_name,omitempty"` @@ -112,6 +117,11 @@ type processRetryTestSource struct { // descendant of the selected root. The envelope fields in processRetryResult // use the same status contract for the root itself. type processRetrySubtreeResult struct { + Exclusions TestExclusions `json:"exclusions,omitempty"` + AssignedDisabled bool `json:"assigned_disabled,omitempty"` + AssignedQuarantined bool `json:"assigned_quarantined,omitempty"` + AssignedAttemptToFix bool `json:"assigned_attempt_to_fix,omitempty"` + TestName string `json:"test_name"` ModuleName string `json:"module_name"` SuiteName string `json:"suite_name"` @@ -260,6 +270,7 @@ func processRetryDirectiveFromMetadata(identity *testIdentity, meta *testExecuti return processRetrySubtreeDirective{TestName: identity.FullName, ModuleName: identity.ModuleName, SuiteName: identity.SuiteName} } return processRetrySubtreeDirective{ + Exclusions: meta.exclusions, AssignedDisabled: meta.assignment.disabled, AssignedQuarantined: meta.assignment.quarantined, AssignedAttemptToFix: meta.assignment.attemptToFix, TestName: identity.FullName, ModuleName: identity.ModuleName, SuiteName: identity.SuiteName, @@ -403,6 +414,34 @@ func (cfg *processRetrySubtreeConfig) resolvedRootDirective() processRetryResolv return resolved } +// resolveChildMetadata keeps cached management resolution continuous through +// ordinary worker carriers as well as reported managed descendants. +func (cfg *processRetrySubtreeConfig) resolveChildMetadata(parent *testExecutionMetadata, moduleName, suiteName, name string) processRetryResolvedDirective { + if name == cfg.SelectedRoot { + return cfg.resolvedRootDirective() + } + resolved := processRetryResolvedDirective{} + if parent != nil && parent.identity != nil { + resolved.directive = processRetryDirectiveFromMetadata(parent.identity, parent) + resolved.attemptOwner = parent.processRetryAttemptOwner + resolved.managed = parent.processRetryManagedDescendant + } else if processRetryNameWithinRoot(name, cfg.SelectedRoot) { + resolved = cfg.resolvedRootDirective() + } + return cfg.resolveChildDirective(resolved, moduleName, suiteName, name) +} + +func applyProcessRetryChildManagement(meta *testExecutionMetadata, resolved processRetryResolvedDirective) { + directive := resolved.directive + meta.assignment = testManagementAssignment{disabled: directive.AssignedDisabled, quarantined: directive.AssignedQuarantined, attemptToFix: directive.AssignedAttemptToFix} + meta.isDisabled, meta.isQuarantined, meta.isAttemptToFix = directive.Disabled, directive.Quarantined, directive.AttemptToFix + meta.hasExplicitDisabled, meta.hasExplicitQuarantined, meta.hasExplicitAttemptToFix = true, true, true + meta.isAModifiedTest = directive.Modified + meta.shouldOrchestrateAttemptToFix = resolved.attemptOwner == directive.TestName + meta.processRetryAttemptOwner = resolved.attemptOwner + meta.processRetryManagedDescendant = resolved.managed +} + // resolveChildDirective preserves the effective state already resolved for the // immediate parent. Callback source files can change at every t.Run boundary, // so re-resolving all path components with the child's suite would lose an @@ -419,6 +458,7 @@ func (cfg *processRetrySubtreeConfig) resolveChildDirective(parent processRetryR resolved.managed = resolved.managed || exact.Disabled || exact.Quarantined || exact.AttemptToFix resolved.directive.Disabled = resolved.directive.Disabled || exact.Disabled resolved.directive.Quarantined = resolved.directive.Quarantined || exact.Quarantined + resolved.directive.AssignedAttemptToFix = exact.AssignedAttemptToFix || exact.AttemptToFix if exact.AttemptToFix { if !parent.directive.AttemptToFix { resolved.attemptOwner = name @@ -447,6 +487,13 @@ func (cfg *processRetrySubtreeConfig) resolveSubtreeResults(results []processRet return nil, fmt.Errorf("%w: subtree parent result missing", errProcessRetryResultInvalid) } resolved[idx] = cfg.resolveChildDirective(parent, result.ModuleName, result.SuiteName, result.TestName) + applyProcessDirectiveExclusions(&resolved[idx].directive, result.Exclusions) + if result.Exclusions.PreventFailureSuppression || result.Exclusions.NoAutoRetry && !resolved[idx].directive.Disabled && !resolved[idx].directive.Quarantined { + resolved[idx].managed = false + } + if result.Exclusions.NoAutoRetry { + resolved[idx].attemptOwner = "" + } seen[result.TestName] = resolved[idx] } return resolved, nil @@ -855,8 +902,7 @@ func applyProcessRetryControlledSubtreeStatus(result *processRetryResult, contro result.SkipReason = "" } -func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), parent *testExecutionMetadata) { - state := parent.quarantinedRaceChild +func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), parent *testExecutionMetadata, state *quarantinedRaceChildState) { if state == nil || state.cfg == nil { original(t) return @@ -865,6 +911,7 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par callbackPC := reflect.ValueOf(original).Pointer() moduleName, suiteName := utils.GetModuleAndSuiteName(callbackPC) sourceFunc := runtime.FuncForPC(callbackPC) + policySource := sourceFunc // Testify's suite keeps its module, suite and method source; another // subtest keeps its parent's module, as a helper package can create it // (dd-trace-go #5401): see quarantinedSubtestIdentity. @@ -872,6 +919,7 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par execMeta := createTestMetadata(t, nil) execMeta.test = parent.test + execMeta.exclusionResults = parent.exclusionResults execMeta.processRetryOwner = parent execMeta.quarantinedRaceChild = state if !processRetryNameWithinRoot(name, state.cfg.SelectedRoot) { @@ -881,27 +929,21 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par } state.beginAggregateCoverage(name) - resolved := state.cfg.resolvedRootDirective() - if name != state.cfg.SelectedRoot { - resolved = state.cfg.resolveChildDirective(processRetryResolvedDirective{ - directive: processRetryDirectiveFromMetadata(parent.identity, parent), - attemptOwner: parent.processRetryAttemptOwner, - managed: parent.processRetryManagedDescendant, - }, moduleName, suiteName, name) + resolved := state.cfg.resolveChildMetadata(parent, moduleName, suiteName, name) + exclusions := exclusionsForChild(t, policySource, parent) + applyProcessDirectiveExclusions(&resolved.directive, exclusions) + if exclusions.PreventFailureSuppression || exclusions.NoAutoRetry && !resolved.directive.Disabled && !resolved.directive.Quarantined { + resolved.managed = false + } + if exclusions.NoAutoRetry { + resolved.attemptOwner = "" } directive, attemptOwner := resolved.directive, resolved.attemptOwner + setExecutionExclusions(t, execMeta, policySource, parent, exclusions) + applyProcessRetryChildManagement(execMeta, resolved) failureSnapshot := captureQuarantinedRaceManagedFailure(t, state.cfg, resolved.managed) maskedByManagedDescendant := resolved.managed - execMeta.isDisabled = directive.Disabled - execMeta.isQuarantined = directive.Quarantined - execMeta.isAttemptToFix = directive.AttemptToFix execMeta.isItrForcedRun = false - execMeta.hasExplicitDisabled = true - execMeta.hasExplicitQuarantined = true - execMeta.hasExplicitAttemptToFix = true - execMeta.shouldOrchestrateAttemptToFix = attemptOwner == name - execMeta.processRetryAttemptOwner = attemptOwner - execMeta.processRetryManagedDescendant = resolved.managed execMeta.identity = newTestIdentity(moduleName, suiteName, name) source := processRetrySourceFromFunc(sourceFunc) @@ -958,23 +1000,33 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par bodyReturned := false defer func() { panicData := recover() + bodyTerminal := panicData + bodyStack := "" + cleanup := &testCleanupResult{observePanics: exclusions.any()} + cleanup.captureShape(t) + execMeta.cleanupResult = cleanup unexpected := panicData == nil && processRetryUnexpectedTestTermination(t, bodyReturned) + var bodyPanicInfo *processRetryErrorInfo if panicData != nil || unexpected { if panicData == nil { panicData = unexpectedTestTerminationMessage } - t.Fail() - execMeta.processRetryPanic.CompareAndSwap(nil, &processRetryErrorInfo{ - Type: "panic", - Message: truncateProcessRetryErrorMessage(toString(panicData)), - Stack: truncateProcessRetryErrorStack(utils.GetStacktrace(1)), - }) + bodyStack = utils.GetStacktrace(1) + // Native cleanup precedes failing a direct body panic unless + // queued children require the recovered-body doPanic path. + if !cleanup.observePanics || cleanup.hadQueuedSubtests { + t.Fail() + } + bodyPanicInfo = &processRetryErrorInfo{ + Type: "panic", Message: truncateProcessRetryErrorMessage(toString(panicData)), + Stack: truncateProcessRetryErrorStack(bodyStack), + } + execMeta.processRetryPanic.CompareAndSwap(nil, bodyPanicInfo) } // Drain queued parallel descendants before user cleanups so the selected // root includes their failures and coverage. Native testing excludes this // scheduler wait from the parent's duration, but cleanup remains active time. - cleanup := &testCleanupResult{} if !activeStart.IsZero() { activeDuration += time.Since(activeStart) activeStart = time.Time{} @@ -986,21 +1038,44 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par // belongs to this unmanaged branch. state.markUnmaskedFailure(moduleName, suiteName, name) } - completeParallelSubtests(t, getTestPrivateFields(t), true) + fieldsBeforeCleanup := getTestPrivateFields(t) + if cleanup.observePanics && (bodyTerminal != nil || unexpected) && cleanup.hadQueuedSubtests && + !directive.Disabled && !directive.Quarantined && fieldsBeforeCleanup != nil && fieldsBeforeCleanup.sub != nil { + // Native doPanic never opens the barrier for these queued children. + // This worker consumes the panic to commit its subtree result, so + // detach the still-paused queue before native tRunner can admit it + // on normal return. Keep the barrier closed: no child has acquired + // its parallel slot or resumed its body. The captured original + // shape still selects native queued-child cleanup precedence. + *fieldsBeforeCleanup.sub = nil + } + finishParallel := completeParallelSubtests(t, fieldsBeforeCleanup, true) activeStart = time.Now() - runTestCleanupCallbacks(t, cleanup) + runTestCleanupCallbacksWithParallelRelease(t, cleanup, finishParallel) activeDuration += time.Since(activeStart) activeStart = time.Time{} - if cleanup.panicData != nil { + terminal, terminalStack := panicData, bodyStack + if cleanup.observePanics { + terminal = resolveTestBodyTerminalAfterCleanup(t, execMeta, bodyTerminal, unexpected) + terminal, terminalStack = resolveTestCleanupDiagnostics(t, execMeta, bodyTerminal, bodyStack, terminal, terminalStack) + } + if terminal == nil && cleanup.panicData != nil { + terminal, terminalStack = cleanup.panicData, cleanup.panicStacktrace + } + var terminalInfo *processRetryErrorInfo + if terminal != nil { t.Fail() - execMeta.processRetryPanic.CompareAndSwap(nil, &processRetryErrorInfo{ - Type: "panic", - Message: truncateProcessRetryErrorMessage(toString(cleanup.panicData)), - Stack: truncateProcessRetryErrorStack(cleanup.panicStacktrace), - }) - // The parent replays this recorded panic. Re-panicking here would let - // testing terminate the child before it can write the subtree result. + terminalInfo = &processRetryErrorInfo{Type: "panic", Message: truncateProcessRetryErrorMessage(toString(terminal)), Stack: truncateProcessRetryErrorStack(terminalStack)} + } + if bodyPanicInfo != nil { + // Update only the body terminal owned by this invocation. A + // separately recorded descendant panic must not be overwritten. + execMeta.processRetryPanic.CompareAndSwap(bodyPanicInfo, terminalInfo) + } else if terminalInfo != nil { + execMeta.processRetryPanic.CompareAndSwap(nil, terminalInfo) } + // The parent replays this recorded panic. Re-panicking here would let + // testing terminate the child before it can write the subtree result. state.finishAggregateCoverage(name) files := collector.Finish() coverageValid = state.coverage.finish(coverageInterval) && coverageValid @@ -1011,6 +1086,7 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par finish := start.Add(activeDuration) fields := getTestPrivateFields(t) result := processRetrySubtreeResult{ + Exclusions: exclusions, AssignedDisabled: directive.AssignedDisabled, AssignedQuarantined: directive.AssignedQuarantined, AssignedAttemptToFix: directive.AssignedAttemptToFix, TestName: name, ModuleName: execMeta.identity.ModuleName, SuiteName: execMeta.identity.SuiteName, @@ -1062,6 +1138,12 @@ func runQuarantinedRaceChildSubtest(t *testing.T, original func(*testing.T), par default: result.Status = processRetryStatusPass } + if result.Failed && exclusions.PreventFailureSuppression { + markProtectedFailureAncestors(t) + } + if result.Failed && !directive.Disabled && !directive.Quarantined && exclusions.any() { + state.markUnsuppressedFailure(name) + } if state.append(result) { // A managed sibling may concurrently restore its own propagated failure. // Reapply this independently owned failure in either completion order. @@ -1347,6 +1429,9 @@ func runQuarantinedRaceProcessIsolation( applyAdditionalFeatureMetadataToExecution(execMeta, featureMeta) propagateTestExecutionMetadataFlags(execMeta, parentExecMeta) execMeta.quarantinedRaceProcess = processCtx + if execMeta.exclusionResults == nil { + execMeta.exclusionResults = &excludedTestResults{} + } execMeta.hasAdditionalFeatureWrapper = true cfg, err := buildProcessRetrySubtreeConfig(processCtx, testInfo, execMeta, parentExecMeta) @@ -1390,7 +1475,7 @@ func runQuarantinedRaceProcessIsolation( invocations := make([]quarantinedRaceInvocation, 0, rootTotal) for idx := 0; idx < rootTotal; idx++ { - attempt := runQuarantinedRaceInvocation(t, processCtx, lease, cfg, cfg.SelectedRoot, idx, parallelBridge) + attempt := runQuarantinedRaceInvocation(t, processCtx, lease, cfg, cfg.SelectedRoot, idx, parallelBridge, execMeta.exclusionResults) if processRetryInfrastructureFailure(attempt) { failQuarantinedRaceIsolation(t, testInfo, execMeta, attempt) return @@ -1406,7 +1491,7 @@ func runQuarantinedRaceProcessIsolation( } stop, failure := continueQuarantinedRaceDescendantFamilies( cfg, attempt, &invocations, func(continuationCfg *processRetrySubtreeConfig, runRoot string, idx int) processRetryAttemptResult { - return runQuarantinedRaceInvocation(t, processCtx, lease, continuationCfg, runRoot, idx, parallelBridge) + return runQuarantinedRaceInvocation(t, processCtx, lease, continuationCfg, runRoot, idx, parallelBridge, execMeta.exclusionResults) }, ) if failure != nil { @@ -1418,11 +1503,35 @@ func runQuarantinedRaceProcessIsolation( } } + if cfg.OwnsAttemptToFix { + protectedFamilyFailure := false + lastRoot := -1 + for idx, invocation := range invocations { + protectedFamilyFailure = protectedFamilyFailure || invocation.attempt.Result.ProtectedDescendantFailure + if invocation.cfg.SelectedRoot == cfg.SelectedRoot { + lastRoot = idx + } + } + if protectedFamilyFailure && lastRoot >= 0 { + // The last root event reports the strict ATF family's final result, + // even if its own native callback recovered an earlier failure. + invocations[lastRoot].attempt.Result.ProtectedDescendantFailure = true + } + } var deferred *quarantinedRaceReplayState if cfg.AncestorAttemptToFix { deferred = quarantinedRaceReplayStateFor(parentExecMeta) } replayQuarantinedRaceResults(testInfo, processCtx, invocations, true, deferred) + if slices.ContainsFunc(invocations, func(invocation quarantinedRaceInvocation) bool { + return invocation.attempt.Result.ProtectedDescendantFailure + }) { + // An executed protected descendant is outside the ancestor's quarantine. + // This path owns only quarantine/ATF, whose failure aggregation is strict. + markProtectedFailureAncestors(t) + t.Fail() + return + } // A valid quarantined failure, panic, or race remains visible in CI // Visibility but is intentionally masked from the package result. An // infrastructure error returns above through Fail and is never quarantined. @@ -1541,16 +1650,18 @@ func runQuarantinedRaceInvocation( runRoot string, attemptIndex int, parallelBridge func() error, + exclusionResults *excludedTestResults, ) processRetryAttemptResult { deadline, deadlineOK := t.Deadline() baseline := captureProcessRetryLaunchBaselineFromTemplate(processCtx.launchTemplate) - // Initial roots use their exact selector. Descendant-owned ATF continuations + // Initial roots keep their native descendant filter. Descendant-owned ATF continuations // re-enter their enclosing root so parallel scheduling siblings exist, while // cfg.SelectedRoot still limits recording to the managed descendant subtree. - baseline.argsSnapshot.runSelector = processRetryExactRunPattern(runRoot) + baseline.argsSnapshot.runSelector = processRetrySubtreeRunPattern(baseline.argsSnapshot.runSelector, runRoot) attempt := runProcessRetryAttemptWithBaselineAndShutdown( context.Background(), processRetryChildConfig{ + ExclusionResults: exclusionResults.snapshotForRoot(cfg.SelectedRoot), TestName: cfg.SelectedRoot, Attempt: attemptIndex + 1, RetryReason: processRetrySubtreeReason, @@ -1564,7 +1675,9 @@ func runQuarantinedRaceInvocation( lease.shutdown, parallelBridge, ) - return finalizeQuarantinedRaceInvocation(cfg, attempt) + attempt = finalizeQuarantinedRaceInvocation(cfg, attempt) + exclusionResults.merge(attempt.Result.ExclusionResults) + return attempt } func finalizeQuarantinedRaceInvocation(cfg *processRetrySubtreeConfig, attempt processRetryAttemptResult) processRetryAttemptResult { @@ -1637,6 +1750,7 @@ func processRetryCommonInfoFromSubtreeResult(result processRetrySubtreeResult) * func (cfg *processRetrySubtreeConfig) forSelectedRoot(result processRetrySubtreeResult) (*processRetrySubtreeConfig, error) { directive := processRetrySubtreeDirective{ + Exclusions: result.Exclusions, AssignedDisabled: result.AssignedDisabled, AssignedQuarantined: result.AssignedQuarantined, AssignedAttemptToFix: result.AssignedAttemptToFix, TestName: result.TestName, ModuleName: result.ModuleName, SuiteName: result.SuiteName, Disabled: result.Disabled, Quarantined: result.Quarantined, AttemptToFix: result.AttemptToFix, Modified: result.Modified, } @@ -1861,6 +1975,8 @@ func replayQuarantinedRaceEvent( attemptIndex = 0 } execMeta := &testExecutionMetadata{ + exclusions: result.Exclusions, + assignment: testManagementAssignment{disabled: result.AssignedDisabled, quarantined: result.AssignedQuarantined, attemptToFix: result.AssignedAttemptToFix}, identity: identity, isQuarantined: result.Quarantined, isDisabled: result.Disabled, @@ -1906,7 +2022,11 @@ func replayQuarantinedRaceEvent( func processRetrySubtreeResultFromEnvelope(result processRetryResult, cfg *processRetrySubtreeConfig) processRetrySubtreeResult { resolved := cfg.resolvedRootDirective() directive, owner := resolved.directive, resolved.attemptOwner + if result.ProtectedDescendantFailure { + directive.Disabled, directive.Quarantined = false, false + } return processRetrySubtreeResult{ + Exclusions: directive.Exclusions, AssignedDisabled: directive.AssignedDisabled, AssignedQuarantined: directive.AssignedQuarantined, AssignedAttemptToFix: directive.AssignedAttemptToFix, TestName: result.TestName, ModuleName: result.ModuleName, SuiteName: result.SuiteName, @@ -2014,3 +2134,20 @@ func quarantinedSubtestIdentity(t *testing.T, state *quarantinedRaceChildState, } return subtestModuleName(fallbackModule, parent), suiteName, sourceFunc } + +func (s *quarantinedRaceChildState) markUnsuppressedFailure(name string) { + if s == nil || s.cfg == nil || !processRetryNameWithinRoot(name, s.cfg.SelectedRoot) { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.unmasked == nil { + s.unmasked = map[string]struct{}{} + } + for candidate := name; ; candidate = candidate[:strings.LastIndexByte(candidate, '/')] { + s.unmasked[candidate] = struct{}{} + if candidate == s.cfg.SelectedRoot { + break + } + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/reflections.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/reflections.go index 8352b7f1a..c5d6305ed 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/reflections.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/reflections.go @@ -1214,3 +1214,26 @@ func getBenchmarkPrivateFieldsReflect(b *testing.B) *benchmarkPrivateFields { return benchFields } + +// getFailureHookPrivateFields accepts native Fail's *testing.common receiver +// without treating the common allocation as a whole testing.T. +func getFailureHookPrivateFields(tb testing.TB) *commonPrivateFields { + if t, ok := tb.(*testing.T); ok { + return getTestPrivateFields(t) + } + value := reflect.ValueOf(tb) + if !value.IsValid() || value.Kind() != reflect.Pointer || value.IsNil() { + return nil + } + typ := value.Type().Elem() + if typ.PkgPath() != "testing" || typ.Name() != "common" { + return nil + } + layout := getTestingInternalsLayout() + if layout == nil || layout.disabled || !allAvailable(layout.common.parent.unsafeField) { + return nil + } + return &commonPrivateFields{ + parent: (*unsafe.Pointer)(fieldRawPtr(value.UnsafePointer(), layout.common.parent.unsafeField)), + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_attempt_integration.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_attempt_integration.go index d5da6c76c..6ae6d6e77 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_attempt_integration.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_attempt_integration.go @@ -146,6 +146,9 @@ func executeFreshRetryAttemptIteration(execOpts *executionOptions) bool { } if execOpts.originalExecutionMetadata != nil { execOpts.originalExecutionMetadata.test = execMeta.test + // Copy the current event binding without importing a historical + // failure bit from the original attempt into this event. + execOpts.originalExecutionMetadata.protectedFailureEvent.Store(execMeta.protectedFailureEvent.Load()) } if execMeta.test == nil && execMeta.identity != nil { log.Debug("execMeta.test nil for %s", execMeta.identity.FullName) diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process.go index 14aa14207..88d25472d 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process.go @@ -103,6 +103,8 @@ func processRetryTimeoutFromEnv() (time.Duration, bool) { } type processRetryChildConfig struct { + ExclusionPolicy *processRetrySubtreeConfig + ExclusionResults map[string]excludedTestResult ResultPath string TestName string Attempt int @@ -152,36 +154,39 @@ const ( ) type processRetryResult struct { - Version int `json:"version"` - TestName string `json:"test_name"` - ModuleName string `json:"module_name,omitempty"` - SuiteName string `json:"suite_name,omitempty"` - Attempt int `json:"attempt"` - RetryReason string `json:"retry_reason"` - MRunEpoch uint64 `json:"m_run_epoch,omitempty"` - InvocationOrdinal uint64 `json:"invocation_ordinal,omitempty"` - Status processRetryStatus `json:"status"` - StartUnixNano int64 `json:"start_unix_nano"` - FinishUnixNano int64 `json:"finish_unix_nano"` - DurationNanos int64 `json:"duration_nanos"` - Failed bool `json:"failed"` - Skipped bool `json:"skipped"` - Panic bool `json:"panic"` - RaceDetected bool `json:"race_detected,omitempty"` - RootParallel bool `json:"root_parallel,omitempty"` - ErrorType string `json:"error_type,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - ErrorStack string `json:"error_stack,omitempty"` - SkipReason string `json:"skip_reason,omitempty"` - ResultError string `json:"result_error,omitempty"` - OutputTail string `json:"output_tail,omitempty"` - OutputTruncated bool `json:"output_truncated,omitempty"` - Source *processRetryTestSource `json:"source,omitempty"` - Coverage []coverage.ProcessTestCoverageFile `json:"coverage,omitempty"` - Subtests []processRetrySubtreeResult `json:"subtests,omitempty"` - SkippedByITR bool `json:"skipped_by_itr,omitempty"` - ITRForcedRun bool `json:"itr_forced_run,omitempty"` - Modified bool `json:"modified,omitempty"` + ExclusionResults map[string]excludedTestResult `json:"exclusion_results,omitempty"` + ExcludedTests []processRetrySubtreeResult `json:"excluded_tests,omitempty"` + ProtectedDescendantFailure bool `json:"protected_descendant_failure,omitempty"` + Version int `json:"version"` + TestName string `json:"test_name"` + ModuleName string `json:"module_name,omitempty"` + SuiteName string `json:"suite_name,omitempty"` + Attempt int `json:"attempt"` + RetryReason string `json:"retry_reason"` + MRunEpoch uint64 `json:"m_run_epoch,omitempty"` + InvocationOrdinal uint64 `json:"invocation_ordinal,omitempty"` + Status processRetryStatus `json:"status"` + StartUnixNano int64 `json:"start_unix_nano"` + FinishUnixNano int64 `json:"finish_unix_nano"` + DurationNanos int64 `json:"duration_nanos"` + Failed bool `json:"failed"` + Skipped bool `json:"skipped"` + Panic bool `json:"panic"` + RaceDetected bool `json:"race_detected,omitempty"` + RootParallel bool `json:"root_parallel,omitempty"` + ErrorType string `json:"error_type,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ErrorStack string `json:"error_stack,omitempty"` + SkipReason string `json:"skip_reason,omitempty"` + ResultError string `json:"result_error,omitempty"` + OutputTail string `json:"output_tail,omitempty"` + OutputTruncated bool `json:"output_truncated,omitempty"` + Source *processRetryTestSource `json:"source,omitempty"` + Coverage []coverage.ProcessTestCoverageFile `json:"coverage,omitempty"` + Subtests []processRetrySubtreeResult `json:"subtests,omitempty"` + SkippedByITR bool `json:"skipped_by_itr,omitempty"` + ITRForcedRun bool `json:"itr_forced_run,omitempty"` + Modified bool `json:"modified,omitempty"` } type processRetryErrorInfo struct { @@ -278,6 +283,8 @@ func enrichProcessRetryChildConfig(cfg processRetryChildConfig, wire processRetr cfg.ParentDeadlineUnixNano = wire.ParentDeadlineUnixNano cfg.ParentDeadlineOK = wire.ParentDeadlineOK cfg.ObservedGOMAXPROCS = wire.ObservedGOMAXPROCS + cfg.ExclusionResults = wire.ExclusionResults + cfg.ExclusionPolicy = wire.ExclusionPolicy cfg.Subtree = wire.Subtree return cfg } @@ -875,6 +882,10 @@ type processRetryEffectiveStatus struct { } type processRetryMetadataSnapshot struct { + protectedDescendantFailure bool + exclusionResults *excludedTestResults + exclusions TestExclusions + assignment testManagementAssignment identity *testIdentity isANewTest bool isAModifiedTest bool @@ -2378,6 +2389,10 @@ func snapshotProcessRetryExecutionMetadata(execMeta *testExecutionMetadata) *pro return nil } return &processRetryMetadataSnapshot{ + protectedDescendantFailure: execMeta.protectedDescendantFailure.Load(), + exclusionResults: execMeta.exclusionResults, + exclusions: execMeta.exclusions, + assignment: execMeta.assignment, identity: execMeta.identity, isANewTest: execMeta.isANewTest, isAModifiedTest: execMeta.isAModifiedTest, @@ -2401,6 +2416,13 @@ func applyProcessRetryMetadataSnapshot(execMeta *testExecutionMetadata, snapshot if execMeta == nil || snapshot == nil || snapshot.identity == nil || snapshot.identity.FullName == "" { return false } + if snapshot.protectedDescendantFailure { + markProtectedFailure(execMeta) + } else { + execMeta.protectedDescendantFailure.Store(false) + } + execMeta.exclusionResults = snapshot.exclusionResults + execMeta.exclusions, execMeta.assignment = snapshot.exclusions, snapshot.assignment execMeta.identity = snapshot.identity execMeta.isANewTest = snapshot.isANewTest execMeta.isAModifiedTest = snapshot.isAModifiedTest @@ -2506,6 +2528,17 @@ func finishProcessRetryTestEvent( } } execMeta.test = test + if len(attempt.Result.ExcludedTests) > 0 || len(attempt.Result.ExclusionResults) > 0 { + replayNewExcludedProcessTests(execMeta, attempt) + } + if attempt.Result.ProtectedDescendantFailure { + markProtectedFailure(execMeta) + } else { + execMeta.protectedDescendantFailure.Store(false) + } + if attempt.Result.ProtectedDescendantFailure { + execMeta.isDisabled, execMeta.isQuarantined = false, false + } coverage.SubmitProcessTestCoverage(session.SessionID(), suite.SuiteID(), test.TestID(), attempt.Result.Coverage) cancelExecution := setTestTagsFromExecutionMetadataNoClose(test, execMeta) // A validated process skip is already the outcome of the execution. In @@ -2513,7 +2546,9 @@ func finishProcessRetryTestEvent( // into the pre-execution metadata cancellation used by in-process tests. disabledSkip := execMeta.isDisabled && !execMeta.isAttemptToFix && attempt.Result.Status == processRetryStatusSkip cancelExecution = cancelExecution && !disabledSkip - test.SetTag(constants.TestRetryExecutionMode, "process") + if !execMeta.exclusions.NoAutoRetry { + test.SetTag(constants.TestRetryExecutionMode, "process") + } if execMeta.isItrForcedRun { test.SetTag(constants.TestForcedToRun, "true") telemetry.ITRForcedRun(telemetry.TestEventType) @@ -2979,7 +3014,7 @@ func instrumentProcessRetryChild(m *testing.M, cfg processRetryChildConfig) (boo defer processRetryChildInstrumentations.mu.Unlock() if state := processRetryChildInstrumentations.states[m]; state != nil { state.multipleMRun.Store(true) - if state.cfg != cfg { + if !reflect.DeepEqual(state.cfg, cfg) { log.Debug("civisibility: conflicting process retry child instrumentation") } _ = state.control.Send(processRetryControlAbort, "testmain_multiple_m_run") @@ -3260,6 +3295,16 @@ func wrapProcessRetryChildTest(original func(*testing.T), cfg processRetryChildC return "testing_t_reflection_drift" } execMeta := createTestMetadata(attempt.test, nil) + execMeta.exclusionResults = &excludedTestResults{items: cfg.ExclusionResults} + if cfg.ExclusionPolicy != nil { + execMeta.exclusionPolicy = newQuarantinedRaceChildState(cfg.ExclusionPolicy) + } + if cfg.Subtree == nil && cfg.ExclusionPolicy != nil { + root := cfg.ExclusionPolicy.Root + execMeta.isDisabled, execMeta.isQuarantined, execMeta.isAttemptToFix = root.Disabled, root.Quarantined, root.AttemptToFix + execMeta.assignment = testManagementAssignment{disabled: root.AssignedDisabled || root.Disabled, quarantined: root.AssignedQuarantined || root.Quarantined, attemptToFix: root.AssignedAttemptToFix || root.AttemptToFix} + } + setExecutionExclusions(attempt.test, execMeta, runtime.FuncForPC(reflect.ValueOf(original).Pointer()), nil, exclusionsForFunc(runtime.FuncForPC(reflect.ValueOf(original).Pointer()))) attempt.metadata = execMeta moduleName, suiteName := "", "" if cfg.Subtree != nil { @@ -3307,6 +3352,8 @@ func wrapProcessRetryChildTest(original func(*testing.T), cfg processRetryChildC if cfg.Subtree != nil && cfg.Subtree.SelectedRoot == topLevelName { childBody = func(t *testing.T) { rootDirective := cfg.Subtree.Root + observation.execMeta.exclusions = rootDirective.Exclusions + observation.execMeta.assignment = testManagementAssignment{disabled: rootDirective.AssignedDisabled, quarantined: rootDirective.AssignedQuarantined, attemptToFix: rootDirective.AssignedAttemptToFix} observation.execMeta.isDisabled = rootDirective.Disabled observation.execMeta.isQuarantined = rootDirective.Quarantined observation.execMeta.isAttemptToFix = rootDirective.AttemptToFix @@ -3410,14 +3457,33 @@ func (o *processRetryChildObservation) writeControlledTerminalResult(control *pr if o.test == nil || o.execMeta == nil { return } + allowRecovery := o.execMeta.exclusionResults.freezeForTerminal() result := o.buildResult(status) - written = o.writer.Write(result) && result.Status == status + written = o.writer.Write(result) && result.Status == status && allowRecovery }) if written && control != nil { _ = control.childControlledTerminal(status) } } +// applyProcessRetryExclusionState carries already recorded exclusion history +// through every executed terminal envelope. It snapshots existing state only: +// early panic publication must not wait for, drain, or complete live tests. +func applyProcessRetryExclusionState(result *processRetryResult, meta *testExecutionMetadata) { + if result == nil || meta == nil { + return + } + result.ExclusionResults = meta.exclusionResults.snapshot() + result.ProtectedDescendantFailure = meta.protectedDescendantFailure.Load() + if meta.exclusionPolicy != nil { + for _, row := range meta.exclusionPolicy.snapshot() { + if row.Exclusions.NoAutoRetry { + result.ExcludedTests = append(result.ExcludedTests, row) + } + } + } +} + func (o *processRetryChildObservation) buildResult(status processRetryStatus) processRetryResult { if o.subtree != nil { if err := o.subtree.waitParallelBridge(); err != nil { @@ -3434,7 +3500,7 @@ func (o *processRetryChildObservation) buildResult(status processRetryStatus) pr // testing reports a race in a subtest for the subtest only; the child runs // one test, so any race that its detector found is that test's. raceDetected := o.result.raceDetected || o.subtree == nil && retryAttemptRaceErrors() > 0 - failed := o.result.failed || raceDetected || panicData != nil + failed := o.result.failed || raceDetected || panicData != nil || o.execMeta.protectedDescendantFailure.Load() rootParallel := false if o.group != nil { o.group.mu.Lock() @@ -3459,6 +3525,7 @@ func (o *processRetryChildObservation) buildResult(status processRetryStatus) pr RaceDetected: raceDetected, RootParallel: rootParallel, } + applyProcessRetryExclusionState(&result, o.execMeta) if result.Failed { if panicInfo := o.execMeta.processRetryPanic.Load(); result.Panic && panicInfo != nil { result.ErrorType = panicInfo.Type @@ -3538,13 +3605,26 @@ func recordProcessRetryChildErrorInfo(tb testing.TB, errType, errMessage string, // T's metadata and name; Fail itself then propagates the failure. func markProcessRetryChildFailed(tb testing.TB) { current := getTestMetadata(tb) - if execMeta := processRetryChildOwnerMetadata(current); execMeta != nil && execMeta.quarantinedRaceChild != nil && - !processRetryChildFailureIsPropagated() && (current == nil || !current.processRetryManagedDescendant) { + execMeta := processRetryChildOwnerMetadata(current) + var state *quarantinedRaceChildState + if execMeta != nil { + state = execMeta.quarantinedRaceChild + if state == nil { + state = execMeta.exclusionPolicy + } + } + propagated := processRetryChildFailureIsPropagated() + if execMeta != nil && current != nil && current.exclusions.PreventFailureSuppression && !propagated { + // An original native Fail is irreversible, even while this callback + // is still running and another child publishes the terminal result. + markProtectedFailure(execMeta) + } + if state != nil && !propagated && (current == nil || !current.processRetryManagedDescendant) { moduleName, suiteName := "", "" if current != nil && current.identity != nil { moduleName, suiteName = current.identity.ModuleName, current.identity.SuiteName } - execMeta.quarantinedRaceChild.markUnmaskedFailure(moduleName, suiteName, tb.Name()) + state.markUnmaskedFailure(moduleName, suiteName, tb.Name()) } t, ok := tb.(*testing.T) if !ok { @@ -3596,19 +3676,98 @@ func instrumentProcessRetryChildSubtest(original func(*testing.T)) func(*testing original(t) return } + source := runtime.FuncForPC(reflect.ValueOf(original).Pointer()) + exclusions := exclusionsForChild(t, source, parent) + declaration := exclusionResultDeclaration(t, source, parent) + if exclusions.NoAutoRetry && owner.exclusionResults.replay(t, declaration) { + return + } + if exclusions.NoAutoRetry { + if !owner.exclusionResults.begin() { + return + } + defer owner.exclusionResults.end() + defer owner.exclusionResults.record(t, exclusions.PreventFailureSuppression, declaration) + } if owner.quarantinedRaceChild != nil { - runQuarantinedRaceChildSubtest(t, original, parent) + runQuarantinedRaceChildSubtest(t, original, parent, owner.quarantinedRaceChild) + return + } + if exclusions.NoAutoRetry && owner.exclusionPolicy != nil { + runQuarantinedRaceChildSubtest(t, original, parent, owner.exclusionPolicy) return } execMeta := createTestMetadata(t, nil) execMeta.test = owner.test + execMeta.exclusionResults = owner.exclusionResults + sourceFunc := runtime.FuncForPC(reflect.ValueOf(original).Pointer()) + setExecutionExclusions(t, execMeta, sourceFunc, parent, exclusionsForChild(t, sourceFunc, parent)) + moduleName, suiteName := utils.GetModuleAndSuiteName(sourceFunc.Entry()) + if state := owner.exclusionPolicy; state != nil && state.cfg != nil { + moduleName, suiteName, _ = quarantinedSubtestIdentity(t, state, parent, moduleName, suiteName, sourceFunc) + execMeta.identity = newTestIdentity(moduleName, suiteName, t.Name()) + resolved := state.cfg.resolveChildMetadata(parent, moduleName, suiteName, t.Name()) + applyProcessDirectiveExclusions(&resolved.directive, execMeta.exclusions) + if execMeta.exclusions.NoAutoRetry { + resolved.attemptOwner = "" + } + applyProcessRetryChildManagement(execMeta, resolved) + } else { + if data := getTestifyTest(t); data != nil { + moduleName, suiteName = data.moduleAndSuite(moduleName, suiteName) + } else { + moduleName = subtestModuleName(moduleName, parent) + } + execMeta.identity = newTestIdentity(moduleName, suiteName, t.Name()) + propagateTestExecutionMetadataFlags(execMeta, parent) + } + registerProtectedChildCleanup(t, execMeta) execMeta.processRetryOwner = owner t.Cleanup(func() { deleteTestMetadata(t) }) bodyReturned := false defer func() { panicData := recover() + if execMeta.exclusions.PreventFailureSuppression { + // Resolve native cleanup before committing protection or a panic + // result: cleanup SkipNow/FailNow can cancel a body panic. + bodyTerminal := panicData + bodyStack := "" + if bodyTerminal != nil { + bodyStack = utils.GetStacktrace(1) + } + unexpected := bodyTerminal == nil && processRetryUnexpectedTestTermination(t, bodyReturned) + execMeta.cleanupResult = &testCleanupResult{} + execMeta.cleanupResult.captureShape(t) + runAndApplyTestCleanupWithDurationOptions(t, execMeta, 0, true, bodyTerminal != nil || unexpected) + terminal := resolveTestBodyTerminalAfterCleanup(t, execMeta, bodyTerminal, unexpected) + terminalStack := bodyStack + if terminal != nil && terminalStack == "" { + terminalStack = utils.GetStacktrace(1) + } + terminal, terminalStack = resolveTestCleanupDiagnostics(t, execMeta, bodyTerminal, bodyStack, terminal, terminalStack) + if terminal != nil || t.Failed() { + markProtectedFailure(owner) + markProtectedFailureAncestors(t) + } + if terminal == nil { + return + } + // This invocation has not published a body-panic record yet; + // retain an independently owned descendant's record if present. + owner.processRetryPanic.CompareAndSwap(nil, &processRetryErrorInfo{ + Type: "panic", + Message: truncateProcessRetryErrorMessage(toString(terminal)), + Stack: truncateProcessRetryErrorStack(terminalStack), + }) + if root, ok := owner.test.(*processRetryNoopTest); ok { + root.writePanicResult(owner.processRetryPanic.Load(), owner) + } + // Only the surviving terminal is rethrown. Native tRunner then + // keeps queued children behind their unopened parent barrier. + panic(terminal) + } unexpectedTermination := false if panicData == nil && processRetryUnexpectedTestTermination(t, bodyReturned) { panicData = unexpectedTestTerminationMessage @@ -3623,7 +3782,7 @@ func instrumentProcessRetryChildSubtest(original func(*testing.T)) func(*testing Stack: truncateProcessRetryErrorStack(utils.GetStacktrace(1)), }) if root, ok := owner.test.(*processRetryNoopTest); ok { - root.writePanicResult(owner.processRetryPanic.Load()) + root.writePanicResult(owner.processRetryPanic.Load(), owner) } if unexpectedTermination { return @@ -3764,11 +3923,12 @@ func marshalProcessRetryResult(result processRetryResult) ([]byte, error) { return nil, err } subtree := len(result.Subtests) > 0 || len(result.Coverage) > 0 || result.Source != nil - limit := processRetryWireMaxBytes(subtree) + extended := subtree || len(result.ExclusionResults) > 0 || len(result.ExcludedTests) > 0 + limit := processRetryWireMaxBytes(extended) if len(payload) <= limit { return payload, nil } - if !subtree { + if !extended { return nil, errors.New("process_retry_result_too_large") } if payload, ok, err := marshalBoundedProcessRetrySubtreeEnvelope(result, limit); err != nil { @@ -3776,6 +3936,12 @@ func marshalProcessRetryResult(result processRetryResult) ([]byte, error) { } else if ok { return payload, nil } + // Keep required excluded execution records intact, including mixed + // envelopes: an oversized core fails instead of dropping executions. + // The existing no-exclusions subtree error protocol remains below. + if !subtree || len(result.ExcludedTests) > 0 { + return nil, errors.New("process_retry_result_too_large") + } // Preserve a valid protocol result when even the mandatory event structure // cannot fit. The parent then fails closed with the explicit reason instead // of misclassifying an absent file as a child-process failure. @@ -3809,7 +3975,7 @@ func marshalBoundedProcessRetrySubtreeEnvelope(result processRetryResult, maxByt continue } - perEventBudget := (maxBytes - len(corePayload)) / (len(result.Subtests) + 1) + perEventBudget := (maxBytes - len(corePayload)) / (len(result.Subtests) + len(result.ExcludedTests) + 1) fieldLimit := perEventBudget / metadataFields for { bounded := cloneProcessRetrySubtreeEnvelope(result, preserveSourceAndCoverage) @@ -3832,6 +3998,7 @@ func marshalBoundedProcessRetrySubtreeEnvelope(result processRetryResult, maxByt func cloneProcessRetrySubtreeEnvelope(result processRetryResult, preserveSourceAndCoverage bool) processRetryResult { result.Subtests = slices.Clone(result.Subtests) + result.ExcludedTests = slices.Clone(result.ExcludedTests) if preserveSourceAndCoverage { return result } @@ -3841,6 +4008,10 @@ func cloneProcessRetrySubtreeEnvelope(result processRetryResult, preserveSourceA result.Subtests[idx].Source = nil result.Subtests[idx].Coverage = nil } + for idx := range result.ExcludedTests { + result.ExcludedTests[idx].Source = nil + result.ExcludedTests[idx].Coverage = nil + } return result } @@ -3857,11 +4028,11 @@ func truncateProcessRetryAggregateMetadata(result *processRetryResult, maxBytes result.ErrorMessage, result.ErrorStack, result.SkipReason, result.OutputTail, result.OutputTruncated = truncate( result.ErrorMessage, result.ErrorStack, result.SkipReason, result.OutputTail, result.OutputTruncated, result.SkippedByITR, ) - for idx := range result.Subtests { - subtest := &result.Subtests[idx] - subtest.ErrorMessage, subtest.ErrorStack, subtest.SkipReason, subtest.OutputTail, subtest.OutputTruncated = truncate( - subtest.ErrorMessage, subtest.ErrorStack, subtest.SkipReason, subtest.OutputTail, subtest.OutputTruncated, subtest.SkippedByITR, - ) + for _, rows := range [][]processRetrySubtreeResult{result.Subtests, result.ExcludedTests} { + for idx := range rows { + row := &rows[idx] + row.ErrorMessage, row.ErrorStack, row.SkipReason, row.OutputTail, row.OutputTruncated = truncate(row.ErrorMessage, row.ErrorStack, row.SkipReason, row.OutputTail, row.OutputTruncated, row.SkippedByITR) + } } } @@ -3875,7 +4046,7 @@ func readProcessRetryResult(resultPath string, expected processRetryChildConfig) } defer file.Close() - limit := processRetryWireMaxBytes(expected.Subtree != nil) + limit := processRetryWireMaxBytes(expected.Subtree != nil || expected.ExclusionPolicy != nil || len(expected.ExclusionResults) > 0) payload, err := io.ReadAll(io.LimitReader(file, int64(limit)+1)) if err != nil { return processRetryResult{}, false, fmt.Errorf("%w: result file unreadable", errProcessRetryResultInvalid) @@ -3928,6 +4099,27 @@ func validateProcessRetryResult(result processRetryResult, expected processRetry !processRetryJSONStringFits(result.OutputTail, processRetrySubtreeOutputMaxBytes) { return fmt.Errorf("%w: metadata field too large", errProcessRetryResultInvalid) } + // Excluded execution records include every generated case as well as its + // declaration. Their encoded frame stays bounded by the existing wire budget; + // the unrelated subtree-node count must not reject otherwise valid events. + if len(result.ExclusionResults) > 4096 { + return errProcessRetryResultInvalid + } + for name := range result.ExclusionResults { + if !excludedResultWithinRoot(name, expected.TestName) { + return errProcessRetryResultInvalid + } + } + seenExcluded := map[string]bool{} + for _, row := range result.ExcludedTests { + if !row.Exclusions.NoAutoRetry || !strings.HasPrefix(row.TestName, expected.TestName+"/") || seenExcluded[row.TestName] || row.AttemptToFix || row.AttemptToFixOwn { + return errProcessRetryResultInvalid + } + seenExcluded[row.TestName] = true + if err := validateProcessRetrySubtreeResult(row, processRetryResolvedDirective{directive: processRetrySubtreeDirective{Disabled: row.Disabled, Quarantined: row.Quarantined}}); err != nil { + return err + } + } if err := validateProcessRetrySubtreeResultEnvelope(result, expected); err != nil { return err } @@ -4003,7 +4195,7 @@ func newProcessRetryNoopTest(root *testing.T, cfg processRetryChildConfig, ident } } -func (t *processRetryNoopTest) writePanicResult(info *processRetryErrorInfo) { +func (t *processRetryNoopTest) writePanicResult(info *processRetryErrorInfo, owner *testExecutionMetadata) { if t == nil || t.writer == nil || info == nil { return } @@ -4031,7 +4223,13 @@ func (t *processRetryNoopTest) writePanicResult(info *processRetryErrorInfo) { result.ModuleName = t.identity.ModuleName result.SuiteName = t.identity.SuiteName } - if t.writer.Write(result) && t.control != nil { + var exclusionResults *excludedTestResults + if owner != nil { + exclusionResults = owner.exclusionResults + } + allowRecovery := exclusionResults.freezeForTerminal() + applyProcessRetryExclusionState(&result, owner) + if t.writer.Write(result) && t.control != nil && allowRecovery { _ = t.control.childControlledTerminal(processRetryStatusControlledPanicReady) } } diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_control.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_control.go index d8dcbdf68..aa1270dcc 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_control.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_control.go @@ -50,19 +50,21 @@ var ( ) type processRetryControlConfig struct { - Version int `json:"version"` - Transport string `json:"transport"` - TestName string `json:"test_name"` - Attempt int `json:"attempt"` - RetryReason string `json:"retry_reason"` - MRunEpoch uint64 `json:"m_run_epoch,omitempty"` - InvocationOrdinal uint64 `json:"invocation_ordinal,omitempty"` - ReadEndpoint uint64 `json:"read_endpoint"` - WriteEndpoint uint64 `json:"write_endpoint"` - ParentDeadlineUnixNano int64 `json:"parent_deadline_unix_nano"` - ParentDeadlineOK bool `json:"parent_deadline_ok"` - ObservedGOMAXPROCS int `json:"observed_gomaxprocs"` - Subtree *processRetrySubtreeConfig `json:"subtree,omitempty"` + ExclusionPolicy *processRetrySubtreeConfig `json:"exclusion_policy,omitempty"` + ExclusionResults map[string]excludedTestResult `json:"exclusion_results,omitempty"` + Version int `json:"version"` + Transport string `json:"transport"` + TestName string `json:"test_name"` + Attempt int `json:"attempt"` + RetryReason string `json:"retry_reason"` + MRunEpoch uint64 `json:"m_run_epoch,omitempty"` + InvocationOrdinal uint64 `json:"invocation_ordinal,omitempty"` + ReadEndpoint uint64 `json:"read_endpoint"` + WriteEndpoint uint64 `json:"write_endpoint"` + ParentDeadlineUnixNano int64 `json:"parent_deadline_unix_nano"` + ParentDeadlineOK bool `json:"parent_deadline_ok"` + ObservedGOMAXPROCS int `json:"observed_gomaxprocs"` + Subtree *processRetrySubtreeConfig `json:"subtree,omitempty"` } type processRetryControlFrame struct { @@ -136,6 +138,8 @@ func newParentProcessRetryControl(cmd *exec.Cmd, cfg processRetryChildConfig) (* transport.config.ParentDeadlineUnixNano = cfg.ParentDeadlineUnixNano transport.config.ParentDeadlineOK = cfg.ParentDeadlineOK transport.config.ObservedGOMAXPROCS = cfg.ObservedGOMAXPROCS + transport.config.ExclusionResults = cfg.ExclusionResults + transport.config.ExclusionPolicy = cfg.ExclusionPolicy transport.config.Subtree = cfg.Subtree if transport.config.ObservedGOMAXPROCS < 1 { transport.config.ObservedGOMAXPROCS = processRetryCurrentCPU() @@ -578,6 +582,8 @@ func writeProcessRetryControlConfig(path string, cfg processRetryControlConfig) RetryReason: cfg.RetryReason, MRunEpoch: cfg.MRunEpoch, InvocationOrdinal: cfg.InvocationOrdinal, + ExclusionPolicy: cfg.ExclusionPolicy, + ExclusionResults: cfg.ExclusionResults, Subtree: cfg.Subtree, }); err != nil { return err @@ -586,7 +592,7 @@ func writeProcessRetryControlConfig(path string, cfg processRetryControlConfig) if err != nil { return err } - if len(payload) > processRetryWireMaxBytes(cfg.Subtree != nil) { + if len(payload) > processRetryWireMaxBytes(cfg.Subtree != nil || cfg.ExclusionPolicy != nil || len(cfg.ExclusionResults) > 0) { return errProcessRetryControlTooLarge } return os.WriteFile(path, payload, 0o600) @@ -601,7 +607,9 @@ func readProcessRetryControlConfig(path string, expected processRetryChildConfig return processRetryControlConfig{}, err } defer file.Close() - limit := processRetryWireMaxBytes(expected.RetryReason == processRetrySubtreeReason || expected.RetryReason == constants.AttemptToFixRetryReason) + // The payload announces whether it carries exclusion state. Read at most + // the existing extended budget, then retain the ordinary limit below. + limit := processRetryWireMaxBytes(true) payload, err := io.ReadAll(io.LimitReader(file, int64(limit)+1)) if err != nil { return processRetryControlConfig{}, err @@ -618,6 +626,10 @@ func readProcessRetryControlConfig(path string, expected processRetryChildConfig if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { return processRetryControlConfig{}, errProcessRetryControlInvalid } + extended := cfg.Subtree != nil || cfg.ExclusionPolicy != nil || len(cfg.ExclusionResults) > 0 || expected.RetryReason == processRetrySubtreeReason || expected.RetryReason == constants.AttemptToFixRetryReason + if len(payload) > processRetryWireMaxBytes(extended) { + return processRetryControlConfig{}, errProcessRetryControlTooLarge + } if err := validateProcessRetryControlConfig(cfg, expected); err != nil { return processRetryControlConfig{}, err } @@ -645,9 +657,23 @@ func validateProcessRetryControlConfig(cfg processRetryControlConfig, expected p cfg.Subtree != nil && cfg.RetryReason != processRetrySubtreeReason && cfg.RetryReason != constants.AttemptToFixRetryReason { return errProcessRetryControlInvalid } + if err := validateProcessRetryExclusionPolicy(cfg.ExclusionPolicy, cfg.TestName); err != nil { + return err + } + if len(cfg.ExclusionResults) > 4096 { + return errors.New("too many excluded retry descendants") + } + for name := range cfg.ExclusionResults { + if !excludedResultWithinRoot(name, cfg.TestName) { + return errors.New("invalid excluded retry descendant") + } + } if err := validateProcessRetrySubtreeConfig(cfg.Subtree, cfg.TestName); err != nil { return errors.Join(errProcessRetryControlInvalid, err) } + if expected.ExclusionResults != nil && !reflect.DeepEqual(cfg.ExclusionResults, expected.ExclusionResults) { + return errProcessRetryControlInvalid + } if expected.Subtree != nil && !reflect.DeepEqual(cfg.Subtree, expected.Subtree) { return errProcessRetryControlInvalid } diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_deferred.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_deferred.go index 77cb80ea9..ef68a98f5 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_deferred.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_deferred.go @@ -76,40 +76,41 @@ const ( ) type deferredProcessRetryGroup struct { - id uint64 - identity testIdentity - testInfo commonInfo - metadata processRetryMetadataSnapshot - launchBaseline *processRetryLaunchBaseline - lease *processRetryGroupLease - parentDeadline time.Time - parentDeadlineOK bool - mRunEpoch uint64 - phaseID uint64 - invocationOrdinal uint64 - executionIndex int - retryCount int64 - initialDuration time.Duration - reservation *flakyRetryBudgetReservation - latest retryAttemptObservation - outcomes retryOutcomeAccumulator - module integrations.TestModule - suite integrations.TestSuite - terminalFailure bool - terminalFailureReason string - truncated bool - tailEvent *deferredProcessRetryEvent - panicPresent bool - panicMessage string - panicStack string - parallelEFD bool - rootParallel bool - nativeMaxParallel int - efdFaultySessionGuard earlyFlakeDetectionFaultySession - raceProcess *quarantinedRaceProcessContext - raceSubtree *processRetrySubtreeConfig - raceDescendantFailed bool - raceReplay *quarantinedRaceReplayState + id uint64 + identity testIdentity + testInfo commonInfo + metadata processRetryMetadataSnapshot + launchBaseline *processRetryLaunchBaseline + lease *processRetryGroupLease + parentDeadline time.Time + parentDeadlineOK bool + mRunEpoch uint64 + phaseID uint64 + invocationOrdinal uint64 + executionIndex int + retryCount int64 + initialDuration time.Duration + reservation *flakyRetryBudgetReservation + latest retryAttemptObservation + outcomes retryOutcomeAccumulator + module integrations.TestModule + suite integrations.TestSuite + terminalFailure bool + terminalFailureReason string + truncated bool + tailEvent *deferredProcessRetryEvent + panicPresent bool + panicMessage string + panicStack string + parallelEFD bool + rootParallel bool + nativeMaxParallel int + efdFaultySessionGuard earlyFlakeDetectionFaultySession + raceProcess *quarantinedRaceProcessContext + raceSubtree *processRetrySubtreeConfig + raceDescendantFailed bool + protectedDescendantFailed bool + raceReplay *quarantinedRaceReplayState } type deferredProcessRetryEvent struct { @@ -1036,6 +1037,8 @@ func runDeferredProcessRetryAttempt(ctx context.Context, group *deferredProcessR attempt := runProcessRetryAttemptWithBaselineAndShutdown( ctx, processRetryChildConfig{ + ExclusionPolicy: processRetryExclusionPolicy(prepared.execMeta), + ExclusionResults: prepared.execMeta.exclusionResults.snapshotForRoot(group.identity.FullName), TestName: group.identity.FullName, Attempt: prepared.index, RetryReason: prepared.retryReason, @@ -1052,6 +1055,9 @@ func runDeferredProcessRetryAttempt(ctx context.Context, group *deferredProcessR if subtree != nil { attempt = finalizeQuarantinedRaceInvocation(subtree, attempt) } + if !processRetryInfrastructureFailure(attempt) { + prepared.execMeta.exclusionResults.merge(attempt.Result.ExclusionResults) + } if subtree == nil || processRetryInfrastructureFailure(attempt) || subtree.AttemptToFixRetries <= 0 || quarantinedRaceFailfastStopsContinuation(attempt) { return attempt } @@ -1059,10 +1065,11 @@ func runDeferredProcessRetryAttempt(ctx context.Context, group *deferredProcessR _, failure := continueQuarantinedRaceDescendantFamilies( subtree, attempt, &invocations, func(continuationCfg *processRetrySubtreeConfig, runRoot string, idx int) processRetryAttemptResult { baseline := captureProcessRetryLaunchBaselineFromTemplate(group.launchBaseline) - baseline.argsSnapshot.runSelector = processRetryExactRunPattern(runRoot) - return finalizeQuarantinedRaceInvocation(continuationCfg, runProcessRetryAttemptWithBaselineAndShutdown( + baseline.argsSnapshot.runSelector = processRetrySubtreeRunPattern(baseline.argsSnapshot.runSelector, runRoot) + next := finalizeQuarantinedRaceInvocation(continuationCfg, runProcessRetryAttemptWithBaselineAndShutdown( ctx, processRetryChildConfig{ + ExclusionResults: prepared.execMeta.exclusionResults.snapshotForRoot(continuationCfg.SelectedRoot), TestName: continuationCfg.SelectedRoot, Attempt: idx + 1, RetryReason: processRetrySubtreeReason, @@ -1076,6 +1083,10 @@ func runDeferredProcessRetryAttempt(ctx context.Context, group *deferredProcessR group.shutdown(), nil, )) + if !processRetryInfrastructureFailure(next) { + prepared.execMeta.exclusionResults.merge(next.Result.ExclusionResults) + } + return next }, ) attempt.quarantinedRaceInvocations = invocations @@ -1093,6 +1104,7 @@ func runDeferredProcessRetryAttempt(ctx context.Context, group *deferredProcessR func (g *deferredProcessRetryGroup) applyCompletedAttempt(completed deferredProcessRetryCompletedAttempt, decideContinuation, continuationAdmitted bool) bool { attempt := completed.result + g.protectedDescendantFailed = g.protectedDescendantFailed || attempt.Result.ProtectedDescendantFailure execMeta := completed.prepared.execMeta // Queue admission commits this continuation to the process backend. By the // time a late setup failure is observable, native M.Run and the original @@ -1109,6 +1121,7 @@ func (g *deferredProcessRetryGroup) applyCompletedAttempt(completed deferredProc }} } for _, invocation := range raceInvocations { + g.protectedDescendantFailed = g.protectedDescendantFailed || invocation.attempt.Result.ProtectedDescendantFailure g.observeQuarantinedRaceSubtree(invocation.attempt.Result.Subtests) } } @@ -1259,18 +1272,29 @@ func consumeDeferredFlakyRetryReservation(group *deferredProcessRetryGroup) { group.reservation = nil } +func (g *deferredProcessRetryGroup) retainedExcludedFailures() (bool, bool) { + if g == nil || g.identity.FullName == "" { + return false, false + } + return g.metadata.exclusionResults.failedDescendants(g.identity.FullName) +} + func (g *deferredProcessRetryGroup) packageFailed() bool { - if g == nil || g.metadata.isDisabled || g.metadata.isQuarantined { + if g == nil { + return false + } + retainedFailure, retainedProtected := g.retainedExcludedFailures() + if (g.metadata.isDisabled || g.metadata.isQuarantined) && !(retainedProtected || g.protectedDescendantFailed || g.metadata.protectedDescendantFailure) { return false } - if g.terminalFailure { + if retainedFailure || g.terminalFailure { return true } if g.raceDescendantFailed { return true } if g.metadata.isAttemptToFix { - return g.outcomes.anyFailed() + return g.outcomes.anyFailed() || g.protectedDescendantFailed || g.metadata.protectedDescendantFailure } if usesEfdRetrySemanticsSnapshot(&g.metadata) { return !g.outcomes.anyPassed() && g.outcomes.anyFailed() @@ -1328,17 +1352,20 @@ func (g *deferredProcessRetryGroup) closeTailEvent(final bool) { return } if final { + retainedFailure, retainedProtected := g.retainedExcludedFailures() + familyFailed := retainedFailure || g.terminalFailure || g.raceDescendantFailed || + g.metadata.isAttemptToFix && (g.protectedDescendantFailed || g.metadata.protectedDescendantFailure) finalStatus := calculateFinalStatus( - g.outcomes.anyPassed() && !g.terminalFailure, - g.outcomes.anyFailed() || g.terminalFailure, + g.outcomes.anyPassed() && !familyFailed, + g.outcomes.anyFailed() || familyFailed, g.latest.skipped, - g.metadata.isQuarantined, - g.metadata.isDisabled, + g.metadata.isQuarantined && !(retainedProtected || g.protectedDescendantFailed || g.metadata.protectedDescendantFailure), + g.metadata.isDisabled && !(retainedProtected || g.protectedDescendantFailed || g.metadata.protectedDescendantFailure), g.metadata.isAttemptToFix, ) g.tailEvent.event.SetTag(constants.TestFinalStatus, finalStatus) if g.metadata.isAttemptToFix { - attemptToFixPassed := g.outcomes.allAttemptsPassed() && !g.truncated && !g.terminalFailure + attemptToFixPassed := g.outcomes.allAttemptsPassed() && !g.truncated && !familyFailed g.tailEvent.event.SetTag(constants.TestAttemptToFixPassed, strconv.FormatBool(attemptToFixPassed)) } if g.tailEvent.failed && g.outcomes.allRetriesFailed() { diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection.go new file mode 100644 index 000000000..88210fd8f --- /dev/null +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection.go @@ -0,0 +1,42 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package gotesting + +import "strings" + +// processRetrySubtreeRunPattern narrows discovery to the managed root without +// admitting descendants that the caller's native -test.run filter excluded. +// Reuse testing's matcher grammar, including slash-separated alternatives, +// rather than interpreting the complete selector as one regular expression. +func processRetrySubtreeRunPattern(original, runRoot string) string { + selection := newTestingSelectionFromPatterns(original, "", "") + if selection == nil { + // testing rejects invalid selectors before execution. Do not broaden one + // if a direct caller nevertheless supplies it here. + return "^$" + } + filters := []testingFilter{selection.filter} + if alternatives, ok := selection.filter.(testingAlternationMatch); ok { + filters = alternatives + } + segments := strings.Split(runRoot, "/") + exact := processRetryExactRunPattern(runRoot) + var patterns []string + for _, filter := range filters { + parts := filter.(testingSimpleMatch) + if ok, _ := parts.matches(segments, selection.matchString); !ok { + continue + } + if len(parts) <= len(segments) { + return exact + } + patterns = append(patterns, exact+"/"+strings.Join(parts[len(segments):], "/")) + } + if len(patterns) == 0 { + return "^$" + } + return strings.Join(patterns, "|") +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection_test.go new file mode 100644 index 000000000..6d60b8ae2 --- /dev/null +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/retry_process_selection_test.go @@ -0,0 +1,53 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package gotesting + +import "testing" + +func TestProcessRetrySubtreeRunPatternPreservesSelection(t *testing.T) { + for _, tc := range []struct{ name, original, root, want string }{ + {"all", "", "TestRoot/managed", "^TestRoot$/^managed$"}, + {"short", "Root", "TestRoot/managed", "^TestRoot$/^managed$"}, + {"descendant", "^TestRoot$/^managed$/^Selected$", "TestRoot/managed", "^TestRoot$/^managed$/^Selected$"}, + {"partial-prefix", "Root/man/Selected", "TestRoot/managed", "^TestRoot$/^managed$/Selected"}, + {"alternatives", "TestOther/man/Other|Root/man/Selected|Root/man/Second", "TestRoot/managed", "^TestRoot$/^managed$/Selected|^TestRoot$/^managed$/Second"}, + {"short-alternative", "Root/man/Selected|Root", "TestRoot/managed", "^TestRoot$/^managed$"}, + {"grouped", "^(TestRoot|TestOther)$/^(managed|other)$/^(Selected|Second)$", "TestRoot/managed", "^TestRoot$/^managed$/^(Selected|Second)$"}, + {"class-delimiters", "Root/[m/|]+/Selected", "TestRoot/m", "^TestRoot$/^m$/Selected"}, + {"grouped-slash", "Root/man/(Selected/row|Second)", "TestRoot/managed", "^TestRoot$/^managed$/(Selected/row|Second)"}, + {"empty-segment", "Root//Selected", "TestRoot/managed", "^TestRoot$/^managed$/Selected"}, + {"native-name-rewrite", "Root/managed row/Selected", "TestRoot/managed_row", "^TestRoot$/^managed_row$/Selected"}, + {"escaped-pipe", `Root/man\|aged/Selected`, "TestRoot/man|aged", `^TestRoot$/^man\|aged$/Selected`}, + {"literal-root", ".*/.*/Selected", "TestRoot/a.b[0]", `^TestRoot$/^a\.b\[0\]$/Selected`}, + {"unselected-root", "^TestOther$/managed/Selected", "TestRoot/managed", "^$"}, + {"invalid", "[", "TestRoot/managed", "^$"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := processRetrySubtreeRunPattern(tc.original, tc.root); got != tc.want { + t.Fatalf("selector=%q want%q", got, tc.want) + } + }) + } +} + +func TestProcessRetrySubtreeRunPatternKeepsDescendantAdmission(t *testing.T) { + for _, original := range []string{"", "Root", "Root//Selected", "Root/man/Selected|Root/man/Second", "Root/man/(Selected|Second)$", "Other/man/Rejected|Root/man/Selected/row", "Root/man/Selected|Root", "Root/man/Selected/row|Root/man/Second/other"} { + t.Run(original, func(t *testing.T) { + native := newTestingSelectionFromPatterns(original, "", "") + narrowed := newTestingSelectionFromPatterns(processRetrySubtreeRunPattern(original, "TestRoot/managed"), "", "") + for _, name := range []string{"TestRoot/managed", "TestRoot/managed/Selected", "TestRoot/managed/Selected/row", "TestRoot/managed/Selected/other", "TestRoot/managed/Second", "TestRoot/managed/Second/row", "TestRoot/managed/Second/other", "TestRoot/managed/Rejected"} { + if got, want := narrowed.selects(name), native.selects(name); got != want { + t.Fatalf("descendant %q admitted=%t native=%t", name, got, want) + } + } + for _, name := range []string{"TestOther", "TestOther/managed/Selected", "TestRoot/other", "TestRoot/other/Selected"} { + if narrowed.selects(name) { + t.Fatalf("narrowed worker admitted unrelated %q", name) + } + } + }) + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/test_execution_finalizer.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/test_execution_finalizer.go index 49dd0eb23..7cf03d6c2 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/test_execution_finalizer.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/test_execution_finalizer.go @@ -28,9 +28,13 @@ func finalizeInstrumentedTestExecution( terminalStack string, markContainersOnTerminal bool, ) { + retainedFailure := applyRetainedExcludedFailure(t, execMeta) if isAnEfdExecution(execMeta) && duration >= bucketBoundFiveM { test.SetTag(constants.TestEarlyFlakeDetectionRetryAborted, "slow") } + if execMeta.protectedDescendantFailure.Load() { + execMeta.isQuarantined, execMeta.isDisabled = false, false + } logTestFinished(t, duration, terminal) collectAndWriteLogs(t, test, attemptOutput) @@ -40,7 +44,10 @@ func finalizeInstrumentedTestExecution( execMeta.panicStacktrace = terminalStack finalExec := isFinalExecution(true, false, execMeta, duration) if finalExec { - finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed, true, false, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) + if execMeta.exclusions.PreventFailureSuppression { + markProtectedFailureAncestors(t) + } + finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed && !retainedFailure, true, false, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) test.SetTag(constants.TestFinalStatus, finalStatusValue(finalStatus)) } if execMeta.isARetry && finalExec { @@ -66,7 +73,10 @@ func finalizeInstrumentedTestExecution( switch { case failed: if finalExec { - finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed, true, false, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) + if execMeta.exclusions.PreventFailureSuppression { + markProtectedFailureAncestors(t) + } + finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed && !retainedFailure, true, false, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) test.SetTag(constants.TestFinalStatus, finalStatusValue(finalStatus)) } if execMeta.isARetry && finalExec { @@ -87,7 +97,7 @@ func finalizeInstrumentedTestExecution( deferOrCloseInstrumentedTestEvent(execMeta, test, integrations.ResultStatusFail, "") case skipped: if finalExec { - finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed, execMeta.anyExecutionFailed, true, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) + finalStatus := calculateFinalStatus(execMeta.anyExecutionPassed && !retainedFailure, execMeta.anyExecutionFailed, true, execMeta.isQuarantined, execMeta.isDisabled, execMeta.isAttemptToFix) test.SetTag(constants.TestFinalStatus, finalStatusValue(finalStatus)) } if execMeta.isAttemptToFix && execMeta.isARetry && finalExec { diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify.go index c1f1f56e9..c99e41598 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify.go @@ -23,7 +23,10 @@ type TestifyTest struct { suiteName string moduleName string methodFunc *runtime.Func - suite reflect.Type + // declarationFunc identifies the method body across promoted wrappers. + // methodFunc retains the existing reflected reporting/source identity. + declarationFunc *runtime.Func + suite reflect.Type } // moduleAndSuite returns the suite's module and suite names, or the given @@ -79,6 +82,17 @@ func getTestifyTest(t *testing.T) *TestifyTest { return bindTestifyTest(t, found) } +// isTestifyMethodBoundary distinguishes a suite method from native descendants +// that retain that method's reporting identity. A nested suite method has its +// own binding even when its parent is already attributed to another method. +func isTestifyMethodBoundary(t *testing.T, data *TestifyTest) bool { + if data == nil { + return false + } + fields := getTestPrivateFields(t) + return fields == nil || fields.parent == nil || getBoundTestifyTest(*fields.parent) != data +} + // testifySuitesRegistered lets ordinary tests skip the parent walk. var testifySuitesRegistered atomic.Bool @@ -196,6 +210,15 @@ func testifySuiteMethods(methodFinder reflect.Type, suiteName, moduleName string // get the file for the method methodFunc := runtime.FuncForPC(uintptr(method.Func.UnsafePointer())) + // Keep reflection's existing source and suite identity. A promoted + // wrapper gets only an exact declaration-policy alias, never another + // receiver's execution metadata or mutable span tags. + declaration := testifyMethodDeclaration(methodFinder, method) + if declaration != nil && methodFunc != nil && declaration.Name() != methodFunc.Name() { + if exclusions := exclusionsForFunc(declaration); exclusions.any() { + RegisterTestExclusionsByName(methodFunc.Name(), exclusions) + } + } var methodFile string if methodFunc != nil { methodFile, _ = methodFunc.FileLine(methodFunc.Entry()) @@ -203,11 +226,12 @@ func testifySuiteMethods(methodFinder reflect.Type, suiteName, moduleName string // append the TestifyTest struct to the tests slice tests = append(tests, TestifyTest{ - methodName: methodName, - suiteName: fmt.Sprintf("%s/%s", filepath.Base(methodFile), suiteName), - moduleName: moduleName, - methodFunc: methodFunc, - suite: methodFinder, + methodName: methodName, + suiteName: fmt.Sprintf("%s/%s", filepath.Base(methodFile), suiteName), + moduleName: moduleName, + methodFunc: methodFunc, + declarationFunc: declaration, + suite: methodFinder, }) } @@ -281,3 +305,73 @@ func setTestifyTestsByParentT(ptr unsafe.Pointer, tests []TestifyTest) { testifyTestsByParentT[ptr] = tests testifySuitesRegistered.Store(true) } + +// Reflection returns a compiler wrapper for a promoted method. Resolve its +// shortest anonymous-field path to the declaring receiver, while an explicit +// override (including a value receiver's pointer wrapper) remains independent. +func testifyMethodDeclaration(receiver reflect.Type, method reflect.Method) *runtime.Func { + direct := func(typ reflect.Type, m reflect.Method) (*runtime.Func, bool) { + if !m.Func.IsValid() { + return nil, false + } + fn := runtime.FuncForPC(m.Func.Pointer()) + if fn == nil { + return nil, false + } + file, _ := fn.FileLine(fn.Entry()) + if file != "" { + return fn, true + } + if typ.Kind() == reflect.Pointer { + if valueMethod, ok := typ.Elem().MethodByName(m.Name); ok && valueMethod.Func.IsValid() { + if valueFn := runtime.FuncForPC(valueMethod.Func.Pointer()); valueFn != nil { + valueFile, _ := valueFn.FileLine(valueFn.Entry()) + if valueFile != "" { + return fn, true + } + } + } + } + return fn, false + } + original, declared := direct(receiver, method) + if declared { + return original + } + pending := []reflect.Type{receiver} + seen := map[reflect.Type]bool{} + for len(pending) > 0 { + current := pending + pending = nil + for _, typ := range current { + if typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if typ.Kind() != reflect.Struct || seen[typ] { + continue + } + seen[typ] = true + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if !field.Anonymous { + continue + } + embedded := field.Type + if embedded.Kind() != reflect.Pointer && embedded.Kind() != reflect.Interface { + embedded = reflect.PointerTo(embedded) + } + if candidate, ok := embedded.MethodByName(method.Name); ok { + // An interface dispatch shadows deeper concrete declarations. + if embedded.Kind() == reflect.Interface { + return original + } + if declaration, declared := direct(embedded, candidate); declared { + return declaration + } + } + pending = append(pending, embedded) + } + } + } + return original +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify_lookup_test.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify_lookup_test.go index 4b97c4945..f0a3ce504 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify_lookup_test.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testify_lookup_test.go @@ -2,6 +2,7 @@ package gotesting import ( "reflect" + "runtime" "testing" "unsafe" ) @@ -128,3 +129,29 @@ func TestTestifyMethodBindingOutlivesSuiteRun(t *testing.T) { }) end() } + +// An interface method shadows a deeper concrete method even though reflection +// exposes no Func for the interface declaration. +type exclusionShadowInterface interface{ TestCritical() } +type exclusionShadowBase struct{} + +func (*exclusionShadowBase) TestCritical() {} + +type exclusionShadowMiddle struct{ exclusionShadowBase } +type exclusionShadowSuite struct { + exclusionShadowInterface + exclusionShadowMiddle +} + +func TestTestifyInterfaceShadowDeclaration(t *testing.T) { + typ := reflect.TypeOf(new(exclusionShadowSuite)) + method, ok := typ.MethodByName("TestCritical") + if !ok { + t.Fatal("missing interface wrapper") + } + got := testifyMethodDeclaration(typ, method) + want := runtime.FuncForPC(method.Func.Pointer()) + if got == nil || got.Name() != want.Name() { + t.Fatalf("resolved shadowed concrete declaration: %v want %v", got, want) + } +} diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go index 2f022a02e..21199ac6a 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testing.go @@ -130,6 +130,7 @@ type ( // commonInfo holds common information about tests and benchmarks. commonInfo struct { + exclusions TestExclusions moduleName string suiteName string testName string @@ -741,6 +742,11 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo, wrapperOpts additional execMeta = createTestMetadata(t, nil) defer deleteTestMetadata(t) } + setExecutionExclusions(t, execMeta, originalFunc, nil, exclusionsForFunc(originalFunc)) + applyExecutionExclusions(execMeta) + if execMeta.exclusions.any() && execMeta.cleanupResult == nil { + execMeta.cleanupResult = &testCleanupResult{} + } execMeta.identity = testInfo.identity execMeta.quarantinedRaceProcess = wrapperOpts.quarantinedRaceProcess @@ -848,16 +854,23 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo, wrapperOpts additional return } - unexpectedTermination := r == nil && processRetryUnexpectedTestTermination(t, bodyReturned) - duration := runAndApplyTestCleanupWithDuration(t, execMeta, bodyDuration) - if unexpectedTermination { - r = unexpectedTestTerminationMessage + bodyTerminal := r + bodyStack := "" + if r != nil && execMeta.cleanupResult != nil && execMeta.exclusions.any() { + bodyStack = utils.GetStacktrace(1) } + unexpectedTermination := r == nil && processRetryUnexpectedTestTermination(t, bodyReturned) + duration := runAndApplyTestCleanupWithDurationOptions(t, execMeta, bodyDuration, execMeta.exclusions.any(), r != nil || unexpectedTermination) + r = resolveTestBodyTerminalAfterCleanup(t, execMeta, r, unexpectedTermination) terminalStack := "" if r != nil { terminalStack = utils.GetStacktrace(1) } + r, terminalStack = resolveTestCleanupDiagnostics(t, execMeta, bodyTerminal, bodyStack, r, terminalStack) finalizeInstrumentedTestExecution(t, execMeta, test, suite, module, duration, nil, r, terminalStack, true) + if r == nil && execMeta.cleanupResult != nil { + r = execMeta.cleanupResult.panicData + } if r != nil && !execMeta.hasAdditionalFeatureWrapper { checkModuleAndSuite(module, suite) if !requestCoverageShutdown(t) { @@ -1345,6 +1358,15 @@ func setTestTagsFromExecutionMetadataNoClose(test integrations.Test, execMeta *t // Set the Test Optimization test to the execution metadata execMeta.test = test + bindProtectedFailureEvent(execMeta, test) + // Preserve effective suppression independently of retained assignment tags. + integrations.SetTestCrashFailureSuppression(test, execMeta.isQuarantined || execMeta.isDisabled) + if execMeta.exclusions.PreventFailureSuppression { + test.SetTag("test.test_management.prevent_failure_suppression", "true") + } + if execMeta.exclusions.NoAutoRetry { + test.SetTag("test.test_management.no_auto_retry", "true") + } if execMeta.identity != nil && len(execMeta.identity.Segments) > 1 { if log.DebugEnabled() { log.Debug("setTestTagsFromExecutionMetadata assigned test for %s", execMeta.identity.FullName) @@ -1382,18 +1404,20 @@ func setTestTagsFromExecutionMetadataNoClose(test integrations.Test, execMeta *t } // If the test is an attempt to fix we tag the test event - if execMeta.isAttemptToFix { + if execMeta.isAttemptToFix || execMeta.assignment.attemptToFix { test.SetTag(constants.TestIsAttempToFix, "true") } // If the test is quarantined we tag the test event - if execMeta.isQuarantined { + if execMeta.isQuarantined || execMeta.assignment.quarantined { test.SetTag(constants.TestIsQuarantined, "true") } // If the test is disabled we tag the test event - if execMeta.isDisabled { + if execMeta.isDisabled || execMeta.assignment.disabled { test.SetTag(constants.TestIsDisabled, "true") + } + if execMeta.isDisabled { if !execMeta.isAttemptToFix { // Disabled test without ATF is always a final execution, set the final status test.SetTag(constants.TestFinalStatus, constants.TestStatusSkip) diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testingF.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testingF.go index 26dc83913..62b26e94b 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testingF.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting/testingF.go @@ -259,7 +259,7 @@ func completeFuzzParallelSeeds(f *testing.F) { fields := getTestPrivateFields(t) state := getFuzzTestState(f) // Unlike tRunner, fRunner releases its slot without reacquiring it. - wait := completeParallelSubtestsWithState(fields, state, false, false) + wait := completeParallelSubtestsWithState(fields, state) if wait == 0 { return } diff --git a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/manual_api_ddtest.go b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/manual_api_ddtest.go index 56ff1b1ce..9e1dba9c3 100644 --- a/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/manual_api_ddtest.go +++ b/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/manual_api_ddtest.go @@ -29,6 +29,14 @@ import ( // Ensures that tslvTest implements the Test interface. var _ Test = (*tslvTest)(nil) +type crashFailurePolicy uint8 + +const ( + crashFailurePolicyUnset crashFailurePolicy = iota + crashFailurePolicyFail + crashFailurePolicySuppress +) + type ( // tslvTest implements the DdTest interface and represents an individual test within a suite. tslvTest struct { @@ -39,6 +47,11 @@ type ( // ended is set by the first Close, which a delayed close finishes // later: another Close does nothing. ended bool + // crashPolicy is immutable once registered from effective execution + // metadata. Assignment tags remain mutable attribution, not policy. + crashPolicy crashFailurePolicy + // protectedFailure is a trusted, durable fact for this event lifetime. + protectedFailure bool } // tslvTestDelayed is a struct that represents a delayed test close operation. @@ -131,6 +144,40 @@ func MarkTestFinished(test Test) { } } +// SetTestCrashFailureSuppression records effective execution policy separately +// from mutable assignment tags. The first registration wins for an open test; +// tests without a registration retain the manual API's tag-based behavior. +func SetTestCrashFailureSuppression(test Test, suppress bool) { + t, ok := test.(*tslvTest) + if !ok || t == nil { + return + } + t.mutex.Lock() + defer t.mutex.Unlock() + if _, open := openTests.Load(t); !open || t.ended || t.closed || t.crashPolicy != crashFailurePolicyUnset { + return + } + t.crashPolicy = crashFailurePolicyFail + if suppress { + t.crashPolicy = crashFailurePolicySuppress + } +} + +// MarkTestProtectedFailure records a trusted failure that must remain blocking +// if a crash closes this event. It does not change the registered base policy. +func MarkTestProtectedFailure(test Test) { + t, ok := test.(*tslvTest) + if !ok || t == nil { + return + } + t.mutex.Lock() + defer t.mutex.Unlock() + if _, open := openTests.Load(t); !open || t.ended || t.closed { + return + } + t.protectedFailure = true +} + // TestID returns the ID of the test. func (t *tslvTest) TestID() uint64 { return t.testID @@ -160,7 +207,8 @@ func (t *tslvTest) failForCrash(errType, message, stack string) bool { return false, "" } final := constants.TestStatusFail - if t.getContextValue(constants.TestIsQuarantined) == "true" || t.getContextValue(constants.TestIsDisabled) == "true" { + if !t.protectedFailure && (t.crashPolicy == crashFailurePolicySuppress || t.crashPolicy == crashFailurePolicyUnset && + (t.getContextValue(constants.TestIsQuarantined) == "true" || t.getContextValue(constants.TestIsDisabled) == "true")) { final = constants.TestStatusSkip } // The test's own SetTag and SetError do not take t.mutex: the span sets diff --git a/ddto/scripts/ci_shards.json b/ddto/scripts/ci_shards.json index f3d7d8252..0f5df9d88 100644 --- a/ddto/scripts/ci_shards.json +++ b/ddto/scripts/ci_shards.json @@ -64,7 +64,8 @@ "TestSharedParityFixtureLifetime", "TestToolBypassHasNativeIdentityAndExit", "TestToolWrapperLifecycle", - "TestToolBypassProcessReplacement" + "TestToolBypassProcessReplacement", + "TestMiniDeclarationExclusions" ], "provision": [ "TestCLIDebugEarlyLocalProvisioning", @@ -167,7 +168,8 @@ "TestTestifyRegularCallerDefaultCoverage", "TestTestifySupportedVersionsAndNativeSemantics", "TestTestifyUnsupportedEntryWarnsAndPreservesClientNames", - "TestTestifyUserOverlay" + "TestTestifyUserOverlay", + "TestMiniTestifyExclusionsKeepSiblingRetries" ], "fuzz": [ "TestAlignMiniTestCommandsRequiresMiniForm", @@ -261,7 +263,9 @@ "TestMiniTemporaryDirectoryInsideModule", "TestMiniVendorSelectionMatchesGo", "TestMiniVendorTreeMatchesGo", - "TestMiniVendorWorkspaceLinksAndReusesBuildCache" + "TestMiniVendorWorkspaceLinksAndReusesBuildCache", + "TestMiniDeclarationExclusions", + "TestMiniTestifyExclusionsKeepSiblingRetries" ], "fuzz": [ "TestMiniFuzzExampleLifecycle", @@ -335,8 +339,10 @@ ], [ "orchestrion", - "sdk-orchestrion", "units" + ], + [ + "sdk-orchestrion" ] ], "differential/ubuntu-latest/race": [ diff --git a/ddto/scripts/test_ci_shards.py b/ddto/scripts/test_ci_shards.py index 8da05a827..32f36f500 100644 --- a/ddto/scripts/test_ci_shards.py +++ b/ddto/scripts/test_ci_shards.py @@ -23,7 +23,7 @@ def setUp(self): def test_matrix_and_platform_inventory(self): differential = ci_shards.matrix(self.groups, "differential")["include"] - self.assertEqual(16, len(differential)) + self.assertEqual(18, len(differential)) self.assertEqual(2, len(ci_shards.matrix(self.groups, "mini")["include"])) self.assertEqual([dict(suite="mini", os="ubuntu-latest", go="tip", mode="normal", shards="units,native,fuzz", job="units-native-fuzz", owner=True)], @@ -36,7 +36,7 @@ def test_matrix_and_platform_inventory(self): owners = [item["owner"] for item in differential if (item["suite"], item["os"], item["go"], item["mode"]) == config] self.assertEqual([True] + [False] * (len(owners) - 1), owners) - for platform, count in (("ubuntu-latest", 192), ("macos-latest", 189), ("windows-latest", 187)): + for platform, count in (("ubuntu-latest", 194), ("macos-latest", 191), ("windows-latest", 189)): tests = [name for shard in ci_shards.SHARDS["differential"][1:] for name in ci_shards.selected_tests(self.groups, "differential", "1.27.x", shard, platform)] self.assertEqual(count, len(tests)) @@ -45,7 +45,7 @@ def test_matrix_and_platform_inventory(self): for changed in (tests[:-1], tests + ["TestNewUnassignedCase"]): with self.assertRaisesRegex(ValueError, "test inventory changed"): ci_shards.validate_inventory(self.groups, changed, platform) - for version, count in (("1.25.x", 61), ("tip", 67)): + for version, count in (("1.25.x", 63), ("tip", 69)): tests = [name for shard in ci_shards.SHARDS["mini"][1:] for name in ci_shards.selected_tests(self.groups, "mini", version, shard, "ubuntu-latest")] self.assertEqual(count, len(tests)) diff --git a/ddto/testopt/exclusions.go b/ddto/testopt/exclusions.go new file mode 100644 index 000000000..64782a33e --- /dev/null +++ b/ddto/testopt/exclusions.go @@ -0,0 +1,22 @@ +package testopt + +import "github.com/DataDog/ddtest/ddto/internal/thirdparty/dd-trace-go/civisibility/integrations/gotesting" + +// TestExclusions describes execution controls on an individual test declaration. +// These controls preserve normal test selection, native skips and native retries. +type TestExclusions = gotesting.TestExclusions + +// RegisterTestExclusions registers a test function or Testify method expression. +// Call it during package initialization, before TestMain or any tests execute. +// ddtest instrument generates these calls for //ddtest:no_auto_retry and +// //ddtest:prevent_failure_suppression comments on test declarations. +func RegisterTestExclusions(fn any, exclusions TestExclusions) { + gotesting.RegisterTestExclusions(fn, exclusions) +} + +// RegisterTestExclusionsByName is used by generated test bootstraps for annotated +// declarations in imported helper packages. Call during package initialization; +// names identify compiled declarations and are unrelated to mutable span tags. +func RegisterTestExclusionsByName(name string, exclusions TestExclusions) { + gotesting.RegisterTestExclusionsByName(name, exclusions) +} diff --git a/internal/testdrive/probe_ownership_test.go b/internal/testdrive/probe_ownership_test.go index f49cb7142..1ff5a68a1 100644 --- a/internal/testdrive/probe_ownership_test.go +++ b/internal/testdrive/probe_ownership_test.go @@ -36,42 +36,75 @@ func TestProbeRecoveryAfterKilledProcess(t *testing.T) { requireWriteFile(t, filepath.Join(root, "abandoned-configuration"), configuration) probe, err := createProbe(root, validationRun{Tests: []jestTest{{File: filepath.Join(root, "real.test.js"), Status: "passed"}}}, "owned test") require.NoError(t, err) - fmt.Println(probe) + if output := os.Getenv("DDTEST_PROBE_STARTUP_OUTPUT"); output != "" { + fmt.Println(output) + } + fmt.Printf("DDTEST_PROBE_READY=%s\n", probe) time.Sleep(time.Minute) // Parent kills us; no deferred cleanup can run. return } - root := t.TempDir() - requireWriteFile(t, filepath.Join(root, "real.test.js"), "customer test") - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProbeRecoveryAfterKilledProcess$") - // This helper's stdout is a protocol, so it must not start a CI test session. - child.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_PROBE_CRASH_HELPER="+root) - stdout, err := child.StdoutPipe() - require.NoError(t, err) - require.NoError(t, child.Start()) - line, err := bufio.NewReader(stdout).ReadString('\n') - require.NoError(t, err) - probe := strings.TrimSpace(line) - require.FileExists(t, probe) - _, err = workspace.AcquireLock(root) - require.ErrorIs(t, err, workspace.ErrBusy, "an active owner must prevent startup recovery") - require.NoError(t, child.Process.Kill()) - require.Error(t, child.Wait()) - lock, err := workspace.AcquireLock(root) - require.NoError(t, err, "the OS releases the killed owner's lock") - defer func() { require.NoError(t, lock.Close()) }() - session, err := os.ReadFile(filepath.Join(root, "abandoned-session")) - require.NoError(t, err) - require.DirExists(t, string(session)) - require.NoError(t, workspace.RecoverSessions(root, &bytes.Buffer{})) - require.NoDirExists(t, string(session)) - configuration, err := os.ReadFile(filepath.Join(root, "abandoned-configuration")) - require.NoError(t, err) - require.NoDirExists(t, string(configuration), "startup recovery reclaims the killed owner’s marked configuration copy") - require.NoError(t, recoverProbes(root)) - require.NoFileExists(t, probe) - require.FileExists(t, filepath.Join(root, "real.test.js")) + for _, startup := range []struct{ name, output string }{ + {name: "quiet"}, + {name: "coverage output", output: "coverage: 0.5% of statements in ./..."}, + {name: "startup output", output: "unrelated helper startup output"}, + } { + t.Run(startup.name, func(t *testing.T) { + t.Setenv("DDTEST_PROBE_STARTUP_OUTPUT", startup.output) + root := t.TempDir() + requireWriteFile(t, filepath.Join(root, "real.test.js"), "customer test") + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProbeRecoveryAfterKilledProcess$") + // This helper's stdout is a protocol, so it must not start a CI test session. + child.Env = append(os.Environ(), "DD_CIVISIBILITY_ENABLED=false", "DDTEST_PROBE_CRASH_HELPER="+root) + stdout, err := child.StdoutPipe() + require.NoError(t, err) + require.NoError(t, child.Start()) + waited := false + t.Cleanup(func() { + if !waited { + _ = child.Process.Kill() + _ = child.Wait() + } + }) + // Instrumented test binaries may print coverage before the helper runs. + // Only the explicit readiness record carries a probe path. + scanner := bufio.NewScanner(stdout) + var startupOutput strings.Builder + var probe string + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "DDTEST_PROBE_READY=") { + probe = strings.TrimPrefix(line, "DDTEST_PROBE_READY=") + break + } + fmt.Fprintln(&startupOutput, line) + } + require.NoError(t, scanner.Err(), "helper stdout before readiness: %s", startupOutput.String()) + require.NotEmpty(t, probe, "helper ended without a readiness record; stdout: %s", startupOutput.String()) + require.FileExists(t, probe) + _, err = workspace.AcquireLock(root) + require.ErrorIs(t, err, workspace.ErrBusy, "an active owner must prevent startup recovery") + require.NoError(t, child.Process.Kill()) + waitErr := child.Wait() + waited = true + require.Error(t, waitErr) + lock, err := workspace.AcquireLock(root) + require.NoError(t, err, "the OS releases the killed owner's lock") + defer func() { require.NoError(t, lock.Close()) }() + session, err := os.ReadFile(filepath.Join(root, "abandoned-session")) + require.NoError(t, err) + require.DirExists(t, string(session)) + require.NoError(t, workspace.RecoverSessions(root, &bytes.Buffer{})) + require.NoDirExists(t, string(session)) + configuration, err := os.ReadFile(filepath.Join(root, "abandoned-configuration")) + require.NoError(t, err) + require.NoDirExists(t, string(configuration), "startup recovery reclaims the killed owner’s marked configuration copy") + require.NoError(t, recoverProbes(root)) + require.NoFileExists(t, probe) + require.FileExists(t, filepath.Join(root, "real.test.js")) + }) + } } func TestProbeRecoveryRequiresExactOwnershipAndUnchangedContents(t *testing.T) { diff --git a/main_test.go b/main_test.go index e14a61e1a..b4adc8f70 100644 --- a/main_test.go +++ b/main_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "runtime" "strings" "testing" @@ -149,11 +150,11 @@ func TestRunUsesOuterValidationStatusInsteadOfBuildExitCode(t *testing.T) { // TestDispatchHandsInstrumentToDdto runs ddtest instrument in a helper // process: ddto handles it, before ddtest sets anything, and exits. func TestDispatchHandsInstrumentToDdto(t *testing.T) { - stdout, _, code := dispatchHelper(t, "instrument", "--version") + stdout, _, code, _ := dispatchHelper(t, "instrument", "--version") if code != 0 || !strings.Contains(stdout, " instrument v") || strings.Contains(stdout, "ddto") { t.Fatalf("instrument --version: exit %d, stdout %q", code, stdout) } - _, stderr, code := dispatchHelper(t, "instrument") + _, stderr, code, _ := dispatchHelper(t, "instrument") if code != 2 || !strings.Contains(stderr, " instrument [go | orchestrion go") { t.Fatalf("instrument without a command: exit %d, stderr %q", code, stderr) } @@ -163,12 +164,21 @@ func TestDispatchHandsInstrumentToDdto(t *testing.T) { // the environment that the process received: the SDK that ddto incorporates // sets its root session variable when it starts. func TestDispatchKeepsCallerEnvironment(t *testing.T) { - // The names of ddto's internal modes stay ddtest's: ddto runs them as ddtest instrument. - for _, command := range []string{"plan", "test-exec", "ci-git-worker", "tool-overlay", "go-shim-exec"} { - stdout, stderr, code := dispatchHelper(t, command) - if code != 0 || stdout != "root session unset\n" { - t.Fatalf("%s: exit %d, stdout %q, stderr %q", command, code, stdout, stderr) - } + for _, startup := range []struct{ name, output string }{ + {name: "quiet"}, + {name: "coverage output", output: "coverage: 0.5% of statements in ./..."}, + {name: "startup output", output: "unrelated helper startup output"}, + } { + t.Run(startup.name, func(t *testing.T) { + t.Setenv("DDTEST_DISPATCH_STARTUP_OUTPUT", startup.output) + // The names of ddto's internal modes stay ddtest's: ddto runs them as ddtest instrument. + for _, command := range []string{"plan", "test-exec", "ci-git-worker", "tool-overlay", "go-shim-exec"} { + stdout, stderr, code, rootSession := dispatchHelper(t, command) + if code != 0 || rootSession != "unset\n" { + t.Fatalf("%s: exit %d, root session %q, stdout %q, stderr %q", command, code, rootSession, stdout, stderr) + } + } + }) } } @@ -177,30 +187,36 @@ func TestDispatchHelper(t *testing.T) { if !ok { t.Skip("runs only as dispatchHelper's process") } + if output := os.Getenv("DDTEST_DISPATCH_STARTUP_OUTPUT"); output != "" { + fmt.Println(output) + } dispatch(append([]string{os.Args[0]}, strings.Split(arguments, "\x1f")...)) + state := "unset\n" if _, set := os.LookupEnv("_DD_ROOT_GO_SESSION_ID"); set { - fmt.Println("root session set") - } else { - fmt.Println("root session unset") + state = "set\n" + } + if err := os.WriteFile(os.Getenv("DDTEST_DISPATCH_ROOT_SESSION_FILE"), []byte(state), 0600); err != nil { + t.Fatal(err) } os.Exit(0) } // dispatchHelper runs dispatch with args in a process of this test binary, // with the environment of a caller that started no root session. -func dispatchHelper(t *testing.T, args ...string) (string, string, int) { +func dispatchHelper(t *testing.T, args ...string) (string, string, int, string) { t.Helper() executable, err := os.Executable() if err != nil { t.Fatal(err) } + rootSessionFile := filepath.Join(t.TempDir(), "root-session") command := exec.Command(executable, "-test.run=^TestDispatchHelper$") for _, entry := range helperEnvironment() { if !strings.HasPrefix(entry, "_DD_ROOT_GO_SESSION_ID=") { command.Env = append(command.Env, entry) } } - command.Env = append(command.Env, "DDTEST_DISPATCH_HELPER="+strings.Join(args, "\x1f")) + command.Env = append(command.Env, "DDTEST_DISPATCH_HELPER="+strings.Join(args, "\x1f"), "DDTEST_DISPATCH_ROOT_SESSION_FILE="+rootSessionFile) var stdout, stderr strings.Builder command.Stdout, command.Stderr = &stdout, &stderr err = command.Run() @@ -208,5 +224,12 @@ func dispatchHelper(t *testing.T, args ...string) (string, string, int) { if err != nil && !errors.As(err, &exitErr) { t.Fatal(err) } - return stdout.String(), stderr.String(), command.ProcessState.ExitCode() + // Instrument commands exit inside dispatch and do not produce this report. + // Keep their stdout intact; startup coverage output must not be mistaken for + // the environment report from commands that return. + rootSession, reportErr := os.ReadFile(rootSessionFile) + if reportErr != nil && !os.IsNotExist(reportErr) { + t.Fatal(reportErr) + } + return stdout.String(), stderr.String(), command.ProcessState.ExitCode(), string(rootSession) } From b41d939f4b3247466a99265059850b45cdbf1a39 Mon Sep 17 00:00:00 2001 From: Calvin Bayer <50014594+calvinbayer@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:46:27 +0100 Subject: [PATCH 2/2] [SDTEST-4060] Register generic Go callbacks by compiled declaration name --- .../mini_exclusions_declarations_test.go | 212 ++++++++++++++++++ ddto/internal/runner/exclusions.go | 8 +- ddto/internal/runner/exclusions_test.go | 54 +++++ 3 files changed, 272 insertions(+), 2 deletions(-) diff --git a/ddto/integration/mini_exclusions_declarations_test.go b/ddto/integration/mini_exclusions_declarations_test.go index 169485c66..4bdff5cc3 100644 --- a/ddto/integration/mini_exclusions_declarations_test.go +++ b/ddto/integration/mini_exclusions_declarations_test.go @@ -291,6 +291,7 @@ func TestAnnotatedRuntimeDependency(t *testing.T){ } func checkMiniGenericDeclarations(t *testing.T) { + t.Run("functions", checkMiniGenericFunctionDeclarations) dir, driver := prepareMiniFixture(t) helper := installExternalTestifyHelper(t, dir) writeBuildFixture(t, helper, map[string]string{"generic.go": `package testkit @@ -428,3 +429,214 @@ func TestImportedGenericString(t *testing.T){recordGeneric(t);suite.Run(t,testki }) } } + +// Generic callback declarations retain one policy across type instantiations, +// while each named child keeps its own retry history. +func checkMiniGenericFunctionDeclarations(t *testing.T) { + for _, context := range []string{"internal", "external", "imported"} { + t.Run(context, func(t *testing.T) { + dir, driver := prepareMiniFixture(t) + callbackDir, callbackPackage, rootPackage, prefix, rootImport := dir, "fixture", "fixture", "", "" + if context == "external" { + rootPackage = "fixture_test" + prefix = "subject." + rootImport = `subject "example.com/dd-ci-testing-fixture"` + } + if context == "imported" { + callbackDir = t.TempDir() + if err := os.WriteFile(filepath.Join(callbackDir, "go.mod"), []byte("module example.com/external-testkit\ngo 1.25.0\n"), 0600); err != nil { + t.Fatal(err) + } + out, stderr, code := command(t, dir, testEnv(), "go", "mod", "edit", "-require=example.com/external-testkit@v0.0.0", "-replace=example.com/external-testkit="+callbackDir) + if code != 0 { + t.Fatalf("generic helper dependency setup%d %s %s", code, out, stderr) + } + callbackPackage = "testkit" + rootPackage = "fixture_test" + prefix = "testkit." + rootImport = `"example.com/external-testkit"` + } + callback := `package PACKAGE +import("testing";"os";"path/filepath";"strings") +func genericFunctionRecord(t *testing.T)int{path:=filepath.Join(os.Getenv("GENERIC_FUNCTION_COUNTER"),strings.ReplaceAll(t.Name(),"/","_"));f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};if _,e=f.WriteString("x");e!=nil{t.Fatal(e)};if e=f.Close();e!=nil{t.Fatal(e)};d,e:=os.ReadFile(path);if e!=nil{t.Fatal(e)};return len(d)} +//ddtest:prevent_failure_suppression +func TestGenericPFS[T any](t *testing.T){genericFunctionRecord(t);t.Error("generic callback failure")} +//ddtest:no_auto_retry +func TestGenericNR[T any](t *testing.T){genericFunctionRecord(t);t.Error("generic callback failure")} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestGenericBoth[A,B any](t *testing.T){genericFunctionRecord(t);t.Error("generic callback failure")} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestGenericUnused[T any](t *testing.T){t.Fatal("unused generic declaration executed")} +func TestGenericOrdinary[T any](t *testing.T){if genericFunctionRecord(t)==1{t.Error("first ordinary generic callback failure")}} +//ddtest:prevent_failure_suppression +func TestDirectPFS(t *testing.T){genericFunctionRecord(t);t.Error("generic callback failure")} +func TestDirectOrdinary(t *testing.T){if genericFunctionRecord(t)==1{t.Error("first ordinary direct callback failure")}} +` + root := `package PACKAGE +import("testing";"os";"path/filepath";IMPORT) +func TestGenericFunctionRoot(t *testing.T){path:=filepath.Join(os.Getenv("GENERIC_FUNCTION_COUNTER"),"root");f,e:=os.OpenFile(path,os.O_CREATE|os.O_APPEND|os.O_WRONLY,0600);if e!=nil{t.Fatal(e)};f.WriteString("x");f.Close() + switch os.Getenv("GENERIC_FUNCTION_KIND"){ + case "pfs":t.Run("int",PREFIXTestGenericPFS[int]);t.Run("string",PREFIXTestGenericPFS[string]) + case "direct-pfs":t.Run("int",PREFIXTestDirectPFS);t.Run("string",PREFIXTestDirectPFS) + case "nr":t.Run("int",PREFIXTestGenericNR[int]);t.Run("string",PREFIXTestGenericNR[string]) + case "both":t.Run("int",PREFIXTestGenericBoth[int,string]);t.Run("string",PREFIXTestGenericBoth[string,int]) + case "ordinary-generic":t.Run("int",PREFIXTestGenericOrdinary[int]);t.Run("string",PREFIXTestGenericOrdinary[string]) + case "ordinary-direct":t.Run("int",PREFIXTestDirectOrdinary);t.Run("string",PREFIXTestDirectOrdinary) + default:t.Fatal("invalid generic function kind") + }} +` + writeBuildFixture(t, callbackDir, map[string]string{"generic_function.go": strings.ReplaceAll(callback, "PACKAGE", callbackPackage)}) + root = strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(root, "PACKAGE", rootPackage), "IMPORT", rootImport), "PREFIX", prefix) + writeBuildFixture(t, dir, map[string]string{"generic_function_test.go": root}) + for _, native := range []bool{false, true} { + bin := filepath.Join(t.TempDir(), executableName("generic-functions.test")) + build := driver + args := exclusionBuildArgs(bin, false) + if native { + build = "go" + } + out, stderr, code := command(t, dir, testEnv("DD_CIVISIBILITY_ENABLED=false"), build, args...) + if code != 0 { + t.Fatalf("compile generic function native%t exit%d %s %s", native, code, out, stderr) + } + pfsFinalShapes := map[string]string{} + kinds := []string{"pfs", "direct-pfs", "nr", "both", "ordinary-generic", "ordinary-direct"} + if native { + kinds = []string{"both"} + } + for _, kind := range kinds { + t.Run(kind+map[bool]string{true: "-native"}[native], func(t *testing.T) { + counters := t.TempDir() + managed := !native && kind == "both" && context != "internal" + receiver, got := runParityCase(t, dir, bin, parityCase{Args: []string{"-test.run=^TestGenericFunctionRoot$"}, Policy: policySettings{Retry: !native, Disabled: managed, Quarantined: managed, AttemptToFix: managed, ManagementTarget: "TestGenericFunctionRoot/int", ManagementSuite: "generic_function.go"}, Env: []string{"GENERIC_FUNCTION_COUNTER=" + counters, "GENERIC_FUNCTION_KIND=" + kind, "DD_CIVISIBILITY_RETRY_EXECUTION_MODE=in_process", "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED=true", "DD_CIVISIBILITY_FLAKY_RETRY_COUNT=2", "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED=false"}}) + rootCalls, childCalls, wantCode := 3, 1, 1 + if kind == "pfs" || kind == "direct-pfs" { + childCalls = 3 + } + if strings.HasPrefix(kind, "ordinary-") { + rootCalls, childCalls, wantCode = 2, 2, 0 + } + if native { + rootCalls = 1 + } + if got.code != wantCode { + t.Fatalf("generic function exit%d want%d %s %s", got.code, wantCode, got.out, got.stderr) + } + requireEarlyMarker(t, filepath.Join(counters, "root"), strings.Repeat("x", rootCalls)) + for _, name := range []string{"int", "string"} { + requireEarlyMarker(t, filepath.Join(counters, "TestGenericFunctionRoot_"+name), strings.Repeat("x", childCalls)) + } + if native { + if len(receiver.events) != 0 { + t.Fatal("native generic callback emitted telemetry") + } + return + } + parents := reviewTestEvents(receiver, "TestGenericFunctionRoot") + if len(parents) != rootCalls { + t.Fatalf("generic root events%d want%d: %v", len(parents), rootCalls, receiver.events) + } + parentFinals, parentRetries := 0, 0 + for _, m := range parents { + if m["test.is_retry"] == "true" { + parentRetries++ + } + wantFinal := "fail" + if wantCode == 0 { + wantFinal = "pass" + } + if m["test.final_status"] == wantFinal { + parentFinals++ + } else if m["test.final_status"] != nil { + t.Fatalf("root final outcome lost: %v", m) + } + for _, flag := range []string{"prevent_failure_suppression", "no_auto_retry", "is_test_disabled", "is_quarantined", "is_attempt_to_fix"} { + if m["test.test_management."+flag] != nil { + t.Fatalf("generic child policy leaked to root: %v", m) + } + } + } + if parentFinals != 1 || parentRetries != rootCalls-1 { + t.Fatalf("root finals%d retries%d want1/%d", parentFinals, parentRetries, rootCalls-1) + } + total := 0 + for _, event := range receiver.events { + if event["type"] == "test" { + total++ + } + } + if total != rootCalls+2*childCalls { + t.Fatalf("generic exact test count%d want%d", total, rootCalls+2*childCalls) + } + for _, name := range []string{"int", "string"} { + children := reviewTestEvents(receiver, "TestGenericFunctionRoot/"+name) + if len(children) != childCalls { + t.Fatalf("generic %s event count%d want%d", name, len(children), childCalls) + } + failed, passed, finals, retries := 0, 0, 0, 0 + finalShape := []any{} + for _, m := range children { + pfs, nr := kind == "pfs" || kind == "direct-pfs" || kind == "both", kind == "nr" || kind == "both" + if (m["test.test_management.prevent_failure_suppression"] == "true") != pfs || (m["test.test_management.no_auto_retry"] == "true") != nr { + t.Fatalf("generic declaration flags lost or shared: %v", m) + } + if nr { + requireReviewNoRetryOwnership(t, m) + } + for _, assignment := range []string{"is_test_disabled", "is_quarantined", "is_attempt_to_fix"} { + if (m["test.test_management."+assignment] == "true") != (managed && name == "int") { + t.Fatalf("generic assignment or sibling isolation lost: %v", m) + } + } + switch m["test.status"] { + case "fail": + failed++ + case "pass": + passed++ + default: + t.Fatalf("generic unexpected status: %v", m) + } + if !strings.HasPrefix(kind, "ordinary-") { + finalShape = append(finalShape, m["test.final_status"]) + if m["test.final_status"] == "fail" { + finals++ + } else if m["test.final_status"] != nil { + t.Fatalf("unexpected generic final status: %v", m) + } + if m["test.is_retry"] == "true" { + retries++ + } + if m["error.message"] != "generic callback failure" { + t.Fatalf("generic protected/NR failure lost: %v", m) + } + } + } + if kind == "pfs" || kind == "direct-pfs" { + encoded, err := json.Marshal(finalShape) + if err != nil { + t.Fatal(err) + } + if kind == "pfs" { + pfsFinalShapes[name] = string(encoded) + } else if string(encoded) != pfsFinalShapes[name] { + t.Fatalf("generic final-status presence differs from ordinary callback %s: generic%s ordinary%s", name, pfsFinalShapes[name], encoded) + } + } + if strings.HasPrefix(kind, "ordinary-") { + if failed != 1 || passed != 1 { + t.Fatalf("ordinary generic recovery failed%d passed%d", failed, passed) + } + } else { + if failed != childCalls || passed != 0 || finals < 1 || (childCalls == 1 && finals != 1) || retries != childCalls-1 { + t.Fatalf("generic failures%d passes%d finals%d retries%d want%d/0/1/%d", failed, passed, finals, retries, childCalls, childCalls-1) + } + } + } + }) + } + } + }) + } +} diff --git a/ddto/internal/runner/exclusions.go b/ddto/internal/runner/exclusions.go index 3b20d3c57..c7e98d874 100644 --- a/ddto/internal/runner/exclusions.go +++ b/ddto/internal/runner/exclusions.go @@ -17,6 +17,7 @@ type exclusionDeclaration struct { receiver string valueReceiver bool genericReceiver bool + genericFunction bool name string prevent, noRetry bool } @@ -52,7 +53,7 @@ func exclusionDeclarations(p goPackage, files []string, replacements map[string] if !ok || fn.Doc == nil || fn.Body == nil { continue } - d := exclusionDeclaration{name: fn.Name.Name, expression: fn.Name.Name} + d := exclusionDeclaration{name: fn.Name.Name, expression: fn.Name.Name, genericFunction: fn.Type.TypeParams != nil && len(fn.Type.TypeParams.List) > 0} for _, comment := range fn.Doc.List { switch strings.TrimSpace(comment.Text) { case "//ddtest:prevent_failure_suppression": @@ -137,7 +138,7 @@ func exclusionRegistrationSourcesForBoundary(p goPackage, replacements map[strin d = resolveExclusionReceiver(d, aliases[pkg]) d.expression = "(*" + d.receiver + ")." + d.name } - if external && pkg == p.Name || d.genericReceiver { + if external && pkg == p.Name || d.genericReceiver || d.genericFunction { declarationPath := p.ImportPath if pkg != p.Name { declarationPath += "_test" @@ -179,6 +180,9 @@ func exclusionRuntimeNames(path string, d exclusionDeclaration) []string { d.receiver += "[...]" } if d.receiver == "" { + if d.genericFunction { + d.name += "[...]" + } return []string{prefix + "." + d.name} } names := []string{prefix + ".(*" + d.receiver + ")." + d.name} diff --git a/ddto/internal/runner/exclusions_test.go b/ddto/internal/runner/exclusions_test.go index 151f5137c..cc3bee623 100644 --- a/ddto/internal/runner/exclusions_test.go +++ b/ddto/internal/runner/exclusions_test.go @@ -78,6 +78,60 @@ func TestExclusionRegistrationsKeepPackagesAndDeclarationsSeparate(t *testing.T) t.Fatalf("generic runtime dependency boundary: %v %v", boundary, err) } }) + t.Run("generic-functions", func(t *testing.T) { + gd := t.TempDir() + source := `package fixture +import "testing" +//ddtest:no_auto_retry +func TestSingle[T any](t *testing.T){} +//ddtest:prevent_failure_suppression +func TestPair[A,B any](t *testing.T){} +//ddtest:prevent_failure_suppression +//ddtest:no_auto_retry +func TestUnused[T any](t *testing.T){} +//ddtest:no_auto_retry +func TestOrdinary(t *testing.T){} +` + if err := os.WriteFile(filepath.Join(gd, "generic.go"), []byte(source), 0600); err != nil { + t.Fatal(err) + } + gp := goPackage{Dir: gd, Name: "fixture", ImportPath: "example.com/functions.v1", GoFiles: []string{"generic.go"}} + generated, err := exclusionRegistrationSources(gp, nil, miniPackage) + if err != nil { + t.Fatal(err) + } + src := generated["fixture"] + for _, name := range []string{"TestSingle", "TestPair", "TestUnused"} { + if !strings.Contains(src, `RegisterTestExclusionsByName("example.com/functions%2ev1.`+name+`[...]"`) || strings.Contains(src, "RegisterTestExclusions("+name) { + t.Fatalf("generic declaration requires type inference or misses runtime name: %s", src) + } + } + if !strings.Contains(src, "RegisterTestExclusions(TestOrdinary") { + t.Fatalf("ordinary declaration registration changed: %s", src) + } + if _, err := parser.ParseFile(token.NewFileSet(), "generated.go", src, 0); err != nil { + t.Fatal(err) + } + gp.Module = &struct { + Path, Version, Dir string + Main bool + Replace *struct{ Path, Dir, Version string } + }{Path: gp.ImportPath} + imported, err := exclusionDependencyRegistrations(goPackage{Imports: []string{gp.ImportPath}}, packageGraph{gp.ImportPath: &gp}, nil, nil) + if err != nil { + t.Fatal(err) + } + boundary, err := exclusionRegistrationSourcesForBoundary(gp, nil, miniPackage, true) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"TestSingle", "TestPair", "TestUnused"} { + needle := `RegisterTestExclusionsByName("example.com/functions%2ev1.` + name + `[...]"` + if !strings.Contains(imported, needle) || !strings.Contains(boundary["fixture_test"], needle) { + t.Fatalf("import/boundary generic declaration missing %s: %s %v", name, imported, boundary) + } + } + }) // An edited overlay and annotation removal replace rather than retain policy. overlay := filepath.Join(dir, "replacement.go") if err := os.WriteFile(overlay, []byte("package fixture\nfunc TestInternal() {}\n"), 0600); err != nil {