From 883b659ed24122ee8abcb41cb295b5e7c9015d61 Mon Sep 17 00:00:00 2001 From: Kevin Jonaitis <266874750+kevinjonaitis-dot@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:19:36 -0600 Subject: [PATCH 1/3] Prevent stale Thrift results from masking successful retries Co-authored-by: Ian Koeppe <13874383+ianvkoeppe@users.noreply.github.com> --- thriftbp/client_middlewares.go | 10 +++- thriftbp/retry_result_test.go | 104 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 thriftbp/retry_result_test.go diff --git a/thriftbp/client_middlewares.go b/thriftbp/client_middlewares.go index 349afb446..24a4db3fa 100644 --- a/thriftbp/client_middlewares.go +++ b/thriftbp/client_middlewares.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "reflect" "slices" "strconv" "sync" @@ -289,15 +290,22 @@ func getClientError(result thrift.TStruct, err error) error { } // Retry returns a thrift.ClientMiddleware that can be used to automatically -// retry thrift requests. +// retry thrift requests. It clears the generated result struct before each +// retry because decoding a response does not clear fields absent from the wire. func Retry(defaults ...retry.Option) thrift.ClientMiddleware { return func(next thrift.TClient) thrift.TClient { return thrift.WrappedTClient{ Wrapped: func(ctx context.Context, method string, args, result thrift.TStruct) (thrift.ResponseMeta, error) { var lastMeta thrift.ResponseMeta + resultValue := reflect.Indirect(reflect.ValueOf(result)) + attempted := false return lastMeta, retrybp.Do( ctx, func() error { + if attempted && resultValue.Kind() == reflect.Struct && resultValue.CanSet() { + resultValue.SetZero() + } + attempted = true var err error lastMeta, err = next.Call(ctx, method, args, result) return getClientError(result, err) diff --git a/thriftbp/retry_result_test.go b/thriftbp/retry_result_test.go new file mode 100644 index 000000000..4d45cf92c --- /dev/null +++ b/thriftbp/retry_result_test.go @@ -0,0 +1,104 @@ +package thriftbp_test + +import ( + "context" + "errors" + "testing" + + "github.com/apache/thrift/lib/go/thrift" + "github.com/avast/retry-go" + baseplatethrift "github.com/reddit/baseplate.go/internal/gen-go/reddit/baseplate" + "github.com/reddit/baseplate.go/thriftbp" +) + +func TestRetryResultLifecycle(t *testing.T) { + tests := []struct { + name string + firstError *baseplatethrift.Error + partialSuccess bool + secondError *baseplatethrift.Error + wantCalls int + }{ + {name: "exception then success", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, wantCalls: 2}, + {name: "partial response then empty response", partialSuccess: true, wantCalls: 2}, + {name: "final exception survives", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, secondError: &baseplatethrift.Error{Message: thrift.Pointer("final error")}, wantCalls: 2}, + {name: "terminal exception is not cleared", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(false)}, wantCalls: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var result retryResult + calls := 0 + client := thrift.WrapClient(thrift.WrappedTClient{ + Wrapped: func(_ context.Context, _ string, _, response thrift.TStruct) (thrift.ResponseMeta, error) { + calls++ + r := response.(*retryResult) + if calls == 1 { + r.Error = tt.firstError + if tt.partialSuccess { + r.Success = thrift.Pointer(false) + return thrift.ResponseMeta{}, thrift.NewTTransportException(thrift.END_OF_FILE, "partial response") + } + } else if tt.secondError != nil { + r.Error = tt.secondError + } + if calls > 1 && tt.secondError == nil && !tt.partialSuccess { + r.Success = thrift.Pointer(true) + } + return thrift.ResponseMeta{}, nil + }, + }, thriftbp.Retry(retry.Attempts(2), retry.Delay(0), retry.RetryIf(func(err error) bool { + var exception *baseplatethrift.Error + if errors.As(err, &exception) { + return exception.GetRetryable() + } + return true + }))) + _, err := client.Call(context.Background(), "test", nil, &result) + if calls != tt.wantCalls { + t.Fatalf("calls = %d, want %d", calls, tt.wantCalls) + } + wantError := tt.secondError + if tt.wantCalls == 1 { + wantError = tt.firstError + } + if wantError != nil { + if !errors.Is(err, wantError) || result.Error != wantError { + t.Fatalf("final exception = %v, error = %v, want %v", result.Error, err, wantError) + } + } else { + if err != nil || result.Error != nil { + t.Fatalf("result = %+v, error = %v, want no exception", result, err) + } + if tt.partialSuccess { + if result.Success != nil { + t.Fatal("empty response retained success from the partial response") + } + } else if result.Success == nil || !*result.Success { + t.Fatalf("result = %+v, want success", result) + } + } + }) + } +} + +func TestRetryOneway(t *testing.T) { + calls := 0 + client := thrift.WrapClient(thrift.WrappedTClient{ + Wrapped: func(context.Context, string, thrift.TStruct, thrift.TStruct) (thrift.ResponseMeta, error) { + calls++ + if calls == 1 { + return thrift.ResponseMeta{}, thrift.NewTTransportException(thrift.NOT_OPEN, "closed") + } + return thrift.ResponseMeta{}, nil + }, + }, thriftbp.Retry(retry.Attempts(2), retry.Delay(0))) + if _, err := client.Call(context.Background(), "oneway", nil, nil); err != nil || calls != 2 { + t.Fatalf("calls = %d, error = %v, want two calls and no error", calls, err) + } +} + +type retryResult struct { + thrift.TStruct + Success *bool + Error *baseplatethrift.Error +} From 10d51db4fdc76746a3fe9a996e5808965a1a4f2f Mon Sep 17 00:00:00 2001 From: Kevin Jonaitis <266874750+kevinjonaitis-dot@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:22:03 -0600 Subject: [PATCH 2/3] Clear Thrift result state before every attempt --- thriftbp/client_middlewares.go | 7 ++----- thriftbp/retry_result_test.go | 10 +++++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/thriftbp/client_middlewares.go b/thriftbp/client_middlewares.go index 24a4db3fa..1d56ee939 100644 --- a/thriftbp/client_middlewares.go +++ b/thriftbp/client_middlewares.go @@ -290,22 +290,19 @@ func getClientError(result thrift.TStruct, err error) error { } // Retry returns a thrift.ClientMiddleware that can be used to automatically -// retry thrift requests. It clears the generated result struct before each -// retry because decoding a response does not clear fields absent from the wire. +// retry thrift requests. func Retry(defaults ...retry.Option) thrift.ClientMiddleware { return func(next thrift.TClient) thrift.TClient { return thrift.WrappedTClient{ Wrapped: func(ctx context.Context, method string, args, result thrift.TStruct) (thrift.ResponseMeta, error) { var lastMeta thrift.ResponseMeta resultValue := reflect.Indirect(reflect.ValueOf(result)) - attempted := false return lastMeta, retrybp.Do( ctx, func() error { - if attempted && resultValue.Kind() == reflect.Struct && resultValue.CanSet() { + if resultValue.Kind() == reflect.Struct && resultValue.CanSet() { resultValue.SetZero() } - attempted = true var err error lastMeta, err = next.Call(ctx, method, args, result) return getClientError(result, err) diff --git a/thriftbp/retry_result_test.go b/thriftbp/retry_result_test.go index 4d45cf92c..3065ec63e 100644 --- a/thriftbp/retry_result_test.go +++ b/thriftbp/retry_result_test.go @@ -14,11 +14,13 @@ import ( func TestRetryResultLifecycle(t *testing.T) { tests := []struct { name string + initialError *baseplatethrift.Error firstError *baseplatethrift.Error partialSuccess bool secondError *baseplatethrift.Error wantCalls int }{ + {name: "success with reused result", initialError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, wantCalls: 1}, {name: "exception then success", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, wantCalls: 2}, {name: "partial response then empty response", partialSuccess: true, wantCalls: 2}, {name: "final exception survives", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, secondError: &baseplatethrift.Error{Message: thrift.Pointer("final error")}, wantCalls: 2}, @@ -26,14 +28,16 @@ func TestRetryResultLifecycle(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var result retryResult + result := retryResult{Error: tt.initialError} calls := 0 client := thrift.WrapClient(thrift.WrappedTClient{ Wrapped: func(_ context.Context, _ string, _, response thrift.TStruct) (thrift.ResponseMeta, error) { calls++ r := response.(*retryResult) if calls == 1 { - r.Error = tt.firstError + if tt.firstError != nil { + r.Error = tt.firstError + } if tt.partialSuccess { r.Success = thrift.Pointer(false) return thrift.ResponseMeta{}, thrift.NewTTransportException(thrift.END_OF_FILE, "partial response") @@ -41,7 +45,7 @@ func TestRetryResultLifecycle(t *testing.T) { } else if tt.secondError != nil { r.Error = tt.secondError } - if calls > 1 && tt.secondError == nil && !tt.partialSuccess { + if (calls > 1 || tt.firstError == nil) && tt.secondError == nil && !tt.partialSuccess { r.Success = thrift.Pointer(true) } return thrift.ResponseMeta{}, nil From 4ce7b858da8d7d79450be8d794a28aa5120f9c52 Mon Sep 17 00:00:00 2001 From: Kevin Jonaitis <266874750+kevinjonaitis-dot@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:26:01 -0600 Subject: [PATCH 3/3] Focus retry regression test on exception followed by success --- thriftbp/retry_result_test.go | 94 ++++++----------------------------- 1 file changed, 14 insertions(+), 80 deletions(-) diff --git a/thriftbp/retry_result_test.go b/thriftbp/retry_result_test.go index 3065ec63e..95b3c81b1 100644 --- a/thriftbp/retry_result_test.go +++ b/thriftbp/retry_result_test.go @@ -2,7 +2,6 @@ package thriftbp_test import ( "context" - "errors" "testing" "github.com/apache/thrift/lib/go/thrift" @@ -11,93 +10,28 @@ import ( "github.com/reddit/baseplate.go/thriftbp" ) -func TestRetryResultLifecycle(t *testing.T) { - tests := []struct { - name string - initialError *baseplatethrift.Error - firstError *baseplatethrift.Error - partialSuccess bool - secondError *baseplatethrift.Error - wantCalls int - }{ - {name: "success with reused result", initialError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, wantCalls: 1}, - {name: "exception then success", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, wantCalls: 2}, - {name: "partial response then empty response", partialSuccess: true, wantCalls: 2}, - {name: "final exception survives", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(true)}, secondError: &baseplatethrift.Error{Message: thrift.Pointer("final error")}, wantCalls: 2}, - {name: "terminal exception is not cleared", firstError: &baseplatethrift.Error{Retryable: thrift.Pointer(false)}, wantCalls: 1}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := retryResult{Error: tt.initialError} - calls := 0 - client := thrift.WrapClient(thrift.WrappedTClient{ - Wrapped: func(_ context.Context, _ string, _, response thrift.TStruct) (thrift.ResponseMeta, error) { - calls++ - r := response.(*retryResult) - if calls == 1 { - if tt.firstError != nil { - r.Error = tt.firstError - } - if tt.partialSuccess { - r.Success = thrift.Pointer(false) - return thrift.ResponseMeta{}, thrift.NewTTransportException(thrift.END_OF_FILE, "partial response") - } - } else if tt.secondError != nil { - r.Error = tt.secondError - } - if (calls > 1 || tt.firstError == nil) && tt.secondError == nil && !tt.partialSuccess { - r.Success = thrift.Pointer(true) - } - return thrift.ResponseMeta{}, nil - }, - }, thriftbp.Retry(retry.Attempts(2), retry.Delay(0), retry.RetryIf(func(err error) bool { - var exception *baseplatethrift.Error - if errors.As(err, &exception) { - return exception.GetRetryable() - } - return true - }))) - _, err := client.Call(context.Background(), "test", nil, &result) - if calls != tt.wantCalls { - t.Fatalf("calls = %d, want %d", calls, tt.wantCalls) - } - wantError := tt.secondError - if tt.wantCalls == 1 { - wantError = tt.firstError - } - if wantError != nil { - if !errors.Is(err, wantError) || result.Error != wantError { - t.Fatalf("final exception = %v, error = %v, want %v", result.Error, err, wantError) - } - } else { - if err != nil || result.Error != nil { - t.Fatalf("result = %+v, error = %v, want no exception", result, err) - } - if tt.partialSuccess { - if result.Success != nil { - t.Fatal("empty response retained success from the partial response") - } - } else if result.Success == nil || !*result.Success { - t.Fatalf("result = %+v, want success", result) - } - } - }) - } -} - -func TestRetryOneway(t *testing.T) { +func TestRetryReturnsSuccessAfterException(t *testing.T) { + var result retryResult calls := 0 client := thrift.WrapClient(thrift.WrappedTClient{ - Wrapped: func(context.Context, string, thrift.TStruct, thrift.TStruct) (thrift.ResponseMeta, error) { + Wrapped: func(_ context.Context, _ string, _, response thrift.TStruct) (thrift.ResponseMeta, error) { calls++ + r := response.(*retryResult) if calls == 1 { - return thrift.ResponseMeta{}, thrift.NewTTransportException(thrift.NOT_OPEN, "closed") + r.Error = &baseplatethrift.Error{Retryable: thrift.Pointer(true)} + } else { + r.Success = thrift.Pointer(true) } return thrift.ResponseMeta{}, nil }, }, thriftbp.Retry(retry.Attempts(2), retry.Delay(0))) - if _, err := client.Call(context.Background(), "oneway", nil, nil); err != nil || calls != 2 { - t.Fatalf("calls = %d, error = %v, want two calls and no error", calls, err) + + _, err := client.Call(context.Background(), "test", nil, &result) + if err != nil || calls != 2 { + t.Fatalf("calls = %d, error = %v, want success after two calls", calls, err) + } + if result.Error != nil || result.Success == nil || !*result.Success { + t.Fatalf("result = %+v, want success without the previous exception", result) } }