diff --git a/clientpool/channel.go b/clientpool/channel.go index 26b20c22f..4d7e2167c 100644 --- a/clientpool/channel.go +++ b/clientpool/channel.go @@ -19,6 +19,7 @@ type channelPool struct { // Make sure channelPool implements Pool interface. var _ Pool = (*channelPool)(nil) +var _ Discarder = (*channelPool)(nil) // NewChannelPool creates a new client pool implemented via channel. func NewChannelPool(ctx context.Context, requiredInitialClients, bestEffortInitialClients, maxClients int, opener ClientOpener) (_ Pool, err error) { @@ -159,6 +160,18 @@ func (cp *channelPool) Release(c Client) error { } } +// Discard closes a checked-out client and removes it from the pool without +// synchronously opening a replacement. A later Get will open a new client if +// no idle client is available. +func (cp *channelPool) Discard(c Client) error { + if c == nil { + return nil + } + + defer cp.numActive.Add(-1) + return c.Close() +} + // Close closes the pool, and all allocated clients. func (cp *channelPool) Close() error { var lastErr error diff --git a/clientpool/channel_test.go b/clientpool/channel_test.go index d3efa0fbd..97c42c0c2 100644 --- a/clientpool/channel_test.go +++ b/clientpool/channel_test.go @@ -113,3 +113,49 @@ func TestChannelPoolWithOpenerFailure(t *testing.T) { }, ) } + +func TestChannelPoolDiscardDoesNotOpenReplacement(t *testing.T) { + var openerCalled atomic.Int32 + opener := func() (clientpool.Client, error) { + openerCalled.Add(1) + return &testClient{}, nil + } + + pool, err := clientpool.NewChannelPool(context.Background(), 1, 1, 1, opener) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := pool.Close(); err != nil { + t.Errorf("pool.Close returned error: %v", err) + } + }) + + c, err := pool.Get() + if err != nil { + t.Fatal(err) + } + discarder := pool.(clientpool.Discarder) + if err := discarder.Discard(c); err != nil { + t.Fatalf("Discard returned error: %v", err) + } + + if got := openerCalled.Load(); got != 1 { + t.Errorf("opener calls after Discard = %d, want 1", got) + } + if c.IsOpen() { + t.Error("Discard did not close the client") + } + checkActiveAndAllocated(t, pool, 0, 0) + + newClient, err := pool.Get() + if err != nil { + t.Fatal(err) + } + if got := openerCalled.Load(); got != 2 { + t.Errorf("opener calls after next Get = %d, want 2", got) + } + if err := pool.Release(newClient); err != nil { + t.Fatalf("pool.Release returned error: %v", err) + } +} diff --git a/clientpool/interface.go b/clientpool/interface.go index 2a906d303..ac44ec6c7 100644 --- a/clientpool/interface.go +++ b/clientpool/interface.go @@ -28,3 +28,9 @@ type Pool interface { NumAllocated() int32 IsExhausted() bool } + +// Discarder is optionally implemented by pools that can discard a checked-out +// client without synchronously replacing it. +type Discarder interface { + Discard(Client) error +} diff --git a/thriftbp/client_middlewares_test.go b/thriftbp/client_middlewares_test.go index a18aff8e3..7599a8231 100644 --- a/thriftbp/client_middlewares_test.go +++ b/thriftbp/client_middlewares_test.go @@ -228,7 +228,9 @@ func TestRetry(t *testing.T) { if err == nil { t.Error("expected an error, got nil") } - const expected = 1 + // The context deadline closes the leased connection and is returned as the + // call error, so the context error filter must prevent another attempt. + const expected = 0 if c.count != expected { t.Errorf("expected middleware to trigger a retry %d times, got %d", expected, c.count) } diff --git a/thriftbp/client_pool.go b/thriftbp/client_pool.go index 680b406fe..f3ea807b0 100644 --- a/thriftbp/client_pool.go +++ b/thriftbp/client_pool.go @@ -8,6 +8,7 @@ import ( "net" "strconv" "strings" + "sync" "time" "github.com/apache/thrift/lib/go/thrift" @@ -135,12 +136,13 @@ type ClientPoolConfig struct { // In most cases, you would want ConnectTimeout to be short, because if you // have problem connecting to the upstream you want to fail fast. // - // For SocketTimeout, the value you should set depends on whether you set a - // deadline to the context object to the client call functions or not. - // If ALL your client calls will have a context object with a deadline - // attached, then it's recommended to set SocketTimeout to a short value, - // as this is the max overhead the client call will take over the set - // deadline, in case the server is not-responding. + // The pool enforces a client call's absolute context deadline by closing its + // leased connection to interrupt any in-flight socket read or write. This is + // needed even with framed transports: although the server buffers a complete + // frame before flushing it, the client receives that frame over a byte stream + // and can still stall while reading partway through it. + // + // SocketTimeout is a fallback for calls without a context deadline. // But if you don't always have a deadline attached to your client calls, // then SocketTimeout needs to be at least your upstream service's p99 latency // SLA. If it's shorter than that you are gonna close connections and fail @@ -648,26 +650,78 @@ func (p *clientPool) pooledCall(ctx context.Context, method string, args, result if err != nil { return thrift.ResponseMeta{}, PoolError{Cause: err} } + + var ( + closeOnce sync.Once + closeErr error + ) + closeClient := func() { + closeOnce.Do(func() { + closeErr = client.Close() + }) + } + cancelCloseDone := make(chan struct{}) + stopCancelClose := context.AfterFunc(ctx, func() { + closeClient() + close(cancelCloseDone) + }) + defer func() { + canceled := false + if !stopCancelClose() { + // The cancellation callback started, so wait until it has finished + // closing the connection before releasing the client to the pool. + <-cancelCloseDone + } + if cause := context.Cause(ctx); cause != nil { + err = cause + canceled = true + } + if shouldCloseConnection(err) { clientPoolClosedConnectionsCounter.With(prometheus.Labels{ "thrift_pool": p.slug, }).Inc() - if e := client.Close(); e != nil { + closeClient() + if closeErr != nil { log.C(ctx).Errorw( "Failed to close client", "pool", p.slug, "origErr", err, - "closeErr", e, + "closeErr", closeErr, ) } } - p.releaseClient(client) + if canceled { + p.discardClient(client) + } else { + p.releaseClient(client) + } }() return client.Call(ctx, method, args, result) } +func (p *clientPool) discardClient(c Client) { + discarder, ok := p.Pool.(clientpool.Discarder) + if !ok { + // Custom Pool implementations predating clientpool.Discarder retain the + // old Release behavior. The built-in channel pool always implements it. + p.releaseClient(c) + return + } + if err := discarder.Discard(c); err != nil { + log.Errorw( + "Failed to discard client from pool", + "pool", p.slug, + "err", err, + ) + clientPoolReleaseErrorCounter.With(prometheus.Labels{ + "thrift_pool": p.slug, + }).Inc() + } +} + func (p *clientPool) getClient() (_ Client, err error) { defer func() { clientPoolGetsCounter.With(prometheus.Labels{ diff --git a/thriftbp/client_pool_internal_test.go b/thriftbp/client_pool_internal_test.go new file mode 100644 index 000000000..d4a736ae0 --- /dev/null +++ b/thriftbp/client_pool_internal_test.go @@ -0,0 +1,162 @@ +package thriftbp + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/apache/thrift/lib/go/thrift" + + "github.com/reddit/baseplate.go/clientpool" +) + +type cancellationTestClient struct { + open atomic.Bool + + callStarted chan struct{} + callReturned chan struct{} + callUnblocked chan struct{} + closeStarted chan struct{} + allowClose chan struct{} + + closeCalls atomic.Int32 +} + +func newCancellationTestClient() *cancellationTestClient { + c := &cancellationTestClient{ + callStarted: make(chan struct{}), + callReturned: make(chan struct{}), + callUnblocked: make(chan struct{}), + closeStarted: make(chan struct{}), + allowClose: make(chan struct{}), + } + c.open.Store(true) + return c +} + +func (c *cancellationTestClient) Call( + context.Context, + string, + thrift.TStruct, + thrift.TStruct, +) (thrift.ResponseMeta, error) { + close(c.callStarted) + <-c.callUnblocked + close(c.callReturned) + return thrift.ResponseMeta{}, errors.New("connection closed") +} + +func (c *cancellationTestClient) Close() error { + c.closeCalls.Add(1) + c.open.Store(false) + close(c.closeStarted) + close(c.callUnblocked) + <-c.allowClose + return nil +} + +func (c *cancellationTestClient) IsOpen() bool { + return c.open.Load() +} + +type cancellationTestPool struct { + client Client + released chan bool + discarded chan bool +} + +func (p *cancellationTestPool) Get() (clientpool.Client, error) { + return p.client, nil +} + +func (p *cancellationTestPool) Release(c clientpool.Client) error { + p.released <- c.IsOpen() + return nil +} + +func (p *cancellationTestPool) Discard(c clientpool.Client) error { + p.discarded <- c.IsOpen() + return nil +} + +func (*cancellationTestPool) Close() error { + return nil +} + +func (*cancellationTestPool) NumActiveClients() int32 { + return 0 +} + +func (*cancellationTestPool) NumAllocated() int32 { + return 1 +} + +func (*cancellationTestPool) IsExhausted() bool { + return false +} + +func TestPooledCallCancellationClosesAndDiscardsClient(t *testing.T) { + client := newCancellationTestClient() + pool := &cancellationTestPool{ + client: client, + released: make(chan bool, 1), + discarded: make(chan bool, 1), + } + p := &clientPool{ + Pool: pool, + slug: "cancellation-test", + } + + ctx, cancel := context.WithCancelCause(context.Background()) + cause := errors.New("test cancellation cause") + result := make(chan error, 1) + go func() { + _, err := p.pooledCall(ctx, "method", nil, nil) + result <- err + }() + + <-client.callStarted + cancel(cause) + <-client.closeStarted + <-client.callReturned + + select { + case <-pool.released: + t.Fatal("client was released before the cancellation callback finished") + case err := <-result: + t.Fatalf("pooledCall returned before the cancellation callback finished: %v", err) + case <-time.After(50 * time.Millisecond): + } + + close(client.allowClose) + + select { + case err := <-result: + if !errors.Is(err, cause) { + t.Fatalf("pooledCall error = %v, want cancellation cause %v", err, cause) + } + case <-time.After(time.Second): + t.Fatal("pooledCall did not return after the cancellation callback finished") + } + + select { + case open := <-pool.discarded: + if open { + t.Error("closed client was released as reusable") + } + default: + t.Fatal("client was not discarded") + } + + select { + case <-pool.released: + t.Fatal("canceled client was released instead of discarded") + default: + } + + if got := client.closeCalls.Load(); got != 1 { + t.Errorf("client Close calls = %d, want 1", got) + } +} diff --git a/thriftbp/ttl_client.go b/thriftbp/ttl_client.go index f8f3a32cc..cfbe5b3a7 100644 --- a/thriftbp/ttl_client.go +++ b/thriftbp/ttl_client.go @@ -2,6 +2,7 @@ package thriftbp import ( "context" + "sync" "sync/atomic" "time" @@ -50,21 +51,44 @@ type ttlClient struct { // state guarded by lock (buffer-1 channel) state chan *ttlClientState + + // Close needs to be able to close the transport used by Call without + // waiting for Call to release state. Closing that transport is what + // interrupts an in-flight socket read. + activeTransportMu sync.Mutex + activeTransport *countingDelegateTransport + closing atomic.Bool + closeOnce sync.Once + closeErr error } // Close implements Client interface. // // It calls underlying TTransport's Close function. func (c *ttlClient) Close() error { - state := <-c.state - defer func() { - c.state <- state - }() - state.closed = true - if state.timer != nil { - state.timer.Stop() - } - return state.transport.Close() + c.closeOnce.Do(func() { + c.closing.Store(true) + + c.activeTransportMu.Lock() + activeTransport := c.activeTransport + if activeTransport != nil { + c.closeErr = activeTransport.Close() + } + c.activeTransportMu.Unlock() + + state := <-c.state + defer func() { + c.state <- state + }() + state.closed = true + if state.timer != nil { + state.timer.Stop() + } + if state.transport != activeTransport { + c.closeErr = state.transport.Close() + } + }) + return c.closeErr } func (c *ttlClient) Call(ctx context.Context, method string, args, result thrift.TStruct) (_ thrift.ResponseMeta, err error) { @@ -83,6 +107,23 @@ func (c *ttlClient) Call(ctx context.Context, method string, args, result thrift clientPayloadSizeRequestBytes.With(labels).Observe(float64(written)) clientPayloadSizeResponseBytes.With(labels).Observe(float64(read)) }() + + c.activeTransportMu.Lock() + if c.closing.Load() { + c.activeTransportMu.Unlock() + return thrift.ResponseMeta{}, thrift.NewTTransportException( + thrift.NOT_OPEN, + "thriftbp: client is closed", + ) + } + c.activeTransport = state.transport + c.activeTransportMu.Unlock() + defer func() { + c.activeTransportMu.Lock() + c.activeTransport = nil + c.activeTransportMu.Unlock() + }() + return state.client.Call(ctx, method, args, result) } @@ -93,11 +134,15 @@ func (c *ttlClient) Call(ctx context.Context, method string, args, result thrift // Otherwise it checks TTL, // returns false if TTL has passed and also close the underlying TTransport. func (c *ttlClient) IsOpen() bool { + if c.closing.Load() { + return false + } + state := <-c.state defer func() { c.state <- state }() - if !state.transport.IsOpen() { + if state.closed || !state.transport.IsOpen() { return false } if !state.expiration.IsZero() && time.Now().After(state.expiration) { @@ -126,7 +171,7 @@ func (c *ttlClient) refresh() { defer func() { c.state <- state }() - if state.closed { + if state.closed || c.closing.Load() { // If Close was called after we entered this function, // close the newly created connection and return early. transport.Close() diff --git a/thriftbp/ttl_client_test.go b/thriftbp/ttl_client_test.go index d86520a77..9fac80df0 100644 --- a/thriftbp/ttl_client_test.go +++ b/thriftbp/ttl_client_test.go @@ -1,7 +1,9 @@ package thriftbp import ( + "context" "errors" + "sync" "sync/atomic" "testing" "time" @@ -237,3 +239,97 @@ func TestCountingDelegateTransport(t *testing.T) { t.Errorf("After reset: Written %d bytes want %d", written, want) } } + +type closeInterruptTransport struct { + thrift.TTransport + + closed chan struct{} + closeOnce sync.Once + closeCalls atomic.Int32 +} + +func (t *closeInterruptTransport) Close() error { + t.closeCalls.Add(1) + t.closeOnce.Do(func() { + close(t.closed) + }) + return nil +} + +type closeInterruptClient struct { + started chan struct{} + transport *closeInterruptTransport +} + +func (c *closeInterruptClient) Call( + context.Context, + string, + thrift.TStruct, + thrift.TStruct, +) (thrift.ResponseMeta, error) { + close(c.started) + <-c.transport.closed + return thrift.ResponseMeta{}, errors.New("transport closed") +} + +func TestTTLClientCloseInterruptsCall(t *testing.T) { + transport := &closeInterruptTransport{ + TTransport: thrift.NewTMemoryBuffer(), + closed: make(chan struct{}), + } + inner := &closeInterruptClient{ + started: make(chan struct{}), + transport: transport, + } + client, err := newTTLClient( + func() (thrift.TClient, *countingDelegateTransport, error) { + return inner, &countingDelegateTransport{TTransport: transport}, nil + }, + -1, + 0, + "", + ) + if err != nil { + t.Fatalf("newTTLClient returned error: %v", err) + } + + callDone := make(chan error, 1) + go func() { + _, err := client.Call(context.Background(), "method", nil, nil) + callDone <- err + }() + <-inner.started + + closeDone := make(chan error, 1) + go func() { + closeDone <- client.Close() + }() + + select { + case err := <-callDone: + if err == nil { + t.Fatal("Call returned nil error after its transport was closed") + } + case <-time.After(time.Second): + t.Fatal("Close did not interrupt the in-flight Call") + } + + select { + case err := <-closeDone: + if err != nil { + t.Fatalf("Close returned error: %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not finish after the in-flight Call returned") + } + + if err := client.Close(); err != nil { + t.Fatalf("second Close returned error: %v", err) + } + if client.IsOpen() { + t.Error("closed client reported itself as reusable") + } + if got := transport.closeCalls.Load(); got != 1 { + t.Errorf("underlying transport Close calls = %d, want 1", got) + } +}