Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions clientpool/channel.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type channelPool struct {

// Make sure channelPool implements Pool interface.
var _ Pool = (*channelPool)(nil)
var _ Discarder = (*channelPool)(nil)

// NewChannelPool creates a new client pool implemented via channel.
func NewChannelPool(ctx context.Context, requiredInitialClients, bestEffortInitialClients, maxClients int, opener ClientOpener) (_ Pool, err error) {
Expand Down Expand Up @@ -159,6 +160,18 @@ func (cp *channelPool) Release(c Client) error {
}
}

// Discard closes a checked-out client and removes it from the pool without
// synchronously opening a replacement. A later Get will open a new client if
// no idle client is available.
func (cp *channelPool) Discard(c Client) error {
if c == nil {
return nil
}

defer cp.numActive.Add(-1)
return c.Close()
}

// Close closes the pool, and all allocated clients.
func (cp *channelPool) Close() error {
var lastErr error
Expand Down
46 changes: 46 additions & 0 deletions clientpool/channel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,49 @@ func TestChannelPoolWithOpenerFailure(t *testing.T) {
},
)
}

func TestChannelPoolDiscardDoesNotOpenReplacement(t *testing.T) {
var openerCalled atomic.Int32
opener := func() (clientpool.Client, error) {
openerCalled.Add(1)
return &testClient{}, nil
}

pool, err := clientpool.NewChannelPool(context.Background(), 1, 1, 1, opener)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := pool.Close(); err != nil {
t.Errorf("pool.Close returned error: %v", err)
}
})

c, err := pool.Get()
if err != nil {
t.Fatal(err)
}
discarder := pool.(clientpool.Discarder)
if err := discarder.Discard(c); err != nil {
t.Fatalf("Discard returned error: %v", err)
}

if got := openerCalled.Load(); got != 1 {
t.Errorf("opener calls after Discard = %d, want 1", got)
}
if c.IsOpen() {
t.Error("Discard did not close the client")
}
checkActiveAndAllocated(t, pool, 0, 0)

newClient, err := pool.Get()
if err != nil {
t.Fatal(err)
}
if got := openerCalled.Load(); got != 2 {
t.Errorf("opener calls after next Get = %d, want 2", got)
}
if err := pool.Release(newClient); err != nil {
t.Fatalf("pool.Release returned error: %v", err)
}
}
6 changes: 6 additions & 0 deletions clientpool/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,9 @@ type Pool interface {
NumAllocated() int32
IsExhausted() bool
}

// Discarder is optionally implemented by pools that can discard a checked-out
// client without synchronously replacing it.
type Discarder interface {
Discard(Client) error
}
4 changes: 3 additions & 1 deletion thriftbp/client_middlewares_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,9 @@ func TestRetry(t *testing.T) {
if err == nil {
t.Error("expected an error, got nil")
}
const expected = 1
// The context deadline closes the leased connection and is returned as the
// call error, so the context error filter must prevent another attempt.
const expected = 0
if c.count != expected {
t.Errorf("expected middleware to trigger a retry %d times, got %d", expected, c.count)
}
Expand Down
72 changes: 63 additions & 9 deletions thriftbp/client_pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"net"
"strconv"
"strings"
"sync"
"time"

"github.com/apache/thrift/lib/go/thrift"
Expand Down Expand Up @@ -135,12 +136,13 @@ type ClientPoolConfig struct {
// In most cases, you would want ConnectTimeout to be short, because if you
// have problem connecting to the upstream you want to fail fast.
//
// For SocketTimeout, the value you should set depends on whether you set a
// deadline to the context object to the client call functions or not.
// If ALL your client calls will have a context object with a deadline
// attached, then it's recommended to set SocketTimeout to a short value,
// as this is the max overhead the client call will take over the set
// deadline, in case the server is not-responding.
// The pool enforces a client call's absolute context deadline by closing its
// leased connection to interrupt any in-flight socket read or write. This is
// needed even with framed transports: although the server buffers a complete
// frame before flushing it, the client receives that frame over a byte stream
// and can still stall while reading partway through it.
//
// SocketTimeout is a fallback for calls without a context deadline.
// But if you don't always have a deadline attached to your client calls,
// then SocketTimeout needs to be at least your upstream service's p99 latency
// SLA. If it's shorter than that you are gonna close connections and fail
Expand Down Expand Up @@ -648,26 +650,78 @@ func (p *clientPool) pooledCall(ctx context.Context, method string, args, result
if err != nil {
return thrift.ResponseMeta{}, PoolError{Cause: err}
}

var (
closeOnce sync.Once
closeErr error
)
closeClient := func() {
closeOnce.Do(func() {
closeErr = client.Close()
})
}
cancelCloseDone := make(chan struct{})
stopCancelClose := context.AfterFunc(ctx, func() {
closeClient()
close(cancelCloseDone)
})

defer func() {
canceled := false
if !stopCancelClose() {
// The cancellation callback started, so wait until it has finished
// closing the connection before releasing the client to the pool.
<-cancelCloseDone
}
if cause := context.Cause(ctx); cause != nil {
err = cause
canceled = true
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this cause a data race between a successful Thrift call but the context being cancelled? For example, the call succeeds, but before pooledCall returns ctx is cancelled. We then override err which was nil to a non-nil error. Based on the description it sounds like we want to interrupt in-flight calls, but here it wouldn't be in-flight anymore, would it? At least it appears that this is changing the previous behavior.

Maybe all this needs is to make sure that we only call this when shouldCloseConnection(err) is true?


if shouldCloseConnection(err) {
clientPoolClosedConnectionsCounter.With(prometheus.Labels{
"thrift_pool": p.slug,
}).Inc()
if e := client.Close(); e != nil {
closeClient()
if closeErr != nil {
log.C(ctx).Errorw(
"Failed to close client",
"pool", p.slug,
"origErr", err,
"closeErr", e,
"closeErr", closeErr,
)
}
}
p.releaseClient(client)
if canceled {
p.discardClient(client)
} else {
p.releaseClient(client)
}
}()

return client.Call(ctx, method, args, result)
}

func (p *clientPool) discardClient(c Client) {
discarder, ok := p.Pool.(clientpool.Discarder)
if !ok {
// Custom Pool implementations predating clientpool.Discarder retain the
// old Release behavior. The built-in channel pool always implements it.
p.releaseClient(c)
return
}
if err := discarder.Discard(c); err != nil {
log.Errorw(
"Failed to discard client from pool",
"pool", p.slug,
"err", err,
)
clientPoolReleaseErrorCounter.With(prometheus.Labels{
"thrift_pool": p.slug,
}).Inc()
}
}

func (p *clientPool) getClient() (_ Client, err error) {
defer func() {
clientPoolGetsCounter.With(prometheus.Labels{
Expand Down
162 changes: 162 additions & 0 deletions thriftbp/client_pool_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package thriftbp

import (
"context"
"errors"
"sync/atomic"
"testing"
"time"

"github.com/apache/thrift/lib/go/thrift"

"github.com/reddit/baseplate.go/clientpool"
)

type cancellationTestClient struct {
open atomic.Bool

callStarted chan struct{}
callReturned chan struct{}
callUnblocked chan struct{}
closeStarted chan struct{}
allowClose chan struct{}

closeCalls atomic.Int32
}

func newCancellationTestClient() *cancellationTestClient {
c := &cancellationTestClient{
callStarted: make(chan struct{}),
callReturned: make(chan struct{}),
callUnblocked: make(chan struct{}),
closeStarted: make(chan struct{}),
allowClose: make(chan struct{}),
}
c.open.Store(true)
return c
}

func (c *cancellationTestClient) Call(
context.Context,
string,
thrift.TStruct,
thrift.TStruct,
) (thrift.ResponseMeta, error) {
close(c.callStarted)
<-c.callUnblocked
close(c.callReturned)
return thrift.ResponseMeta{}, errors.New("connection closed")
}

func (c *cancellationTestClient) Close() error {
c.closeCalls.Add(1)
c.open.Store(false)
close(c.closeStarted)
close(c.callUnblocked)
<-c.allowClose
return nil
}

func (c *cancellationTestClient) IsOpen() bool {
return c.open.Load()
}

type cancellationTestPool struct {
client Client
released chan bool
discarded chan bool
}

func (p *cancellationTestPool) Get() (clientpool.Client, error) {
return p.client, nil
}

func (p *cancellationTestPool) Release(c clientpool.Client) error {
p.released <- c.IsOpen()
return nil
}

func (p *cancellationTestPool) Discard(c clientpool.Client) error {
p.discarded <- c.IsOpen()
return nil
}

func (*cancellationTestPool) Close() error {
return nil
}

func (*cancellationTestPool) NumActiveClients() int32 {
return 0
}

func (*cancellationTestPool) NumAllocated() int32 {
return 1
}

func (*cancellationTestPool) IsExhausted() bool {
return false
}

func TestPooledCallCancellationClosesAndDiscardsClient(t *testing.T) {
client := newCancellationTestClient()
pool := &cancellationTestPool{
client: client,
released: make(chan bool, 1),
discarded: make(chan bool, 1),
}
p := &clientPool{
Pool: pool,
slug: "cancellation-test",
}

ctx, cancel := context.WithCancelCause(context.Background())
cause := errors.New("test cancellation cause")
result := make(chan error, 1)
go func() {
_, err := p.pooledCall(ctx, "method", nil, nil)
result <- err
}()

<-client.callStarted
cancel(cause)
<-client.closeStarted
<-client.callReturned

select {
case <-pool.released:
t.Fatal("client was released before the cancellation callback finished")
case err := <-result:
t.Fatalf("pooledCall returned before the cancellation callback finished: %v", err)
case <-time.After(50 * time.Millisecond):
}

close(client.allowClose)

select {
case err := <-result:
if !errors.Is(err, cause) {
t.Fatalf("pooledCall error = %v, want cancellation cause %v", err, cause)
}
case <-time.After(time.Second):
t.Fatal("pooledCall did not return after the cancellation callback finished")
}

select {
case open := <-pool.discarded:
if open {
t.Error("closed client was released as reusable")
}
default:
t.Fatal("client was not discarded")
}

select {
case <-pool.released:
t.Fatal("canceled client was released instead of discarded")
default:
}

if got := client.closeCalls.Load(); got != 1 {
t.Errorf("client Close calls = %d, want 1", got)
}
}
Loading
Loading