Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 60 additions & 3 deletions internal/bridges/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ type Manager struct {
newNode func(bridge config.Bridge, stateDir string, userLogf, debugLogf func(string, ...any)) tailnetNode
}

const (
bridgeDNSRetryWindow = 5 * time.Second
bridgeDNSRetryInterval = 250 * time.Millisecond
)

type nodeRuntime struct {
node tailnetNode
proxies map[string]*proxyRuntime
Expand Down Expand Up @@ -255,14 +260,21 @@ func startProxy(node tailnetNode, target *url.URL, logf func(string), debug bool
if debug {
logf(fmt.Sprintf("Bridge dialing network=%s address=%s", network, address))
}
conn, err := node.DialContext(ctx, network, address)
conn, attempts, err := dialWithDNSRetry(
ctx,
node.DialContext,
network,
address,
bridgeDNSRetryWindow,
bridgeDNSRetryInterval,
)
elapsed := time.Since(start).Round(time.Millisecond)
if err != nil {
logf(fmt.Sprintf("Bridge dial failed: network=%s address=%s elapsed=%s error=%T: %v", network, address, elapsed, err, err))
logf(fmt.Sprintf("Bridge dial failed: network=%s address=%s attempts=%d elapsed=%s error=%T: %v", network, address, attempts, elapsed, err, err))
return nil, err
}
if debug {
logf(fmt.Sprintf("Bridge dial connected: address=%s remote=%s elapsed=%s", address, conn.RemoteAddr(), elapsed))
logf(fmt.Sprintf("Bridge dial connected: address=%s remote=%s attempts=%d elapsed=%s", address, conn.RemoteAddr(), attempts, elapsed))
}
return conn, nil
}
Expand Down Expand Up @@ -291,6 +303,51 @@ func startProxy(node tailnetNode, target *url.URL, logf func(string), debug bool
}, nil
}

type bridgeDialFunc func(context.Context, string, string) (net.Conn, error)

// dialWithDNSRetry gives an embedded tsnet node a short window to receive the
// target's peer map after Up reports Running. Until that map arrives, tsnet's
// MagicDNS lookup falls through to the host resolver and returns a DNSError.
// Non-DNS failures are returned immediately.
func dialWithDNSRetry(
ctx context.Context,
dial bridgeDialFunc,
network, address string,
retryWindow, retryInterval time.Duration,
) (net.Conn, int, error) {
deadline := time.Now().Add(retryWindow)
attempts := 0
for {
conn, err := dial(ctx, network, address)
attempts++
if err == nil {
return conn, attempts, nil
}
if ctxErr := ctx.Err(); ctxErr != nil {
return nil, attempts, ctxErr
}
var dnsErr *net.DNSError
if !errors.As(err, &dnsErr) || retryWindow <= 0 || retryInterval <= 0 {
return nil, attempts, err
}

remaining := time.Until(deadline)
if remaining <= 0 {
return nil, attempts, err
}
if retryInterval > remaining {
retryInterval = remaining
}
timer := time.NewTimer(retryInterval)
select {
case <-ctx.Done():
timer.Stop()
return nil, attempts, ctx.Err()
case <-timer.C:
}
}
}

func logBridgeStatus(logf func(string), status *ipnstate.Status, target *url.URL) {
if status == nil {
logf("Bridge network status is unavailable.")
Expand Down
157 changes: 157 additions & 0 deletions internal/bridges/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ import (
"net/http/httptest"
"net/netip"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/tailscale/aperture-cli/internal/config"
"tailscale.com/ipn/ipnstate"
Expand All @@ -21,6 +23,7 @@ type fakeNode struct {
upErr error
statusErr error
dialErr error
dialFn bridgeDialFunc
up int
closed bool
}
Expand All @@ -35,6 +38,9 @@ func (n *fakeNode) Status(context.Context) (*ipnstate.Status, error) {
}

func (n *fakeNode) DialContext(ctx context.Context, network, _ string) (net.Conn, error) {
if n.dialFn != nil {
return n.dialFn(ctx, network, n.backendAddr)
}
if n.dialErr != nil {
return nil, n.dialErr
}
Expand Down Expand Up @@ -169,6 +175,157 @@ func TestActivateNormalLoggingOmitsDebugDiagnostics(t *testing.T) {
}
}

func TestActivateRetriesDNSWhilePeerMapArrives(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer backend.Close()

var attempts atomic.Int32
node := &fakeNode{
backendAddr: strings.TrimPrefix(backend.URL, "http://"),
status: &ipnstate.Status{
BackendState: "Running",
TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1")},
},
}
node.dialFn = func(ctx context.Context, network, address string) (net.Conn, error) {
if attempts.Add(1) == 1 {
return nil, &net.DNSError{
Err: "server misbehaving",
Name: "ai",
Server: "127.0.0.53:53",
IsTemporary: true,
}
}
var d net.Dialer
return d.DialContext(ctx, network, address)
}

m := NewManager(true)
m.newNode = func(_ config.Bridge, _ string, _ func(string, ...any), _ func(string, ...any)) tailnetNode {
return node
}
defer m.Close()

var logs []string
localURL, err := m.Activate(
context.Background(),
config.Bridge{ID: "bridge-abcdef", Name: "Work"},
"http://ai",
func(line string) { logs = append(logs, line) },
)
if err != nil {
t.Fatal(err)
}

resp, err := http.Get(localURL + "/v1/models")
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
}
if got := attempts.Load(); got != 2 {
t.Fatalf("dial attempts = %d, want 2", got)
}
if got := strings.Join(logs, "\n"); !strings.Contains(got, "attempts=2") {
t.Fatalf("logs missing recovered dial attempt count:\n%s", got)
}
}

func TestDialWithDNSRetry(t *testing.T) {
t.Run("recovers when embedded DNS receives the target", func(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
defer backend.Close()

attempts := 0
dial := func(ctx context.Context, network, address string) (net.Conn, error) {
attempts++
if attempts == 1 {
return nil, &net.DNSError{
Err: "server misbehaving",
Name: "ai",
Server: "127.0.0.53:53",
IsTemporary: true,
}
}
var d net.Dialer
return d.DialContext(ctx, network, strings.TrimPrefix(backend.URL, "http://"))
}

conn, gotAttempts, err := dialWithDNSRetry(
context.Background(), dial, "tcp", "ai:80", time.Second, time.Millisecond,
)
if err != nil {
t.Fatal(err)
}
conn.Close()
if gotAttempts != 2 {
t.Fatalf("attempts = %d, want 2", gotAttempts)
}
})

t.Run("stops when the retry window expires", func(t *testing.T) {
wantErr := &net.DNSError{Err: "server misbehaving", Name: "ai"}
attempts := 0
dial := func(context.Context, string, string) (net.Conn, error) {
attempts++
return nil, wantErr
}

_, gotAttempts, err := dialWithDNSRetry(
context.Background(), dial, "tcp", "ai:80", 5*time.Millisecond, time.Hour,
)
if !errors.Is(err, wantErr) {
t.Fatalf("error = %v, want %v", err, wantErr)
}
if gotAttempts < 2 || gotAttempts != attempts {
t.Fatalf("attempts = %d/%d, want at least 2 matching attempts", gotAttempts, attempts)
}
})

t.Run("does not retry non-DNS failures", func(t *testing.T) {
wantErr := errors.New("connection refused")
attempts := 0
dial := func(context.Context, string, string) (net.Conn, error) {
attempts++
return nil, wantErr
}

_, gotAttempts, err := dialWithDNSRetry(
context.Background(), dial, "tcp", "ai:80", time.Second, time.Millisecond,
)
if !errors.Is(err, wantErr) {
t.Fatalf("error = %v, want %v", err, wantErr)
}
if gotAttempts != 1 || attempts != 1 {
t.Fatalf("attempts = %d/%d, want 1/1", gotAttempts, attempts)
}
})

t.Run("stops when activation is canceled", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
attempts := 0
dial := func(context.Context, string, string) (net.Conn, error) {
attempts++
cancel()
return nil, &net.DNSError{Err: "server misbehaving", Name: "ai"}
}

_, gotAttempts, err := dialWithDNSRetry(
ctx, dial, "tcp", "ai:80", time.Second, time.Second,
)
if !errors.Is(err, context.Canceled) {
t.Fatalf("error = %v, want context canceled", err)
}
if gotAttempts != 1 || attempts != 1 {
t.Fatalf("attempts = %d/%d, want 1/1", gotAttempts, attempts)
}
})
}

func (n *fakeNode) Close() error {
n.closed = true
return nil
Expand Down
Loading
Loading