guestnet: add a private userspace network for each VM guest

go.mod

@@ -11,14 +11,17 @@ require (
 	github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900
 	github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b
 	github.com/cespare/xxhash/v2 v2.3.0
-	github.com/go-jose/go-jose/v4 v4.1.3
+	github.com/go-jose/go-jose/v4 v4.1.4
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/go-cmp v0.7.0
+	github.com/insomniacslk/dhcp v0.0.0-20260901064844-234b97448fae
 	github.com/pierrec/lz4/v4 v4.1.26
 	github.com/prometheus/client_golang v1.24.1
 	github.com/prometheus/client_model v0.6.3
+	golang.org/x/net v0.59.0
 	golang.org/x/sync v0.23.0
 	golang.org/x/time v0.16.0
+	gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72
 	modernc.org/sqlite v1.51.0
 	tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef
 )
@@ -47,6 +50,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.3 // indirect
 	github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect
+	github.com/google/btree v1.1.3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hdevalence/ed25519consensus v0.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -63,12 +67,12 @@ require (
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect
 	go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect
 	golang.org/x/crypto v0.57.0 // indirect
 	golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect
-	golang.org/x/net v0.59.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.zx2c4.com/wireguard/windows v1.0.1 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect

go.sum

@@ -79,8 +79,8 @@ github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng
 github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
 github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI=
 github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE=
 github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
@@ -91,6 +91,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
+github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
+github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

guestnet/addr.go

@@ -0,0 +1,28 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package guestnet
+
+import (
+	"net"
+	"net/netip"
+)
+
+var netipBroadcast = netip.AddrFrom4([4]byte{255, 255, 255, 255})
+
+func netipFromIP(ip net.IP) (netip.Addr, bool) {
+	if ip4 := ip.To4(); ip4 != nil {
+		return netip.AddrFrom4([4]byte(ip4)), true
+	}
+	return netip.AddrFromSlice(ip)
+}
+
+// broadcastOf returns the directed broadcast address of an IPv4 prefix.
+func broadcastOf(p netip.Prefix) net.IP {
+	a := p.Masked().Addr().As4()
+	bits := p.Bits()
+	for i := bits; i < 32; i++ {
+		a[i/8] |= 1 << (7 - uint(i%8))
+	}
+	return net.IP(a[:])
+}

guestnet/arp_test.go

@@ -0,0 +1,95 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package guestnet
+
+import (
+	"net"
+	"net/netip"
+	"testing"
+	"time"
+
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/header"
+)
+
+// buildARPRequest returns an Ethernet broadcast ARP request from mac/srcIP
+// asking for dstIP. An ARP probe (RFC 5227) has an unspecified srcIP.
+func buildARPRequest(mac net.HardwareAddr, srcIP, dstIP netip.Addr) []byte {
+	frame := make([]byte, header.EthernetMinimumSize+header.ARPSize)
+	header.Ethernet(frame).Encode(&header.EthernetFields{
+		SrcAddr: tcpip.LinkAddress(mac),
+		DstAddr: tcpip.LinkAddress("\xff\xff\xff\xff\xff\xff"),
+		Type:    header.ARPProtocolNumber,
+	})
+	arp := header.ARP(frame[header.EthernetMinimumSize:])
+	arp.SetIPv4OverEthernet()
+	arp.SetOp(header.ARPRequest)
+	copy(arp.HardwareAddressSender(), mac)
+	src, dst := srcIP.As4(), dstIP.As4()
+	copy(arp.ProtocolAddressSender(), src[:])
+	copy(arp.ProtocolAddressTarget(), dst[:])
+	return frame
+}
+
+// TestARPOnlyForOurAddresses checks that we answer ARP only for our own
+// addresses. In particular the guest's DHCP client probes its offered
+// address (RFC 5227) and declines the lease if anything answers; macOS does
+// this, and a promiscuous gVisor NIC would answer for every address.
+func TestARPOnlyForOurAddresses(t *testing.T) {
+	host, peer := NewMemLinkPair()
+	n, err := New(Config{Link: host, Logf: t.Logf})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer n.Close()
+
+	replies := make(chan netip.Addr, 16)
+	go func() {
+		buf := make([]byte, 65536)
+		for {
+			m, err := peer.ReadFrame(buf)
+			if err != nil {
+				return
+			}
+			frame := buf[:m]
+			if header.Ethernet(frame).Type() != header.ARPProtocolNumber {
+				continue
+			}
+			arp := header.ARP(frame[header.EthernetMinimumSize:])
+			if arp.IsValid() && arp.Op() == header.ARPReply {
+				replies <- netip.AddrFrom4([4]byte(arp.ProtocolAddressSender()))
+			}
+		}
+	}()
+
+	mac := DefaultGuestMAC
+	unspec := netip.AddrFrom4([4]byte{})
+	for _, tt := range []struct {
+		src, dst netip.Addr
+		want     bool
+	}{
+		{unspec, DefaultGuestIP, false},         // DHCP client's duplicate address probe
+		{DefaultGuestIP, DefaultGuestIP, false}, // gratuitous ARP
+		{DefaultGuestIP, netip.MustParseAddr("192.168.64.77"), false},
+		{DefaultGuestIP, DefaultGatewayIP, true},
+		{DefaultGuestIP, IMDSAddr, true},
+		{DefaultGuestIP, DNSAddr, true},
+	} {
+		if err := peer.WriteFrame(buildARPRequest(mac, tt.src, tt.dst)); err != nil {
+			t.Fatal(err)
+		}
+		var got bool
+		select {
+		case from := <-replies:
+			got = true
+			if from != tt.dst {
+				t.Errorf("ARP for %v: reply claims %v", tt.dst, from)
+			}
+		case <-time.After(300 * time.Millisecond):
+		}
+		if got != tt.want {
+			t.Errorf("ARP from %v for %v: replied = %v, want %v", tt.src, tt.dst, got, tt.want)
+		}
+	}
+}

guestnet/close_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package guestnet
+
+import (
+	"net"
+	"net/netip"
+	"testing"
+	"time"
+
+	"gvisor.dev/gvisor/pkg/tcpip"
+	"gvisor.dev/gvisor/pkg/tcpip/checksum"
+	"gvisor.dev/gvisor/pkg/tcpip/header"
+)
+
+// buildTCPSYNFrame assembles an Ethernet/IPv4/TCP SYN frame with checksums.
+func buildTCPSYNFrame(srcMAC, dstMAC net.HardwareAddr, src, dst netip.AddrPort) []byte {
+	ipLen := header.IPv4MinimumSize + header.TCPMinimumSize
+	frame := make([]byte, header.EthernetMinimumSize+ipLen)
+	header.Ethernet(frame).Encode(&header.EthernetFields{
+		SrcAddr: tcpip.LinkAddress(srcMAC),
+		DstAddr: tcpip.LinkAddress(dstMAC),
+		Type:    header.IPv4ProtocolNumber,
+	})
+	srcIP, dstIP := tcpip.AddrFrom4(src.Addr().As4()), tcpip.AddrFrom4(dst.Addr().As4())
+	ip := header.IPv4(frame[header.EthernetMinimumSize:])
+	ip.Encode(&header.IPv4Fields{
+		TotalLength: uint16(ipLen),
+		TTL:         64,
+		Protocol:    uint8(header.TCPProtocolNumber),
+		SrcAddr:     srcIP,
+		DstAddr:     dstIP,
+	})
+	ip.SetChecksum(^ip.CalculateChecksum())
+	tcp := header.TCP(ip.Payload())
+	tcp.Encode(&header.TCPFields{
+		SrcPort:    src.Port(),
+		DstPort:    dst.Port(),
+		SeqNum:     1000,
+		DataOffset: header.TCPMinimumSize,
+		Flags:      header.TCPFlagSyn,
+		WindowSize: 65535,
+	})
+	xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, srcIP, dstIP, header.TCPMinimumSize)
+	tcp.SetChecksum(^checksum.Checksum(tcp, xsum))
+	return frame
+}
+
+// TestCloseDuringHandshake checks that Close doesn't wait for a TCP
+// handshake the guest will never finish. A guest that is stopped just after
+// sending a SYN (the VM stopping mid-connection) leaves the forwarder waiting
+// in the handshake; Close used to wait for gVisor's handshake timeout (30s
+// or more) before tearing down.
+func TestCloseDuringHandshake(t *testing.T) {
+	target, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer target.Close()
+	accepted := make(chan net.Conn, 1)
+	go func() {
+		c, err := target.Accept()
+		if err == nil {
+			accepted <- c
+		}
+	}()
+
+	host, peer := NewMemLinkPair()
+	n, err := New(Config{Link: host, Dialer: anyDialer{tcpTarget: target.Addr().String()}, Logf: t.Logf})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Answer ARP for the guest's address, as a real guest would, so the
+	// SYN-ACK goes out, but never ACK it: the guest has stopped.
+	go func() {
+		buf := make([]byte, 65536)
+		for {
+			m, err := peer.ReadFrame(buf)
+			if err != nil {
+				return
+			}
+			frame := buf[:m]
+			if header.Ethernet(frame).Type() != header.ARPProtocolNumber {
+				continue
+			}
+			req := header.ARP(frame[header.EthernetMinimumSize:])
+			if !req.IsValid() || req.Op() != header.ARPRequest {
+				continue
+			}
+			reply := make([]byte, header.EthernetMinimumSize+header.ARPSize)
+			header.Ethernet(reply).Encode(&header.EthernetFields{
+				SrcAddr: tcpip.LinkAddress(DefaultGuestMAC),
+				DstAddr: tcpip.LinkAddress(req.HardwareAddressSender()),
+				Type:    header.ARPProtocolNumber,
+			})
+			arp := header.ARP(reply[header.EthernetMinimumSize:])
+			arp.SetIPv4OverEthernet()
+			arp.SetOp(header.ARPReply)
+			copy(arp.HardwareAddressSender(), DefaultGuestMAC)
+			copy(arp.ProtocolAddressSender(), req.ProtocolAddressTarget())
+			copy(arp.HardwareAddressTarget(), req.HardwareAddressSender())
+			copy(arp.ProtocolAddressTarget(), req.ProtocolAddressSender())
+			peer.WriteFrame(reply)
+		}
+	}()
+
+	src := netip.AddrPortFrom(DefaultGuestIP, 40000)
+	dst := netip.MustParseAddrPort("198.51.100.7:443")
+	if err := peer.WriteFrame(buildTCPSYNFrame(DefaultGuestMAC, DefaultGatewayMAC, src, dst)); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case c := <-accepted:
+		defer c.Close()
+	case <-time.After(5 * time.Second):
+		t.Fatal("egress dial never happened")
+	}
+	time.Sleep(100 * time.Millisecond) // let the forwarder enter the handshake
+
+	done := make(chan struct{})
+	t0 := time.Now()
+	go func() {
+		n.Close()
+		close(done)
+	}()
+	select {
+	case <-done:
+		t.Logf("Close took %v", time.Since(t0))
+	case <-time.After(5 * time.Second):
+		t.Fatal("Close waited for the guest's unfinished TCP handshake")
+	}
+}