From 1f75e37b14d061f21359ba195c84f7c695e7d97b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Nov 2026 18:15:38 +0000 Subject: [PATCH] guestnet: add a private userspace network for each VM guest This is the network for mantsana, a macOS CI VM backend replacing Tart (see the DESIGN.md that follows), but nothing in it is macOS-specific. A guest's virtual NIC hands us raw Ethernet frames (on macOS over a SOCK_DGRAM socketpair to a VZFileHandleNetworkDeviceAttachment) and they all terminate in a gVisor netstack owned by the guest's Net, so a guest can only reach what we connect it to, and can't see other guests: - The gateway, IMDS (169.254.169.254) and DNS (169.254.169.253) addresses are on our NIC, so gVisor answers ARP and ICMP for them, and the guest needs no route or DNS changes to reach them. ARP for any other address is dropped: the NIC is promiscuous, and answering the guest's DHCP duplicate-address probe makes it decline its lease. - DHCP is answered from the frame loop with one fixed lease, the same in every slot, so a VM memory snapshot restores anywhere. - Each TCP SYN and UDP flow is dispatched by destination to a registered Service (a guest-facing server, or a forward to a host-side address), to DNS (any port 53) or NTP (any port 123) interception, or to egress dialed from the host, subject to a deny list. Every flow and DNS query is logged. - The host reaches the guest only by DialGuest; there is no host route to it. guestnettest provides a fake guest stack for tests of code built on it. Updates tailscale/corp#39994 Signed-off-by: Brad Fitzpatrick Change-Id: I5b359a459ae52218e97cef3b74ecc14a806e0cc7 --- go.mod | 8 +- go.sum | 6 +- guestnet/addr.go | 28 ++ guestnet/arp_test.go | 95 +++++ guestnet/close_test.go | 133 +++++++ guestnet/dhcp.go | 198 ++++++++++ guestnet/dns.go | 299 ++++++++++++++ guestnet/guestnet.go | 468 ++++++++++++++++++++++ guestnet/guestnet_test.go | 544 ++++++++++++++++++++++++++ guestnet/guestnettest/guestnettest.go | 131 +++++++ guestnet/link.go | 114 ++++++ guestnet/log.go | 120 ++++++ guestnet/ntp.go | 67 ++++ guestnet/services.go | 130 ++++++ guestnet/socketpair.go | 61 +++ guestnet/socketpair_test.go | 35 ++ guestnet/tcp.go | 226 +++++++++++ guestnet/udp.go | 146 +++++++ 18 files changed, 2805 insertions(+), 4 deletions(-) create mode 100644 guestnet/addr.go create mode 100644 guestnet/arp_test.go create mode 100644 guestnet/close_test.go create mode 100644 guestnet/dhcp.go create mode 100644 guestnet/dns.go create mode 100644 guestnet/guestnet.go create mode 100644 guestnet/guestnet_test.go create mode 100644 guestnet/guestnettest/guestnettest.go create mode 100644 guestnet/link.go create mode 100644 guestnet/log.go create mode 100644 guestnet/ntp.go create mode 100644 guestnet/services.go create mode 100644 guestnet/socketpair.go create mode 100644 guestnet/socketpair_test.go create mode 100644 guestnet/tcp.go create mode 100644 guestnet/udp.go diff --git a/go.mod b/go.mod index c781e5a..2caf8f2 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,17 @@ require ( github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900 github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b github.com/cespare/xxhash/v2 v2.3.0 - github.com/go-jose/go-jose/v4 v4.1.3 + github.com/go-jose/go-jose/v4 v4.1.4 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 + github.com/insomniacslk/dhcp v0.0.0-20260901064844-234b97448fae github.com/pierrec/lz4/v4 v4.1.26 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 + golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 golang.org/x/time v0.16.0 + gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 modernc.org/sqlite v1.51.0 tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef ) @@ -47,6 +50,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.3 // indirect github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -63,12 +67,12 @@ require ( github.com/prometheus/procfs v0.21.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect + github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/x448/float16 v0.8.4 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.59.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect diff --git a/go.sum b/go.sum index fff738a..180ac20 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI= github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -91,6 +91,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/guestnet/addr.go b/guestnet/addr.go new file mode 100644 index 0000000..7c70db4 --- /dev/null +++ b/guestnet/addr.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" +) + +var netipBroadcast = netip.AddrFrom4([4]byte{255, 255, 255, 255}) + +func netipFromIP(ip net.IP) (netip.Addr, bool) { + if ip4 := ip.To4(); ip4 != nil { + return netip.AddrFrom4([4]byte(ip4)), true + } + return netip.AddrFromSlice(ip) +} + +// broadcastOf returns the directed broadcast address of an IPv4 prefix. +func broadcastOf(p netip.Prefix) net.IP { + a := p.Masked().Addr().As4() + bits := p.Bits() + for i := bits; i < 32; i++ { + a[i/8] |= 1 << (7 - uint(i%8)) + } + return net.IP(a[:]) +} diff --git a/guestnet/arp_test.go b/guestnet/arp_test.go new file mode 100644 index 0000000..69b10de --- /dev/null +++ b/guestnet/arp_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildARPRequest returns an Ethernet broadcast ARP request from mac/srcIP +// asking for dstIP. An ARP probe (RFC 5227) has an unspecified srcIP. +func buildARPRequest(mac net.HardwareAddr, srcIP, dstIP netip.Addr) []byte { + frame := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(mac), + DstAddr: tcpip.LinkAddress("\xff\xff\xff\xff\xff\xff"), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(frame[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPRequest) + copy(arp.HardwareAddressSender(), mac) + src, dst := srcIP.As4(), dstIP.As4() + copy(arp.ProtocolAddressSender(), src[:]) + copy(arp.ProtocolAddressTarget(), dst[:]) + return frame +} + +// TestARPOnlyForOurAddresses checks that we answer ARP only for our own +// addresses. In particular the guest's DHCP client probes its offered +// address (RFC 5227) and declines the lease if anything answers; macOS does +// this, and a promiscuous gVisor NIC would answer for every address. +func TestARPOnlyForOurAddresses(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + replies := make(chan netip.Addr, 16) + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + arp := header.ARP(frame[header.EthernetMinimumSize:]) + if arp.IsValid() && arp.Op() == header.ARPReply { + replies <- netip.AddrFrom4([4]byte(arp.ProtocolAddressSender())) + } + } + }() + + mac := DefaultGuestMAC + unspec := netip.AddrFrom4([4]byte{}) + for _, tt := range []struct { + src, dst netip.Addr + want bool + }{ + {unspec, DefaultGuestIP, false}, // DHCP client's duplicate address probe + {DefaultGuestIP, DefaultGuestIP, false}, // gratuitous ARP + {DefaultGuestIP, netip.MustParseAddr("192.168.64.77"), false}, + {DefaultGuestIP, DefaultGatewayIP, true}, + {DefaultGuestIP, IMDSAddr, true}, + {DefaultGuestIP, DNSAddr, true}, + } { + if err := peer.WriteFrame(buildARPRequest(mac, tt.src, tt.dst)); err != nil { + t.Fatal(err) + } + var got bool + select { + case from := <-replies: + got = true + if from != tt.dst { + t.Errorf("ARP for %v: reply claims %v", tt.dst, from) + } + case <-time.After(300 * time.Millisecond): + } + if got != tt.want { + t.Errorf("ARP from %v for %v: replied = %v, want %v", tt.src, tt.dst, got, tt.want) + } + } +} diff --git a/guestnet/close_test.go b/guestnet/close_test.go new file mode 100644 index 0000000..81840b3 --- /dev/null +++ b/guestnet/close_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildTCPSYNFrame assembles an Ethernet/IPv4/TCP SYN frame with checksums. +func buildTCPSYNFrame(srcMAC, dstMAC net.HardwareAddr, src, dst netip.AddrPort) []byte { + ipLen := header.IPv4MinimumSize + header.TCPMinimumSize + frame := make([]byte, header.EthernetMinimumSize+ipLen) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + srcIP, dstIP := tcpip.AddrFrom4(src.Addr().As4()), tcpip.AddrFrom4(dst.Addr().As4()) + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.TCPProtocolNumber), + SrcAddr: srcIP, + DstAddr: dstIP, + }) + ip.SetChecksum(^ip.CalculateChecksum()) + tcp := header.TCP(ip.Payload()) + tcp.Encode(&header.TCPFields{ + SrcPort: src.Port(), + DstPort: dst.Port(), + SeqNum: 1000, + DataOffset: header.TCPMinimumSize, + Flags: header.TCPFlagSyn, + WindowSize: 65535, + }) + xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, srcIP, dstIP, header.TCPMinimumSize) + tcp.SetChecksum(^checksum.Checksum(tcp, xsum)) + return frame +} + +// TestCloseDuringHandshake checks that Close doesn't wait for a TCP +// handshake the guest will never finish. A guest that is stopped just after +// sending a SYN (the VM stopping mid-connection) leaves the forwarder waiting +// in the handshake; Close used to wait for gVisor's handshake timeout (30s +// or more) before tearing down. +func TestCloseDuringHandshake(t *testing.T) { + target, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer target.Close() + accepted := make(chan net.Conn, 1) + go func() { + c, err := target.Accept() + if err == nil { + accepted <- c + } + }() + + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Dialer: anyDialer{tcpTarget: target.Addr().String()}, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + // Answer ARP for the guest's address, as a real guest would, so the + // SYN-ACK goes out, but never ACK it: the guest has stopped. + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + req := header.ARP(frame[header.EthernetMinimumSize:]) + if !req.IsValid() || req.Op() != header.ARPRequest { + continue + } + reply := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(reply).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(DefaultGuestMAC), + DstAddr: tcpip.LinkAddress(req.HardwareAddressSender()), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(reply[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPReply) + copy(arp.HardwareAddressSender(), DefaultGuestMAC) + copy(arp.ProtocolAddressSender(), req.ProtocolAddressTarget()) + copy(arp.HardwareAddressTarget(), req.HardwareAddressSender()) + copy(arp.ProtocolAddressTarget(), req.ProtocolAddressSender()) + peer.WriteFrame(reply) + } + }() + + src := netip.AddrPortFrom(DefaultGuestIP, 40000) + dst := netip.MustParseAddrPort("198.51.100.7:443") + if err := peer.WriteFrame(buildTCPSYNFrame(DefaultGuestMAC, DefaultGatewayMAC, src, dst)); err != nil { + t.Fatal(err) + } + select { + case c := <-accepted: + defer c.Close() + case <-time.After(5 * time.Second): + t.Fatal("egress dial never happened") + } + time.Sleep(100 * time.Millisecond) // let the forwarder enter the handshake + + done := make(chan struct{}) + t0 := time.Now() + go func() { + n.Close() + close(done) + }() + select { + case <-done: + t.Logf("Close took %v", time.Since(t0)) + case <-time.After(5 * time.Second): + t.Fatal("Close waited for the guest's unfinished TCP handshake") + } +} diff --git a/guestnet/dhcp.go b/guestnet/dhcp.go new file mode 100644 index 0000000..ec4d25b --- /dev/null +++ b/guestnet/dhcp.go @@ -0,0 +1,198 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// handleDHCP answers DHCPv4 requests found in frame. It reports whether the +// frame was a DHCP request (and so must not be injected into the stack). +// +// DHCP is answered outside gVisor because the client has no address yet and +// the reply rules (unicast to the client MAC vs. broadcast) are about the +// link layer, which is simplest to control when we build the frame ourselves. +func (n *Net) handleDHCP(frame []byte) bool { + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if len(ip) < header.IPv4MinimumSize || !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return false + } + if ip.More() || ip.FragmentOffset() != 0 { + return false + } + udpPkt := header.UDP(ip.Payload()) + if len(udpPkt) < header.UDPMinimumSize || udpPkt.DestinationPort() != dhcpv4.ServerPort { + return false + } + req, err := dhcpv4.FromBytes(udpPkt.Payload()) + if err != nil { + n.cfg.Logf("guestnet: dhcp: bad request: %v", err) + return true + } + if req.OpCode != dhcpv4.OpcodeBootRequest { + return true + } + + var replyType dhcpv4.MessageType + switch req.MessageType() { + case dhcpv4.MessageTypeDiscover: + replyType = dhcpv4.MessageTypeOffer + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-discover", Msg: req.ClientHWAddr.String()}) + case dhcpv4.MessageTypeRequest: + // A REQUEST for some other address (a lease carried over from + // elsewhere) gets a NAK so the client starts over and takes ours. + if want := req.RequestedIPAddress(); want != nil && !want.IsUnspecified() && !want.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else if !req.ClientIPAddr.IsUnspecified() && !req.ClientIPAddr.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else { + replyType = dhcpv4.MessageTypeAck + } + case dhcpv4.MessageTypeInform: + replyType = dhcpv4.MessageTypeAck + case dhcpv4.MessageTypeDecline: + n.cfg.Logf("guestnet: dhcp: guest declined %v", n.cfg.GuestIP) + return true + default: + return true + } + + resp, err := n.buildDHCPReply(req, replyType) + if err != nil { + n.cfg.Logf("guestnet: dhcp: build %v: %v", replyType, err) + return true + } + + // Where to send it (RFC 2131 section 4.1). A NAK is always broadcast. + // A client that already has an address (renewing) is answered unicast. + // Otherwise honor the broadcast flag; a client without the flag is + // answered unicast to its MAC with its new address as the IP destination. + dstMAC := net.HardwareAddr(req.ClientHWAddr) + dstIP := n.cfg.GuestIP + switch { + case replyType == dhcpv4.MessageTypeNak, req.IsBroadcast() && req.ClientIPAddr.IsUnspecified(): + dstMAC = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + dstIP = netipBroadcast + case !req.ClientIPAddr.IsUnspecified(): + if a, ok := netipFromIP(req.ClientIPAddr); ok { + dstIP = a + } + } + + srcIP := n.cfg.GatewayIP + if replyType == dhcpv4.MessageTypeAck || replyType == dhcpv4.MessageTypeOffer { + if replyType == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeRequest { + n.dhcpLeasedOnce.Do(func() { close(n.dhcpLeased) }) + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-lease", Msg: fmt.Sprintf("%v -> %v", req.ClientHWAddr, n.cfg.GuestIP)}) + } + } + n.writeFrame(buildUDPFrame(n.cfg.GatewayMAC, dstMAC, srcIP.As4(), dstIP.As4(), dhcpv4.ServerPort, dhcpv4.ClientPort, resp.ToBytes())) + return true +} + +func (n *Net) buildDHCPReply(req *dhcpv4.DHCPv4, mt dhcpv4.MessageType) (*dhcpv4.DHCPv4, error) { + gw := net.IP(n.cfg.GatewayIP.AsSlice()) + mods := []dhcpv4.Modifier{ + dhcpv4.WithMessageType(mt), + dhcpv4.WithServerIP(gw), + dhcpv4.WithOption(dhcpv4.OptServerIdentifier(gw)), + } + if mt != dhcpv4.MessageTypeNak { + var mtu [2]byte + binary.BigEndian.PutUint16(mtu[:], uint16(n.cfg.MTU)) + mods = append(mods, + dhcpv4.WithYourIP(n.cfg.GuestIP.AsSlice()), + dhcpv4.WithOption(dhcpv4.OptSubnetMask(net.CIDRMask(n.cfg.Subnet.Bits(), 32))), + dhcpv4.WithOption(dhcpv4.OptRouter(gw)), + dhcpv4.WithOption(dhcpv4.OptDNS(net.IP(DNSAddr.AsSlice()))), + dhcpv4.WithOption(dhcpv4.OptBroadcastAddress(broadcastOf(n.cfg.Subnet))), + dhcpv4.WithOption(dhcpv4.OptIPAddressLeaseTime(n.cfg.LeaseTime)), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRenewTimeValue, secs(n.cfg.LeaseTime/2))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRebindingTimeValue, secs(n.cfg.LeaseTime*7/8))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, mtu[:])), + ) + if mt == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeInform { + // INFORM replies must not carry a lease. + mods = append(mods, func(d *dhcpv4.DHCPv4) { + d.YourIPAddr = net.IPv4zero + d.DeleteOption(dhcpv4.OptionIPAddressLeaseTime) + d.DeleteOption(dhcpv4.OptionRenewTimeValue) + d.DeleteOption(dhcpv4.OptionRebindingTimeValue) + }) + } + } + return dhcpv4.NewReplyFromRequest(req, mods...) +} + +func secs(d time.Duration) []byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], uint32(d/time.Second)) + return b[:] +} + +// buildUDPFrame assembles an Ethernet/IPv4/UDP frame with checksums. +func buildUDPFrame(srcMAC, dstMAC net.HardwareAddr, srcIP, dstIP [4]byte, srcPort, dstPort uint16, payload []byte) []byte { + udpLen := header.UDPMinimumSize + len(payload) + ipLen := header.IPv4MinimumSize + udpLen + frame := make([]byte, header.EthernetMinimumSize+ipLen) + + eth := header.Ethernet(frame) + eth.Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.UDPProtocolNumber), + SrcAddr: tcpip.AddrFrom4(srcIP), + DstAddr: tcpip.AddrFrom4(dstIP), + }) + ip.SetChecksum(^ip.CalculateChecksum()) + + u := header.UDP(ip.Payload()) + u.Encode(&header.UDPFields{ + SrcPort: srcPort, + DstPort: dstPort, + Length: uint16(udpLen), + }) + copy(u.Payload(), payload) + xsum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, tcpip.AddrFrom4(srcIP), tcpip.AddrFrom4(dstIP), uint16(udpLen)) + xsum = checksum.Checksum(payload, xsum) + u.SetChecksum(^u.CalculateChecksum(xsum)) + return frame +} + +// parseUDPFrame extracts the addressing and payload of an IPv4 UDP frame. +// It is used by tests and by the DHCP path's counterpart in a fake guest. +func parseUDPFrame(frame []byte) (src, dst [4]byte, srcPort, dstPort uint16, payload []byte, ok bool) { + if len(frame) < header.EthernetMinimumSize+header.IPv4MinimumSize+header.UDPMinimumSize { + return + } + eth := header.Ethernet(frame) + if eth.Type() != header.IPv4ProtocolNumber { + return + } + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return + } + u := header.UDP(ip.Payload()) + if len(u) < header.UDPMinimumSize { + return + } + return ip.SourceAddress().As4(), ip.DestinationAddress().As4(), u.SourcePort(), u.DestinationPort(), bytes.Clone(u.Payload()), true +} diff --git a/guestnet/dns.go b/guestnet/dns.go new file mode 100644 index 0000000..ccd2c76 --- /dev/null +++ b/guestnet/dns.go @@ -0,0 +1,299 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +// DefaultUpstreams are the resolvers used when none are configured. They +// match corp CI's defaults. +var DefaultUpstreams = []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:53"), + netip.MustParseAddrPort("1.1.1.1:53"), +} + +// dnsQueryTimeout bounds one upstream exchange. +const dnsQueryTimeout = 5 * time.Second + +// dnsIdleTimeout ends a guest DNS session (one source port) after this long +// without a query; the guest's resolver typically uses a fresh port per +// query anyway. +const dnsIdleTimeout = 10 * time.Second + +// Resolver answers DNS queries on the guest's behalf. +type Resolver interface { + // Exchange sends the wire-format query and returns the wire-format + // response and a description of who answered (for logs). + Exchange(ctx context.Context, query []byte) (resp []byte, upstream string, err error) +} + +// UpstreamResolver forwards queries to upstream servers in order, over UDP +// with TCP fallback on truncation. +type UpstreamResolver struct { + Servers []netip.AddrPort + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// NewUpstreamResolver returns an UpstreamResolver for servers. +func NewUpstreamResolver(servers ...netip.AddrPort) *UpstreamResolver { + return &UpstreamResolver{Servers: servers} +} + +// Exchange implements Resolver. +func (r *UpstreamResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + d := r.Dialer + if d == nil { + d = &net.Dialer{} + } + var lastErr error + for _, s := range r.Servers { + resp, err := exchangeUDP(ctx, d, s, query) + if err == nil && len(resp) >= 12 && resp[2]&0x02 != 0 { // TC bit set + resp, err = exchangeTCP(ctx, d, s, query) + } + if err == nil { + return resp, s.String(), nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("no upstream resolvers configured") + } + return nil, "", lastErr +} + +func exchangeUDP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "udp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if _, err := conn.Write(query); err != nil { + return nil, err + } + buf := make([]byte, 4096) + for { + m, err := conn.Read(buf) + if err != nil { + return nil, err + } + // Match the transaction ID; ignore stray datagrams. + if m >= 2 && len(query) >= 2 && buf[0] == query[0] && buf[1] == query[1] { + return buf[:m], nil + } + } +} + +func exchangeTCP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "tcp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if err := writeDNSTCP(conn, query); err != nil { + return nil, err + } + return readDNSTCP(conn) +} + +func writeDNSTCP(w io.Writer, msg []byte) error { + buf := make([]byte, 2+len(msg)) + binary.BigEndian.PutUint16(buf, uint16(len(msg))) + copy(buf[2:], msg) + _, err := w.Write(buf) + return err +} + +func readDNSTCP(r io.Reader) ([]byte, error) { + var lenBuf [2]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, err + } + msg := make([]byte, binary.BigEndian.Uint16(lenBuf[:])) + if _, err := io.ReadFull(r, msg); err != nil { + return nil, err + } + return msg, nil +} + +// serveDNSUDP answers queries arriving on one guest UDP session. +func (n *Net) serveDNSUDP(conn net.Conn, src, dst netip.AddrPort) error { + buf := make([]byte, 4096) + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + resp := n.resolve(buf[:m], src, dst, "udp") + if resp != nil { + if _, err := conn.Write(resp); err != nil { + return err + } + } + } +} + +// serveDNSTCP answers length-prefixed queries on one guest TCP connection. +func (n *Net) serveDNSTCP(conn net.Conn, src, dst netip.AddrPort) error { + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + q, err := readDNSTCP(conn) + if err != nil { + return nil + } + resp := n.resolve(q, src, dst, "tcp") + if resp == nil { + return nil + } + if err := writeDNSTCP(conn, resp); err != nil { + return err + } + } +} + +// resolve answers one query via the configured Resolver and logs it. On +// upstream failure it returns a SERVFAIL so the guest doesn't wait. +func (n *Net) resolve(query []byte, src, dst netip.AddrPort, proto string) []byte { + ev := DNSEvent{Time: n.cfg.Now(), Src: src, Dst: dst, Proto: proto} + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + ev.Err = "unparseable query" + n.DNS.Add(ev) + return nil + } + if q, err := p.Question(); err == nil { + ev.Name = q.Name.String() + ev.Type = q.Type.String() + } + + ctx, cancel := context.WithTimeout(n.ctx, dnsQueryTimeout) + defer cancel() + resp, upstream, err := n.cfg.Resolver.Exchange(ctx, query) + ev.Duration = n.cfg.Now().Sub(ev.Time) + ev.Upstream = upstream + if err != nil { + ev.Err = err.Error() + ev.RCode = dnsmessage.RCodeServerFailure.String() + n.DNS.Add(ev) + return servfail(hdr, query) + } + var rp dnsmessage.Parser + if rh, err := rp.Start(resp); err == nil { + ev.RCode = rh.RCode.String() + rp.SkipAllQuestions() + if answers, err := rp.AllAnswers(); err == nil { + ev.Answers = len(answers) + } + } + n.DNS.Add(ev) + return resp +} + +// servfail builds a SERVFAIL response echoing the query's ID and question. +func servfail(hdr dnsmessage.Header, query []byte) []byte { + resp := make([]byte, len(query)) + copy(resp, query) + if len(resp) < 12 { + return nil + } + resp[2] = 0x80 | (resp[2] & 0x79) // QR=1, keep opcode and RD + resp[3] = byte(dnsmessage.RCodeServerFailure) + // Zero the answer, authority and additional counts; keep QDCOUNT. + binary.BigEndian.PutUint16(resp[6:], 0) + binary.BigEndian.PutUint16(resp[8:], 0) + binary.BigEndian.PutUint16(resp[10:], 0) + _ = hdr + return resp[:12+questionLen(resp[12:])] +} + +// questionLen returns the encoded length of the first question section +// entry in b, or 0 if malformed. +func questionLen(b []byte) int { + i := 0 + for i < len(b) { + l := int(b[i]) + if l == 0 { + i++ + break + } + if l&0xC0 == 0xC0 { + i += 2 + break + } + i += 1 + l + } + if i+4 > len(b) { + return 0 + } + return i + 4 +} + +// StaticResolver answers every A query with a fixed address and everything +// else with NXDOMAIN. It exists for tests and for air-gapped experiments. +type StaticResolver struct { + Addr netip.Addr +} + +// Exchange implements Resolver. +func (r StaticResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + return nil, "", err + } + q, err := p.Question() + if err != nil { + return nil, "", err + } + rh := dnsmessage.Header{ID: hdr.ID, Response: true, Authoritative: true, RecursionDesired: hdr.RecursionDesired, RecursionAvailable: true} + b := dnsmessage.NewBuilder(nil, rh) + b.EnableCompression() + if err := b.StartQuestions(); err != nil { + return nil, "", err + } + if err := b.Question(q); err != nil { + return nil, "", err + } + if q.Type == dnsmessage.TypeA && r.Addr.Is4() { + if err := b.StartAnswers(); err != nil { + return nil, "", err + } + if err := b.AResource(dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 60}, dnsmessage.AResource{A: r.Addr.As4()}); err != nil { + return nil, "", err + } + } else { + rh.RCode = dnsmessage.RCodeNameError + } + out, err := b.Finish() + if err != nil { + return nil, "", fmt.Errorf("build response: %w", err) + } + if rh.RCode != 0 { + out[3] = out[3]&0xF0 | byte(rh.RCode) + } + return out, "static", nil +} diff --git a/guestnet/guestnet.go b/guestnet/guestnet.go new file mode 100644 index 0000000..12e3ae2 --- /dev/null +++ b/guestnet/guestnet.go @@ -0,0 +1,468 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnet is a private userspace network for one guest VM. +// +// The guest's virtual NIC hands us raw Ethernet frames (on macOS via a +// VZFileHandleNetworkDeviceAttachment over a SOCK_DGRAM socketpair; in tests +// via an in-memory link). Every frame terminates in a gVisor netstack owned by +// this package, so a guest can only ever reach what this package chooses to +// connect it to: +// +// - The gateway, IMDS and DNS addresses live on our NIC, so ARP and ICMP +// echo for them are answered by gVisor itself. +// - DHCP is answered directly from the frame loop with a fixed lease, so a +// memory snapshot restored in another slot sees the same addresses. +// - Every TCP SYN and UDP flow is dispatched by destination: a registered +// [Service] (IMDS, the caller's NFS or proxies), an intercepted protocol +// (DNS on any port 53, NTP on any port 123), or egress dialed from the +// host. Each flow is logged. +// - The host reaches the guest only by dialing into this stack with +// [Net.DialGuest]; there is no host interface routed to the guest. +// +// One [Net] per VM. Nothing here is process-global, so many guests can share +// a process without seeing each other. +package guestnet + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "sync" + "time" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "tailscale.com/types/logger" +) + +// Well-known addresses. They are the same for every guest because each guest +// has its own L2 domain; that is what makes memory snapshots slot-independent. +var ( + // DefaultSubnet is the guest-visible subnet. + DefaultSubnet = netip.MustParsePrefix("192.168.64.0/24") + // DefaultGatewayIP is our address on the guest subnet and the guest's + // default router. + DefaultGatewayIP = netip.MustParseAddr("192.168.64.1") + // DefaultGuestIP is the one address the DHCP server hands out. + DefaultGuestIP = netip.MustParseAddr("192.168.64.2") + // IMDSAddr is the metadata service address, matching cloud convention and + // the address corp CI already uses. + IMDSAddr = netip.MustParseAddr("169.254.169.254") + // DNSAddr is the resolver address handed out by DHCP, matching corp CI. + DNSAddr = netip.MustParseAddr("169.254.169.253") + + // DefaultGatewayMAC is our MAC. It is constant so a restored guest's ARP + // cache stays valid. + DefaultGatewayMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x01} + // DefaultGuestMAC is the guest MAC used when an image doesn't specify one. + DefaultGuestMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} +) + +// DefaultDeny is the egress deny list applied when [Config.Deny] is nil: the +// tailnet CGNAT range and its IPv6 counterpart (so a guest can't reach the +// host's tailnet), loopback, and the link-local range (IMDS and DNS are +// reached through the service table before policy applies). +var DefaultDeny = []netip.Prefix{ + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("fd7a:115c:a1e0::/48"), + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("0.0.0.0/8"), +} + +const nicID tcpip.NICID = 1 + +// Dialer dials egress connections on behalf of the guest. *net.Dialer +// satisfies it. +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +// Config configures a guest network. The zero value is usable with a link. +type Config struct { + // Link carries raw Ethernet frames to and from the guest. Required. + Link FrameLink + + // GuestMAC is the guest's MAC address, used as the DHCP client identity + // check and as the destination of DHCP replies. Defaults to + // [DefaultGuestMAC]. + GuestMAC net.HardwareAddr + // GatewayMAC is our MAC. Defaults to [DefaultGatewayMAC]. + GatewayMAC net.HardwareAddr + + // Subnet, GatewayIP and GuestIP default to the Default* values. + Subnet netip.Prefix + GatewayIP netip.Addr + GuestIP netip.Addr + + // MTU is the link MTU advertised to the guest via DHCP. Defaults to 1500. + MTU uint32 + + // LeaseTime is the DHCP lease duration. Defaults to 24h. Restored + // snapshots renew against this server, so long leases are fine. + LeaseTime time.Duration + + // Dialer dials egress TCP and UDP. Defaults to a net.Dialer. + Dialer Dialer + + // Resolver answers the guest's DNS queries. Defaults to + // [NewUpstreamResolver] with [DefaultUpstreams]. + Resolver Resolver + + // Deny lists destinations the guest may not reach via egress. nil means + // [DefaultDeny]; an empty non-nil slice denies nothing. + Deny []netip.Prefix + + // Services maps guest-visible destinations to handlers. The destination + // address must be one of ours (gateway, IMDS, DNS) or the lookup happens + // before egress policy and would let a guest hijack a public address. + Services map[netip.AddrPort]Service + + // Now returns the current time, for NTP and logs. Defaults to time.Now. + Now func() time.Time + + // Logf logs. Defaults to logger.Discard. + Logf logger.Logf +} + +func (c *Config) withDefaults() { + if c.GuestMAC == nil { + c.GuestMAC = DefaultGuestMAC + } + if c.GatewayMAC == nil { + c.GatewayMAC = DefaultGatewayMAC + } + if !c.Subnet.IsValid() { + c.Subnet = DefaultSubnet + } + if !c.GatewayIP.IsValid() { + c.GatewayIP = DefaultGatewayIP + } + if !c.GuestIP.IsValid() { + c.GuestIP = DefaultGuestIP + } + if c.MTU == 0 { + c.MTU = 1500 + } + if c.LeaseTime == 0 { + c.LeaseTime = 24 * time.Hour + } + if c.Dialer == nil { + c.Dialer = &net.Dialer{Timeout: 10 * time.Second} + } + if c.Resolver == nil { + c.Resolver = NewUpstreamResolver(DefaultUpstreams...) + } + if c.Deny == nil { + c.Deny = DefaultDeny + } + if c.Now == nil { + c.Now = time.Now + } + if c.Logf == nil { + c.Logf = logger.Discard + } +} + +// Net is one guest's network. Create it with [New] and shut it down with +// [Net.Close]. +type Net struct { + cfg Config + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + stack *stack.Stack + linkEP *channel.Endpoint + + // Flows is the TCP/UDP flow log. + Flows *Log[Flow] + // DNS is the DNS query log. + DNS *Log[DNSEvent] + // Events records notable network events (DHCP lease, blocked flows). + Events *Log[Event] + + // dhcpLeased is closed the first time the guest acknowledges a lease. + dhcpLeased chan struct{} + dhcpLeasedOnce sync.Once + + mu sync.Mutex + services map[netip.AddrPort]Service +} + +// New starts a guest network on cfg.Link. It returns once the stack is up; +// frames are pumped on background goroutines until [Net.Close]. +func New(cfg Config) (*Net, error) { + if cfg.Link == nil { + return nil, errors.New("guestnet: Config.Link is required") + } + cfg.withDefaults() + if !cfg.Subnet.Contains(cfg.GatewayIP) || !cfg.Subnet.Contains(cfg.GuestIP) { + return nil, fmt.Errorf("guestnet: gateway %v and guest %v must be within %v", cfg.GatewayIP, cfg.GuestIP, cfg.Subnet) + } + if !cfg.GatewayIP.Is4() { + return nil, errors.New("guestnet: only IPv4 is supported") + } + + ctx, cancel := context.WithCancel(context.Background()) + n := &Net{ + cfg: cfg, + ctx: ctx, + cancel: cancel, + Flows: NewLog[Flow](1024), + DNS: NewLog[DNSEvent](1024), + Events: NewLog[Event](256), + dhcpLeased: make(chan struct{}), + services: make(map[netip.AddrPort]Service), + } + for ap, svc := range cfg.Services { + n.services[ap] = svc + } + + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + sack := tcpip.TCPSACKEnabled(true) + if err := s.SetTransportProtocolOption(tcp.ProtocolNumber, &sack); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: enable SACK: %v", err) + } + + // The channel endpoint carries whole Ethernet frames; the ethernet wrapper + // adds and strips link headers so gVisor handles ARP for us. + n.linkEP = channel.New(4096, cfg.MTU, tcpip.LinkAddress(cfg.GatewayMAC)) + if err := s.CreateNIC(nicID, ethernet.New(n.linkEP)); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: CreateNIC: %v", err) + } + // Accept packets for any destination so the forwarders see every flow, + // and allow replies from any source address. + if err := s.SetPromiscuousMode(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetPromiscuousMode: %v", err) + } + if err := s.SetSpoofing(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetSpoofing: %v", err) + } + + // Our addresses. gVisor answers ARP and ICMP echo for these. The + // link-local ones are on the NIC precisely so the guest's on-link ARP for + // them (169.254/16 is APIPA and never routed) gets an answer. + addrs := []struct { + addr netip.Addr + bits int + }{ + {cfg.GatewayIP, cfg.Subnet.Bits()}, + {IMDSAddr, 32}, + {DNSAddr, 32}, + } + for _, a := range addrs { + pa := tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom4(a.addr.As4()), + PrefixLen: a.bits, + }, + } + if err := s.AddProtocolAddress(nicID, pa, stack.AddressProperties{}); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: AddProtocolAddress(%v): %v", a.addr, err) + } + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + NIC: nicID, + }}) + + tcpFwd := tcp.NewForwarder(s, 0, 4096, n.handleTCP) + s.SetTransportProtocolHandler(tcp.ProtocolNumber, tcpFwd.HandlePacket) + udpFwd := udp.NewForwarder(s, n.handleUDP) + s.SetTransportProtocolHandler(udp.ProtocolNumber, udpFwd.HandlePacket) + + n.stack = s + n.wg.Add(2) + go n.pumpFromGuest() + go n.pumpToGuest() + return n, nil +} + +// Close stops the network and closes the link. Open flows are torn down. +func (n *Net) Close() error { + n.cancel() + err := n.cfg.Link.Close() + n.linkEP.Close() + // Close every endpoint before waiting for the flow goroutines: one may be + // in a TCP handshake with a guest that will never finish it (the VM + // stopped just after its SYN), which only aborts when its endpoint is + // closed, or after gVisor's handshake timeout of 30s or more. + n.stack.Close() + n.wg.Wait() + n.stack.Destroy() + return err +} + +// GuestIP returns the address the guest is leased. +func (n *Net) GuestIP() netip.Addr { return n.cfg.GuestIP } + +// GatewayIP returns our address on the guest subnet. +func (n *Net) GatewayIP() netip.Addr { return n.cfg.GatewayIP } + +// Leased returns a channel that is closed once the guest has acknowledged a +// DHCP lease. For a restored snapshot this may not fire until the guest +// renews, so callers should also consider the guest reachable if +// [Net.DialGuest] succeeds. +func (n *Net) Leased() <-chan struct{} { return n.dhcpLeased } + +// SetService registers or replaces the handler for a guest-visible +// destination. Passing a nil svc removes it. +func (n *Net) SetService(dst netip.AddrPort, svc Service) { + n.mu.Lock() + defer n.mu.Unlock() + if svc == nil { + delete(n.services, dst) + } else { + n.services[dst] = svc + } +} + +func (n *Net) service(dst netip.AddrPort) (Service, bool) { + n.mu.Lock() + defer n.mu.Unlock() + svc, ok := n.services[dst] + return svc, ok +} + +// DialGuest opens a TCP connection from the gateway address to port on the +// guest. It is how the host reaches sshd in the guest. +func (n *Net) DialGuest(ctx context.Context, port uint16) (net.Conn, error) { + local := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GatewayIP.As4())} + remote := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GuestIP.As4()), Port: port} + return gonet.DialTCPWithBind(ctx, n.stack, local, remote, ipv4.ProtocolNumber) +} + +// pumpFromGuest reads frames from the link, answers DHCP directly, and injects +// everything else into gVisor. +func (n *Net) pumpFromGuest() { + defer n.wg.Done() + buf := make([]byte, 65536+header.EthernetMinimumSize) + for { + m, err := n.cfg.Link.ReadFrame(buf) + if err != nil { + if n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link read: %v", err) + n.cancel() + } + return + } + frame := buf[:m] + if len(frame) < header.EthernetMinimumSize { + continue + } + eth := header.Ethernet(frame) + switch eth.Type() { + case header.IPv4ProtocolNumber: + if n.handleDHCP(frame) { + continue + } + case header.ARPProtocolNumber: + if !n.arpForUs(frame[header.EthernetMinimumSize:]) { + continue + } + default: + // IPv6 and everything else is dropped for now. + continue + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithData(append([]byte(nil), frame...)), + }) + // The ethernet wrapper parses the real EtherType from the frame; the + // protocol argument is ignored. + n.linkEP.InjectInbound(eth.Type(), pkt) + pkt.DecRef() + } +} + +// arpForUs reports whether an ARP packet from the guest should reach gVisor: +// replies (to our requests for the guest's address), and requests for one of +// our own addresses. Our NIC is promiscuous, so gVisor would otherwise answer +// requests for every address, including the guest's own; the guest's DHCP +// client then sees its duplicate address probe answered and declines the +// lease. +func (n *Net) arpForUs(b []byte) bool { + arp := header.ARP(b) + if !arp.IsValid() { + return false + } + if arp.Op() != header.ARPRequest { + return true + } + target := netip.AddrFrom4([4]byte(arp.ProtocolAddressTarget())) + return target == n.cfg.GatewayIP || target == IMDSAddr || target == DNSAddr +} + +// pumpToGuest writes frames produced by gVisor to the link. +func (n *Net) pumpToGuest() { + defer n.wg.Done() + for { + pkt := n.linkEP.ReadContext(n.ctx) + if pkt == nil { + return + } + v := pkt.ToView() + err := n.cfg.Link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + if err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } + } +} + +// writeFrame sends a frame we built ourselves (DHCP replies) to the guest. +func (n *Net) writeFrame(frame []byte) { + if err := n.cfg.Link.WriteFrame(frame); err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } +} + +// egressAllowed reports whether policy lets the guest reach dst. +func (n *Net) egressAllowed(dst netip.Addr) bool { + if dst.IsMulticast() || dst.IsUnspecified() || dst == netip.AddrFrom4([4]byte{255, 255, 255, 255}) { + return false + } + if n.cfg.Subnet.Contains(dst) { + // Only the gateway (via services) and the guest itself exist here. + return false + } + for _, p := range n.cfg.Deny { + if p.Contains(dst) { + return false + } + } + return true +} + +func addrFromTCPIP(a tcpip.Address) netip.Addr { + switch a.Len() { + case 4: + return netip.AddrFrom4(a.As4()) + case 16: + return netip.AddrFrom16(a.As16()) + } + return netip.Addr{} +} diff --git a/guestnet/guestnet_test.go b/guestnet/guestnet_test.go new file mode 100644 index 0000000..40fdd03 --- /dev/null +++ b/guestnet/guestnet_test.go @@ -0,0 +1,544 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "strings" + "testing" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tailscale/tb/guestnet/guestnettest" + "golang.org/x/net/dns/dnsmessage" +) + +// anyDialer dials every address to a single local target, standing in for +// the internet in tests. +type anyDialer struct { + tcpTarget string + udpTarget string +} + +func (d anyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + var nd net.Dialer + switch { + case strings.HasPrefix(network, "tcp"): + return nd.DialContext(ctx, network, d.tcpTarget) + case strings.HasPrefix(network, "udp"): + return nd.DialContext(ctx, network, d.udpTarget) + } + return nil, fmt.Errorf("unsupported network %q", network) +} + +type env struct { + n *Net + guest *guestnettest.Guest + peer *MemLink +} + +func newEnv(t testing.TB, cfg Config) *env { + t.Helper() + host, peer := NewMemLinkPair() + cfg.Link = host + cfg.Logf = t.Logf + n, err := New(cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { n.Close() }) + g := guestnettest.New(t, peer, guestnettest.Config{ + IP: n.cfg.GuestIP, + Subnet: n.cfg.Subnet, + Gateway: n.cfg.GatewayIP, + MAC: n.cfg.GuestMAC, + }) + return &env{n: n, guest: g, peer: peer} +} + +func testContext(t testing.TB) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + return ctx +} + +// dhcpExchange writes a DHCP message from an addressless client and returns +// the reply. +func dhcpExchange(t testing.TB, peer *MemLink, mac net.HardwareAddr, msg *dhcpv4.DHCPv4) *dhcpv4.DHCPv4 { + t.Helper() + frame := buildUDPFrame(mac, net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, [4]byte{}, [4]byte{255, 255, 255, 255}, dhcpv4.ClientPort, dhcpv4.ServerPort, msg.ToBytes()) + if err := peer.WriteFrame(frame); err != nil { + t.Fatal(err) + } + buf := make([]byte, 65536) + deadline := time.After(5 * time.Second) + for { + select { + case <-deadline: + t.Fatal("timed out waiting for DHCP reply") + default: + } + n, err := peer.ReadFrame(buf) + if err != nil { + t.Fatal(err) + } + _, _, srcPort, dstPort, payload, ok := parseUDPFrame(buf[:n]) + if !ok || srcPort != dhcpv4.ServerPort || dstPort != dhcpv4.ClientPort { + continue + } + reply, err := dhcpv4.FromBytes(payload) + if err != nil { + t.Fatalf("bad DHCP reply: %v", err) + } + return reply + } +} + +func TestDHCP(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf, MTU: 9000}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + mac := DefaultGuestMAC + disc, err := dhcpv4.NewDiscovery(mac, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + offer := dhcpExchange(t, peer, mac, disc) + if offer.MessageType() != dhcpv4.MessageTypeOffer { + t.Fatalf("got %v, want OFFER", offer.MessageType()) + } + if got := offer.YourIPAddr.String(); got != DefaultGuestIP.String() { + t.Errorf("yiaddr = %s, want %v", got, DefaultGuestIP) + } + if got := offer.Router(); len(got) != 1 || got[0].String() != DefaultGatewayIP.String() { + t.Errorf("router = %v, want %v", got, DefaultGatewayIP) + } + if got := offer.DNS(); len(got) != 1 || got[0].String() != DNSAddr.String() { + t.Errorf("dns = %v, want %v", got, DNSAddr) + } + if mtu := offer.Options.Get(dhcpv4.OptionInterfaceMTU); len(mtu) != 2 || binary.BigEndian.Uint16(mtu) != 9000 { + t.Errorf("mtu option = %x, want 9000", mtu) + } + if lt := offer.IPAddressLeaseTime(0); lt != 24*time.Hour { + t.Errorf("lease = %v, want 24h", lt) + } + + select { + case <-n.Leased(): + t.Fatal("Leased closed after OFFER; should wait for ACK") + default: + } + + req, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + ack := dhcpExchange(t, peer, mac, req) + if ack.MessageType() != dhcpv4.MessageTypeAck { + t.Fatalf("got %v, want ACK", ack.MessageType()) + } + select { + case <-n.Leased(): + case <-time.After(time.Second): + t.Fatal("Leased not closed after ACK") + } + + // A request for a foreign address is refused. + bad, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true), dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(net.IPv4(10, 1, 2, 3)))) + if err != nil { + t.Fatal(err) + } + if nak := dhcpExchange(t, peer, mac, bad); nak.MessageType() != dhcpv4.MessageTypeNak { + t.Errorf("got %v for foreign address, want NAK", nak.MessageType()) + } + + var kinds []string + for _, ev := range n.Events.Recent() { + kinds = append(kinds, ev.Kind) + } + if got := strings.Join(kinds, ","); got != "dhcp-discover,dhcp-lease" { + t.Errorf("events = %q", got) + } +} + +func TestTCPEgress(t *testing.T) { + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello %s", r.Host) + })} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go srv.Serve(ln) + + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: ln.Addr().String()}}) + ctx := testContext(t) + + dst := netip.MustParseAddrPort("93.184.216.34:80") + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET / HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "hello example.com" { + t.Fatalf("body = %q", body) + } + c.Close() + + flows := waitForClosedFlow(t, e.n, dst) + if flows.Kind != "egress" { + t.Errorf("kind = %q, want egress", flows.Kind) + } + if flows.BytesOut == 0 || flows.BytesIn == 0 { + t.Errorf("bytes in/out = %d/%d, want both > 0", flows.BytesIn, flows.BytesOut) + } + if flows.Src.Addr() != DefaultGuestIP { + t.Errorf("src = %v, want guest IP", flows.Src) + } +} + +func waitForClosedFlow(t testing.TB, n *Net, dst netip.AddrPort) Flow { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + for _, f := range n.Flows.Recent() { + if f.Dst == dst && f.Closed() { + return f + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("no closed flow to %v; flows: %+v", dst, n.Flows.Recent()) + return Flow{} +} + +func TestTCPBlocked(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ctx := testContext(t) + for _, dst := range []netip.AddrPort{ + netip.MustParseAddrPort("100.100.100.100:80"), // tailnet CGNAT + netip.MustParseAddrPort("192.168.64.7:80"), // guest subnet, nonexistent + netip.MustParseAddrPort("192.168.64.1:80"), // gateway with no service + } { + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Errorf("dial %v succeeded, want refusal", dst) + continue + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "blocked" { + t.Errorf("%v: kind = %q, want blocked", dst, f.Kind) + } + } +} + +func TestTCPDialError(t *testing.T) { + // Point egress at a closed port so the host-side dial fails; the guest + // must see a refused connection, not an accepted one. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: addr}}) + ctx := testContext(t) + dst := netip.MustParseAddrPort("93.184.216.34:443") + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Fatal("dial succeeded, want refusal") + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "egress" || f.Err == "" { + t.Errorf("flow = %+v, want egress with error", f) + } +} + +func TestService(t *testing.T) { + imds := NewListenerService("imds") + go http.Serve(imds, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "meta for %s", r.RemoteAddr) + })) + defer imds.Close() + + e := newEnv(t, Config{ + Dialer: anyDialer{}, + Services: map[netip.AddrPort]Service{netip.AddrPortFrom(IMDSAddr, 80): imds}, + }) + ctx := testContext(t) + dst := netip.AddrPortFrom(IMDSAddr, 80) + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial IMDS: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET /v0/meta HTTP/1.1\r\nHost: 169.254.169.254\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + if !strings.HasPrefix(string(body), "meta for 192.168.64.2:") { + t.Fatalf("body = %q", body) + } + c.Close() + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "service" || f.Service != "imds" { + t.Errorf("flow = %+v, want service imds", f) + } + + // A late-registered forwarding service on the gateway works too. + echo, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer echo.Close() + go func() { + for { + c, err := echo.Accept() + if err != nil { + return + } + go func() { io.Copy(c, c); c.Close() }() + } + }() + nfs := netip.AddrPortFrom(DefaultGatewayIP, 2049) + e.n.SetService(nfs, &ForwardService{ServiceName: "nfs", Network: "tcp", Address: echo.Addr().String()}) + c2, err := e.guest.DialTCP(ctx, nfs) + if err != nil { + t.Fatalf("guest dial nfs: %v", err) + } + defer c2.Close() + if _, err := c2.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c2, buf); err != nil || string(buf) != "ping" { + t.Fatalf("echo = %q, %v", buf, err) + } +} + +func TestDNS(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}, Resolver: StaticResolver{Addr: netip.MustParseAddr("10.9.8.7")}}) + + // Guest queries a public resolver; we intercept regardless of address. + dst := netip.MustParseAddrPort("8.8.8.8:53") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + q := dnsmessage.Message{ + Header: dnsmessage.Header{ID: 0xBEEF, RecursionDesired: true}, + Questions: []dnsmessage.Question{{Name: dnsmessage.MustNewName("example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, + } + qb, err := q.Pack() + if err != nil { + t.Fatal(err) + } + if _, err := c.Write(qb); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 4096) + n, err := c.Read(buf) + if err != nil { + t.Fatalf("read DNS response: %v", err) + } + var m dnsmessage.Message + if err := m.Unpack(buf[:n]); err != nil { + t.Fatal(err) + } + if m.ID != 0xBEEF || len(m.Answers) != 1 { + t.Fatalf("response = %+v", m) + } + if a, ok := m.Answers[0].Body.(*dnsmessage.AResource); !ok || a.A != [4]byte{10, 9, 8, 7} { + t.Fatalf("answer = %+v", m.Answers[0].Body) + } + + // Also over TCP, to the advertised resolver address. + tc, err := e.guest.DialTCP(testContext(t), netip.AddrPortFrom(DNSAddr, 53)) + if err != nil { + t.Fatal(err) + } + defer tc.Close() + if err := writeDNSTCP(tc, qb); err != nil { + t.Fatal(err) + } + tc.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := readDNSTCP(tc) + if err != nil { + t.Fatalf("TCP DNS: %v", err) + } + if err := m.Unpack(resp); err != nil || len(m.Answers) != 1 { + t.Fatalf("TCP response = %+v, %v", m, err) + } + + events := e.n.DNS.Recent() + if len(events) != 2 { + t.Fatalf("got %d DNS events, want 2: %+v", len(events), events) + } + for _, ev := range events { + if ev.Name != "example.com." || ev.Type != "TypeA" || ev.Answers != 1 || ev.Upstream != "static" { + t.Errorf("event = %+v", ev) + } + } + if events[0].Proto != "udp" || events[1].Proto != "tcp" { + t.Errorf("protos = %s, %s", events[0].Proto, events[1].Proto) + } +} + +func TestNTP(t *testing.T) { + now := time.Date(2026, 9, 29, 12, 0, 0, 500_000_000, time.UTC) + e := newEnv(t, Config{Dialer: anyDialer{}, Now: func() time.Time { return now }}) + c, err := e.guest.DialUDP(netip.MustParseAddrPort("17.253.2.1:123")) + if err != nil { + t.Fatal(err) + } + defer c.Close() + req := make([]byte, 48) + req[0] = 4<<3 | 3 // v4, client + binary.BigEndian.PutUint64(req[40:], 0x1122334455667788) + if _, err := c.Write(req); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp := make([]byte, 64) + n, err := c.Read(resp) + if err != nil { + t.Fatal(err) + } + if n != 48 || resp[0]&0x07 != 4 { + t.Fatalf("resp len %d mode %d", n, resp[0]&0x07) + } + if binary.BigEndian.Uint64(resp[24:]) != 0x1122334455667788 { + t.Error("originate timestamp not echoed") + } + ts := binary.BigEndian.Uint64(resp[40:]) + secs := int64(ts>>32) - ntpEpochOffset + frac := float64(ts&0xffffffff) / (1 << 32) + got := time.Unix(secs, int64(frac*1e9)) + if d := got.Sub(now); d < -time.Millisecond || d > time.Millisecond { + t.Errorf("transmit = %v, want %v", got, now) + } +} + +func TestUDPEgress(t *testing.T) { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + go func() { + buf := make([]byte, 1500) + for { + n, addr, err := pc.ReadFrom(buf) + if err != nil { + return + } + pc.WriteTo(append([]byte("echo:"), buf[:n]...), addr) + } + }() + e := newEnv(t, Config{Dialer: anyDialer{udpTarget: pc.LocalAddr().String()}}) + dst := netip.MustParseAddrPort("203.0.113.9:4000") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if _, err := c.Write([]byte("hi")); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 100) + n, err := c.Read(buf) + if err != nil || string(buf[:n]) != "echo:hi" { + t.Fatalf("got %q, %v", buf[:n], err) + } + var open bool + for _, f := range e.n.Flows.Recent() { + if f.Dst == dst && f.Kind == "egress" { + open = true + } + } + if !open { + t.Errorf("no egress flow record for %v", dst) + } +} + +func TestDialGuest(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ln, err := e.guest.ListenTCP(22) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + fmt.Fprint(c, "SSH-2.0-fake\r\n") + c.Close() + }() + ctx := testContext(t) + c, err := e.n.DialGuest(ctx, 22) + if err != nil { + t.Fatalf("DialGuest: %v", err) + } + defer c.Close() + line, err := bufio.NewReader(c).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if line != "SSH-2.0-fake\r\n" { + t.Fatalf("banner = %q", line) + } + if !strings.HasPrefix(c.LocalAddr().String(), DefaultGatewayIP.String()+":") { + t.Errorf("local addr = %v, want gateway", c.LocalAddr()) + } +} + +func TestCloseUnblocks(t *testing.T) { + host, _ := NewMemLinkPair() + n, err := New(Config{Link: host}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close hung") + } + if _, err := New(Config{}); !errors.Is(err, err) || err == nil { + t.Fatal("New without a link should fail") + } +} diff --git a/guestnet/guestnettest/guestnettest.go b/guestnet/guestnettest/guestnettest.go new file mode 100644 index 0000000..3f5906f --- /dev/null +++ b/guestnet/guestnettest/guestnettest.go @@ -0,0 +1,131 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnettest provides a fake guest for testing code built on +// guestnet: a gVisor stack with a static address attached to the far end of +// a frame link, standing in for a VM (as one restored from a snapshot would +// look, already holding its lease). +// +// It deliberately does not import guestnet, so guestnet's own tests can use +// it. +package guestnettest + +import ( + "context" + "net" + "net/netip" + "testing" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" +) + +// FrameLink is the subset of guestnet.FrameLink the fake guest needs. +type FrameLink interface { + ReadFrame(buf []byte) (int, error) + WriteFrame(frame []byte) error +} + +// Guest is a fake guest VM's network stack. +type Guest struct { + s *stack.Stack + ep *channel.Endpoint +} + +// Config configures a Guest. +type Config struct { + IP netip.Addr + Subnet netip.Prefix + Gateway netip.Addr + MAC net.HardwareAddr + MTU uint32 // defaults to 1500 +} + +// New starts a fake guest on link. It is torn down by t.Cleanup; closing the +// link is the caller's job. +func New(t testing.TB, link FrameLink, cfg Config) *Guest { + t.Helper() + if cfg.MTU == 0 { + cfg.MTU = 1500 + } + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + ep := channel.New(1024, cfg.MTU, tcpip.LinkAddress(cfg.MAC)) + if err := s.CreateNIC(1, ethernet.New(ep)); err != nil { + t.Fatalf("guest CreateNIC: %v", err) + } + if err := s.AddProtocolAddress(1, tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{Address: tcpip.AddrFrom4(cfg.IP.As4()), PrefixLen: cfg.Subnet.Bits()}, + }, stack.AddressProperties{}); err != nil { + t.Fatalf("guest AddProtocolAddress: %v", err) + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + Gateway: tcpip.AddrFrom4(cfg.Gateway.As4()), + NIC: 1, + }}) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + buf := make([]byte, 65536) + for { + n, err := link.ReadFrame(buf) + if err != nil { + return + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(append([]byte(nil), buf[:n]...))}) + ep.InjectInbound(header.Ethernet(buf).Type(), pkt) + pkt.DecRef() + } + }() + go func() { + for { + pkt := ep.ReadContext(ctx) + if pkt == nil { + return + } + v := pkt.ToView() + link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + } + }() + t.Cleanup(func() { + cancel() + ep.Close() + s.Destroy() + }) + return &Guest{s: s, ep: ep} +} + +func fullAddr(ap netip.AddrPort) tcpip.FullAddress { + return tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4(ap.Addr().As4()), Port: ap.Port()} +} + +// DialTCP dials dst from the guest. +func (g *Guest) DialTCP(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { + return gonet.DialContextTCP(ctx, g.s, fullAddr(dst), ipv4.ProtocolNumber) +} + +// DialUDP opens a UDP socket from the guest connected to dst. +func (g *Guest) DialUDP(dst netip.AddrPort) (net.Conn, error) { + fa := fullAddr(dst) + return gonet.DialUDP(g.s, nil, &fa, ipv4.ProtocolNumber) +} + +// ListenTCP listens on port on the guest. +func (g *Guest) ListenTCP(port uint16) (net.Listener, error) { + return gonet.ListenTCP(g.s, tcpip.FullAddress{NIC: 1, Port: port}, ipv4.ProtocolNumber) +} diff --git a/guestnet/link.go b/guestnet/link.go new file mode 100644 index 0000000..11d1f05 --- /dev/null +++ b/guestnet/link.go @@ -0,0 +1,114 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "errors" + "io" + "os" + "sync" +) + +// FrameLink carries raw Ethernet frames, one per call, between a guest NIC +// and a [Net]. +type FrameLink interface { + // ReadFrame reads one frame into buf and returns its length. + ReadFrame(buf []byte) (int, error) + // WriteFrame writes one frame. + WriteFrame(frame []byte) error + // Close closes the link and unblocks readers. + Close() error +} + +// FileLink is a FrameLink over a datagram socket held as an *os.File, such +// as one end of a SOCK_DGRAM socketpair whose other end is attached to a +// VZFileHandleNetworkDeviceAttachment. Each Read returns one datagram, which +// is one Ethernet frame. +type FileLink struct { + f *os.File +} + +// NewFileLink wraps a datagram socket. It takes ownership of f. The socket +// should have been non-blocking when f was made with os.NewFile; otherwise +// Close does not unblock a pending ReadFrame. +func NewFileLink(f *os.File) *FileLink { return &FileLink{f: f} } + +// ReadFrame implements FrameLink. +func (l *FileLink) ReadFrame(buf []byte) (int, error) { + for { + n, err := l.f.Read(buf) + if err != nil { + return 0, err + } + if n == 0 { + // Zero-length datagrams are not frames. + continue + } + return n, nil + } +} + +// WriteFrame implements FrameLink. +func (l *FileLink) WriteFrame(frame []byte) error { + _, err := l.f.Write(frame) + return err +} + +// Close implements FrameLink. +func (l *FileLink) Close() error { return l.f.Close() } + +// MemLink is an in-memory FrameLink for tests. Create a connected pair with +// [NewMemLinkPair]. +type MemLink struct { + in chan []byte + out chan []byte + done chan struct{} + once sync.Once + peer *MemLink +} + +// NewMemLinkPair returns two links where frames written to one are read from +// the other. Closing either end unblocks both. +func NewMemLinkPair() (a, b *MemLink) { + ab := make(chan []byte, 1024) + ba := make(chan []byte, 1024) + done := make(chan struct{}) + a = &MemLink{in: ba, out: ab, done: done} + b = &MemLink{in: ab, out: ba, done: done} + a.peer, b.peer = b, a + return a, b +} + +// ReadFrame implements FrameLink. +func (l *MemLink) ReadFrame(buf []byte) (int, error) { + select { + case f := <-l.in: + if len(f) > len(buf) { + return 0, io.ErrShortBuffer + } + return copy(buf, f), nil + case <-l.done: + return 0, errors.New("memlink: closed") + } +} + +// WriteFrame implements FrameLink. +func (l *MemLink) WriteFrame(frame []byte) error { + f := append([]byte(nil), frame...) + select { + case l.out <- f: + return nil + case <-l.done: + return errors.New("memlink: closed") + } +} + +// Close implements FrameLink. It closes both ends of the pair. +func (l *MemLink) Close() error { + l.once.Do(func() { close(l.done) }) + if l.peer != nil { + l.peer.once.Do(func() {}) + } + return nil +} diff --git a/guestnet/log.go b/guestnet/log.go new file mode 100644 index 0000000..dd75c1f --- /dev/null +++ b/guestnet/log.go @@ -0,0 +1,120 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "net/netip" + "sync" + "time" +) + +// Flow is one TCP connection or UDP session from the guest. +type Flow struct { + // ID is unique within a Net and identical for the open and close records + // of one flow. + ID uint64 `json:"id"` + Proto string `json:"proto"` // "tcp" or "udp" + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Start time.Time `json:"start"` + // End is zero for the open record. + End time.Time `json:"end,omitzero"` + // Kind says how the flow was handled: "egress", "service", "dns", "ntp", + // or "blocked". + Kind string `json:"kind"` + // Service names the registered service for Kind "service". + Service string `json:"service,omitempty"` + // BytesIn counts bytes sent to the guest, BytesOut from the guest. + BytesIn int64 `json:"bytes_in"` + BytesOut int64 `json:"bytes_out"` + Err string `json:"err,omitempty"` +} + +// Closed reports whether f is a close record. +func (f Flow) Closed() bool { return !f.End.IsZero() } + +// DNSEvent is one DNS query from the guest and how it was answered. +type DNSEvent struct { + Time time.Time `json:"time"` + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Proto string `json:"proto"` + Name string `json:"name"` + Type string `json:"type"` + RCode string `json:"rcode,omitempty"` + Answers int `json:"answers"` + Upstream string `json:"upstream,omitempty"` + Duration time.Duration `json:"duration"` + Err string `json:"err,omitempty"` +} + +// Event is a notable network event. +type Event struct { + Time time.Time `json:"time"` + // Kind is "dhcp-discover", "dhcp-lease", "blocked", "dial-error", ... + Kind string `json:"kind"` + Msg string `json:"msg"` +} + +// Log is a bounded ring of records with live subscribers. +type Log[T any] struct { + mu sync.Mutex + ring []T + next int + full bool + subs map[chan T]struct{} +} + +// NewLog returns a Log keeping the last n records. +func NewLog[T any](n int) *Log[T] { + return &Log[T]{ring: make([]T, n), subs: make(map[chan T]struct{})} +} + +// Add appends a record and delivers it to subscribers that are keeping up. +// Slow subscribers miss records rather than block the network. +func (l *Log[T]) Add(v T) { + l.mu.Lock() + defer l.mu.Unlock() + l.ring[l.next] = v + l.next++ + if l.next == len(l.ring) { + l.next = 0 + l.full = true + } + for ch := range l.subs { + select { + case ch <- v: + default: + } + } +} + +// Recent returns the retained records, oldest first. +func (l *Log[T]) Recent() []T { + l.mu.Lock() + defer l.mu.Unlock() + if !l.full { + return append([]T(nil), l.ring[:l.next]...) + } + out := make([]T, 0, len(l.ring)) + out = append(out, l.ring[l.next:]...) + out = append(out, l.ring[:l.next]...) + return out +} + +// Subscribe returns a channel receiving new records until ctx is done. +func (l *Log[T]) Subscribe(ctx context.Context) <-chan T { + ch := make(chan T, 256) + l.mu.Lock() + l.subs[ch] = struct{}{} + l.mu.Unlock() + go func() { + <-ctx.Done() + l.mu.Lock() + delete(l.subs, ch) + l.mu.Unlock() + }() + return ch +} diff --git a/guestnet/ntp.go b/guestnet/ntp.go new file mode 100644 index 0000000..8a888fb --- /dev/null +++ b/guestnet/ntp.go @@ -0,0 +1,67 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "encoding/binary" + "net" + "time" +) + +// ntpEpochOffset is the number of seconds between the NTP epoch (1900) and +// the Unix epoch (1970). +const ntpEpochOffset = 2208988800 + +// ntpIdleTimeout ends an NTP session after this long without a request. +const ntpIdleTimeout = 30 * time.Second + +// serveNTP answers SNTP requests on one guest UDP session with the host's +// time. A guest restored from a memory snapshot has a clock that is stale by +// the snapshot's age; macOS's timed corrects it on its next exchange with +// what it believes is time.apple.com, which this intercepts. +func (n *Net) serveNTP(conn net.Conn) error { + buf := make([]byte, 128) + for { + conn.SetReadDeadline(time.Now().Add(ntpIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + if m < 48 { + continue + } + resp := ntpResponse(buf[:48], n.cfg.Now()) + if _, err := conn.Write(resp); err != nil { + return err + } + } +} + +// ntpResponse builds a 48-byte server reply to req with all server +// timestamps set to now. +func ntpResponse(req []byte, now time.Time) []byte { + resp := make([]byte, 48) + vn := (req[0] >> 3) & 0x07 + if vn < 3 || vn > 4 { + vn = 4 + } + resp[0] = vn<<3 | 4 // LI=0, mode 4 (server) + resp[1] = 2 // stratum + resp[2] = req[2] // poll, echoed + resp[3] = 0xEC // precision: about 2^-20 seconds + // Root delay and dispersion: zero. + copy(resp[12:16], "LOCL") + ts := ntpTimestamp(now) + binary.BigEndian.PutUint64(resp[16:], ts) // reference + copy(resp[24:32], req[40:48]) // originate = client transmit + binary.BigEndian.PutUint64(resp[32:], ts) // receive + binary.BigEndian.PutUint64(resp[40:], ts) // transmit + return resp +} + +func ntpTimestamp(t time.Time) uint64 { + secs := uint64(t.Unix()) + ntpEpochOffset + frac := uint64(t.Nanosecond()) * (1 << 32) / 1e9 + return secs<<32 | frac +} diff --git a/guestnet/services.go b/guestnet/services.go new file mode 100644 index 0000000..fe4bf6c --- /dev/null +++ b/guestnet/services.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "errors" + "net" + "sync" +) + +// Service handles guest TCP connections to a registered destination. +type Service interface { + // Name identifies the service in flow logs. + Name() string + // HandleConn serves one guest connection. It owns conn and must close it. + // ctx is done when the Net closes. + HandleConn(ctx context.Context, conn net.Conn) +} + +// ServiceFunc adapts a function to a Service. +type ServiceFunc struct { + ServiceName string + Handle func(ctx context.Context, conn net.Conn) +} + +// Name implements Service. +func (s ServiceFunc) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s ServiceFunc) HandleConn(ctx context.Context, conn net.Conn) { s.Handle(ctx, conn) } + +// ForwardService relays guest connections to a host-side address, such as +// "tcp:127.0.0.1:2049" or "unix:/var/run/foo.sock". It replaces the pf rdr +// rules corp CI uses on macOS today. +type ForwardService struct { + ServiceName string + // Network and Address are passed to Dialer.DialContext. + Network, Address string + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// Name implements Service. +func (s *ForwardService) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s *ForwardService) HandleConn(ctx context.Context, conn net.Conn) { + defer conn.Close() + d := s.Dialer + if d == nil { + d = &net.Dialer{} + } + remote, err := d.DialContext(ctx, s.Network, s.Address) + if err != nil { + return + } + defer remote.Close() + splice(conn, remote) +} + +// ListenerService is a Service that exposes guest connections as a +// net.Listener, so an http.Server (or anything else that Accepts) can serve +// the guest without knowing about gVisor. +type ListenerService struct { + ServiceName string + + once sync.Once + conns chan net.Conn + closed chan struct{} + cmu sync.Once +} + +// NewListenerService returns a ListenerService named name. +func NewListenerService(name string) *ListenerService { + s := &ListenerService{ServiceName: name} + s.init() + return s +} + +func (s *ListenerService) init() { + s.once.Do(func() { + s.conns = make(chan net.Conn) + s.closed = make(chan struct{}) + }) +} + +// Name implements Service. +func (s *ListenerService) Name() string { return s.ServiceName } + +// HandleConn implements Service by handing conn to Accept. +func (s *ListenerService) HandleConn(ctx context.Context, conn net.Conn) { + s.init() + select { + case s.conns <- conn: + case <-s.closed: + conn.Close() + case <-ctx.Done(): + conn.Close() + } +} + +// Accept implements net.Listener. +func (s *ListenerService) Accept() (net.Conn, error) { + s.init() + select { + case c := <-s.conns: + return c, nil + case <-s.closed: + return nil, errListenerClosed + } +} + +// Close implements net.Listener. +func (s *ListenerService) Close() error { + s.init() + s.cmu.Do(func() { close(s.closed) }) + return nil +} + +// Addr implements net.Listener. +func (s *ListenerService) Addr() net.Addr { return serviceAddr(s.ServiceName) } + +var errListenerClosed = errors.New("guestnet: listener closed") + +type serviceAddr string + +func (a serviceAddr) Network() string { return "guestnet" } +func (a serviceAddr) String() string { return string(a) } diff --git a/guestnet/socketpair.go b/guestnet/socketpair.go new file mode 100644 index 0000000..ca55377 --- /dev/null +++ b/guestnet/socketpair.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "fmt" + "os" + "syscall" +) + +// Buffer sizes for the guest datagram link. Apple documents +// (VZFileHandleNetworkDeviceAttachment.maximumTransmissionUnit) that +// SO_RCVBUF must be at least twice SO_SNDBUF, and recommends four times. +// The absolute sizes are our guess, room for a few hundred full-size frames, +// and well under macOS's default kern.ipc.maxsockbuf of 8 MiB. Tune them +// with throughput measurements. +const ( + linkSendBuf = 512 << 10 + linkRecvBuf = 4 * linkSendBuf +) + +// NewSocketPairLink creates a connected SOCK_DGRAM unix socketpair. The +// returned *os.File is for the VM side (hand it to the network device +// attachment; the framework dups it, so close it after the VM is created). +// The returned FileLink is our side. +func NewSocketPairLink() (vmSide *os.File, host *FileLink, err error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + return nil, nil, fmt.Errorf("socketpair: %w", err) + } + for _, fd := range fds { + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, linkSendBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_SNDBUF: %w", err) + } + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, linkRecvBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_RCVBUF: %w", err) + } + syscall.CloseOnExec(fd) + } + // Our end must be non-blocking before os.NewFile sees it, so the file + // uses the runtime poller and Close unblocks a pending ReadFrame. The + // VM end stays blocking, as the framework expects. + if err := syscall.SetNonblock(fds[1], true); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SetNonblock: %w", err) + } + vmSide = os.NewFile(uintptr(fds[0]), "guestnet-vm") + hostFile := os.NewFile(uintptr(fds[1]), "guestnet-host") + return vmSide, NewFileLink(hostFile), nil +} + +func closeFDs(fds [2]int) { + for _, fd := range fds { + syscall.Close(fd) + } +} diff --git a/guestnet/socketpair_test.go b/guestnet/socketpair_test.go new file mode 100644 index 0000000..0872baf --- /dev/null +++ b/guestnet/socketpair_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "testing" + "time" +) + +// TestSocketPairCloseUnblocks checks that a Net on a real socketpair shuts +// down while its frame reader is blocked, which requires our end of the +// pair to use the runtime poller. +func TestSocketPairCloseUnblocks(t *testing.T) { + vmSide, link, err := NewSocketPairLink() + if err != nil { + t.Fatal(err) + } + defer vmSide.Close() + n, err := New(Config{Link: link}) + if err != nil { + t.Fatal(err) + } + // Let the frame reader block in ReadFrame. + time.Sleep(50 * time.Millisecond) + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close hung with a blocked reader") + } +} diff --git a/guestnet/tcp.go b/guestnet/tcp.go new file mode 100644 index 0000000..b2b99bd --- /dev/null +++ b/guestnet/tcp.go @@ -0,0 +1,226 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "fmt" + "io" + "net" + "net/netip" + "sync" + "sync/atomic" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// egressDialTimeout bounds how long the guest's SYN waits for the host-side +// dial before we answer with a RST. +const egressDialTimeout = 10 * time.Second + +var flowIDs atomic.Uint64 + +// handleTCP is called by the gVisor forwarder for every SYN from the guest. +func (n *Net) handleTCP(r *tcp.ForwarderRequest) { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "tcp", Src: src, Dst: dst, Start: n.cfg.Now()} + + if svc, ok := n.service(dst); ok { + flow.Kind, flow.Service = "service", svc.Name() + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + // The service owns conn and records the flow's close when it + // closes it. + svc.HandleConn(n.ctx, conn) + }() + return + } + + if dst.Port() == 53 { + flow.Kind = "dns" + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSTCP(conn, src, dst)) + }() + return + } + + if !n.egressAllowed(dst.Addr()) { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("tcp %v -> %v", src, dst)}) + r.Complete(true) + return + } + + // Egress: dial first so a refused or unreachable destination becomes a + // RST to the guest instead of an accepted-then-closed connection. The + // forwarder drops the guest's retransmitted SYNs meanwhile. + flow.Kind = "egress" + n.wg.Add(1) + go func() { + defer n.wg.Done() + ctx, cancel := context.WithTimeout(n.ctx, egressDialTimeout) + defer cancel() + remote, err := n.cfg.Dialer.DialContext(ctx, "tcp", dst.String()) + if err != nil { + flow.Err = err.Error() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.End, Kind: "dial-error", Msg: fmt.Sprintf("tcp %v: %v", dst, err)}) + r.Complete(true) + return + } + conn := n.acceptTCP(r, flow) + if conn == nil { + remote.Close() + return + } + defer n.track(conn)() + defer n.track(remote)() + splice(conn, remote) + conn.Close() + }() +} + +// acceptTCP completes the handshake and returns the guest-side connection +// wrapped for flow accounting, recording the flow's open record. It returns +// nil (and records the failure) if the endpoint could not be created. +func (n *Net) acceptTCP(r *tcp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + r.Complete(true) + return nil + } + ep.SocketOptions().SetKeepAlive(true) + r.Complete(false) + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewTCPConn(&wq, ep), flow) +} + +// track arranges for conn to be closed when the Net closes, so no goroutine +// outlives it. The returned func cancels that arrangement; call it when the +// conn is done. +func (n *Net) track(conn net.Conn) func() { + stop := context.AfterFunc(n.ctx, func() { + // gonet conns don't reliably wake a blocked Read on Close alone; a + // past deadline does. + conn.SetReadDeadline(time.Unix(1, 0)) + conn.Close() + }) + return func() { stop() } +} + +// flowConn wraps a guest-side connection to count bytes and record the +// flow's close record exactly once, when whoever owns it closes it. +type flowConn struct { + net.Conn + n *Net + flow Flow + in atomic.Int64 // bytes written to the guest + out atomic.Int64 // bytes read from the guest + once sync.Once +} + +func newFlowConn(n *Net, c net.Conn, flow Flow) *flowConn { + return &flowConn{Conn: c, n: n, flow: flow} +} + +func (c *flowConn) Read(b []byte) (int, error) { + m, err := c.Conn.Read(b) + c.out.Add(int64(m)) + return m, err +} + +func (c *flowConn) Write(b []byte) (int, error) { + m, err := c.Conn.Write(b) + c.in.Add(int64(m)) + return m, err +} + +// CloseWrite half-closes the guest side, for splice. +func (c *flowConn) CloseWrite() error { + if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return nil +} + +// Close implements net.Conn and records the flow's close record. +func (c *flowConn) Close() error { return c.closeWithErr(nil) } + +// closeWithErr closes the conn, recording err (if any) in the flow's close +// record. Only the first close records anything. +func (c *flowConn) closeWithErr(err error) error { + cerr := c.Conn.Close() + c.once.Do(func() { + f := c.flow + f.End = c.n.cfg.Now() + f.BytesIn, f.BytesOut = c.in.Load(), c.out.Load() + if err != nil { + f.Err = err.Error() + } + c.n.Flows.Add(f) + }) + return cerr +} + +// splice copies bidirectionally between a and b with half-close +// propagation, returning when both directions are done. +// +// When one direction hits EOF it half-closes the other side, then waits a +// bounded time for the remaining direction. gVisor's gonet.TCPConn does not +// reliably wake a blocked Read on Close, so after the grace period both +// connections get a past read deadline before Close to guarantee the second +// copier exits and the endpoint is released. +func splice(a, b net.Conn) { + done := make(chan struct{}, 2) + cp := func(dst, src net.Conn) { + io.Copy(dst, src) + if cw, ok := dst.(interface{ CloseWrite() error }); ok { + cw.CloseWrite() + } + done <- struct{}{} + } + go cp(a, b) + go cp(b, a) + + <-done + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case <-done: + case <-t.C: + past := time.Unix(1, 0) + a.SetReadDeadline(past) + b.SetReadDeadline(past) + a.Close() + b.Close() + <-done + } + a.Close() + b.Close() +} diff --git a/guestnet/udp.go b/guestnet/udp.go new file mode 100644 index 0000000..29609ad --- /dev/null +++ b/guestnet/udp.go @@ -0,0 +1,146 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "fmt" + "net" + "net/netip" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// udpIdleTimeout ends a UDP session after this long without traffic in +// either direction. +const udpIdleTimeout = 60 * time.Second + +// udpMaxPacket is the largest UDP payload we relay. +const udpMaxPacket = 65535 + +// handleUDP is called by the gVisor forwarder for the first packet of every +// UDP 4-tuple from the guest. Returning false drops the packet (and gVisor +// answers with ICMP port unreachable). +func (n *Net) handleUDP(r *udp.ForwarderRequest) bool { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "udp", Src: src, Dst: dst, Start: n.cfg.Now()} + + switch dst.Port() { + case 53: + flow.Kind = "dns" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSUDP(conn, src, dst)) + }() + return true + case 123: + flow.Kind = "ntp" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveNTP(conn)) + }() + return true + } + + if !n.egressAllowed(dst.Addr()) { + // Multicast (mDNS, SSDP) and broadcast chatter is dropped silently; + // unicast blocks are worth a record. + if !dst.Addr().IsMulticast() && dst.Addr() != netipBroadcast { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("udp %v -> %v", src, dst)}) + } + return false + } + + flow.Kind = "egress" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.relayUDP(conn, dst)) + }() + return true +} + +// acceptUDP creates the gVisor endpoint for the session (bound to the +// guest's chosen destination and connected to the guest) and records the +// flow's open record. +func (n *Net) acceptUDP(r *udp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + return nil + } + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewUDPConn(&wq, ep), flow) +} + +// relayUDP shuttles datagrams between the guest session and a host-side UDP +// socket to dst until idle. Byte counts are kept by the flowConn. +func (n *Net) relayUDP(guest net.Conn, dst netip.AddrPort) error { + remote, err := n.cfg.Dialer.DialContext(n.ctx, "udp", dst.String()) + if err != nil { + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dial-error", Msg: fmt.Sprintf("udp %v: %v", dst, err)}) + return err + } + defer remote.Close() + defer n.track(remote)() + + done := make(chan struct{}) + go func() { + defer close(done) + buf := make([]byte, udpMaxPacket) + for { + remote.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := remote.Read(buf) + if err != nil { + guest.Close() + return + } + if _, err := guest.Write(buf[:m]); err != nil { + return + } + } + }() + buf := make([]byte, udpMaxPacket) + for { + guest.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := guest.Read(buf) + if err != nil { + // Idle timeout, guest close, or Net shutdown: all normal. + remote.Close() + <-done + return nil + } + if _, err := remote.Write(buf[:m]); err != nil { + <-done + return err + } + } +}