diff --git a/go.mod b/go.mod index c781e5a..2caf8f2 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,17 @@ require ( github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900 github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b github.com/cespare/xxhash/v2 v2.3.0 - github.com/go-jose/go-jose/v4 v4.1.3 + github.com/go-jose/go-jose/v4 v4.1.4 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 + github.com/insomniacslk/dhcp v0.0.0-20260901064844-234b97448fae github.com/pierrec/lz4/v4 v4.1.26 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 + golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 golang.org/x/time v0.16.0 + gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 modernc.org/sqlite v1.51.0 tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef ) @@ -47,6 +50,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.3 // indirect github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -63,12 +67,12 @@ require ( github.com/prometheus/procfs v0.21.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect + github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/x448/float16 v0.8.4 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.59.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect diff --git a/go.sum b/go.sum index fff738a..180ac20 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI= github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -91,6 +91,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/guestnet/addr.go b/guestnet/addr.go new file mode 100644 index 0000000..7c70db4 --- /dev/null +++ b/guestnet/addr.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" +) + +var netipBroadcast = netip.AddrFrom4([4]byte{255, 255, 255, 255}) + +func netipFromIP(ip net.IP) (netip.Addr, bool) { + if ip4 := ip.To4(); ip4 != nil { + return netip.AddrFrom4([4]byte(ip4)), true + } + return netip.AddrFromSlice(ip) +} + +// broadcastOf returns the directed broadcast address of an IPv4 prefix. +func broadcastOf(p netip.Prefix) net.IP { + a := p.Masked().Addr().As4() + bits := p.Bits() + for i := bits; i < 32; i++ { + a[i/8] |= 1 << (7 - uint(i%8)) + } + return net.IP(a[:]) +} diff --git a/guestnet/arp_test.go b/guestnet/arp_test.go new file mode 100644 index 0000000..69b10de --- /dev/null +++ b/guestnet/arp_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildARPRequest returns an Ethernet broadcast ARP request from mac/srcIP +// asking for dstIP. An ARP probe (RFC 5227) has an unspecified srcIP. +func buildARPRequest(mac net.HardwareAddr, srcIP, dstIP netip.Addr) []byte { + frame := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(mac), + DstAddr: tcpip.LinkAddress("\xff\xff\xff\xff\xff\xff"), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(frame[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPRequest) + copy(arp.HardwareAddressSender(), mac) + src, dst := srcIP.As4(), dstIP.As4() + copy(arp.ProtocolAddressSender(), src[:]) + copy(arp.ProtocolAddressTarget(), dst[:]) + return frame +} + +// TestARPOnlyForOurAddresses checks that we answer ARP only for our own +// addresses. In particular the guest's DHCP client probes its offered +// address (RFC 5227) and declines the lease if anything answers; macOS does +// this, and a promiscuous gVisor NIC would answer for every address. +func TestARPOnlyForOurAddresses(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + replies := make(chan netip.Addr, 16) + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + arp := header.ARP(frame[header.EthernetMinimumSize:]) + if arp.IsValid() && arp.Op() == header.ARPReply { + replies <- netip.AddrFrom4([4]byte(arp.ProtocolAddressSender())) + } + } + }() + + mac := DefaultGuestMAC + unspec := netip.AddrFrom4([4]byte{}) + for _, tt := range []struct { + src, dst netip.Addr + want bool + }{ + {unspec, DefaultGuestIP, false}, // DHCP client's duplicate address probe + {DefaultGuestIP, DefaultGuestIP, false}, // gratuitous ARP + {DefaultGuestIP, netip.MustParseAddr("192.168.64.77"), false}, + {DefaultGuestIP, DefaultGatewayIP, true}, + {DefaultGuestIP, IMDSAddr, true}, + {DefaultGuestIP, DNSAddr, true}, + } { + if err := peer.WriteFrame(buildARPRequest(mac, tt.src, tt.dst)); err != nil { + t.Fatal(err) + } + var got bool + select { + case from := <-replies: + got = true + if from != tt.dst { + t.Errorf("ARP for %v: reply claims %v", tt.dst, from) + } + case <-time.After(300 * time.Millisecond): + } + if got != tt.want { + t.Errorf("ARP from %v for %v: replied = %v, want %v", tt.src, tt.dst, got, tt.want) + } + } +} diff --git a/guestnet/close_test.go b/guestnet/close_test.go new file mode 100644 index 0000000..81840b3 --- /dev/null +++ b/guestnet/close_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildTCPSYNFrame assembles an Ethernet/IPv4/TCP SYN frame with checksums. +func buildTCPSYNFrame(srcMAC, dstMAC net.HardwareAddr, src, dst netip.AddrPort) []byte { + ipLen := header.IPv4MinimumSize + header.TCPMinimumSize + frame := make([]byte, header.EthernetMinimumSize+ipLen) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + srcIP, dstIP := tcpip.AddrFrom4(src.Addr().As4()), tcpip.AddrFrom4(dst.Addr().As4()) + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.TCPProtocolNumber), + SrcAddr: srcIP, + DstAddr: dstIP, + }) + ip.SetChecksum(^ip.CalculateChecksum()) + tcp := header.TCP(ip.Payload()) + tcp.Encode(&header.TCPFields{ + SrcPort: src.Port(), + DstPort: dst.Port(), + SeqNum: 1000, + DataOffset: header.TCPMinimumSize, + Flags: header.TCPFlagSyn, + WindowSize: 65535, + }) + xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, srcIP, dstIP, header.TCPMinimumSize) + tcp.SetChecksum(^checksum.Checksum(tcp, xsum)) + return frame +} + +// TestCloseDuringHandshake checks that Close doesn't wait for a TCP +// handshake the guest will never finish. A guest that is stopped just after +// sending a SYN (the VM stopping mid-connection) leaves the forwarder waiting +// in the handshake; Close used to wait for gVisor's handshake timeout (30s +// or more) before tearing down. +func TestCloseDuringHandshake(t *testing.T) { + target, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer target.Close() + accepted := make(chan net.Conn, 1) + go func() { + c, err := target.Accept() + if err == nil { + accepted <- c + } + }() + + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Dialer: anyDialer{tcpTarget: target.Addr().String()}, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + // Answer ARP for the guest's address, as a real guest would, so the + // SYN-ACK goes out, but never ACK it: the guest has stopped. + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + req := header.ARP(frame[header.EthernetMinimumSize:]) + if !req.IsValid() || req.Op() != header.ARPRequest { + continue + } + reply := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(reply).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(DefaultGuestMAC), + DstAddr: tcpip.LinkAddress(req.HardwareAddressSender()), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(reply[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPReply) + copy(arp.HardwareAddressSender(), DefaultGuestMAC) + copy(arp.ProtocolAddressSender(), req.ProtocolAddressTarget()) + copy(arp.HardwareAddressTarget(), req.HardwareAddressSender()) + copy(arp.ProtocolAddressTarget(), req.ProtocolAddressSender()) + peer.WriteFrame(reply) + } + }() + + src := netip.AddrPortFrom(DefaultGuestIP, 40000) + dst := netip.MustParseAddrPort("198.51.100.7:443") + if err := peer.WriteFrame(buildTCPSYNFrame(DefaultGuestMAC, DefaultGatewayMAC, src, dst)); err != nil { + t.Fatal(err) + } + select { + case c := <-accepted: + defer c.Close() + case <-time.After(5 * time.Second): + t.Fatal("egress dial never happened") + } + time.Sleep(100 * time.Millisecond) // let the forwarder enter the handshake + + done := make(chan struct{}) + t0 := time.Now() + go func() { + n.Close() + close(done) + }() + select { + case <-done: + t.Logf("Close took %v", time.Since(t0)) + case <-time.After(5 * time.Second): + t.Fatal("Close waited for the guest's unfinished TCP handshake") + } +} diff --git a/guestnet/dhcp.go b/guestnet/dhcp.go new file mode 100644 index 0000000..ec4d25b --- /dev/null +++ b/guestnet/dhcp.go @@ -0,0 +1,198 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// handleDHCP answers DHCPv4 requests found in frame. It reports whether the +// frame was a DHCP request (and so must not be injected into the stack). +// +// DHCP is answered outside gVisor because the client has no address yet and +// the reply rules (unicast to the client MAC vs. broadcast) are about the +// link layer, which is simplest to control when we build the frame ourselves. +func (n *Net) handleDHCP(frame []byte) bool { + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if len(ip) < header.IPv4MinimumSize || !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return false + } + if ip.More() || ip.FragmentOffset() != 0 { + return false + } + udpPkt := header.UDP(ip.Payload()) + if len(udpPkt) < header.UDPMinimumSize || udpPkt.DestinationPort() != dhcpv4.ServerPort { + return false + } + req, err := dhcpv4.FromBytes(udpPkt.Payload()) + if err != nil { + n.cfg.Logf("guestnet: dhcp: bad request: %v", err) + return true + } + if req.OpCode != dhcpv4.OpcodeBootRequest { + return true + } + + var replyType dhcpv4.MessageType + switch req.MessageType() { + case dhcpv4.MessageTypeDiscover: + replyType = dhcpv4.MessageTypeOffer + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-discover", Msg: req.ClientHWAddr.String()}) + case dhcpv4.MessageTypeRequest: + // A REQUEST for some other address (a lease carried over from + // elsewhere) gets a NAK so the client starts over and takes ours. + if want := req.RequestedIPAddress(); want != nil && !want.IsUnspecified() && !want.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else if !req.ClientIPAddr.IsUnspecified() && !req.ClientIPAddr.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else { + replyType = dhcpv4.MessageTypeAck + } + case dhcpv4.MessageTypeInform: + replyType = dhcpv4.MessageTypeAck + case dhcpv4.MessageTypeDecline: + n.cfg.Logf("guestnet: dhcp: guest declined %v", n.cfg.GuestIP) + return true + default: + return true + } + + resp, err := n.buildDHCPReply(req, replyType) + if err != nil { + n.cfg.Logf("guestnet: dhcp: build %v: %v", replyType, err) + return true + } + + // Where to send it (RFC 2131 section 4.1). A NAK is always broadcast. + // A client that already has an address (renewing) is answered unicast. + // Otherwise honor the broadcast flag; a client without the flag is + // answered unicast to its MAC with its new address as the IP destination. + dstMAC := net.HardwareAddr(req.ClientHWAddr) + dstIP := n.cfg.GuestIP + switch { + case replyType == dhcpv4.MessageTypeNak, req.IsBroadcast() && req.ClientIPAddr.IsUnspecified(): + dstMAC = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + dstIP = netipBroadcast + case !req.ClientIPAddr.IsUnspecified(): + if a, ok := netipFromIP(req.ClientIPAddr); ok { + dstIP = a + } + } + + srcIP := n.cfg.GatewayIP + if replyType == dhcpv4.MessageTypeAck || replyType == dhcpv4.MessageTypeOffer { + if replyType == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeRequest { + n.dhcpLeasedOnce.Do(func() { close(n.dhcpLeased) }) + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-lease", Msg: fmt.Sprintf("%v -> %v", req.ClientHWAddr, n.cfg.GuestIP)}) + } + } + n.writeFrame(buildUDPFrame(n.cfg.GatewayMAC, dstMAC, srcIP.As4(), dstIP.As4(), dhcpv4.ServerPort, dhcpv4.ClientPort, resp.ToBytes())) + return true +} + +func (n *Net) buildDHCPReply(req *dhcpv4.DHCPv4, mt dhcpv4.MessageType) (*dhcpv4.DHCPv4, error) { + gw := net.IP(n.cfg.GatewayIP.AsSlice()) + mods := []dhcpv4.Modifier{ + dhcpv4.WithMessageType(mt), + dhcpv4.WithServerIP(gw), + dhcpv4.WithOption(dhcpv4.OptServerIdentifier(gw)), + } + if mt != dhcpv4.MessageTypeNak { + var mtu [2]byte + binary.BigEndian.PutUint16(mtu[:], uint16(n.cfg.MTU)) + mods = append(mods, + dhcpv4.WithYourIP(n.cfg.GuestIP.AsSlice()), + dhcpv4.WithOption(dhcpv4.OptSubnetMask(net.CIDRMask(n.cfg.Subnet.Bits(), 32))), + dhcpv4.WithOption(dhcpv4.OptRouter(gw)), + dhcpv4.WithOption(dhcpv4.OptDNS(net.IP(DNSAddr.AsSlice()))), + dhcpv4.WithOption(dhcpv4.OptBroadcastAddress(broadcastOf(n.cfg.Subnet))), + dhcpv4.WithOption(dhcpv4.OptIPAddressLeaseTime(n.cfg.LeaseTime)), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRenewTimeValue, secs(n.cfg.LeaseTime/2))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRebindingTimeValue, secs(n.cfg.LeaseTime*7/8))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, mtu[:])), + ) + if mt == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeInform { + // INFORM replies must not carry a lease. + mods = append(mods, func(d *dhcpv4.DHCPv4) { + d.YourIPAddr = net.IPv4zero + d.DeleteOption(dhcpv4.OptionIPAddressLeaseTime) + d.DeleteOption(dhcpv4.OptionRenewTimeValue) + d.DeleteOption(dhcpv4.OptionRebindingTimeValue) + }) + } + } + return dhcpv4.NewReplyFromRequest(req, mods...) +} + +func secs(d time.Duration) []byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], uint32(d/time.Second)) + return b[:] +} + +// buildUDPFrame assembles an Ethernet/IPv4/UDP frame with checksums. +func buildUDPFrame(srcMAC, dstMAC net.HardwareAddr, srcIP, dstIP [4]byte, srcPort, dstPort uint16, payload []byte) []byte { + udpLen := header.UDPMinimumSize + len(payload) + ipLen := header.IPv4MinimumSize + udpLen + frame := make([]byte, header.EthernetMinimumSize+ipLen) + + eth := header.Ethernet(frame) + eth.Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.UDPProtocolNumber), + SrcAddr: tcpip.AddrFrom4(srcIP), + DstAddr: tcpip.AddrFrom4(dstIP), + }) + ip.SetChecksum(^ip.CalculateChecksum()) + + u := header.UDP(ip.Payload()) + u.Encode(&header.UDPFields{ + SrcPort: srcPort, + DstPort: dstPort, + Length: uint16(udpLen), + }) + copy(u.Payload(), payload) + xsum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, tcpip.AddrFrom4(srcIP), tcpip.AddrFrom4(dstIP), uint16(udpLen)) + xsum = checksum.Checksum(payload, xsum) + u.SetChecksum(^u.CalculateChecksum(xsum)) + return frame +} + +// parseUDPFrame extracts the addressing and payload of an IPv4 UDP frame. +// It is used by tests and by the DHCP path's counterpart in a fake guest. +func parseUDPFrame(frame []byte) (src, dst [4]byte, srcPort, dstPort uint16, payload []byte, ok bool) { + if len(frame) < header.EthernetMinimumSize+header.IPv4MinimumSize+header.UDPMinimumSize { + return + } + eth := header.Ethernet(frame) + if eth.Type() != header.IPv4ProtocolNumber { + return + } + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return + } + u := header.UDP(ip.Payload()) + if len(u) < header.UDPMinimumSize { + return + } + return ip.SourceAddress().As4(), ip.DestinationAddress().As4(), u.SourcePort(), u.DestinationPort(), bytes.Clone(u.Payload()), true +} diff --git a/guestnet/dns.go b/guestnet/dns.go new file mode 100644 index 0000000..ccd2c76 --- /dev/null +++ b/guestnet/dns.go @@ -0,0 +1,299 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +// DefaultUpstreams are the resolvers used when none are configured. They +// match corp CI's defaults. +var DefaultUpstreams = []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:53"), + netip.MustParseAddrPort("1.1.1.1:53"), +} + +// dnsQueryTimeout bounds one upstream exchange. +const dnsQueryTimeout = 5 * time.Second + +// dnsIdleTimeout ends a guest DNS session (one source port) after this long +// without a query; the guest's resolver typically uses a fresh port per +// query anyway. +const dnsIdleTimeout = 10 * time.Second + +// Resolver answers DNS queries on the guest's behalf. +type Resolver interface { + // Exchange sends the wire-format query and returns the wire-format + // response and a description of who answered (for logs). + Exchange(ctx context.Context, query []byte) (resp []byte, upstream string, err error) +} + +// UpstreamResolver forwards queries to upstream servers in order, over UDP +// with TCP fallback on truncation. +type UpstreamResolver struct { + Servers []netip.AddrPort + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// NewUpstreamResolver returns an UpstreamResolver for servers. +func NewUpstreamResolver(servers ...netip.AddrPort) *UpstreamResolver { + return &UpstreamResolver{Servers: servers} +} + +// Exchange implements Resolver. +func (r *UpstreamResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + d := r.Dialer + if d == nil { + d = &net.Dialer{} + } + var lastErr error + for _, s := range r.Servers { + resp, err := exchangeUDP(ctx, d, s, query) + if err == nil && len(resp) >= 12 && resp[2]&0x02 != 0 { // TC bit set + resp, err = exchangeTCP(ctx, d, s, query) + } + if err == nil { + return resp, s.String(), nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("no upstream resolvers configured") + } + return nil, "", lastErr +} + +func exchangeUDP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "udp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if _, err := conn.Write(query); err != nil { + return nil, err + } + buf := make([]byte, 4096) + for { + m, err := conn.Read(buf) + if err != nil { + return nil, err + } + // Match the transaction ID; ignore stray datagrams. + if m >= 2 && len(query) >= 2 && buf[0] == query[0] && buf[1] == query[1] { + return buf[:m], nil + } + } +} + +func exchangeTCP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "tcp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if err := writeDNSTCP(conn, query); err != nil { + return nil, err + } + return readDNSTCP(conn) +} + +func writeDNSTCP(w io.Writer, msg []byte) error { + buf := make([]byte, 2+len(msg)) + binary.BigEndian.PutUint16(buf, uint16(len(msg))) + copy(buf[2:], msg) + _, err := w.Write(buf) + return err +} + +func readDNSTCP(r io.Reader) ([]byte, error) { + var lenBuf [2]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, err + } + msg := make([]byte, binary.BigEndian.Uint16(lenBuf[:])) + if _, err := io.ReadFull(r, msg); err != nil { + return nil, err + } + return msg, nil +} + +// serveDNSUDP answers queries arriving on one guest UDP session. +func (n *Net) serveDNSUDP(conn net.Conn, src, dst netip.AddrPort) error { + buf := make([]byte, 4096) + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + resp := n.resolve(buf[:m], src, dst, "udp") + if resp != nil { + if _, err := conn.Write(resp); err != nil { + return err + } + } + } +} + +// serveDNSTCP answers length-prefixed queries on one guest TCP connection. +func (n *Net) serveDNSTCP(conn net.Conn, src, dst netip.AddrPort) error { + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + q, err := readDNSTCP(conn) + if err != nil { + return nil + } + resp := n.resolve(q, src, dst, "tcp") + if resp == nil { + return nil + } + if err := writeDNSTCP(conn, resp); err != nil { + return err + } + } +} + +// resolve answers one query via the configured Resolver and logs it. On +// upstream failure it returns a SERVFAIL so the guest doesn't wait. +func (n *Net) resolve(query []byte, src, dst netip.AddrPort, proto string) []byte { + ev := DNSEvent{Time: n.cfg.Now(), Src: src, Dst: dst, Proto: proto} + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + ev.Err = "unparseable query" + n.DNS.Add(ev) + return nil + } + if q, err := p.Question(); err == nil { + ev.Name = q.Name.String() + ev.Type = q.Type.String() + } + + ctx, cancel := context.WithTimeout(n.ctx, dnsQueryTimeout) + defer cancel() + resp, upstream, err := n.cfg.Resolver.Exchange(ctx, query) + ev.Duration = n.cfg.Now().Sub(ev.Time) + ev.Upstream = upstream + if err != nil { + ev.Err = err.Error() + ev.RCode = dnsmessage.RCodeServerFailure.String() + n.DNS.Add(ev) + return servfail(hdr, query) + } + var rp dnsmessage.Parser + if rh, err := rp.Start(resp); err == nil { + ev.RCode = rh.RCode.String() + rp.SkipAllQuestions() + if answers, err := rp.AllAnswers(); err == nil { + ev.Answers = len(answers) + } + } + n.DNS.Add(ev) + return resp +} + +// servfail builds a SERVFAIL response echoing the query's ID and question. +func servfail(hdr dnsmessage.Header, query []byte) []byte { + resp := make([]byte, len(query)) + copy(resp, query) + if len(resp) < 12 { + return nil + } + resp[2] = 0x80 | (resp[2] & 0x79) // QR=1, keep opcode and RD + resp[3] = byte(dnsmessage.RCodeServerFailure) + // Zero the answer, authority and additional counts; keep QDCOUNT. + binary.BigEndian.PutUint16(resp[6:], 0) + binary.BigEndian.PutUint16(resp[8:], 0) + binary.BigEndian.PutUint16(resp[10:], 0) + _ = hdr + return resp[:12+questionLen(resp[12:])] +} + +// questionLen returns the encoded length of the first question section +// entry in b, or 0 if malformed. +func questionLen(b []byte) int { + i := 0 + for i < len(b) { + l := int(b[i]) + if l == 0 { + i++ + break + } + if l&0xC0 == 0xC0 { + i += 2 + break + } + i += 1 + l + } + if i+4 > len(b) { + return 0 + } + return i + 4 +} + +// StaticResolver answers every A query with a fixed address and everything +// else with NXDOMAIN. It exists for tests and for air-gapped experiments. +type StaticResolver struct { + Addr netip.Addr +} + +// Exchange implements Resolver. +func (r StaticResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + return nil, "", err + } + q, err := p.Question() + if err != nil { + return nil, "", err + } + rh := dnsmessage.Header{ID: hdr.ID, Response: true, Authoritative: true, RecursionDesired: hdr.RecursionDesired, RecursionAvailable: true} + b := dnsmessage.NewBuilder(nil, rh) + b.EnableCompression() + if err := b.StartQuestions(); err != nil { + return nil, "", err + } + if err := b.Question(q); err != nil { + return nil, "", err + } + if q.Type == dnsmessage.TypeA && r.Addr.Is4() { + if err := b.StartAnswers(); err != nil { + return nil, "", err + } + if err := b.AResource(dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 60}, dnsmessage.AResource{A: r.Addr.As4()}); err != nil { + return nil, "", err + } + } else { + rh.RCode = dnsmessage.RCodeNameError + } + out, err := b.Finish() + if err != nil { + return nil, "", fmt.Errorf("build response: %w", err) + } + if rh.RCode != 0 { + out[3] = out[3]&0xF0 | byte(rh.RCode) + } + return out, "static", nil +} diff --git a/guestnet/guestnet.go b/guestnet/guestnet.go new file mode 100644 index 0000000..12e3ae2 --- /dev/null +++ b/guestnet/guestnet.go @@ -0,0 +1,468 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnet is a private userspace network for one guest VM. +// +// The guest's virtual NIC hands us raw Ethernet frames (on macOS via a +// VZFileHandleNetworkDeviceAttachment over a SOCK_DGRAM socketpair; in tests +// via an in-memory link). Every frame terminates in a gVisor netstack owned by +// this package, so a guest can only ever reach what this package chooses to +// connect it to: +// +// - The gateway, IMDS and DNS addresses live on our NIC, so ARP and ICMP +// echo for them are answered by gVisor itself. +// - DHCP is answered directly from the frame loop with a fixed lease, so a +// memory snapshot restored in another slot sees the same addresses. +// - Every TCP SYN and UDP flow is dispatched by destination: a registered +// [Service] (IMDS, the caller's NFS or proxies), an intercepted protocol +// (DNS on any port 53, NTP on any port 123), or egress dialed from the +// host. Each flow is logged. +// - The host reaches the guest only by dialing into this stack with +// [Net.DialGuest]; there is no host interface routed to the guest. +// +// One [Net] per VM. Nothing here is process-global, so many guests can share +// a process without seeing each other. +package guestnet + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "sync" + "time" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "tailscale.com/types/logger" +) + +// Well-known addresses. They are the same for every guest because each guest +// has its own L2 domain; that is what makes memory snapshots slot-independent. +var ( + // DefaultSubnet is the guest-visible subnet. + DefaultSubnet = netip.MustParsePrefix("192.168.64.0/24") + // DefaultGatewayIP is our address on the guest subnet and the guest's + // default router. + DefaultGatewayIP = netip.MustParseAddr("192.168.64.1") + // DefaultGuestIP is the one address the DHCP server hands out. + DefaultGuestIP = netip.MustParseAddr("192.168.64.2") + // IMDSAddr is the metadata service address, matching cloud convention and + // the address corp CI already uses. + IMDSAddr = netip.MustParseAddr("169.254.169.254") + // DNSAddr is the resolver address handed out by DHCP, matching corp CI. + DNSAddr = netip.MustParseAddr("169.254.169.253") + + // DefaultGatewayMAC is our MAC. It is constant so a restored guest's ARP + // cache stays valid. + DefaultGatewayMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x01} + // DefaultGuestMAC is the guest MAC used when an image doesn't specify one. + DefaultGuestMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} +) + +// DefaultDeny is the egress deny list applied when [Config.Deny] is nil: the +// tailnet CGNAT range and its IPv6 counterpart (so a guest can't reach the +// host's tailnet), loopback, and the link-local range (IMDS and DNS are +// reached through the service table before policy applies). +var DefaultDeny = []netip.Prefix{ + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("fd7a:115c:a1e0::/48"), + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("0.0.0.0/8"), +} + +const nicID tcpip.NICID = 1 + +// Dialer dials egress connections on behalf of the guest. *net.Dialer +// satisfies it. +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +// Config configures a guest network. The zero value is usable with a link. +type Config struct { + // Link carries raw Ethernet frames to and from the guest. Required. + Link FrameLink + + // GuestMAC is the guest's MAC address, used as the DHCP client identity + // check and as the destination of DHCP replies. Defaults to + // [DefaultGuestMAC]. + GuestMAC net.HardwareAddr + // GatewayMAC is our MAC. Defaults to [DefaultGatewayMAC]. + GatewayMAC net.HardwareAddr + + // Subnet, GatewayIP and GuestIP default to the Default* values. + Subnet netip.Prefix + GatewayIP netip.Addr + GuestIP netip.Addr + + // MTU is the link MTU advertised to the guest via DHCP. Defaults to 1500. + MTU uint32 + + // LeaseTime is the DHCP lease duration. Defaults to 24h. Restored + // snapshots renew against this server, so long leases are fine. + LeaseTime time.Duration + + // Dialer dials egress TCP and UDP. Defaults to a net.Dialer. + Dialer Dialer + + // Resolver answers the guest's DNS queries. Defaults to + // [NewUpstreamResolver] with [DefaultUpstreams]. + Resolver Resolver + + // Deny lists destinations the guest may not reach via egress. nil means + // [DefaultDeny]; an empty non-nil slice denies nothing. + Deny []netip.Prefix + + // Services maps guest-visible destinations to handlers. The destination + // address must be one of ours (gateway, IMDS, DNS) or the lookup happens + // before egress policy and would let a guest hijack a public address. + Services map[netip.AddrPort]Service + + // Now returns the current time, for NTP and logs. Defaults to time.Now. + Now func() time.Time + + // Logf logs. Defaults to logger.Discard. + Logf logger.Logf +} + +func (c *Config) withDefaults() { + if c.GuestMAC == nil { + c.GuestMAC = DefaultGuestMAC + } + if c.GatewayMAC == nil { + c.GatewayMAC = DefaultGatewayMAC + } + if !c.Subnet.IsValid() { + c.Subnet = DefaultSubnet + } + if !c.GatewayIP.IsValid() { + c.GatewayIP = DefaultGatewayIP + } + if !c.GuestIP.IsValid() { + c.GuestIP = DefaultGuestIP + } + if c.MTU == 0 { + c.MTU = 1500 + } + if c.LeaseTime == 0 { + c.LeaseTime = 24 * time.Hour + } + if c.Dialer == nil { + c.Dialer = &net.Dialer{Timeout: 10 * time.Second} + } + if c.Resolver == nil { + c.Resolver = NewUpstreamResolver(DefaultUpstreams...) + } + if c.Deny == nil { + c.Deny = DefaultDeny + } + if c.Now == nil { + c.Now = time.Now + } + if c.Logf == nil { + c.Logf = logger.Discard + } +} + +// Net is one guest's network. Create it with [New] and shut it down with +// [Net.Close]. +type Net struct { + cfg Config + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + stack *stack.Stack + linkEP *channel.Endpoint + + // Flows is the TCP/UDP flow log. + Flows *Log[Flow] + // DNS is the DNS query log. + DNS *Log[DNSEvent] + // Events records notable network events (DHCP lease, blocked flows). + Events *Log[Event] + + // dhcpLeased is closed the first time the guest acknowledges a lease. + dhcpLeased chan struct{} + dhcpLeasedOnce sync.Once + + mu sync.Mutex + services map[netip.AddrPort]Service +} + +// New starts a guest network on cfg.Link. It returns once the stack is up; +// frames are pumped on background goroutines until [Net.Close]. +func New(cfg Config) (*Net, error) { + if cfg.Link == nil { + return nil, errors.New("guestnet: Config.Link is required") + } + cfg.withDefaults() + if !cfg.Subnet.Contains(cfg.GatewayIP) || !cfg.Subnet.Contains(cfg.GuestIP) { + return nil, fmt.Errorf("guestnet: gateway %v and guest %v must be within %v", cfg.GatewayIP, cfg.GuestIP, cfg.Subnet) + } + if !cfg.GatewayIP.Is4() { + return nil, errors.New("guestnet: only IPv4 is supported") + } + + ctx, cancel := context.WithCancel(context.Background()) + n := &Net{ + cfg: cfg, + ctx: ctx, + cancel: cancel, + Flows: NewLog[Flow](1024), + DNS: NewLog[DNSEvent](1024), + Events: NewLog[Event](256), + dhcpLeased: make(chan struct{}), + services: make(map[netip.AddrPort]Service), + } + for ap, svc := range cfg.Services { + n.services[ap] = svc + } + + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + sack := tcpip.TCPSACKEnabled(true) + if err := s.SetTransportProtocolOption(tcp.ProtocolNumber, &sack); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: enable SACK: %v", err) + } + + // The channel endpoint carries whole Ethernet frames; the ethernet wrapper + // adds and strips link headers so gVisor handles ARP for us. + n.linkEP = channel.New(4096, cfg.MTU, tcpip.LinkAddress(cfg.GatewayMAC)) + if err := s.CreateNIC(nicID, ethernet.New(n.linkEP)); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: CreateNIC: %v", err) + } + // Accept packets for any destination so the forwarders see every flow, + // and allow replies from any source address. + if err := s.SetPromiscuousMode(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetPromiscuousMode: %v", err) + } + if err := s.SetSpoofing(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetSpoofing: %v", err) + } + + // Our addresses. gVisor answers ARP and ICMP echo for these. The + // link-local ones are on the NIC precisely so the guest's on-link ARP for + // them (169.254/16 is APIPA and never routed) gets an answer. + addrs := []struct { + addr netip.Addr + bits int + }{ + {cfg.GatewayIP, cfg.Subnet.Bits()}, + {IMDSAddr, 32}, + {DNSAddr, 32}, + } + for _, a := range addrs { + pa := tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom4(a.addr.As4()), + PrefixLen: a.bits, + }, + } + if err := s.AddProtocolAddress(nicID, pa, stack.AddressProperties{}); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: AddProtocolAddress(%v): %v", a.addr, err) + } + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + NIC: nicID, + }}) + + tcpFwd := tcp.NewForwarder(s, 0, 4096, n.handleTCP) + s.SetTransportProtocolHandler(tcp.ProtocolNumber, tcpFwd.HandlePacket) + udpFwd := udp.NewForwarder(s, n.handleUDP) + s.SetTransportProtocolHandler(udp.ProtocolNumber, udpFwd.HandlePacket) + + n.stack = s + n.wg.Add(2) + go n.pumpFromGuest() + go n.pumpToGuest() + return n, nil +} + +// Close stops the network and closes the link. Open flows are torn down. +func (n *Net) Close() error { + n.cancel() + err := n.cfg.Link.Close() + n.linkEP.Close() + // Close every endpoint before waiting for the flow goroutines: one may be + // in a TCP handshake with a guest that will never finish it (the VM + // stopped just after its SYN), which only aborts when its endpoint is + // closed, or after gVisor's handshake timeout of 30s or more. + n.stack.Close() + n.wg.Wait() + n.stack.Destroy() + return err +} + +// GuestIP returns the address the guest is leased. +func (n *Net) GuestIP() netip.Addr { return n.cfg.GuestIP } + +// GatewayIP returns our address on the guest subnet. +func (n *Net) GatewayIP() netip.Addr { return n.cfg.GatewayIP } + +// Leased returns a channel that is closed once the guest has acknowledged a +// DHCP lease. For a restored snapshot this may not fire until the guest +// renews, so callers should also consider the guest reachable if +// [Net.DialGuest] succeeds. +func (n *Net) Leased() <-chan struct{} { return n.dhcpLeased } + +// SetService registers or replaces the handler for a guest-visible +// destination. Passing a nil svc removes it. +func (n *Net) SetService(dst netip.AddrPort, svc Service) { + n.mu.Lock() + defer n.mu.Unlock() + if svc == nil { + delete(n.services, dst) + } else { + n.services[dst] = svc + } +} + +func (n *Net) service(dst netip.AddrPort) (Service, bool) { + n.mu.Lock() + defer n.mu.Unlock() + svc, ok := n.services[dst] + return svc, ok +} + +// DialGuest opens a TCP connection from the gateway address to port on the +// guest. It is how the host reaches sshd in the guest. +func (n *Net) DialGuest(ctx context.Context, port uint16) (net.Conn, error) { + local := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GatewayIP.As4())} + remote := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GuestIP.As4()), Port: port} + return gonet.DialTCPWithBind(ctx, n.stack, local, remote, ipv4.ProtocolNumber) +} + +// pumpFromGuest reads frames from the link, answers DHCP directly, and injects +// everything else into gVisor. +func (n *Net) pumpFromGuest() { + defer n.wg.Done() + buf := make([]byte, 65536+header.EthernetMinimumSize) + for { + m, err := n.cfg.Link.ReadFrame(buf) + if err != nil { + if n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link read: %v", err) + n.cancel() + } + return + } + frame := buf[:m] + if len(frame) < header.EthernetMinimumSize { + continue + } + eth := header.Ethernet(frame) + switch eth.Type() { + case header.IPv4ProtocolNumber: + if n.handleDHCP(frame) { + continue + } + case header.ARPProtocolNumber: + if !n.arpForUs(frame[header.EthernetMinimumSize:]) { + continue + } + default: + // IPv6 and everything else is dropped for now. + continue + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithData(append([]byte(nil), frame...)), + }) + // The ethernet wrapper parses the real EtherType from the frame; the + // protocol argument is ignored. + n.linkEP.InjectInbound(eth.Type(), pkt) + pkt.DecRef() + } +} + +// arpForUs reports whether an ARP packet from the guest should reach gVisor: +// replies (to our requests for the guest's address), and requests for one of +// our own addresses. Our NIC is promiscuous, so gVisor would otherwise answer +// requests for every address, including the guest's own; the guest's DHCP +// client then sees its duplicate address probe answered and declines the +// lease. +func (n *Net) arpForUs(b []byte) bool { + arp := header.ARP(b) + if !arp.IsValid() { + return false + } + if arp.Op() != header.ARPRequest { + return true + } + target := netip.AddrFrom4([4]byte(arp.ProtocolAddressTarget())) + return target == n.cfg.GatewayIP || target == IMDSAddr || target == DNSAddr +} + +// pumpToGuest writes frames produced by gVisor to the link. +func (n *Net) pumpToGuest() { + defer n.wg.Done() + for { + pkt := n.linkEP.ReadContext(n.ctx) + if pkt == nil { + return + } + v := pkt.ToView() + err := n.cfg.Link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + if err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } + } +} + +// writeFrame sends a frame we built ourselves (DHCP replies) to the guest. +func (n *Net) writeFrame(frame []byte) { + if err := n.cfg.Link.WriteFrame(frame); err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } +} + +// egressAllowed reports whether policy lets the guest reach dst. +func (n *Net) egressAllowed(dst netip.Addr) bool { + if dst.IsMulticast() || dst.IsUnspecified() || dst == netip.AddrFrom4([4]byte{255, 255, 255, 255}) { + return false + } + if n.cfg.Subnet.Contains(dst) { + // Only the gateway (via services) and the guest itself exist here. + return false + } + for _, p := range n.cfg.Deny { + if p.Contains(dst) { + return false + } + } + return true +} + +func addrFromTCPIP(a tcpip.Address) netip.Addr { + switch a.Len() { + case 4: + return netip.AddrFrom4(a.As4()) + case 16: + return netip.AddrFrom16(a.As16()) + } + return netip.Addr{} +} diff --git a/guestnet/guestnet_test.go b/guestnet/guestnet_test.go new file mode 100644 index 0000000..40fdd03 --- /dev/null +++ b/guestnet/guestnet_test.go @@ -0,0 +1,544 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "strings" + "testing" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tailscale/tb/guestnet/guestnettest" + "golang.org/x/net/dns/dnsmessage" +) + +// anyDialer dials every address to a single local target, standing in for +// the internet in tests. +type anyDialer struct { + tcpTarget string + udpTarget string +} + +func (d anyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + var nd net.Dialer + switch { + case strings.HasPrefix(network, "tcp"): + return nd.DialContext(ctx, network, d.tcpTarget) + case strings.HasPrefix(network, "udp"): + return nd.DialContext(ctx, network, d.udpTarget) + } + return nil, fmt.Errorf("unsupported network %q", network) +} + +type env struct { + n *Net + guest *guestnettest.Guest + peer *MemLink +} + +func newEnv(t testing.TB, cfg Config) *env { + t.Helper() + host, peer := NewMemLinkPair() + cfg.Link = host + cfg.Logf = t.Logf + n, err := New(cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { n.Close() }) + g := guestnettest.New(t, peer, guestnettest.Config{ + IP: n.cfg.GuestIP, + Subnet: n.cfg.Subnet, + Gateway: n.cfg.GatewayIP, + MAC: n.cfg.GuestMAC, + }) + return &env{n: n, guest: g, peer: peer} +} + +func testContext(t testing.TB) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + return ctx +} + +// dhcpExchange writes a DHCP message from an addressless client and returns +// the reply. +func dhcpExchange(t testing.TB, peer *MemLink, mac net.HardwareAddr, msg *dhcpv4.DHCPv4) *dhcpv4.DHCPv4 { + t.Helper() + frame := buildUDPFrame(mac, net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, [4]byte{}, [4]byte{255, 255, 255, 255}, dhcpv4.ClientPort, dhcpv4.ServerPort, msg.ToBytes()) + if err := peer.WriteFrame(frame); err != nil { + t.Fatal(err) + } + buf := make([]byte, 65536) + deadline := time.After(5 * time.Second) + for { + select { + case <-deadline: + t.Fatal("timed out waiting for DHCP reply") + default: + } + n, err := peer.ReadFrame(buf) + if err != nil { + t.Fatal(err) + } + _, _, srcPort, dstPort, payload, ok := parseUDPFrame(buf[:n]) + if !ok || srcPort != dhcpv4.ServerPort || dstPort != dhcpv4.ClientPort { + continue + } + reply, err := dhcpv4.FromBytes(payload) + if err != nil { + t.Fatalf("bad DHCP reply: %v", err) + } + return reply + } +} + +func TestDHCP(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf, MTU: 9000}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + mac := DefaultGuestMAC + disc, err := dhcpv4.NewDiscovery(mac, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + offer := dhcpExchange(t, peer, mac, disc) + if offer.MessageType() != dhcpv4.MessageTypeOffer { + t.Fatalf("got %v, want OFFER", offer.MessageType()) + } + if got := offer.YourIPAddr.String(); got != DefaultGuestIP.String() { + t.Errorf("yiaddr = %s, want %v", got, DefaultGuestIP) + } + if got := offer.Router(); len(got) != 1 || got[0].String() != DefaultGatewayIP.String() { + t.Errorf("router = %v, want %v", got, DefaultGatewayIP) + } + if got := offer.DNS(); len(got) != 1 || got[0].String() != DNSAddr.String() { + t.Errorf("dns = %v, want %v", got, DNSAddr) + } + if mtu := offer.Options.Get(dhcpv4.OptionInterfaceMTU); len(mtu) != 2 || binary.BigEndian.Uint16(mtu) != 9000 { + t.Errorf("mtu option = %x, want 9000", mtu) + } + if lt := offer.IPAddressLeaseTime(0); lt != 24*time.Hour { + t.Errorf("lease = %v, want 24h", lt) + } + + select { + case <-n.Leased(): + t.Fatal("Leased closed after OFFER; should wait for ACK") + default: + } + + req, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + ack := dhcpExchange(t, peer, mac, req) + if ack.MessageType() != dhcpv4.MessageTypeAck { + t.Fatalf("got %v, want ACK", ack.MessageType()) + } + select { + case <-n.Leased(): + case <-time.After(time.Second): + t.Fatal("Leased not closed after ACK") + } + + // A request for a foreign address is refused. + bad, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true), dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(net.IPv4(10, 1, 2, 3)))) + if err != nil { + t.Fatal(err) + } + if nak := dhcpExchange(t, peer, mac, bad); nak.MessageType() != dhcpv4.MessageTypeNak { + t.Errorf("got %v for foreign address, want NAK", nak.MessageType()) + } + + var kinds []string + for _, ev := range n.Events.Recent() { + kinds = append(kinds, ev.Kind) + } + if got := strings.Join(kinds, ","); got != "dhcp-discover,dhcp-lease" { + t.Errorf("events = %q", got) + } +} + +func TestTCPEgress(t *testing.T) { + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello %s", r.Host) + })} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go srv.Serve(ln) + + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: ln.Addr().String()}}) + ctx := testContext(t) + + dst := netip.MustParseAddrPort("93.184.216.34:80") + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET / HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "hello example.com" { + t.Fatalf("body = %q", body) + } + c.Close() + + flows := waitForClosedFlow(t, e.n, dst) + if flows.Kind != "egress" { + t.Errorf("kind = %q, want egress", flows.Kind) + } + if flows.BytesOut == 0 || flows.BytesIn == 0 { + t.Errorf("bytes in/out = %d/%d, want both > 0", flows.BytesIn, flows.BytesOut) + } + if flows.Src.Addr() != DefaultGuestIP { + t.Errorf("src = %v, want guest IP", flows.Src) + } +} + +func waitForClosedFlow(t testing.TB, n *Net, dst netip.AddrPort) Flow { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + for _, f := range n.Flows.Recent() { + if f.Dst == dst && f.Closed() { + return f + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("no closed flow to %v; flows: %+v", dst, n.Flows.Recent()) + return Flow{} +} + +func TestTCPBlocked(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ctx := testContext(t) + for _, dst := range []netip.AddrPort{ + netip.MustParseAddrPort("100.100.100.100:80"), // tailnet CGNAT + netip.MustParseAddrPort("192.168.64.7:80"), // guest subnet, nonexistent + netip.MustParseAddrPort("192.168.64.1:80"), // gateway with no service + } { + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Errorf("dial %v succeeded, want refusal", dst) + continue + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "blocked" { + t.Errorf("%v: kind = %q, want blocked", dst, f.Kind) + } + } +} + +func TestTCPDialError(t *testing.T) { + // Point egress at a closed port so the host-side dial fails; the guest + // must see a refused connection, not an accepted one. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: addr}}) + ctx := testContext(t) + dst := netip.MustParseAddrPort("93.184.216.34:443") + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Fatal("dial succeeded, want refusal") + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "egress" || f.Err == "" { + t.Errorf("flow = %+v, want egress with error", f) + } +} + +func TestService(t *testing.T) { + imds := NewListenerService("imds") + go http.Serve(imds, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "meta for %s", r.RemoteAddr) + })) + defer imds.Close() + + e := newEnv(t, Config{ + Dialer: anyDialer{}, + Services: map[netip.AddrPort]Service{netip.AddrPortFrom(IMDSAddr, 80): imds}, + }) + ctx := testContext(t) + dst := netip.AddrPortFrom(IMDSAddr, 80) + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial IMDS: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET /v0/meta HTTP/1.1\r\nHost: 169.254.169.254\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + if !strings.HasPrefix(string(body), "meta for 192.168.64.2:") { + t.Fatalf("body = %q", body) + } + c.Close() + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "service" || f.Service != "imds" { + t.Errorf("flow = %+v, want service imds", f) + } + + // A late-registered forwarding service on the gateway works too. + echo, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer echo.Close() + go func() { + for { + c, err := echo.Accept() + if err != nil { + return + } + go func() { io.Copy(c, c); c.Close() }() + } + }() + nfs := netip.AddrPortFrom(DefaultGatewayIP, 2049) + e.n.SetService(nfs, &ForwardService{ServiceName: "nfs", Network: "tcp", Address: echo.Addr().String()}) + c2, err := e.guest.DialTCP(ctx, nfs) + if err != nil { + t.Fatalf("guest dial nfs: %v", err) + } + defer c2.Close() + if _, err := c2.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c2, buf); err != nil || string(buf) != "ping" { + t.Fatalf("echo = %q, %v", buf, err) + } +} + +func TestDNS(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}, Resolver: StaticResolver{Addr: netip.MustParseAddr("10.9.8.7")}}) + + // Guest queries a public resolver; we intercept regardless of address. + dst := netip.MustParseAddrPort("8.8.8.8:53") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + q := dnsmessage.Message{ + Header: dnsmessage.Header{ID: 0xBEEF, RecursionDesired: true}, + Questions: []dnsmessage.Question{{Name: dnsmessage.MustNewName("example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, + } + qb, err := q.Pack() + if err != nil { + t.Fatal(err) + } + if _, err := c.Write(qb); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 4096) + n, err := c.Read(buf) + if err != nil { + t.Fatalf("read DNS response: %v", err) + } + var m dnsmessage.Message + if err := m.Unpack(buf[:n]); err != nil { + t.Fatal(err) + } + if m.ID != 0xBEEF || len(m.Answers) != 1 { + t.Fatalf("response = %+v", m) + } + if a, ok := m.Answers[0].Body.(*dnsmessage.AResource); !ok || a.A != [4]byte{10, 9, 8, 7} { + t.Fatalf("answer = %+v", m.Answers[0].Body) + } + + // Also over TCP, to the advertised resolver address. + tc, err := e.guest.DialTCP(testContext(t), netip.AddrPortFrom(DNSAddr, 53)) + if err != nil { + t.Fatal(err) + } + defer tc.Close() + if err := writeDNSTCP(tc, qb); err != nil { + t.Fatal(err) + } + tc.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := readDNSTCP(tc) + if err != nil { + t.Fatalf("TCP DNS: %v", err) + } + if err := m.Unpack(resp); err != nil || len(m.Answers) != 1 { + t.Fatalf("TCP response = %+v, %v", m, err) + } + + events := e.n.DNS.Recent() + if len(events) != 2 { + t.Fatalf("got %d DNS events, want 2: %+v", len(events), events) + } + for _, ev := range events { + if ev.Name != "example.com." || ev.Type != "TypeA" || ev.Answers != 1 || ev.Upstream != "static" { + t.Errorf("event = %+v", ev) + } + } + if events[0].Proto != "udp" || events[1].Proto != "tcp" { + t.Errorf("protos = %s, %s", events[0].Proto, events[1].Proto) + } +} + +func TestNTP(t *testing.T) { + now := time.Date(2026, 9, 29, 12, 0, 0, 500_000_000, time.UTC) + e := newEnv(t, Config{Dialer: anyDialer{}, Now: func() time.Time { return now }}) + c, err := e.guest.DialUDP(netip.MustParseAddrPort("17.253.2.1:123")) + if err != nil { + t.Fatal(err) + } + defer c.Close() + req := make([]byte, 48) + req[0] = 4<<3 | 3 // v4, client + binary.BigEndian.PutUint64(req[40:], 0x1122334455667788) + if _, err := c.Write(req); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp := make([]byte, 64) + n, err := c.Read(resp) + if err != nil { + t.Fatal(err) + } + if n != 48 || resp[0]&0x07 != 4 { + t.Fatalf("resp len %d mode %d", n, resp[0]&0x07) + } + if binary.BigEndian.Uint64(resp[24:]) != 0x1122334455667788 { + t.Error("originate timestamp not echoed") + } + ts := binary.BigEndian.Uint64(resp[40:]) + secs := int64(ts>>32) - ntpEpochOffset + frac := float64(ts&0xffffffff) / (1 << 32) + got := time.Unix(secs, int64(frac*1e9)) + if d := got.Sub(now); d < -time.Millisecond || d > time.Millisecond { + t.Errorf("transmit = %v, want %v", got, now) + } +} + +func TestUDPEgress(t *testing.T) { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + go func() { + buf := make([]byte, 1500) + for { + n, addr, err := pc.ReadFrom(buf) + if err != nil { + return + } + pc.WriteTo(append([]byte("echo:"), buf[:n]...), addr) + } + }() + e := newEnv(t, Config{Dialer: anyDialer{udpTarget: pc.LocalAddr().String()}}) + dst := netip.MustParseAddrPort("203.0.113.9:4000") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if _, err := c.Write([]byte("hi")); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 100) + n, err := c.Read(buf) + if err != nil || string(buf[:n]) != "echo:hi" { + t.Fatalf("got %q, %v", buf[:n], err) + } + var open bool + for _, f := range e.n.Flows.Recent() { + if f.Dst == dst && f.Kind == "egress" { + open = true + } + } + if !open { + t.Errorf("no egress flow record for %v", dst) + } +} + +func TestDialGuest(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ln, err := e.guest.ListenTCP(22) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + fmt.Fprint(c, "SSH-2.0-fake\r\n") + c.Close() + }() + ctx := testContext(t) + c, err := e.n.DialGuest(ctx, 22) + if err != nil { + t.Fatalf("DialGuest: %v", err) + } + defer c.Close() + line, err := bufio.NewReader(c).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if line != "SSH-2.0-fake\r\n" { + t.Fatalf("banner = %q", line) + } + if !strings.HasPrefix(c.LocalAddr().String(), DefaultGatewayIP.String()+":") { + t.Errorf("local addr = %v, want gateway", c.LocalAddr()) + } +} + +func TestCloseUnblocks(t *testing.T) { + host, _ := NewMemLinkPair() + n, err := New(Config{Link: host}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close hung") + } + if _, err := New(Config{}); !errors.Is(err, err) || err == nil { + t.Fatal("New without a link should fail") + } +} diff --git a/guestnet/guestnettest/guestnettest.go b/guestnet/guestnettest/guestnettest.go new file mode 100644 index 0000000..3f5906f --- /dev/null +++ b/guestnet/guestnettest/guestnettest.go @@ -0,0 +1,131 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnettest provides a fake guest for testing code built on +// guestnet: a gVisor stack with a static address attached to the far end of +// a frame link, standing in for a VM (as one restored from a snapshot would +// look, already holding its lease). +// +// It deliberately does not import guestnet, so guestnet's own tests can use +// it. +package guestnettest + +import ( + "context" + "net" + "net/netip" + "testing" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" +) + +// FrameLink is the subset of guestnet.FrameLink the fake guest needs. +type FrameLink interface { + ReadFrame(buf []byte) (int, error) + WriteFrame(frame []byte) error +} + +// Guest is a fake guest VM's network stack. +type Guest struct { + s *stack.Stack + ep *channel.Endpoint +} + +// Config configures a Guest. +type Config struct { + IP netip.Addr + Subnet netip.Prefix + Gateway netip.Addr + MAC net.HardwareAddr + MTU uint32 // defaults to 1500 +} + +// New starts a fake guest on link. It is torn down by t.Cleanup; closing the +// link is the caller's job. +func New(t testing.TB, link FrameLink, cfg Config) *Guest { + t.Helper() + if cfg.MTU == 0 { + cfg.MTU = 1500 + } + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + ep := channel.New(1024, cfg.MTU, tcpip.LinkAddress(cfg.MAC)) + if err := s.CreateNIC(1, ethernet.New(ep)); err != nil { + t.Fatalf("guest CreateNIC: %v", err) + } + if err := s.AddProtocolAddress(1, tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{Address: tcpip.AddrFrom4(cfg.IP.As4()), PrefixLen: cfg.Subnet.Bits()}, + }, stack.AddressProperties{}); err != nil { + t.Fatalf("guest AddProtocolAddress: %v", err) + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + Gateway: tcpip.AddrFrom4(cfg.Gateway.As4()), + NIC: 1, + }}) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + buf := make([]byte, 65536) + for { + n, err := link.ReadFrame(buf) + if err != nil { + return + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(append([]byte(nil), buf[:n]...))}) + ep.InjectInbound(header.Ethernet(buf).Type(), pkt) + pkt.DecRef() + } + }() + go func() { + for { + pkt := ep.ReadContext(ctx) + if pkt == nil { + return + } + v := pkt.ToView() + link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + } + }() + t.Cleanup(func() { + cancel() + ep.Close() + s.Destroy() + }) + return &Guest{s: s, ep: ep} +} + +func fullAddr(ap netip.AddrPort) tcpip.FullAddress { + return tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4(ap.Addr().As4()), Port: ap.Port()} +} + +// DialTCP dials dst from the guest. +func (g *Guest) DialTCP(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { + return gonet.DialContextTCP(ctx, g.s, fullAddr(dst), ipv4.ProtocolNumber) +} + +// DialUDP opens a UDP socket from the guest connected to dst. +func (g *Guest) DialUDP(dst netip.AddrPort) (net.Conn, error) { + fa := fullAddr(dst) + return gonet.DialUDP(g.s, nil, &fa, ipv4.ProtocolNumber) +} + +// ListenTCP listens on port on the guest. +func (g *Guest) ListenTCP(port uint16) (net.Listener, error) { + return gonet.ListenTCP(g.s, tcpip.FullAddress{NIC: 1, Port: port}, ipv4.ProtocolNumber) +} diff --git a/guestnet/link.go b/guestnet/link.go new file mode 100644 index 0000000..11d1f05 --- /dev/null +++ b/guestnet/link.go @@ -0,0 +1,114 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "errors" + "io" + "os" + "sync" +) + +// FrameLink carries raw Ethernet frames, one per call, between a guest NIC +// and a [Net]. +type FrameLink interface { + // ReadFrame reads one frame into buf and returns its length. + ReadFrame(buf []byte) (int, error) + // WriteFrame writes one frame. + WriteFrame(frame []byte) error + // Close closes the link and unblocks readers. + Close() error +} + +// FileLink is a FrameLink over a datagram socket held as an *os.File, such +// as one end of a SOCK_DGRAM socketpair whose other end is attached to a +// VZFileHandleNetworkDeviceAttachment. Each Read returns one datagram, which +// is one Ethernet frame. +type FileLink struct { + f *os.File +} + +// NewFileLink wraps a datagram socket. It takes ownership of f. The socket +// should have been non-blocking when f was made with os.NewFile; otherwise +// Close does not unblock a pending ReadFrame. +func NewFileLink(f *os.File) *FileLink { return &FileLink{f: f} } + +// ReadFrame implements FrameLink. +func (l *FileLink) ReadFrame(buf []byte) (int, error) { + for { + n, err := l.f.Read(buf) + if err != nil { + return 0, err + } + if n == 0 { + // Zero-length datagrams are not frames. + continue + } + return n, nil + } +} + +// WriteFrame implements FrameLink. +func (l *FileLink) WriteFrame(frame []byte) error { + _, err := l.f.Write(frame) + return err +} + +// Close implements FrameLink. +func (l *FileLink) Close() error { return l.f.Close() } + +// MemLink is an in-memory FrameLink for tests. Create a connected pair with +// [NewMemLinkPair]. +type MemLink struct { + in chan []byte + out chan []byte + done chan struct{} + once sync.Once + peer *MemLink +} + +// NewMemLinkPair returns two links where frames written to one are read from +// the other. Closing either end unblocks both. +func NewMemLinkPair() (a, b *MemLink) { + ab := make(chan []byte, 1024) + ba := make(chan []byte, 1024) + done := make(chan struct{}) + a = &MemLink{in: ba, out: ab, done: done} + b = &MemLink{in: ab, out: ba, done: done} + a.peer, b.peer = b, a + return a, b +} + +// ReadFrame implements FrameLink. +func (l *MemLink) ReadFrame(buf []byte) (int, error) { + select { + case f := <-l.in: + if len(f) > len(buf) { + return 0, io.ErrShortBuffer + } + return copy(buf, f), nil + case <-l.done: + return 0, errors.New("memlink: closed") + } +} + +// WriteFrame implements FrameLink. +func (l *MemLink) WriteFrame(frame []byte) error { + f := append([]byte(nil), frame...) + select { + case l.out <- f: + return nil + case <-l.done: + return errors.New("memlink: closed") + } +} + +// Close implements FrameLink. It closes both ends of the pair. +func (l *MemLink) Close() error { + l.once.Do(func() { close(l.done) }) + if l.peer != nil { + l.peer.once.Do(func() {}) + } + return nil +} diff --git a/guestnet/log.go b/guestnet/log.go new file mode 100644 index 0000000..dd75c1f --- /dev/null +++ b/guestnet/log.go @@ -0,0 +1,120 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "net/netip" + "sync" + "time" +) + +// Flow is one TCP connection or UDP session from the guest. +type Flow struct { + // ID is unique within a Net and identical for the open and close records + // of one flow. + ID uint64 `json:"id"` + Proto string `json:"proto"` // "tcp" or "udp" + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Start time.Time `json:"start"` + // End is zero for the open record. + End time.Time `json:"end,omitzero"` + // Kind says how the flow was handled: "egress", "service", "dns", "ntp", + // or "blocked". + Kind string `json:"kind"` + // Service names the registered service for Kind "service". + Service string `json:"service,omitempty"` + // BytesIn counts bytes sent to the guest, BytesOut from the guest. + BytesIn int64 `json:"bytes_in"` + BytesOut int64 `json:"bytes_out"` + Err string `json:"err,omitempty"` +} + +// Closed reports whether f is a close record. +func (f Flow) Closed() bool { return !f.End.IsZero() } + +// DNSEvent is one DNS query from the guest and how it was answered. +type DNSEvent struct { + Time time.Time `json:"time"` + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Proto string `json:"proto"` + Name string `json:"name"` + Type string `json:"type"` + RCode string `json:"rcode,omitempty"` + Answers int `json:"answers"` + Upstream string `json:"upstream,omitempty"` + Duration time.Duration `json:"duration"` + Err string `json:"err,omitempty"` +} + +// Event is a notable network event. +type Event struct { + Time time.Time `json:"time"` + // Kind is "dhcp-discover", "dhcp-lease", "blocked", "dial-error", ... + Kind string `json:"kind"` + Msg string `json:"msg"` +} + +// Log is a bounded ring of records with live subscribers. +type Log[T any] struct { + mu sync.Mutex + ring []T + next int + full bool + subs map[chan T]struct{} +} + +// NewLog returns a Log keeping the last n records. +func NewLog[T any](n int) *Log[T] { + return &Log[T]{ring: make([]T, n), subs: make(map[chan T]struct{})} +} + +// Add appends a record and delivers it to subscribers that are keeping up. +// Slow subscribers miss records rather than block the network. +func (l *Log[T]) Add(v T) { + l.mu.Lock() + defer l.mu.Unlock() + l.ring[l.next] = v + l.next++ + if l.next == len(l.ring) { + l.next = 0 + l.full = true + } + for ch := range l.subs { + select { + case ch <- v: + default: + } + } +} + +// Recent returns the retained records, oldest first. +func (l *Log[T]) Recent() []T { + l.mu.Lock() + defer l.mu.Unlock() + if !l.full { + return append([]T(nil), l.ring[:l.next]...) + } + out := make([]T, 0, len(l.ring)) + out = append(out, l.ring[l.next:]...) + out = append(out, l.ring[:l.next]...) + return out +} + +// Subscribe returns a channel receiving new records until ctx is done. +func (l *Log[T]) Subscribe(ctx context.Context) <-chan T { + ch := make(chan T, 256) + l.mu.Lock() + l.subs[ch] = struct{}{} + l.mu.Unlock() + go func() { + <-ctx.Done() + l.mu.Lock() + delete(l.subs, ch) + l.mu.Unlock() + }() + return ch +} diff --git a/guestnet/ntp.go b/guestnet/ntp.go new file mode 100644 index 0000000..8a888fb --- /dev/null +++ b/guestnet/ntp.go @@ -0,0 +1,67 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "encoding/binary" + "net" + "time" +) + +// ntpEpochOffset is the number of seconds between the NTP epoch (1900) and +// the Unix epoch (1970). +const ntpEpochOffset = 2208988800 + +// ntpIdleTimeout ends an NTP session after this long without a request. +const ntpIdleTimeout = 30 * time.Second + +// serveNTP answers SNTP requests on one guest UDP session with the host's +// time. A guest restored from a memory snapshot has a clock that is stale by +// the snapshot's age; macOS's timed corrects it on its next exchange with +// what it believes is time.apple.com, which this intercepts. +func (n *Net) serveNTP(conn net.Conn) error { + buf := make([]byte, 128) + for { + conn.SetReadDeadline(time.Now().Add(ntpIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + if m < 48 { + continue + } + resp := ntpResponse(buf[:48], n.cfg.Now()) + if _, err := conn.Write(resp); err != nil { + return err + } + } +} + +// ntpResponse builds a 48-byte server reply to req with all server +// timestamps set to now. +func ntpResponse(req []byte, now time.Time) []byte { + resp := make([]byte, 48) + vn := (req[0] >> 3) & 0x07 + if vn < 3 || vn > 4 { + vn = 4 + } + resp[0] = vn<<3 | 4 // LI=0, mode 4 (server) + resp[1] = 2 // stratum + resp[2] = req[2] // poll, echoed + resp[3] = 0xEC // precision: about 2^-20 seconds + // Root delay and dispersion: zero. + copy(resp[12:16], "LOCL") + ts := ntpTimestamp(now) + binary.BigEndian.PutUint64(resp[16:], ts) // reference + copy(resp[24:32], req[40:48]) // originate = client transmit + binary.BigEndian.PutUint64(resp[32:], ts) // receive + binary.BigEndian.PutUint64(resp[40:], ts) // transmit + return resp +} + +func ntpTimestamp(t time.Time) uint64 { + secs := uint64(t.Unix()) + ntpEpochOffset + frac := uint64(t.Nanosecond()) * (1 << 32) / 1e9 + return secs<<32 | frac +} diff --git a/guestnet/services.go b/guestnet/services.go new file mode 100644 index 0000000..fe4bf6c --- /dev/null +++ b/guestnet/services.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "errors" + "net" + "sync" +) + +// Service handles guest TCP connections to a registered destination. +type Service interface { + // Name identifies the service in flow logs. + Name() string + // HandleConn serves one guest connection. It owns conn and must close it. + // ctx is done when the Net closes. + HandleConn(ctx context.Context, conn net.Conn) +} + +// ServiceFunc adapts a function to a Service. +type ServiceFunc struct { + ServiceName string + Handle func(ctx context.Context, conn net.Conn) +} + +// Name implements Service. +func (s ServiceFunc) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s ServiceFunc) HandleConn(ctx context.Context, conn net.Conn) { s.Handle(ctx, conn) } + +// ForwardService relays guest connections to a host-side address, such as +// "tcp:127.0.0.1:2049" or "unix:/var/run/foo.sock". It replaces the pf rdr +// rules corp CI uses on macOS today. +type ForwardService struct { + ServiceName string + // Network and Address are passed to Dialer.DialContext. + Network, Address string + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// Name implements Service. +func (s *ForwardService) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s *ForwardService) HandleConn(ctx context.Context, conn net.Conn) { + defer conn.Close() + d := s.Dialer + if d == nil { + d = &net.Dialer{} + } + remote, err := d.DialContext(ctx, s.Network, s.Address) + if err != nil { + return + } + defer remote.Close() + splice(conn, remote) +} + +// ListenerService is a Service that exposes guest connections as a +// net.Listener, so an http.Server (or anything else that Accepts) can serve +// the guest without knowing about gVisor. +type ListenerService struct { + ServiceName string + + once sync.Once + conns chan net.Conn + closed chan struct{} + cmu sync.Once +} + +// NewListenerService returns a ListenerService named name. +func NewListenerService(name string) *ListenerService { + s := &ListenerService{ServiceName: name} + s.init() + return s +} + +func (s *ListenerService) init() { + s.once.Do(func() { + s.conns = make(chan net.Conn) + s.closed = make(chan struct{}) + }) +} + +// Name implements Service. +func (s *ListenerService) Name() string { return s.ServiceName } + +// HandleConn implements Service by handing conn to Accept. +func (s *ListenerService) HandleConn(ctx context.Context, conn net.Conn) { + s.init() + select { + case s.conns <- conn: + case <-s.closed: + conn.Close() + case <-ctx.Done(): + conn.Close() + } +} + +// Accept implements net.Listener. +func (s *ListenerService) Accept() (net.Conn, error) { + s.init() + select { + case c := <-s.conns: + return c, nil + case <-s.closed: + return nil, errListenerClosed + } +} + +// Close implements net.Listener. +func (s *ListenerService) Close() error { + s.init() + s.cmu.Do(func() { close(s.closed) }) + return nil +} + +// Addr implements net.Listener. +func (s *ListenerService) Addr() net.Addr { return serviceAddr(s.ServiceName) } + +var errListenerClosed = errors.New("guestnet: listener closed") + +type serviceAddr string + +func (a serviceAddr) Network() string { return "guestnet" } +func (a serviceAddr) String() string { return string(a) } diff --git a/guestnet/socketpair.go b/guestnet/socketpair.go new file mode 100644 index 0000000..ca55377 --- /dev/null +++ b/guestnet/socketpair.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "fmt" + "os" + "syscall" +) + +// Buffer sizes for the guest datagram link. Apple documents +// (VZFileHandleNetworkDeviceAttachment.maximumTransmissionUnit) that +// SO_RCVBUF must be at least twice SO_SNDBUF, and recommends four times. +// The absolute sizes are our guess, room for a few hundred full-size frames, +// and well under macOS's default kern.ipc.maxsockbuf of 8 MiB. Tune them +// with throughput measurements. +const ( + linkSendBuf = 512 << 10 + linkRecvBuf = 4 * linkSendBuf +) + +// NewSocketPairLink creates a connected SOCK_DGRAM unix socketpair. The +// returned *os.File is for the VM side (hand it to the network device +// attachment; the framework dups it, so close it after the VM is created). +// The returned FileLink is our side. +func NewSocketPairLink() (vmSide *os.File, host *FileLink, err error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + return nil, nil, fmt.Errorf("socketpair: %w", err) + } + for _, fd := range fds { + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, linkSendBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_SNDBUF: %w", err) + } + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, linkRecvBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_RCVBUF: %w", err) + } + syscall.CloseOnExec(fd) + } + // Our end must be non-blocking before os.NewFile sees it, so the file + // uses the runtime poller and Close unblocks a pending ReadFrame. The + // VM end stays blocking, as the framework expects. + if err := syscall.SetNonblock(fds[1], true); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SetNonblock: %w", err) + } + vmSide = os.NewFile(uintptr(fds[0]), "guestnet-vm") + hostFile := os.NewFile(uintptr(fds[1]), "guestnet-host") + return vmSide, NewFileLink(hostFile), nil +} + +func closeFDs(fds [2]int) { + for _, fd := range fds { + syscall.Close(fd) + } +} diff --git a/guestnet/socketpair_test.go b/guestnet/socketpair_test.go new file mode 100644 index 0000000..0872baf --- /dev/null +++ b/guestnet/socketpair_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "testing" + "time" +) + +// TestSocketPairCloseUnblocks checks that a Net on a real socketpair shuts +// down while its frame reader is blocked, which requires our end of the +// pair to use the runtime poller. +func TestSocketPairCloseUnblocks(t *testing.T) { + vmSide, link, err := NewSocketPairLink() + if err != nil { + t.Fatal(err) + } + defer vmSide.Close() + n, err := New(Config{Link: link}) + if err != nil { + t.Fatal(err) + } + // Let the frame reader block in ReadFrame. + time.Sleep(50 * time.Millisecond) + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close hung with a blocked reader") + } +} diff --git a/guestnet/tcp.go b/guestnet/tcp.go new file mode 100644 index 0000000..b2b99bd --- /dev/null +++ b/guestnet/tcp.go @@ -0,0 +1,226 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "fmt" + "io" + "net" + "net/netip" + "sync" + "sync/atomic" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// egressDialTimeout bounds how long the guest's SYN waits for the host-side +// dial before we answer with a RST. +const egressDialTimeout = 10 * time.Second + +var flowIDs atomic.Uint64 + +// handleTCP is called by the gVisor forwarder for every SYN from the guest. +func (n *Net) handleTCP(r *tcp.ForwarderRequest) { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "tcp", Src: src, Dst: dst, Start: n.cfg.Now()} + + if svc, ok := n.service(dst); ok { + flow.Kind, flow.Service = "service", svc.Name() + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + // The service owns conn and records the flow's close when it + // closes it. + svc.HandleConn(n.ctx, conn) + }() + return + } + + if dst.Port() == 53 { + flow.Kind = "dns" + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSTCP(conn, src, dst)) + }() + return + } + + if !n.egressAllowed(dst.Addr()) { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("tcp %v -> %v", src, dst)}) + r.Complete(true) + return + } + + // Egress: dial first so a refused or unreachable destination becomes a + // RST to the guest instead of an accepted-then-closed connection. The + // forwarder drops the guest's retransmitted SYNs meanwhile. + flow.Kind = "egress" + n.wg.Add(1) + go func() { + defer n.wg.Done() + ctx, cancel := context.WithTimeout(n.ctx, egressDialTimeout) + defer cancel() + remote, err := n.cfg.Dialer.DialContext(ctx, "tcp", dst.String()) + if err != nil { + flow.Err = err.Error() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.End, Kind: "dial-error", Msg: fmt.Sprintf("tcp %v: %v", dst, err)}) + r.Complete(true) + return + } + conn := n.acceptTCP(r, flow) + if conn == nil { + remote.Close() + return + } + defer n.track(conn)() + defer n.track(remote)() + splice(conn, remote) + conn.Close() + }() +} + +// acceptTCP completes the handshake and returns the guest-side connection +// wrapped for flow accounting, recording the flow's open record. It returns +// nil (and records the failure) if the endpoint could not be created. +func (n *Net) acceptTCP(r *tcp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + r.Complete(true) + return nil + } + ep.SocketOptions().SetKeepAlive(true) + r.Complete(false) + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewTCPConn(&wq, ep), flow) +} + +// track arranges for conn to be closed when the Net closes, so no goroutine +// outlives it. The returned func cancels that arrangement; call it when the +// conn is done. +func (n *Net) track(conn net.Conn) func() { + stop := context.AfterFunc(n.ctx, func() { + // gonet conns don't reliably wake a blocked Read on Close alone; a + // past deadline does. + conn.SetReadDeadline(time.Unix(1, 0)) + conn.Close() + }) + return func() { stop() } +} + +// flowConn wraps a guest-side connection to count bytes and record the +// flow's close record exactly once, when whoever owns it closes it. +type flowConn struct { + net.Conn + n *Net + flow Flow + in atomic.Int64 // bytes written to the guest + out atomic.Int64 // bytes read from the guest + once sync.Once +} + +func newFlowConn(n *Net, c net.Conn, flow Flow) *flowConn { + return &flowConn{Conn: c, n: n, flow: flow} +} + +func (c *flowConn) Read(b []byte) (int, error) { + m, err := c.Conn.Read(b) + c.out.Add(int64(m)) + return m, err +} + +func (c *flowConn) Write(b []byte) (int, error) { + m, err := c.Conn.Write(b) + c.in.Add(int64(m)) + return m, err +} + +// CloseWrite half-closes the guest side, for splice. +func (c *flowConn) CloseWrite() error { + if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return nil +} + +// Close implements net.Conn and records the flow's close record. +func (c *flowConn) Close() error { return c.closeWithErr(nil) } + +// closeWithErr closes the conn, recording err (if any) in the flow's close +// record. Only the first close records anything. +func (c *flowConn) closeWithErr(err error) error { + cerr := c.Conn.Close() + c.once.Do(func() { + f := c.flow + f.End = c.n.cfg.Now() + f.BytesIn, f.BytesOut = c.in.Load(), c.out.Load() + if err != nil { + f.Err = err.Error() + } + c.n.Flows.Add(f) + }) + return cerr +} + +// splice copies bidirectionally between a and b with half-close +// propagation, returning when both directions are done. +// +// When one direction hits EOF it half-closes the other side, then waits a +// bounded time for the remaining direction. gVisor's gonet.TCPConn does not +// reliably wake a blocked Read on Close, so after the grace period both +// connections get a past read deadline before Close to guarantee the second +// copier exits and the endpoint is released. +func splice(a, b net.Conn) { + done := make(chan struct{}, 2) + cp := func(dst, src net.Conn) { + io.Copy(dst, src) + if cw, ok := dst.(interface{ CloseWrite() error }); ok { + cw.CloseWrite() + } + done <- struct{}{} + } + go cp(a, b) + go cp(b, a) + + <-done + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case <-done: + case <-t.C: + past := time.Unix(1, 0) + a.SetReadDeadline(past) + b.SetReadDeadline(past) + a.Close() + b.Close() + <-done + } + a.Close() + b.Close() +} diff --git a/guestnet/udp.go b/guestnet/udp.go new file mode 100644 index 0000000..29609ad --- /dev/null +++ b/guestnet/udp.go @@ -0,0 +1,146 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "fmt" + "net" + "net/netip" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// udpIdleTimeout ends a UDP session after this long without traffic in +// either direction. +const udpIdleTimeout = 60 * time.Second + +// udpMaxPacket is the largest UDP payload we relay. +const udpMaxPacket = 65535 + +// handleUDP is called by the gVisor forwarder for the first packet of every +// UDP 4-tuple from the guest. Returning false drops the packet (and gVisor +// answers with ICMP port unreachable). +func (n *Net) handleUDP(r *udp.ForwarderRequest) bool { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "udp", Src: src, Dst: dst, Start: n.cfg.Now()} + + switch dst.Port() { + case 53: + flow.Kind = "dns" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSUDP(conn, src, dst)) + }() + return true + case 123: + flow.Kind = "ntp" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveNTP(conn)) + }() + return true + } + + if !n.egressAllowed(dst.Addr()) { + // Multicast (mDNS, SSDP) and broadcast chatter is dropped silently; + // unicast blocks are worth a record. + if !dst.Addr().IsMulticast() && dst.Addr() != netipBroadcast { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("udp %v -> %v", src, dst)}) + } + return false + } + + flow.Kind = "egress" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.relayUDP(conn, dst)) + }() + return true +} + +// acceptUDP creates the gVisor endpoint for the session (bound to the +// guest's chosen destination and connected to the guest) and records the +// flow's open record. +func (n *Net) acceptUDP(r *udp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + return nil + } + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewUDPConn(&wq, ep), flow) +} + +// relayUDP shuttles datagrams between the guest session and a host-side UDP +// socket to dst until idle. Byte counts are kept by the flowConn. +func (n *Net) relayUDP(guest net.Conn, dst netip.AddrPort) error { + remote, err := n.cfg.Dialer.DialContext(n.ctx, "udp", dst.String()) + if err != nil { + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dial-error", Msg: fmt.Sprintf("udp %v: %v", dst, err)}) + return err + } + defer remote.Close() + defer n.track(remote)() + + done := make(chan struct{}) + go func() { + defer close(done) + buf := make([]byte, udpMaxPacket) + for { + remote.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := remote.Read(buf) + if err != nil { + guest.Close() + return + } + if _, err := guest.Write(buf[:m]); err != nil { + return + } + } + }() + buf := make([]byte, udpMaxPacket) + for { + guest.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := guest.Read(buf) + if err != nil { + // Idle timeout, guest close, or Net shutdown: all normal. + remote.Close() + <-done + return nil + } + if _, err := remote.Write(buf[:m]); err != nil { + <-done + return err + } + } +}