From 1f75e37b14d061f21359ba195c84f7c695e7d97b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Nov 2026 18:15:38 +0000 Subject: [PATCH 1/3] guestnet: add a private userspace network for each VM guest This is the network for mantsana, a macOS CI VM backend replacing Tart (see the DESIGN.md that follows), but nothing in it is macOS-specific. A guest's virtual NIC hands us raw Ethernet frames (on macOS over a SOCK_DGRAM socketpair to a VZFileHandleNetworkDeviceAttachment) and they all terminate in a gVisor netstack owned by the guest's Net, so a guest can only reach what we connect it to, and can't see other guests: - The gateway, IMDS (169.254.169.254) and DNS (169.254.169.253) addresses are on our NIC, so gVisor answers ARP and ICMP for them, and the guest needs no route or DNS changes to reach them. ARP for any other address is dropped: the NIC is promiscuous, and answering the guest's DHCP duplicate-address probe makes it decline its lease. - DHCP is answered from the frame loop with one fixed lease, the same in every slot, so a VM memory snapshot restores anywhere. - Each TCP SYN and UDP flow is dispatched by destination to a registered Service (a guest-facing server, or a forward to a host-side address), to DNS (any port 53) or NTP (any port 123) interception, or to egress dialed from the host, subject to a deny list. Every flow and DNS query is logged. - The host reaches the guest only by DialGuest; there is no host route to it. guestnettest provides a fake guest stack for tests of code built on it. Updates tailscale/corp#39994 Signed-off-by: Brad Fitzpatrick Change-Id: I5b359a459ae52218e97cef3b74ecc14a806e0cc7 --- go.mod | 8 +- go.sum | 6 +- guestnet/addr.go | 28 ++ guestnet/arp_test.go | 95 +++++ guestnet/close_test.go | 133 +++++++ guestnet/dhcp.go | 198 ++++++++++ guestnet/dns.go | 299 ++++++++++++++ guestnet/guestnet.go | 468 ++++++++++++++++++++++ guestnet/guestnet_test.go | 544 ++++++++++++++++++++++++++ guestnet/guestnettest/guestnettest.go | 131 +++++++ guestnet/link.go | 114 ++++++ guestnet/log.go | 120 ++++++ guestnet/ntp.go | 67 ++++ guestnet/services.go | 130 ++++++ guestnet/socketpair.go | 61 +++ guestnet/socketpair_test.go | 35 ++ guestnet/tcp.go | 226 +++++++++++ guestnet/udp.go | 146 +++++++ 18 files changed, 2805 insertions(+), 4 deletions(-) create mode 100644 guestnet/addr.go create mode 100644 guestnet/arp_test.go create mode 100644 guestnet/close_test.go create mode 100644 guestnet/dhcp.go create mode 100644 guestnet/dns.go create mode 100644 guestnet/guestnet.go create mode 100644 guestnet/guestnet_test.go create mode 100644 guestnet/guestnettest/guestnettest.go create mode 100644 guestnet/link.go create mode 100644 guestnet/log.go create mode 100644 guestnet/ntp.go create mode 100644 guestnet/services.go create mode 100644 guestnet/socketpair.go create mode 100644 guestnet/socketpair_test.go create mode 100644 guestnet/tcp.go create mode 100644 guestnet/udp.go diff --git a/go.mod b/go.mod index c781e5a..2caf8f2 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,17 @@ require ( github.com/bradfitz/parentdeath v0.0.0-20260315043412-764506aeb900 github.com/bradfitz/qcow2 v0.0.0-20260303185237-93afc730382b github.com/cespare/xxhash/v2 v2.3.0 - github.com/go-jose/go-jose/v4 v4.1.3 + github.com/go-jose/go-jose/v4 v4.1.4 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/go-cmp v0.7.0 + github.com/insomniacslk/dhcp v0.0.0-20260901064844-234b97448fae github.com/pierrec/lz4/v4 v4.1.26 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 + golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 golang.org/x/time v0.16.0 + gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 modernc.org/sqlite v1.51.0 tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef ) @@ -47,6 +50,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fxamacker/cbor/v2 v2.9.3 // indirect github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -63,12 +67,12 @@ require ( github.com/prometheus/procfs v0.21.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect + github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/x448/float16 v0.8.4 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/net v0.59.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect diff --git a/go.sum b/go.sum index fff738a..180ac20 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI= github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE= github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -91,6 +91,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/guestnet/addr.go b/guestnet/addr.go new file mode 100644 index 0000000..7c70db4 --- /dev/null +++ b/guestnet/addr.go @@ -0,0 +1,28 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" +) + +var netipBroadcast = netip.AddrFrom4([4]byte{255, 255, 255, 255}) + +func netipFromIP(ip net.IP) (netip.Addr, bool) { + if ip4 := ip.To4(); ip4 != nil { + return netip.AddrFrom4([4]byte(ip4)), true + } + return netip.AddrFromSlice(ip) +} + +// broadcastOf returns the directed broadcast address of an IPv4 prefix. +func broadcastOf(p netip.Prefix) net.IP { + a := p.Masked().Addr().As4() + bits := p.Bits() + for i := bits; i < 32; i++ { + a[i/8] |= 1 << (7 - uint(i%8)) + } + return net.IP(a[:]) +} diff --git a/guestnet/arp_test.go b/guestnet/arp_test.go new file mode 100644 index 0000000..69b10de --- /dev/null +++ b/guestnet/arp_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildARPRequest returns an Ethernet broadcast ARP request from mac/srcIP +// asking for dstIP. An ARP probe (RFC 5227) has an unspecified srcIP. +func buildARPRequest(mac net.HardwareAddr, srcIP, dstIP netip.Addr) []byte { + frame := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(mac), + DstAddr: tcpip.LinkAddress("\xff\xff\xff\xff\xff\xff"), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(frame[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPRequest) + copy(arp.HardwareAddressSender(), mac) + src, dst := srcIP.As4(), dstIP.As4() + copy(arp.ProtocolAddressSender(), src[:]) + copy(arp.ProtocolAddressTarget(), dst[:]) + return frame +} + +// TestARPOnlyForOurAddresses checks that we answer ARP only for our own +// addresses. In particular the guest's DHCP client probes its offered +// address (RFC 5227) and declines the lease if anything answers; macOS does +// this, and a promiscuous gVisor NIC would answer for every address. +func TestARPOnlyForOurAddresses(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + replies := make(chan netip.Addr, 16) + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + arp := header.ARP(frame[header.EthernetMinimumSize:]) + if arp.IsValid() && arp.Op() == header.ARPReply { + replies <- netip.AddrFrom4([4]byte(arp.ProtocolAddressSender())) + } + } + }() + + mac := DefaultGuestMAC + unspec := netip.AddrFrom4([4]byte{}) + for _, tt := range []struct { + src, dst netip.Addr + want bool + }{ + {unspec, DefaultGuestIP, false}, // DHCP client's duplicate address probe + {DefaultGuestIP, DefaultGuestIP, false}, // gratuitous ARP + {DefaultGuestIP, netip.MustParseAddr("192.168.64.77"), false}, + {DefaultGuestIP, DefaultGatewayIP, true}, + {DefaultGuestIP, IMDSAddr, true}, + {DefaultGuestIP, DNSAddr, true}, + } { + if err := peer.WriteFrame(buildARPRequest(mac, tt.src, tt.dst)); err != nil { + t.Fatal(err) + } + var got bool + select { + case from := <-replies: + got = true + if from != tt.dst { + t.Errorf("ARP for %v: reply claims %v", tt.dst, from) + } + case <-time.After(300 * time.Millisecond): + } + if got != tt.want { + t.Errorf("ARP from %v for %v: replied = %v, want %v", tt.src, tt.dst, got, tt.want) + } + } +} diff --git a/guestnet/close_test.go b/guestnet/close_test.go new file mode 100644 index 0000000..81840b3 --- /dev/null +++ b/guestnet/close_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "net" + "net/netip" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// buildTCPSYNFrame assembles an Ethernet/IPv4/TCP SYN frame with checksums. +func buildTCPSYNFrame(srcMAC, dstMAC net.HardwareAddr, src, dst netip.AddrPort) []byte { + ipLen := header.IPv4MinimumSize + header.TCPMinimumSize + frame := make([]byte, header.EthernetMinimumSize+ipLen) + header.Ethernet(frame).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + srcIP, dstIP := tcpip.AddrFrom4(src.Addr().As4()), tcpip.AddrFrom4(dst.Addr().As4()) + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.TCPProtocolNumber), + SrcAddr: srcIP, + DstAddr: dstIP, + }) + ip.SetChecksum(^ip.CalculateChecksum()) + tcp := header.TCP(ip.Payload()) + tcp.Encode(&header.TCPFields{ + SrcPort: src.Port(), + DstPort: dst.Port(), + SeqNum: 1000, + DataOffset: header.TCPMinimumSize, + Flags: header.TCPFlagSyn, + WindowSize: 65535, + }) + xsum := header.PseudoHeaderChecksum(header.TCPProtocolNumber, srcIP, dstIP, header.TCPMinimumSize) + tcp.SetChecksum(^checksum.Checksum(tcp, xsum)) + return frame +} + +// TestCloseDuringHandshake checks that Close doesn't wait for a TCP +// handshake the guest will never finish. A guest that is stopped just after +// sending a SYN (the VM stopping mid-connection) leaves the forwarder waiting +// in the handshake; Close used to wait for gVisor's handshake timeout (30s +// or more) before tearing down. +func TestCloseDuringHandshake(t *testing.T) { + target, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer target.Close() + accepted := make(chan net.Conn, 1) + go func() { + c, err := target.Accept() + if err == nil { + accepted <- c + } + }() + + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Dialer: anyDialer{tcpTarget: target.Addr().String()}, Logf: t.Logf}) + if err != nil { + t.Fatal(err) + } + // Answer ARP for the guest's address, as a real guest would, so the + // SYN-ACK goes out, but never ACK it: the guest has stopped. + go func() { + buf := make([]byte, 65536) + for { + m, err := peer.ReadFrame(buf) + if err != nil { + return + } + frame := buf[:m] + if header.Ethernet(frame).Type() != header.ARPProtocolNumber { + continue + } + req := header.ARP(frame[header.EthernetMinimumSize:]) + if !req.IsValid() || req.Op() != header.ARPRequest { + continue + } + reply := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(reply).Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(DefaultGuestMAC), + DstAddr: tcpip.LinkAddress(req.HardwareAddressSender()), + Type: header.ARPProtocolNumber, + }) + arp := header.ARP(reply[header.EthernetMinimumSize:]) + arp.SetIPv4OverEthernet() + arp.SetOp(header.ARPReply) + copy(arp.HardwareAddressSender(), DefaultGuestMAC) + copy(arp.ProtocolAddressSender(), req.ProtocolAddressTarget()) + copy(arp.HardwareAddressTarget(), req.HardwareAddressSender()) + copy(arp.ProtocolAddressTarget(), req.ProtocolAddressSender()) + peer.WriteFrame(reply) + } + }() + + src := netip.AddrPortFrom(DefaultGuestIP, 40000) + dst := netip.MustParseAddrPort("198.51.100.7:443") + if err := peer.WriteFrame(buildTCPSYNFrame(DefaultGuestMAC, DefaultGatewayMAC, src, dst)); err != nil { + t.Fatal(err) + } + select { + case c := <-accepted: + defer c.Close() + case <-time.After(5 * time.Second): + t.Fatal("egress dial never happened") + } + time.Sleep(100 * time.Millisecond) // let the forwarder enter the handshake + + done := make(chan struct{}) + t0 := time.Now() + go func() { + n.Close() + close(done) + }() + select { + case <-done: + t.Logf("Close took %v", time.Since(t0)) + case <-time.After(5 * time.Second): + t.Fatal("Close waited for the guest's unfinished TCP handshake") + } +} diff --git a/guestnet/dhcp.go b/guestnet/dhcp.go new file mode 100644 index 0000000..ec4d25b --- /dev/null +++ b/guestnet/dhcp.go @@ -0,0 +1,198 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +// handleDHCP answers DHCPv4 requests found in frame. It reports whether the +// frame was a DHCP request (and so must not be injected into the stack). +// +// DHCP is answered outside gVisor because the client has no address yet and +// the reply rules (unicast to the client MAC vs. broadcast) are about the +// link layer, which is simplest to control when we build the frame ourselves. +func (n *Net) handleDHCP(frame []byte) bool { + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if len(ip) < header.IPv4MinimumSize || !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return false + } + if ip.More() || ip.FragmentOffset() != 0 { + return false + } + udpPkt := header.UDP(ip.Payload()) + if len(udpPkt) < header.UDPMinimumSize || udpPkt.DestinationPort() != dhcpv4.ServerPort { + return false + } + req, err := dhcpv4.FromBytes(udpPkt.Payload()) + if err != nil { + n.cfg.Logf("guestnet: dhcp: bad request: %v", err) + return true + } + if req.OpCode != dhcpv4.OpcodeBootRequest { + return true + } + + var replyType dhcpv4.MessageType + switch req.MessageType() { + case dhcpv4.MessageTypeDiscover: + replyType = dhcpv4.MessageTypeOffer + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-discover", Msg: req.ClientHWAddr.String()}) + case dhcpv4.MessageTypeRequest: + // A REQUEST for some other address (a lease carried over from + // elsewhere) gets a NAK so the client starts over and takes ours. + if want := req.RequestedIPAddress(); want != nil && !want.IsUnspecified() && !want.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else if !req.ClientIPAddr.IsUnspecified() && !req.ClientIPAddr.Equal(n.cfg.GuestIP.AsSlice()) { + replyType = dhcpv4.MessageTypeNak + } else { + replyType = dhcpv4.MessageTypeAck + } + case dhcpv4.MessageTypeInform: + replyType = dhcpv4.MessageTypeAck + case dhcpv4.MessageTypeDecline: + n.cfg.Logf("guestnet: dhcp: guest declined %v", n.cfg.GuestIP) + return true + default: + return true + } + + resp, err := n.buildDHCPReply(req, replyType) + if err != nil { + n.cfg.Logf("guestnet: dhcp: build %v: %v", replyType, err) + return true + } + + // Where to send it (RFC 2131 section 4.1). A NAK is always broadcast. + // A client that already has an address (renewing) is answered unicast. + // Otherwise honor the broadcast flag; a client without the flag is + // answered unicast to its MAC with its new address as the IP destination. + dstMAC := net.HardwareAddr(req.ClientHWAddr) + dstIP := n.cfg.GuestIP + switch { + case replyType == dhcpv4.MessageTypeNak, req.IsBroadcast() && req.ClientIPAddr.IsUnspecified(): + dstMAC = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + dstIP = netipBroadcast + case !req.ClientIPAddr.IsUnspecified(): + if a, ok := netipFromIP(req.ClientIPAddr); ok { + dstIP = a + } + } + + srcIP := n.cfg.GatewayIP + if replyType == dhcpv4.MessageTypeAck || replyType == dhcpv4.MessageTypeOffer { + if replyType == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeRequest { + n.dhcpLeasedOnce.Do(func() { close(n.dhcpLeased) }) + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dhcp-lease", Msg: fmt.Sprintf("%v -> %v", req.ClientHWAddr, n.cfg.GuestIP)}) + } + } + n.writeFrame(buildUDPFrame(n.cfg.GatewayMAC, dstMAC, srcIP.As4(), dstIP.As4(), dhcpv4.ServerPort, dhcpv4.ClientPort, resp.ToBytes())) + return true +} + +func (n *Net) buildDHCPReply(req *dhcpv4.DHCPv4, mt dhcpv4.MessageType) (*dhcpv4.DHCPv4, error) { + gw := net.IP(n.cfg.GatewayIP.AsSlice()) + mods := []dhcpv4.Modifier{ + dhcpv4.WithMessageType(mt), + dhcpv4.WithServerIP(gw), + dhcpv4.WithOption(dhcpv4.OptServerIdentifier(gw)), + } + if mt != dhcpv4.MessageTypeNak { + var mtu [2]byte + binary.BigEndian.PutUint16(mtu[:], uint16(n.cfg.MTU)) + mods = append(mods, + dhcpv4.WithYourIP(n.cfg.GuestIP.AsSlice()), + dhcpv4.WithOption(dhcpv4.OptSubnetMask(net.CIDRMask(n.cfg.Subnet.Bits(), 32))), + dhcpv4.WithOption(dhcpv4.OptRouter(gw)), + dhcpv4.WithOption(dhcpv4.OptDNS(net.IP(DNSAddr.AsSlice()))), + dhcpv4.WithOption(dhcpv4.OptBroadcastAddress(broadcastOf(n.cfg.Subnet))), + dhcpv4.WithOption(dhcpv4.OptIPAddressLeaseTime(n.cfg.LeaseTime)), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRenewTimeValue, secs(n.cfg.LeaseTime/2))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionRebindingTimeValue, secs(n.cfg.LeaseTime*7/8))), + dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, mtu[:])), + ) + if mt == dhcpv4.MessageTypeAck && req.MessageType() == dhcpv4.MessageTypeInform { + // INFORM replies must not carry a lease. + mods = append(mods, func(d *dhcpv4.DHCPv4) { + d.YourIPAddr = net.IPv4zero + d.DeleteOption(dhcpv4.OptionIPAddressLeaseTime) + d.DeleteOption(dhcpv4.OptionRenewTimeValue) + d.DeleteOption(dhcpv4.OptionRebindingTimeValue) + }) + } + } + return dhcpv4.NewReplyFromRequest(req, mods...) +} + +func secs(d time.Duration) []byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], uint32(d/time.Second)) + return b[:] +} + +// buildUDPFrame assembles an Ethernet/IPv4/UDP frame with checksums. +func buildUDPFrame(srcMAC, dstMAC net.HardwareAddr, srcIP, dstIP [4]byte, srcPort, dstPort uint16, payload []byte) []byte { + udpLen := header.UDPMinimumSize + len(payload) + ipLen := header.IPv4MinimumSize + udpLen + frame := make([]byte, header.EthernetMinimumSize+ipLen) + + eth := header.Ethernet(frame) + eth.Encode(&header.EthernetFields{ + SrcAddr: tcpip.LinkAddress(srcMAC), + DstAddr: tcpip.LinkAddress(dstMAC), + Type: header.IPv4ProtocolNumber, + }) + + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + ip.Encode(&header.IPv4Fields{ + TotalLength: uint16(ipLen), + TTL: 64, + Protocol: uint8(header.UDPProtocolNumber), + SrcAddr: tcpip.AddrFrom4(srcIP), + DstAddr: tcpip.AddrFrom4(dstIP), + }) + ip.SetChecksum(^ip.CalculateChecksum()) + + u := header.UDP(ip.Payload()) + u.Encode(&header.UDPFields{ + SrcPort: srcPort, + DstPort: dstPort, + Length: uint16(udpLen), + }) + copy(u.Payload(), payload) + xsum := header.PseudoHeaderChecksum(header.UDPProtocolNumber, tcpip.AddrFrom4(srcIP), tcpip.AddrFrom4(dstIP), uint16(udpLen)) + xsum = checksum.Checksum(payload, xsum) + u.SetChecksum(^u.CalculateChecksum(xsum)) + return frame +} + +// parseUDPFrame extracts the addressing and payload of an IPv4 UDP frame. +// It is used by tests and by the DHCP path's counterpart in a fake guest. +func parseUDPFrame(frame []byte) (src, dst [4]byte, srcPort, dstPort uint16, payload []byte, ok bool) { + if len(frame) < header.EthernetMinimumSize+header.IPv4MinimumSize+header.UDPMinimumSize { + return + } + eth := header.Ethernet(frame) + if eth.Type() != header.IPv4ProtocolNumber { + return + } + ip := header.IPv4(frame[header.EthernetMinimumSize:]) + if !ip.IsValid(len(ip)) || ip.Protocol() != uint8(header.UDPProtocolNumber) { + return + } + u := header.UDP(ip.Payload()) + if len(u) < header.UDPMinimumSize { + return + } + return ip.SourceAddress().As4(), ip.DestinationAddress().As4(), u.SourcePort(), u.DestinationPort(), bytes.Clone(u.Payload()), true +} diff --git a/guestnet/dns.go b/guestnet/dns.go new file mode 100644 index 0000000..ccd2c76 --- /dev/null +++ b/guestnet/dns.go @@ -0,0 +1,299 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "time" + + "golang.org/x/net/dns/dnsmessage" +) + +// DefaultUpstreams are the resolvers used when none are configured. They +// match corp CI's defaults. +var DefaultUpstreams = []netip.AddrPort{ + netip.MustParseAddrPort("8.8.8.8:53"), + netip.MustParseAddrPort("1.1.1.1:53"), +} + +// dnsQueryTimeout bounds one upstream exchange. +const dnsQueryTimeout = 5 * time.Second + +// dnsIdleTimeout ends a guest DNS session (one source port) after this long +// without a query; the guest's resolver typically uses a fresh port per +// query anyway. +const dnsIdleTimeout = 10 * time.Second + +// Resolver answers DNS queries on the guest's behalf. +type Resolver interface { + // Exchange sends the wire-format query and returns the wire-format + // response and a description of who answered (for logs). + Exchange(ctx context.Context, query []byte) (resp []byte, upstream string, err error) +} + +// UpstreamResolver forwards queries to upstream servers in order, over UDP +// with TCP fallback on truncation. +type UpstreamResolver struct { + Servers []netip.AddrPort + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// NewUpstreamResolver returns an UpstreamResolver for servers. +func NewUpstreamResolver(servers ...netip.AddrPort) *UpstreamResolver { + return &UpstreamResolver{Servers: servers} +} + +// Exchange implements Resolver. +func (r *UpstreamResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + d := r.Dialer + if d == nil { + d = &net.Dialer{} + } + var lastErr error + for _, s := range r.Servers { + resp, err := exchangeUDP(ctx, d, s, query) + if err == nil && len(resp) >= 12 && resp[2]&0x02 != 0 { // TC bit set + resp, err = exchangeTCP(ctx, d, s, query) + } + if err == nil { + return resp, s.String(), nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("no upstream resolvers configured") + } + return nil, "", lastErr +} + +func exchangeUDP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "udp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if _, err := conn.Write(query); err != nil { + return nil, err + } + buf := make([]byte, 4096) + for { + m, err := conn.Read(buf) + if err != nil { + return nil, err + } + // Match the transaction ID; ignore stray datagrams. + if m >= 2 && len(query) >= 2 && buf[0] == query[0] && buf[1] == query[1] { + return buf[:m], nil + } + } +} + +func exchangeTCP(ctx context.Context, d Dialer, server netip.AddrPort, query []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, dnsQueryTimeout) + defer cancel() + conn, err := d.DialContext(ctx, "tcp", server.String()) + if err != nil { + return nil, err + } + defer conn.Close() + if dl, ok := ctx.Deadline(); ok { + conn.SetDeadline(dl) + } + if err := writeDNSTCP(conn, query); err != nil { + return nil, err + } + return readDNSTCP(conn) +} + +func writeDNSTCP(w io.Writer, msg []byte) error { + buf := make([]byte, 2+len(msg)) + binary.BigEndian.PutUint16(buf, uint16(len(msg))) + copy(buf[2:], msg) + _, err := w.Write(buf) + return err +} + +func readDNSTCP(r io.Reader) ([]byte, error) { + var lenBuf [2]byte + if _, err := io.ReadFull(r, lenBuf[:]); err != nil { + return nil, err + } + msg := make([]byte, binary.BigEndian.Uint16(lenBuf[:])) + if _, err := io.ReadFull(r, msg); err != nil { + return nil, err + } + return msg, nil +} + +// serveDNSUDP answers queries arriving on one guest UDP session. +func (n *Net) serveDNSUDP(conn net.Conn, src, dst netip.AddrPort) error { + buf := make([]byte, 4096) + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + resp := n.resolve(buf[:m], src, dst, "udp") + if resp != nil { + if _, err := conn.Write(resp); err != nil { + return err + } + } + } +} + +// serveDNSTCP answers length-prefixed queries on one guest TCP connection. +func (n *Net) serveDNSTCP(conn net.Conn, src, dst netip.AddrPort) error { + for { + conn.SetReadDeadline(time.Now().Add(dnsIdleTimeout)) + q, err := readDNSTCP(conn) + if err != nil { + return nil + } + resp := n.resolve(q, src, dst, "tcp") + if resp == nil { + return nil + } + if err := writeDNSTCP(conn, resp); err != nil { + return err + } + } +} + +// resolve answers one query via the configured Resolver and logs it. On +// upstream failure it returns a SERVFAIL so the guest doesn't wait. +func (n *Net) resolve(query []byte, src, dst netip.AddrPort, proto string) []byte { + ev := DNSEvent{Time: n.cfg.Now(), Src: src, Dst: dst, Proto: proto} + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + ev.Err = "unparseable query" + n.DNS.Add(ev) + return nil + } + if q, err := p.Question(); err == nil { + ev.Name = q.Name.String() + ev.Type = q.Type.String() + } + + ctx, cancel := context.WithTimeout(n.ctx, dnsQueryTimeout) + defer cancel() + resp, upstream, err := n.cfg.Resolver.Exchange(ctx, query) + ev.Duration = n.cfg.Now().Sub(ev.Time) + ev.Upstream = upstream + if err != nil { + ev.Err = err.Error() + ev.RCode = dnsmessage.RCodeServerFailure.String() + n.DNS.Add(ev) + return servfail(hdr, query) + } + var rp dnsmessage.Parser + if rh, err := rp.Start(resp); err == nil { + ev.RCode = rh.RCode.String() + rp.SkipAllQuestions() + if answers, err := rp.AllAnswers(); err == nil { + ev.Answers = len(answers) + } + } + n.DNS.Add(ev) + return resp +} + +// servfail builds a SERVFAIL response echoing the query's ID and question. +func servfail(hdr dnsmessage.Header, query []byte) []byte { + resp := make([]byte, len(query)) + copy(resp, query) + if len(resp) < 12 { + return nil + } + resp[2] = 0x80 | (resp[2] & 0x79) // QR=1, keep opcode and RD + resp[3] = byte(dnsmessage.RCodeServerFailure) + // Zero the answer, authority and additional counts; keep QDCOUNT. + binary.BigEndian.PutUint16(resp[6:], 0) + binary.BigEndian.PutUint16(resp[8:], 0) + binary.BigEndian.PutUint16(resp[10:], 0) + _ = hdr + return resp[:12+questionLen(resp[12:])] +} + +// questionLen returns the encoded length of the first question section +// entry in b, or 0 if malformed. +func questionLen(b []byte) int { + i := 0 + for i < len(b) { + l := int(b[i]) + if l == 0 { + i++ + break + } + if l&0xC0 == 0xC0 { + i += 2 + break + } + i += 1 + l + } + if i+4 > len(b) { + return 0 + } + return i + 4 +} + +// StaticResolver answers every A query with a fixed address and everything +// else with NXDOMAIN. It exists for tests and for air-gapped experiments. +type StaticResolver struct { + Addr netip.Addr +} + +// Exchange implements Resolver. +func (r StaticResolver) Exchange(ctx context.Context, query []byte) ([]byte, string, error) { + var p dnsmessage.Parser + hdr, err := p.Start(query) + if err != nil { + return nil, "", err + } + q, err := p.Question() + if err != nil { + return nil, "", err + } + rh := dnsmessage.Header{ID: hdr.ID, Response: true, Authoritative: true, RecursionDesired: hdr.RecursionDesired, RecursionAvailable: true} + b := dnsmessage.NewBuilder(nil, rh) + b.EnableCompression() + if err := b.StartQuestions(); err != nil { + return nil, "", err + } + if err := b.Question(q); err != nil { + return nil, "", err + } + if q.Type == dnsmessage.TypeA && r.Addr.Is4() { + if err := b.StartAnswers(); err != nil { + return nil, "", err + } + if err := b.AResource(dnsmessage.ResourceHeader{Name: q.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 60}, dnsmessage.AResource{A: r.Addr.As4()}); err != nil { + return nil, "", err + } + } else { + rh.RCode = dnsmessage.RCodeNameError + } + out, err := b.Finish() + if err != nil { + return nil, "", fmt.Errorf("build response: %w", err) + } + if rh.RCode != 0 { + out[3] = out[3]&0xF0 | byte(rh.RCode) + } + return out, "static", nil +} diff --git a/guestnet/guestnet.go b/guestnet/guestnet.go new file mode 100644 index 0000000..12e3ae2 --- /dev/null +++ b/guestnet/guestnet.go @@ -0,0 +1,468 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnet is a private userspace network for one guest VM. +// +// The guest's virtual NIC hands us raw Ethernet frames (on macOS via a +// VZFileHandleNetworkDeviceAttachment over a SOCK_DGRAM socketpair; in tests +// via an in-memory link). Every frame terminates in a gVisor netstack owned by +// this package, so a guest can only ever reach what this package chooses to +// connect it to: +// +// - The gateway, IMDS and DNS addresses live on our NIC, so ARP and ICMP +// echo for them are answered by gVisor itself. +// - DHCP is answered directly from the frame loop with a fixed lease, so a +// memory snapshot restored in another slot sees the same addresses. +// - Every TCP SYN and UDP flow is dispatched by destination: a registered +// [Service] (IMDS, the caller's NFS or proxies), an intercepted protocol +// (DNS on any port 53, NTP on any port 123), or egress dialed from the +// host. Each flow is logged. +// - The host reaches the guest only by dialing into this stack with +// [Net.DialGuest]; there is no host interface routed to the guest. +// +// One [Net] per VM. Nothing here is process-global, so many guests can share +// a process without seeing each other. +package guestnet + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "sync" + "time" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "tailscale.com/types/logger" +) + +// Well-known addresses. They are the same for every guest because each guest +// has its own L2 domain; that is what makes memory snapshots slot-independent. +var ( + // DefaultSubnet is the guest-visible subnet. + DefaultSubnet = netip.MustParsePrefix("192.168.64.0/24") + // DefaultGatewayIP is our address on the guest subnet and the guest's + // default router. + DefaultGatewayIP = netip.MustParseAddr("192.168.64.1") + // DefaultGuestIP is the one address the DHCP server hands out. + DefaultGuestIP = netip.MustParseAddr("192.168.64.2") + // IMDSAddr is the metadata service address, matching cloud convention and + // the address corp CI already uses. + IMDSAddr = netip.MustParseAddr("169.254.169.254") + // DNSAddr is the resolver address handed out by DHCP, matching corp CI. + DNSAddr = netip.MustParseAddr("169.254.169.253") + + // DefaultGatewayMAC is our MAC. It is constant so a restored guest's ARP + // cache stays valid. + DefaultGatewayMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x01} + // DefaultGuestMAC is the guest MAC used when an image doesn't specify one. + DefaultGuestMAC = net.HardwareAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} +) + +// DefaultDeny is the egress deny list applied when [Config.Deny] is nil: the +// tailnet CGNAT range and its IPv6 counterpart (so a guest can't reach the +// host's tailnet), loopback, and the link-local range (IMDS and DNS are +// reached through the service table before policy applies). +var DefaultDeny = []netip.Prefix{ + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("fd7a:115c:a1e0::/48"), + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("0.0.0.0/8"), +} + +const nicID tcpip.NICID = 1 + +// Dialer dials egress connections on behalf of the guest. *net.Dialer +// satisfies it. +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +// Config configures a guest network. The zero value is usable with a link. +type Config struct { + // Link carries raw Ethernet frames to and from the guest. Required. + Link FrameLink + + // GuestMAC is the guest's MAC address, used as the DHCP client identity + // check and as the destination of DHCP replies. Defaults to + // [DefaultGuestMAC]. + GuestMAC net.HardwareAddr + // GatewayMAC is our MAC. Defaults to [DefaultGatewayMAC]. + GatewayMAC net.HardwareAddr + + // Subnet, GatewayIP and GuestIP default to the Default* values. + Subnet netip.Prefix + GatewayIP netip.Addr + GuestIP netip.Addr + + // MTU is the link MTU advertised to the guest via DHCP. Defaults to 1500. + MTU uint32 + + // LeaseTime is the DHCP lease duration. Defaults to 24h. Restored + // snapshots renew against this server, so long leases are fine. + LeaseTime time.Duration + + // Dialer dials egress TCP and UDP. Defaults to a net.Dialer. + Dialer Dialer + + // Resolver answers the guest's DNS queries. Defaults to + // [NewUpstreamResolver] with [DefaultUpstreams]. + Resolver Resolver + + // Deny lists destinations the guest may not reach via egress. nil means + // [DefaultDeny]; an empty non-nil slice denies nothing. + Deny []netip.Prefix + + // Services maps guest-visible destinations to handlers. The destination + // address must be one of ours (gateway, IMDS, DNS) or the lookup happens + // before egress policy and would let a guest hijack a public address. + Services map[netip.AddrPort]Service + + // Now returns the current time, for NTP and logs. Defaults to time.Now. + Now func() time.Time + + // Logf logs. Defaults to logger.Discard. + Logf logger.Logf +} + +func (c *Config) withDefaults() { + if c.GuestMAC == nil { + c.GuestMAC = DefaultGuestMAC + } + if c.GatewayMAC == nil { + c.GatewayMAC = DefaultGatewayMAC + } + if !c.Subnet.IsValid() { + c.Subnet = DefaultSubnet + } + if !c.GatewayIP.IsValid() { + c.GatewayIP = DefaultGatewayIP + } + if !c.GuestIP.IsValid() { + c.GuestIP = DefaultGuestIP + } + if c.MTU == 0 { + c.MTU = 1500 + } + if c.LeaseTime == 0 { + c.LeaseTime = 24 * time.Hour + } + if c.Dialer == nil { + c.Dialer = &net.Dialer{Timeout: 10 * time.Second} + } + if c.Resolver == nil { + c.Resolver = NewUpstreamResolver(DefaultUpstreams...) + } + if c.Deny == nil { + c.Deny = DefaultDeny + } + if c.Now == nil { + c.Now = time.Now + } + if c.Logf == nil { + c.Logf = logger.Discard + } +} + +// Net is one guest's network. Create it with [New] and shut it down with +// [Net.Close]. +type Net struct { + cfg Config + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + stack *stack.Stack + linkEP *channel.Endpoint + + // Flows is the TCP/UDP flow log. + Flows *Log[Flow] + // DNS is the DNS query log. + DNS *Log[DNSEvent] + // Events records notable network events (DHCP lease, blocked flows). + Events *Log[Event] + + // dhcpLeased is closed the first time the guest acknowledges a lease. + dhcpLeased chan struct{} + dhcpLeasedOnce sync.Once + + mu sync.Mutex + services map[netip.AddrPort]Service +} + +// New starts a guest network on cfg.Link. It returns once the stack is up; +// frames are pumped on background goroutines until [Net.Close]. +func New(cfg Config) (*Net, error) { + if cfg.Link == nil { + return nil, errors.New("guestnet: Config.Link is required") + } + cfg.withDefaults() + if !cfg.Subnet.Contains(cfg.GatewayIP) || !cfg.Subnet.Contains(cfg.GuestIP) { + return nil, fmt.Errorf("guestnet: gateway %v and guest %v must be within %v", cfg.GatewayIP, cfg.GuestIP, cfg.Subnet) + } + if !cfg.GatewayIP.Is4() { + return nil, errors.New("guestnet: only IPv4 is supported") + } + + ctx, cancel := context.WithCancel(context.Background()) + n := &Net{ + cfg: cfg, + ctx: ctx, + cancel: cancel, + Flows: NewLog[Flow](1024), + DNS: NewLog[DNSEvent](1024), + Events: NewLog[Event](256), + dhcpLeased: make(chan struct{}), + services: make(map[netip.AddrPort]Service), + } + for ap, svc := range cfg.Services { + n.services[ap] = svc + } + + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + sack := tcpip.TCPSACKEnabled(true) + if err := s.SetTransportProtocolOption(tcp.ProtocolNumber, &sack); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: enable SACK: %v", err) + } + + // The channel endpoint carries whole Ethernet frames; the ethernet wrapper + // adds and strips link headers so gVisor handles ARP for us. + n.linkEP = channel.New(4096, cfg.MTU, tcpip.LinkAddress(cfg.GatewayMAC)) + if err := s.CreateNIC(nicID, ethernet.New(n.linkEP)); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: CreateNIC: %v", err) + } + // Accept packets for any destination so the forwarders see every flow, + // and allow replies from any source address. + if err := s.SetPromiscuousMode(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetPromiscuousMode: %v", err) + } + if err := s.SetSpoofing(nicID, true); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: SetSpoofing: %v", err) + } + + // Our addresses. gVisor answers ARP and ICMP echo for these. The + // link-local ones are on the NIC precisely so the guest's on-link ARP for + // them (169.254/16 is APIPA and never routed) gets an answer. + addrs := []struct { + addr netip.Addr + bits int + }{ + {cfg.GatewayIP, cfg.Subnet.Bits()}, + {IMDSAddr, 32}, + {DNSAddr, 32}, + } + for _, a := range addrs { + pa := tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{ + Address: tcpip.AddrFrom4(a.addr.As4()), + PrefixLen: a.bits, + }, + } + if err := s.AddProtocolAddress(nicID, pa, stack.AddressProperties{}); err != nil { + cancel() + return nil, fmt.Errorf("guestnet: AddProtocolAddress(%v): %v", a.addr, err) + } + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + NIC: nicID, + }}) + + tcpFwd := tcp.NewForwarder(s, 0, 4096, n.handleTCP) + s.SetTransportProtocolHandler(tcp.ProtocolNumber, tcpFwd.HandlePacket) + udpFwd := udp.NewForwarder(s, n.handleUDP) + s.SetTransportProtocolHandler(udp.ProtocolNumber, udpFwd.HandlePacket) + + n.stack = s + n.wg.Add(2) + go n.pumpFromGuest() + go n.pumpToGuest() + return n, nil +} + +// Close stops the network and closes the link. Open flows are torn down. +func (n *Net) Close() error { + n.cancel() + err := n.cfg.Link.Close() + n.linkEP.Close() + // Close every endpoint before waiting for the flow goroutines: one may be + // in a TCP handshake with a guest that will never finish it (the VM + // stopped just after its SYN), which only aborts when its endpoint is + // closed, or after gVisor's handshake timeout of 30s or more. + n.stack.Close() + n.wg.Wait() + n.stack.Destroy() + return err +} + +// GuestIP returns the address the guest is leased. +func (n *Net) GuestIP() netip.Addr { return n.cfg.GuestIP } + +// GatewayIP returns our address on the guest subnet. +func (n *Net) GatewayIP() netip.Addr { return n.cfg.GatewayIP } + +// Leased returns a channel that is closed once the guest has acknowledged a +// DHCP lease. For a restored snapshot this may not fire until the guest +// renews, so callers should also consider the guest reachable if +// [Net.DialGuest] succeeds. +func (n *Net) Leased() <-chan struct{} { return n.dhcpLeased } + +// SetService registers or replaces the handler for a guest-visible +// destination. Passing a nil svc removes it. +func (n *Net) SetService(dst netip.AddrPort, svc Service) { + n.mu.Lock() + defer n.mu.Unlock() + if svc == nil { + delete(n.services, dst) + } else { + n.services[dst] = svc + } +} + +func (n *Net) service(dst netip.AddrPort) (Service, bool) { + n.mu.Lock() + defer n.mu.Unlock() + svc, ok := n.services[dst] + return svc, ok +} + +// DialGuest opens a TCP connection from the gateway address to port on the +// guest. It is how the host reaches sshd in the guest. +func (n *Net) DialGuest(ctx context.Context, port uint16) (net.Conn, error) { + local := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GatewayIP.As4())} + remote := tcpip.FullAddress{NIC: nicID, Addr: tcpip.AddrFrom4(n.cfg.GuestIP.As4()), Port: port} + return gonet.DialTCPWithBind(ctx, n.stack, local, remote, ipv4.ProtocolNumber) +} + +// pumpFromGuest reads frames from the link, answers DHCP directly, and injects +// everything else into gVisor. +func (n *Net) pumpFromGuest() { + defer n.wg.Done() + buf := make([]byte, 65536+header.EthernetMinimumSize) + for { + m, err := n.cfg.Link.ReadFrame(buf) + if err != nil { + if n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link read: %v", err) + n.cancel() + } + return + } + frame := buf[:m] + if len(frame) < header.EthernetMinimumSize { + continue + } + eth := header.Ethernet(frame) + switch eth.Type() { + case header.IPv4ProtocolNumber: + if n.handleDHCP(frame) { + continue + } + case header.ARPProtocolNumber: + if !n.arpForUs(frame[header.EthernetMinimumSize:]) { + continue + } + default: + // IPv6 and everything else is dropped for now. + continue + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{ + Payload: buffer.MakeWithData(append([]byte(nil), frame...)), + }) + // The ethernet wrapper parses the real EtherType from the frame; the + // protocol argument is ignored. + n.linkEP.InjectInbound(eth.Type(), pkt) + pkt.DecRef() + } +} + +// arpForUs reports whether an ARP packet from the guest should reach gVisor: +// replies (to our requests for the guest's address), and requests for one of +// our own addresses. Our NIC is promiscuous, so gVisor would otherwise answer +// requests for every address, including the guest's own; the guest's DHCP +// client then sees its duplicate address probe answered and declines the +// lease. +func (n *Net) arpForUs(b []byte) bool { + arp := header.ARP(b) + if !arp.IsValid() { + return false + } + if arp.Op() != header.ARPRequest { + return true + } + target := netip.AddrFrom4([4]byte(arp.ProtocolAddressTarget())) + return target == n.cfg.GatewayIP || target == IMDSAddr || target == DNSAddr +} + +// pumpToGuest writes frames produced by gVisor to the link. +func (n *Net) pumpToGuest() { + defer n.wg.Done() + for { + pkt := n.linkEP.ReadContext(n.ctx) + if pkt == nil { + return + } + v := pkt.ToView() + err := n.cfg.Link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + if err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } + } +} + +// writeFrame sends a frame we built ourselves (DHCP replies) to the guest. +func (n *Net) writeFrame(frame []byte) { + if err := n.cfg.Link.WriteFrame(frame); err != nil && n.ctx.Err() == nil { + n.cfg.Logf("guestnet: link write: %v", err) + } +} + +// egressAllowed reports whether policy lets the guest reach dst. +func (n *Net) egressAllowed(dst netip.Addr) bool { + if dst.IsMulticast() || dst.IsUnspecified() || dst == netip.AddrFrom4([4]byte{255, 255, 255, 255}) { + return false + } + if n.cfg.Subnet.Contains(dst) { + // Only the gateway (via services) and the guest itself exist here. + return false + } + for _, p := range n.cfg.Deny { + if p.Contains(dst) { + return false + } + } + return true +} + +func addrFromTCPIP(a tcpip.Address) netip.Addr { + switch a.Len() { + case 4: + return netip.AddrFrom4(a.As4()) + case 16: + return netip.AddrFrom16(a.As16()) + } + return netip.Addr{} +} diff --git a/guestnet/guestnet_test.go b/guestnet/guestnet_test.go new file mode 100644 index 0000000..40fdd03 --- /dev/null +++ b/guestnet/guestnet_test.go @@ -0,0 +1,544 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "strings" + "testing" + "time" + + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tailscale/tb/guestnet/guestnettest" + "golang.org/x/net/dns/dnsmessage" +) + +// anyDialer dials every address to a single local target, standing in for +// the internet in tests. +type anyDialer struct { + tcpTarget string + udpTarget string +} + +func (d anyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + var nd net.Dialer + switch { + case strings.HasPrefix(network, "tcp"): + return nd.DialContext(ctx, network, d.tcpTarget) + case strings.HasPrefix(network, "udp"): + return nd.DialContext(ctx, network, d.udpTarget) + } + return nil, fmt.Errorf("unsupported network %q", network) +} + +type env struct { + n *Net + guest *guestnettest.Guest + peer *MemLink +} + +func newEnv(t testing.TB, cfg Config) *env { + t.Helper() + host, peer := NewMemLinkPair() + cfg.Link = host + cfg.Logf = t.Logf + n, err := New(cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { n.Close() }) + g := guestnettest.New(t, peer, guestnettest.Config{ + IP: n.cfg.GuestIP, + Subnet: n.cfg.Subnet, + Gateway: n.cfg.GatewayIP, + MAC: n.cfg.GuestMAC, + }) + return &env{n: n, guest: g, peer: peer} +} + +func testContext(t testing.TB) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + return ctx +} + +// dhcpExchange writes a DHCP message from an addressless client and returns +// the reply. +func dhcpExchange(t testing.TB, peer *MemLink, mac net.HardwareAddr, msg *dhcpv4.DHCPv4) *dhcpv4.DHCPv4 { + t.Helper() + frame := buildUDPFrame(mac, net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, [4]byte{}, [4]byte{255, 255, 255, 255}, dhcpv4.ClientPort, dhcpv4.ServerPort, msg.ToBytes()) + if err := peer.WriteFrame(frame); err != nil { + t.Fatal(err) + } + buf := make([]byte, 65536) + deadline := time.After(5 * time.Second) + for { + select { + case <-deadline: + t.Fatal("timed out waiting for DHCP reply") + default: + } + n, err := peer.ReadFrame(buf) + if err != nil { + t.Fatal(err) + } + _, _, srcPort, dstPort, payload, ok := parseUDPFrame(buf[:n]) + if !ok || srcPort != dhcpv4.ServerPort || dstPort != dhcpv4.ClientPort { + continue + } + reply, err := dhcpv4.FromBytes(payload) + if err != nil { + t.Fatalf("bad DHCP reply: %v", err) + } + return reply + } +} + +func TestDHCP(t *testing.T) { + host, peer := NewMemLinkPair() + n, err := New(Config{Link: host, Logf: t.Logf, MTU: 9000}) + if err != nil { + t.Fatal(err) + } + defer n.Close() + + mac := DefaultGuestMAC + disc, err := dhcpv4.NewDiscovery(mac, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + offer := dhcpExchange(t, peer, mac, disc) + if offer.MessageType() != dhcpv4.MessageTypeOffer { + t.Fatalf("got %v, want OFFER", offer.MessageType()) + } + if got := offer.YourIPAddr.String(); got != DefaultGuestIP.String() { + t.Errorf("yiaddr = %s, want %v", got, DefaultGuestIP) + } + if got := offer.Router(); len(got) != 1 || got[0].String() != DefaultGatewayIP.String() { + t.Errorf("router = %v, want %v", got, DefaultGatewayIP) + } + if got := offer.DNS(); len(got) != 1 || got[0].String() != DNSAddr.String() { + t.Errorf("dns = %v, want %v", got, DNSAddr) + } + if mtu := offer.Options.Get(dhcpv4.OptionInterfaceMTU); len(mtu) != 2 || binary.BigEndian.Uint16(mtu) != 9000 { + t.Errorf("mtu option = %x, want 9000", mtu) + } + if lt := offer.IPAddressLeaseTime(0); lt != 24*time.Hour { + t.Errorf("lease = %v, want 24h", lt) + } + + select { + case <-n.Leased(): + t.Fatal("Leased closed after OFFER; should wait for ACK") + default: + } + + req, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true)) + if err != nil { + t.Fatal(err) + } + ack := dhcpExchange(t, peer, mac, req) + if ack.MessageType() != dhcpv4.MessageTypeAck { + t.Fatalf("got %v, want ACK", ack.MessageType()) + } + select { + case <-n.Leased(): + case <-time.After(time.Second): + t.Fatal("Leased not closed after ACK") + } + + // A request for a foreign address is refused. + bad, err := dhcpv4.NewRequestFromOffer(offer, dhcpv4.WithBroadcast(true), dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(net.IPv4(10, 1, 2, 3)))) + if err != nil { + t.Fatal(err) + } + if nak := dhcpExchange(t, peer, mac, bad); nak.MessageType() != dhcpv4.MessageTypeNak { + t.Errorf("got %v for foreign address, want NAK", nak.MessageType()) + } + + var kinds []string + for _, ev := range n.Events.Recent() { + kinds = append(kinds, ev.Kind) + } + if got := strings.Join(kinds, ","); got != "dhcp-discover,dhcp-lease" { + t.Errorf("events = %q", got) + } +} + +func TestTCPEgress(t *testing.T) { + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello %s", r.Host) + })} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go srv.Serve(ln) + + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: ln.Addr().String()}}) + ctx := testContext(t) + + dst := netip.MustParseAddrPort("93.184.216.34:80") + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET / HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + body, _ := io.ReadAll(resp.Body) + if string(body) != "hello example.com" { + t.Fatalf("body = %q", body) + } + c.Close() + + flows := waitForClosedFlow(t, e.n, dst) + if flows.Kind != "egress" { + t.Errorf("kind = %q, want egress", flows.Kind) + } + if flows.BytesOut == 0 || flows.BytesIn == 0 { + t.Errorf("bytes in/out = %d/%d, want both > 0", flows.BytesIn, flows.BytesOut) + } + if flows.Src.Addr() != DefaultGuestIP { + t.Errorf("src = %v, want guest IP", flows.Src) + } +} + +func waitForClosedFlow(t testing.TB, n *Net, dst netip.AddrPort) Flow { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + for _, f := range n.Flows.Recent() { + if f.Dst == dst && f.Closed() { + return f + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("no closed flow to %v; flows: %+v", dst, n.Flows.Recent()) + return Flow{} +} + +func TestTCPBlocked(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ctx := testContext(t) + for _, dst := range []netip.AddrPort{ + netip.MustParseAddrPort("100.100.100.100:80"), // tailnet CGNAT + netip.MustParseAddrPort("192.168.64.7:80"), // guest subnet, nonexistent + netip.MustParseAddrPort("192.168.64.1:80"), // gateway with no service + } { + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Errorf("dial %v succeeded, want refusal", dst) + continue + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "blocked" { + t.Errorf("%v: kind = %q, want blocked", dst, f.Kind) + } + } +} + +func TestTCPDialError(t *testing.T) { + // Point egress at a closed port so the host-side dial fails; the guest + // must see a refused connection, not an accepted one. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + ln.Close() + e := newEnv(t, Config{Dialer: anyDialer{tcpTarget: addr}}) + ctx := testContext(t) + dst := netip.MustParseAddrPort("93.184.216.34:443") + c, err := e.guest.DialTCP(ctx, dst) + if err == nil { + c.Close() + t.Fatal("dial succeeded, want refusal") + } + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "egress" || f.Err == "" { + t.Errorf("flow = %+v, want egress with error", f) + } +} + +func TestService(t *testing.T) { + imds := NewListenerService("imds") + go http.Serve(imds, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "meta for %s", r.RemoteAddr) + })) + defer imds.Close() + + e := newEnv(t, Config{ + Dialer: anyDialer{}, + Services: map[netip.AddrPort]Service{netip.AddrPortFrom(IMDSAddr, 80): imds}, + }) + ctx := testContext(t) + dst := netip.AddrPortFrom(IMDSAddr, 80) + c, err := e.guest.DialTCP(ctx, dst) + if err != nil { + t.Fatalf("guest dial IMDS: %v", err) + } + defer c.Close() + fmt.Fprintf(c, "GET /v0/meta HTTP/1.1\r\nHost: 169.254.169.254\r\nConnection: close\r\n\r\n") + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + if !strings.HasPrefix(string(body), "meta for 192.168.64.2:") { + t.Fatalf("body = %q", body) + } + c.Close() + f := waitForClosedFlow(t, e.n, dst) + if f.Kind != "service" || f.Service != "imds" { + t.Errorf("flow = %+v, want service imds", f) + } + + // A late-registered forwarding service on the gateway works too. + echo, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer echo.Close() + go func() { + for { + c, err := echo.Accept() + if err != nil { + return + } + go func() { io.Copy(c, c); c.Close() }() + } + }() + nfs := netip.AddrPortFrom(DefaultGatewayIP, 2049) + e.n.SetService(nfs, &ForwardService{ServiceName: "nfs", Network: "tcp", Address: echo.Addr().String()}) + c2, err := e.guest.DialTCP(ctx, nfs) + if err != nil { + t.Fatalf("guest dial nfs: %v", err) + } + defer c2.Close() + if _, err := c2.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(c2, buf); err != nil || string(buf) != "ping" { + t.Fatalf("echo = %q, %v", buf, err) + } +} + +func TestDNS(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}, Resolver: StaticResolver{Addr: netip.MustParseAddr("10.9.8.7")}}) + + // Guest queries a public resolver; we intercept regardless of address. + dst := netip.MustParseAddrPort("8.8.8.8:53") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + q := dnsmessage.Message{ + Header: dnsmessage.Header{ID: 0xBEEF, RecursionDesired: true}, + Questions: []dnsmessage.Question{{Name: dnsmessage.MustNewName("example.com."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}, + } + qb, err := q.Pack() + if err != nil { + t.Fatal(err) + } + if _, err := c.Write(qb); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 4096) + n, err := c.Read(buf) + if err != nil { + t.Fatalf("read DNS response: %v", err) + } + var m dnsmessage.Message + if err := m.Unpack(buf[:n]); err != nil { + t.Fatal(err) + } + if m.ID != 0xBEEF || len(m.Answers) != 1 { + t.Fatalf("response = %+v", m) + } + if a, ok := m.Answers[0].Body.(*dnsmessage.AResource); !ok || a.A != [4]byte{10, 9, 8, 7} { + t.Fatalf("answer = %+v", m.Answers[0].Body) + } + + // Also over TCP, to the advertised resolver address. + tc, err := e.guest.DialTCP(testContext(t), netip.AddrPortFrom(DNSAddr, 53)) + if err != nil { + t.Fatal(err) + } + defer tc.Close() + if err := writeDNSTCP(tc, qb); err != nil { + t.Fatal(err) + } + tc.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp, err := readDNSTCP(tc) + if err != nil { + t.Fatalf("TCP DNS: %v", err) + } + if err := m.Unpack(resp); err != nil || len(m.Answers) != 1 { + t.Fatalf("TCP response = %+v, %v", m, err) + } + + events := e.n.DNS.Recent() + if len(events) != 2 { + t.Fatalf("got %d DNS events, want 2: %+v", len(events), events) + } + for _, ev := range events { + if ev.Name != "example.com." || ev.Type != "TypeA" || ev.Answers != 1 || ev.Upstream != "static" { + t.Errorf("event = %+v", ev) + } + } + if events[0].Proto != "udp" || events[1].Proto != "tcp" { + t.Errorf("protos = %s, %s", events[0].Proto, events[1].Proto) + } +} + +func TestNTP(t *testing.T) { + now := time.Date(2026, 9, 29, 12, 0, 0, 500_000_000, time.UTC) + e := newEnv(t, Config{Dialer: anyDialer{}, Now: func() time.Time { return now }}) + c, err := e.guest.DialUDP(netip.MustParseAddrPort("17.253.2.1:123")) + if err != nil { + t.Fatal(err) + } + defer c.Close() + req := make([]byte, 48) + req[0] = 4<<3 | 3 // v4, client + binary.BigEndian.PutUint64(req[40:], 0x1122334455667788) + if _, err := c.Write(req); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + resp := make([]byte, 64) + n, err := c.Read(resp) + if err != nil { + t.Fatal(err) + } + if n != 48 || resp[0]&0x07 != 4 { + t.Fatalf("resp len %d mode %d", n, resp[0]&0x07) + } + if binary.BigEndian.Uint64(resp[24:]) != 0x1122334455667788 { + t.Error("originate timestamp not echoed") + } + ts := binary.BigEndian.Uint64(resp[40:]) + secs := int64(ts>>32) - ntpEpochOffset + frac := float64(ts&0xffffffff) / (1 << 32) + got := time.Unix(secs, int64(frac*1e9)) + if d := got.Sub(now); d < -time.Millisecond || d > time.Millisecond { + t.Errorf("transmit = %v, want %v", got, now) + } +} + +func TestUDPEgress(t *testing.T) { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer pc.Close() + go func() { + buf := make([]byte, 1500) + for { + n, addr, err := pc.ReadFrom(buf) + if err != nil { + return + } + pc.WriteTo(append([]byte("echo:"), buf[:n]...), addr) + } + }() + e := newEnv(t, Config{Dialer: anyDialer{udpTarget: pc.LocalAddr().String()}}) + dst := netip.MustParseAddrPort("203.0.113.9:4000") + c, err := e.guest.DialUDP(dst) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if _, err := c.Write([]byte("hi")); err != nil { + t.Fatal(err) + } + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 100) + n, err := c.Read(buf) + if err != nil || string(buf[:n]) != "echo:hi" { + t.Fatalf("got %q, %v", buf[:n], err) + } + var open bool + for _, f := range e.n.Flows.Recent() { + if f.Dst == dst && f.Kind == "egress" { + open = true + } + } + if !open { + t.Errorf("no egress flow record for %v", dst) + } +} + +func TestDialGuest(t *testing.T) { + e := newEnv(t, Config{Dialer: anyDialer{}}) + ln, err := e.guest.ListenTCP(22) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + fmt.Fprint(c, "SSH-2.0-fake\r\n") + c.Close() + }() + ctx := testContext(t) + c, err := e.n.DialGuest(ctx, 22) + if err != nil { + t.Fatalf("DialGuest: %v", err) + } + defer c.Close() + line, err := bufio.NewReader(c).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if line != "SSH-2.0-fake\r\n" { + t.Fatalf("banner = %q", line) + } + if !strings.HasPrefix(c.LocalAddr().String(), DefaultGatewayIP.String()+":") { + t.Errorf("local addr = %v, want gateway", c.LocalAddr()) + } +} + +func TestCloseUnblocks(t *testing.T) { + host, _ := NewMemLinkPair() + n, err := New(Config{Link: host}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close hung") + } + if _, err := New(Config{}); !errors.Is(err, err) || err == nil { + t.Fatal("New without a link should fail") + } +} diff --git a/guestnet/guestnettest/guestnettest.go b/guestnet/guestnettest/guestnettest.go new file mode 100644 index 0000000..3f5906f --- /dev/null +++ b/guestnet/guestnettest/guestnettest.go @@ -0,0 +1,131 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package guestnettest provides a fake guest for testing code built on +// guestnet: a gVisor stack with a static address attached to the far end of +// a frame link, standing in for a VM (as one restored from a snapshot would +// look, already holding its lease). +// +// It deliberately does not import guestnet, so guestnet's own tests can use +// it. +package guestnettest + +import ( + "context" + "net" + "net/netip" + "testing" + + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" +) + +// FrameLink is the subset of guestnet.FrameLink the fake guest needs. +type FrameLink interface { + ReadFrame(buf []byte) (int, error) + WriteFrame(frame []byte) error +} + +// Guest is a fake guest VM's network stack. +type Guest struct { + s *stack.Stack + ep *channel.Endpoint +} + +// Config configures a Guest. +type Config struct { + IP netip.Addr + Subnet netip.Prefix + Gateway netip.Addr + MAC net.HardwareAddr + MTU uint32 // defaults to 1500 +} + +// New starts a fake guest on link. It is torn down by t.Cleanup; closing the +// link is the caller's job. +func New(t testing.TB, link FrameLink, cfg Config) *Guest { + t.Helper() + if cfg.MTU == 0 { + cfg.MTU = 1500 + } + s := stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4}, + }) + ep := channel.New(1024, cfg.MTU, tcpip.LinkAddress(cfg.MAC)) + if err := s.CreateNIC(1, ethernet.New(ep)); err != nil { + t.Fatalf("guest CreateNIC: %v", err) + } + if err := s.AddProtocolAddress(1, tcpip.ProtocolAddress{ + Protocol: ipv4.ProtocolNumber, + AddressWithPrefix: tcpip.AddressWithPrefix{Address: tcpip.AddrFrom4(cfg.IP.As4()), PrefixLen: cfg.Subnet.Bits()}, + }, stack.AddressProperties{}); err != nil { + t.Fatalf("guest AddProtocolAddress: %v", err) + } + s.SetRouteTable([]tcpip.Route{{ + Destination: header.IPv4EmptySubnet, + Gateway: tcpip.AddrFrom4(cfg.Gateway.As4()), + NIC: 1, + }}) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + buf := make([]byte, 65536) + for { + n, err := link.ReadFrame(buf) + if err != nil { + return + } + pkt := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(append([]byte(nil), buf[:n]...))}) + ep.InjectInbound(header.Ethernet(buf).Type(), pkt) + pkt.DecRef() + } + }() + go func() { + for { + pkt := ep.ReadContext(ctx) + if pkt == nil { + return + } + v := pkt.ToView() + link.WriteFrame(v.AsSlice()) + v.Release() + pkt.DecRef() + } + }() + t.Cleanup(func() { + cancel() + ep.Close() + s.Destroy() + }) + return &Guest{s: s, ep: ep} +} + +func fullAddr(ap netip.AddrPort) tcpip.FullAddress { + return tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4(ap.Addr().As4()), Port: ap.Port()} +} + +// DialTCP dials dst from the guest. +func (g *Guest) DialTCP(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { + return gonet.DialContextTCP(ctx, g.s, fullAddr(dst), ipv4.ProtocolNumber) +} + +// DialUDP opens a UDP socket from the guest connected to dst. +func (g *Guest) DialUDP(dst netip.AddrPort) (net.Conn, error) { + fa := fullAddr(dst) + return gonet.DialUDP(g.s, nil, &fa, ipv4.ProtocolNumber) +} + +// ListenTCP listens on port on the guest. +func (g *Guest) ListenTCP(port uint16) (net.Listener, error) { + return gonet.ListenTCP(g.s, tcpip.FullAddress{NIC: 1, Port: port}, ipv4.ProtocolNumber) +} diff --git a/guestnet/link.go b/guestnet/link.go new file mode 100644 index 0000000..11d1f05 --- /dev/null +++ b/guestnet/link.go @@ -0,0 +1,114 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "errors" + "io" + "os" + "sync" +) + +// FrameLink carries raw Ethernet frames, one per call, between a guest NIC +// and a [Net]. +type FrameLink interface { + // ReadFrame reads one frame into buf and returns its length. + ReadFrame(buf []byte) (int, error) + // WriteFrame writes one frame. + WriteFrame(frame []byte) error + // Close closes the link and unblocks readers. + Close() error +} + +// FileLink is a FrameLink over a datagram socket held as an *os.File, such +// as one end of a SOCK_DGRAM socketpair whose other end is attached to a +// VZFileHandleNetworkDeviceAttachment. Each Read returns one datagram, which +// is one Ethernet frame. +type FileLink struct { + f *os.File +} + +// NewFileLink wraps a datagram socket. It takes ownership of f. The socket +// should have been non-blocking when f was made with os.NewFile; otherwise +// Close does not unblock a pending ReadFrame. +func NewFileLink(f *os.File) *FileLink { return &FileLink{f: f} } + +// ReadFrame implements FrameLink. +func (l *FileLink) ReadFrame(buf []byte) (int, error) { + for { + n, err := l.f.Read(buf) + if err != nil { + return 0, err + } + if n == 0 { + // Zero-length datagrams are not frames. + continue + } + return n, nil + } +} + +// WriteFrame implements FrameLink. +func (l *FileLink) WriteFrame(frame []byte) error { + _, err := l.f.Write(frame) + return err +} + +// Close implements FrameLink. +func (l *FileLink) Close() error { return l.f.Close() } + +// MemLink is an in-memory FrameLink for tests. Create a connected pair with +// [NewMemLinkPair]. +type MemLink struct { + in chan []byte + out chan []byte + done chan struct{} + once sync.Once + peer *MemLink +} + +// NewMemLinkPair returns two links where frames written to one are read from +// the other. Closing either end unblocks both. +func NewMemLinkPair() (a, b *MemLink) { + ab := make(chan []byte, 1024) + ba := make(chan []byte, 1024) + done := make(chan struct{}) + a = &MemLink{in: ba, out: ab, done: done} + b = &MemLink{in: ab, out: ba, done: done} + a.peer, b.peer = b, a + return a, b +} + +// ReadFrame implements FrameLink. +func (l *MemLink) ReadFrame(buf []byte) (int, error) { + select { + case f := <-l.in: + if len(f) > len(buf) { + return 0, io.ErrShortBuffer + } + return copy(buf, f), nil + case <-l.done: + return 0, errors.New("memlink: closed") + } +} + +// WriteFrame implements FrameLink. +func (l *MemLink) WriteFrame(frame []byte) error { + f := append([]byte(nil), frame...) + select { + case l.out <- f: + return nil + case <-l.done: + return errors.New("memlink: closed") + } +} + +// Close implements FrameLink. It closes both ends of the pair. +func (l *MemLink) Close() error { + l.once.Do(func() { close(l.done) }) + if l.peer != nil { + l.peer.once.Do(func() {}) + } + return nil +} diff --git a/guestnet/log.go b/guestnet/log.go new file mode 100644 index 0000000..dd75c1f --- /dev/null +++ b/guestnet/log.go @@ -0,0 +1,120 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "net/netip" + "sync" + "time" +) + +// Flow is one TCP connection or UDP session from the guest. +type Flow struct { + // ID is unique within a Net and identical for the open and close records + // of one flow. + ID uint64 `json:"id"` + Proto string `json:"proto"` // "tcp" or "udp" + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Start time.Time `json:"start"` + // End is zero for the open record. + End time.Time `json:"end,omitzero"` + // Kind says how the flow was handled: "egress", "service", "dns", "ntp", + // or "blocked". + Kind string `json:"kind"` + // Service names the registered service for Kind "service". + Service string `json:"service,omitempty"` + // BytesIn counts bytes sent to the guest, BytesOut from the guest. + BytesIn int64 `json:"bytes_in"` + BytesOut int64 `json:"bytes_out"` + Err string `json:"err,omitempty"` +} + +// Closed reports whether f is a close record. +func (f Flow) Closed() bool { return !f.End.IsZero() } + +// DNSEvent is one DNS query from the guest and how it was answered. +type DNSEvent struct { + Time time.Time `json:"time"` + Src netip.AddrPort `json:"src"` + Dst netip.AddrPort `json:"dst"` + Proto string `json:"proto"` + Name string `json:"name"` + Type string `json:"type"` + RCode string `json:"rcode,omitempty"` + Answers int `json:"answers"` + Upstream string `json:"upstream,omitempty"` + Duration time.Duration `json:"duration"` + Err string `json:"err,omitempty"` +} + +// Event is a notable network event. +type Event struct { + Time time.Time `json:"time"` + // Kind is "dhcp-discover", "dhcp-lease", "blocked", "dial-error", ... + Kind string `json:"kind"` + Msg string `json:"msg"` +} + +// Log is a bounded ring of records with live subscribers. +type Log[T any] struct { + mu sync.Mutex + ring []T + next int + full bool + subs map[chan T]struct{} +} + +// NewLog returns a Log keeping the last n records. +func NewLog[T any](n int) *Log[T] { + return &Log[T]{ring: make([]T, n), subs: make(map[chan T]struct{})} +} + +// Add appends a record and delivers it to subscribers that are keeping up. +// Slow subscribers miss records rather than block the network. +func (l *Log[T]) Add(v T) { + l.mu.Lock() + defer l.mu.Unlock() + l.ring[l.next] = v + l.next++ + if l.next == len(l.ring) { + l.next = 0 + l.full = true + } + for ch := range l.subs { + select { + case ch <- v: + default: + } + } +} + +// Recent returns the retained records, oldest first. +func (l *Log[T]) Recent() []T { + l.mu.Lock() + defer l.mu.Unlock() + if !l.full { + return append([]T(nil), l.ring[:l.next]...) + } + out := make([]T, 0, len(l.ring)) + out = append(out, l.ring[l.next:]...) + out = append(out, l.ring[:l.next]...) + return out +} + +// Subscribe returns a channel receiving new records until ctx is done. +func (l *Log[T]) Subscribe(ctx context.Context) <-chan T { + ch := make(chan T, 256) + l.mu.Lock() + l.subs[ch] = struct{}{} + l.mu.Unlock() + go func() { + <-ctx.Done() + l.mu.Lock() + delete(l.subs, ch) + l.mu.Unlock() + }() + return ch +} diff --git a/guestnet/ntp.go b/guestnet/ntp.go new file mode 100644 index 0000000..8a888fb --- /dev/null +++ b/guestnet/ntp.go @@ -0,0 +1,67 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "encoding/binary" + "net" + "time" +) + +// ntpEpochOffset is the number of seconds between the NTP epoch (1900) and +// the Unix epoch (1970). +const ntpEpochOffset = 2208988800 + +// ntpIdleTimeout ends an NTP session after this long without a request. +const ntpIdleTimeout = 30 * time.Second + +// serveNTP answers SNTP requests on one guest UDP session with the host's +// time. A guest restored from a memory snapshot has a clock that is stale by +// the snapshot's age; macOS's timed corrects it on its next exchange with +// what it believes is time.apple.com, which this intercepts. +func (n *Net) serveNTP(conn net.Conn) error { + buf := make([]byte, 128) + for { + conn.SetReadDeadline(time.Now().Add(ntpIdleTimeout)) + m, err := conn.Read(buf) + if err != nil { + return nil + } + if m < 48 { + continue + } + resp := ntpResponse(buf[:48], n.cfg.Now()) + if _, err := conn.Write(resp); err != nil { + return err + } + } +} + +// ntpResponse builds a 48-byte server reply to req with all server +// timestamps set to now. +func ntpResponse(req []byte, now time.Time) []byte { + resp := make([]byte, 48) + vn := (req[0] >> 3) & 0x07 + if vn < 3 || vn > 4 { + vn = 4 + } + resp[0] = vn<<3 | 4 // LI=0, mode 4 (server) + resp[1] = 2 // stratum + resp[2] = req[2] // poll, echoed + resp[3] = 0xEC // precision: about 2^-20 seconds + // Root delay and dispersion: zero. + copy(resp[12:16], "LOCL") + ts := ntpTimestamp(now) + binary.BigEndian.PutUint64(resp[16:], ts) // reference + copy(resp[24:32], req[40:48]) // originate = client transmit + binary.BigEndian.PutUint64(resp[32:], ts) // receive + binary.BigEndian.PutUint64(resp[40:], ts) // transmit + return resp +} + +func ntpTimestamp(t time.Time) uint64 { + secs := uint64(t.Unix()) + ntpEpochOffset + frac := uint64(t.Nanosecond()) * (1 << 32) / 1e9 + return secs<<32 | frac +} diff --git a/guestnet/services.go b/guestnet/services.go new file mode 100644 index 0000000..fe4bf6c --- /dev/null +++ b/guestnet/services.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "errors" + "net" + "sync" +) + +// Service handles guest TCP connections to a registered destination. +type Service interface { + // Name identifies the service in flow logs. + Name() string + // HandleConn serves one guest connection. It owns conn and must close it. + // ctx is done when the Net closes. + HandleConn(ctx context.Context, conn net.Conn) +} + +// ServiceFunc adapts a function to a Service. +type ServiceFunc struct { + ServiceName string + Handle func(ctx context.Context, conn net.Conn) +} + +// Name implements Service. +func (s ServiceFunc) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s ServiceFunc) HandleConn(ctx context.Context, conn net.Conn) { s.Handle(ctx, conn) } + +// ForwardService relays guest connections to a host-side address, such as +// "tcp:127.0.0.1:2049" or "unix:/var/run/foo.sock". It replaces the pf rdr +// rules corp CI uses on macOS today. +type ForwardService struct { + ServiceName string + // Network and Address are passed to Dialer.DialContext. + Network, Address string + // Dialer defaults to a net.Dialer. + Dialer Dialer +} + +// Name implements Service. +func (s *ForwardService) Name() string { return s.ServiceName } + +// HandleConn implements Service. +func (s *ForwardService) HandleConn(ctx context.Context, conn net.Conn) { + defer conn.Close() + d := s.Dialer + if d == nil { + d = &net.Dialer{} + } + remote, err := d.DialContext(ctx, s.Network, s.Address) + if err != nil { + return + } + defer remote.Close() + splice(conn, remote) +} + +// ListenerService is a Service that exposes guest connections as a +// net.Listener, so an http.Server (or anything else that Accepts) can serve +// the guest without knowing about gVisor. +type ListenerService struct { + ServiceName string + + once sync.Once + conns chan net.Conn + closed chan struct{} + cmu sync.Once +} + +// NewListenerService returns a ListenerService named name. +func NewListenerService(name string) *ListenerService { + s := &ListenerService{ServiceName: name} + s.init() + return s +} + +func (s *ListenerService) init() { + s.once.Do(func() { + s.conns = make(chan net.Conn) + s.closed = make(chan struct{}) + }) +} + +// Name implements Service. +func (s *ListenerService) Name() string { return s.ServiceName } + +// HandleConn implements Service by handing conn to Accept. +func (s *ListenerService) HandleConn(ctx context.Context, conn net.Conn) { + s.init() + select { + case s.conns <- conn: + case <-s.closed: + conn.Close() + case <-ctx.Done(): + conn.Close() + } +} + +// Accept implements net.Listener. +func (s *ListenerService) Accept() (net.Conn, error) { + s.init() + select { + case c := <-s.conns: + return c, nil + case <-s.closed: + return nil, errListenerClosed + } +} + +// Close implements net.Listener. +func (s *ListenerService) Close() error { + s.init() + s.cmu.Do(func() { close(s.closed) }) + return nil +} + +// Addr implements net.Listener. +func (s *ListenerService) Addr() net.Addr { return serviceAddr(s.ServiceName) } + +var errListenerClosed = errors.New("guestnet: listener closed") + +type serviceAddr string + +func (a serviceAddr) Network() string { return "guestnet" } +func (a serviceAddr) String() string { return string(a) } diff --git a/guestnet/socketpair.go b/guestnet/socketpair.go new file mode 100644 index 0000000..ca55377 --- /dev/null +++ b/guestnet/socketpair.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "fmt" + "os" + "syscall" +) + +// Buffer sizes for the guest datagram link. Apple documents +// (VZFileHandleNetworkDeviceAttachment.maximumTransmissionUnit) that +// SO_RCVBUF must be at least twice SO_SNDBUF, and recommends four times. +// The absolute sizes are our guess, room for a few hundred full-size frames, +// and well under macOS's default kern.ipc.maxsockbuf of 8 MiB. Tune them +// with throughput measurements. +const ( + linkSendBuf = 512 << 10 + linkRecvBuf = 4 * linkSendBuf +) + +// NewSocketPairLink creates a connected SOCK_DGRAM unix socketpair. The +// returned *os.File is for the VM side (hand it to the network device +// attachment; the framework dups it, so close it after the VM is created). +// The returned FileLink is our side. +func NewSocketPairLink() (vmSide *os.File, host *FileLink, err error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + return nil, nil, fmt.Errorf("socketpair: %w", err) + } + for _, fd := range fds { + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, linkSendBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_SNDBUF: %w", err) + } + if err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, linkRecvBuf); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SO_RCVBUF: %w", err) + } + syscall.CloseOnExec(fd) + } + // Our end must be non-blocking before os.NewFile sees it, so the file + // uses the runtime poller and Close unblocks a pending ReadFrame. The + // VM end stays blocking, as the framework expects. + if err := syscall.SetNonblock(fds[1], true); err != nil { + closeFDs(fds) + return nil, nil, fmt.Errorf("SetNonblock: %w", err) + } + vmSide = os.NewFile(uintptr(fds[0]), "guestnet-vm") + hostFile := os.NewFile(uintptr(fds[1]), "guestnet-host") + return vmSide, NewFileLink(hostFile), nil +} + +func closeFDs(fds [2]int) { + for _, fd := range fds { + syscall.Close(fd) + } +} diff --git a/guestnet/socketpair_test.go b/guestnet/socketpair_test.go new file mode 100644 index 0000000..0872baf --- /dev/null +++ b/guestnet/socketpair_test.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guestnet + +import ( + "testing" + "time" +) + +// TestSocketPairCloseUnblocks checks that a Net on a real socketpair shuts +// down while its frame reader is blocked, which requires our end of the +// pair to use the runtime poller. +func TestSocketPairCloseUnblocks(t *testing.T) { + vmSide, link, err := NewSocketPairLink() + if err != nil { + t.Fatal(err) + } + defer vmSide.Close() + n, err := New(Config{Link: link}) + if err != nil { + t.Fatal(err) + } + // Let the frame reader block in ReadFrame. + time.Sleep(50 * time.Millisecond) + done := make(chan error, 1) + go func() { done <- n.Close() }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close hung with a blocked reader") + } +} diff --git a/guestnet/tcp.go b/guestnet/tcp.go new file mode 100644 index 0000000..b2b99bd --- /dev/null +++ b/guestnet/tcp.go @@ -0,0 +1,226 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "context" + "fmt" + "io" + "net" + "net/netip" + "sync" + "sync/atomic" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// egressDialTimeout bounds how long the guest's SYN waits for the host-side +// dial before we answer with a RST. +const egressDialTimeout = 10 * time.Second + +var flowIDs atomic.Uint64 + +// handleTCP is called by the gVisor forwarder for every SYN from the guest. +func (n *Net) handleTCP(r *tcp.ForwarderRequest) { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "tcp", Src: src, Dst: dst, Start: n.cfg.Now()} + + if svc, ok := n.service(dst); ok { + flow.Kind, flow.Service = "service", svc.Name() + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + // The service owns conn and records the flow's close when it + // closes it. + svc.HandleConn(n.ctx, conn) + }() + return + } + + if dst.Port() == 53 { + flow.Kind = "dns" + conn := n.acceptTCP(r, flow) + if conn == nil { + return + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSTCP(conn, src, dst)) + }() + return + } + + if !n.egressAllowed(dst.Addr()) { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("tcp %v -> %v", src, dst)}) + r.Complete(true) + return + } + + // Egress: dial first so a refused or unreachable destination becomes a + // RST to the guest instead of an accepted-then-closed connection. The + // forwarder drops the guest's retransmitted SYNs meanwhile. + flow.Kind = "egress" + n.wg.Add(1) + go func() { + defer n.wg.Done() + ctx, cancel := context.WithTimeout(n.ctx, egressDialTimeout) + defer cancel() + remote, err := n.cfg.Dialer.DialContext(ctx, "tcp", dst.String()) + if err != nil { + flow.Err = err.Error() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.End, Kind: "dial-error", Msg: fmt.Sprintf("tcp %v: %v", dst, err)}) + r.Complete(true) + return + } + conn := n.acceptTCP(r, flow) + if conn == nil { + remote.Close() + return + } + defer n.track(conn)() + defer n.track(remote)() + splice(conn, remote) + conn.Close() + }() +} + +// acceptTCP completes the handshake and returns the guest-side connection +// wrapped for flow accounting, recording the flow's open record. It returns +// nil (and records the failure) if the endpoint could not be created. +func (n *Net) acceptTCP(r *tcp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + r.Complete(true) + return nil + } + ep.SocketOptions().SetKeepAlive(true) + r.Complete(false) + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewTCPConn(&wq, ep), flow) +} + +// track arranges for conn to be closed when the Net closes, so no goroutine +// outlives it. The returned func cancels that arrangement; call it when the +// conn is done. +func (n *Net) track(conn net.Conn) func() { + stop := context.AfterFunc(n.ctx, func() { + // gonet conns don't reliably wake a blocked Read on Close alone; a + // past deadline does. + conn.SetReadDeadline(time.Unix(1, 0)) + conn.Close() + }) + return func() { stop() } +} + +// flowConn wraps a guest-side connection to count bytes and record the +// flow's close record exactly once, when whoever owns it closes it. +type flowConn struct { + net.Conn + n *Net + flow Flow + in atomic.Int64 // bytes written to the guest + out atomic.Int64 // bytes read from the guest + once sync.Once +} + +func newFlowConn(n *Net, c net.Conn, flow Flow) *flowConn { + return &flowConn{Conn: c, n: n, flow: flow} +} + +func (c *flowConn) Read(b []byte) (int, error) { + m, err := c.Conn.Read(b) + c.out.Add(int64(m)) + return m, err +} + +func (c *flowConn) Write(b []byte) (int, error) { + m, err := c.Conn.Write(b) + c.in.Add(int64(m)) + return m, err +} + +// CloseWrite half-closes the guest side, for splice. +func (c *flowConn) CloseWrite() error { + if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return nil +} + +// Close implements net.Conn and records the flow's close record. +func (c *flowConn) Close() error { return c.closeWithErr(nil) } + +// closeWithErr closes the conn, recording err (if any) in the flow's close +// record. Only the first close records anything. +func (c *flowConn) closeWithErr(err error) error { + cerr := c.Conn.Close() + c.once.Do(func() { + f := c.flow + f.End = c.n.cfg.Now() + f.BytesIn, f.BytesOut = c.in.Load(), c.out.Load() + if err != nil { + f.Err = err.Error() + } + c.n.Flows.Add(f) + }) + return cerr +} + +// splice copies bidirectionally between a and b with half-close +// propagation, returning when both directions are done. +// +// When one direction hits EOF it half-closes the other side, then waits a +// bounded time for the remaining direction. gVisor's gonet.TCPConn does not +// reliably wake a blocked Read on Close, so after the grace period both +// connections get a past read deadline before Close to guarantee the second +// copier exits and the endpoint is released. +func splice(a, b net.Conn) { + done := make(chan struct{}, 2) + cp := func(dst, src net.Conn) { + io.Copy(dst, src) + if cw, ok := dst.(interface{ CloseWrite() error }); ok { + cw.CloseWrite() + } + done <- struct{}{} + } + go cp(a, b) + go cp(b, a) + + <-done + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case <-done: + case <-t.C: + past := time.Unix(1, 0) + a.SetReadDeadline(past) + b.SetReadDeadline(past) + a.Close() + b.Close() + <-done + } + a.Close() + b.Close() +} diff --git a/guestnet/udp.go b/guestnet/udp.go new file mode 100644 index 0000000..29609ad --- /dev/null +++ b/guestnet/udp.go @@ -0,0 +1,146 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package guestnet + +import ( + "fmt" + "net" + "net/netip" + "time" + + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "gvisor.dev/gvisor/pkg/waiter" +) + +// udpIdleTimeout ends a UDP session after this long without traffic in +// either direction. +const udpIdleTimeout = 60 * time.Second + +// udpMaxPacket is the largest UDP payload we relay. +const udpMaxPacket = 65535 + +// handleUDP is called by the gVisor forwarder for the first packet of every +// UDP 4-tuple from the guest. Returning false drops the packet (and gVisor +// answers with ICMP port unreachable). +func (n *Net) handleUDP(r *udp.ForwarderRequest) bool { + id := r.ID() + dst := netip.AddrPortFrom(addrFromTCPIP(id.LocalAddress), id.LocalPort) + src := netip.AddrPortFrom(addrFromTCPIP(id.RemoteAddress), id.RemotePort) + flow := Flow{ID: flowIDs.Add(1), Proto: "udp", Src: src, Dst: dst, Start: n.cfg.Now()} + + switch dst.Port() { + case 53: + flow.Kind = "dns" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveDNSUDP(conn, src, dst)) + }() + return true + case 123: + flow.Kind = "ntp" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.serveNTP(conn)) + }() + return true + } + + if !n.egressAllowed(dst.Addr()) { + // Multicast (mDNS, SSDP) and broadcast chatter is dropped silently; + // unicast blocks are worth a record. + if !dst.Addr().IsMulticast() && dst.Addr() != netipBroadcast { + flow.Kind = "blocked" + flow.End = flow.Start + n.Flows.Add(flow) + n.Events.Add(Event{Time: flow.Start, Kind: "blocked", Msg: fmt.Sprintf("udp %v -> %v", src, dst)}) + } + return false + } + + flow.Kind = "egress" + conn := n.acceptUDP(r, flow) + if conn == nil { + return true + } + n.wg.Add(1) + go func() { + defer n.wg.Done() + defer n.track(conn)() + conn.closeWithErr(n.relayUDP(conn, dst)) + }() + return true +} + +// acceptUDP creates the gVisor endpoint for the session (bound to the +// guest's chosen destination and connected to the guest) and records the +// flow's open record. +func (n *Net) acceptUDP(r *udp.ForwarderRequest, flow Flow) *flowConn { + var wq waiter.Queue + ep, err := r.CreateEndpoint(&wq) + if err != nil { + flow.Err = err.String() + flow.End = n.cfg.Now() + n.Flows.Add(flow) + return nil + } + n.Flows.Add(flow) + return newFlowConn(n, gonet.NewUDPConn(&wq, ep), flow) +} + +// relayUDP shuttles datagrams between the guest session and a host-side UDP +// socket to dst until idle. Byte counts are kept by the flowConn. +func (n *Net) relayUDP(guest net.Conn, dst netip.AddrPort) error { + remote, err := n.cfg.Dialer.DialContext(n.ctx, "udp", dst.String()) + if err != nil { + n.Events.Add(Event{Time: n.cfg.Now(), Kind: "dial-error", Msg: fmt.Sprintf("udp %v: %v", dst, err)}) + return err + } + defer remote.Close() + defer n.track(remote)() + + done := make(chan struct{}) + go func() { + defer close(done) + buf := make([]byte, udpMaxPacket) + for { + remote.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := remote.Read(buf) + if err != nil { + guest.Close() + return + } + if _, err := guest.Write(buf[:m]); err != nil { + return + } + } + }() + buf := make([]byte, udpMaxPacket) + for { + guest.SetReadDeadline(time.Now().Add(udpIdleTimeout)) + m, err := guest.Read(buf) + if err != nil { + // Idle timeout, guest close, or Net shutdown: all normal. + remote.Close() + <-done + return nil + } + if _, err := remote.Write(buf[:m]); err != nil { + <-done + return err + } + } +} From 9f4198674ab0d618e0b27ea7fa0194837b83ad12 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Nov 2026 18:17:20 +0000 Subject: [PATCH 2/3] mantsana/image, cmd/oci-img-dl: add VM images and an OCI image downloader These are the on-disk VM images for mantsana, a macOS CI VM backend replacing Tart (see the DESIGN.md that follows), and a way to get them for now. An image is a directory of plain files: a manifest (hardware model, machine identifier, CPUs, memory, display, MTU), a raw sparse disk, and the macOS auxiliary storage (NVRAM). A warm subdirectory holds a saved VM memory snapshot and the disk and NVRAM it goes with, for restoring in seconds instead of cold booting. Snapshots are only usable on the host OS build and at the VM size they were saved with, so WarmUsable checks, and a failed restore can mark one stale. cmd/oci-img-dl (package ocidl) downloads the macOS VM images corp CI uses today, published to OCI registries such as ghcr.io/cirruslabs/macos-tahoe-xcode, straight into such an image. Their disks are 512 MiB chunks in Apple's LZ4 stream framing, each annotated with its uncompressed SHA-256, which is verified. Chunks download in parallel, all-zero pages are left as holes, and interrupted downloads resume. It's for development and the transition; mantsana's own image distribution will be different. Updates tailscale/corp#39994 Signed-off-by: Brad Fitzpatrick Change-Id: I54ff615054bbe0ff0e1fecfa920ffa2c947bd6e7 --- cmd/oci-img-dl/main.go | 52 +++ cmd/oci-img-dl/ocidl/applelz4.go | 81 ++++ cmd/oci-img-dl/ocidl/ocidl.go | 503 ++++++++++++++++++++++ cmd/oci-img-dl/ocidl/ocidl_test.go | 377 ++++++++++++++++ cmd/oci-img-dl/ocidl/registry.go | 209 +++++++++ cmd/oci-img-dl/ocidl/sparse_other_test.go | 11 + cmd/oci-img-dl/ocidl/sparse_unix_test.go | 21 + go.mod | 2 +- mantsana/image/clone_darwin.go | 25 ++ mantsana/image/clone_other.go | 10 + mantsana/image/copy.go | 30 ++ mantsana/image/image.go | 407 +++++++++++++++++ mantsana/image/image_test.go | 155 +++++++ 13 files changed, 1882 insertions(+), 1 deletion(-) create mode 100644 cmd/oci-img-dl/main.go create mode 100644 cmd/oci-img-dl/ocidl/applelz4.go create mode 100644 cmd/oci-img-dl/ocidl/ocidl.go create mode 100644 cmd/oci-img-dl/ocidl/ocidl_test.go create mode 100644 cmd/oci-img-dl/ocidl/registry.go create mode 100644 cmd/oci-img-dl/ocidl/sparse_other_test.go create mode 100644 cmd/oci-img-dl/ocidl/sparse_unix_test.go create mode 100644 mantsana/image/clone_darwin.go create mode 100644 mantsana/image/clone_other.go create mode 100644 mantsana/image/copy.go create mode 100644 mantsana/image/image.go create mode 100644 mantsana/image/image_test.go diff --git a/cmd/oci-img-dl/main.go b/cmd/oci-img-dl/main.go new file mode 100644 index 0000000..742649d --- /dev/null +++ b/cmd/oci-img-dl/main.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The oci-img-dl command downloads a macOS VM image published to an OCI +// registry (for example ghcr.io/cirruslabs/macos-tahoe-xcode:26.5) into a +// mantsana image directory. See package ocidl. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "github.com/tailscale/tb/cmd/oci-img-dl/ocidl" + "github.com/tailscale/tb/mantsana/image" +) + +var parallel = flag.Int("j", 8, "number of chunks to download at once") + +func main() { + log.SetFlags(log.Ltime) + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: oci-img-dl [-j N] <[registry/]repo[:tag|@digest]> \n\n") + fmt.Fprintf(os.Stderr, "A bare image name is created in %s.\n\n", image.DefaultStoreDir()) + fmt.Fprintf(os.Stderr, "example: oci-img-dl ghcr.io/cirruslabs/macos-tahoe-xcode:26.5 tahoe-xcode-26.5\n\n") + flag.PrintDefaults() + } + flag.Parse() + if flag.NArg() != 2 { + flag.Usage() + os.Exit(2) + } + r, err := ocidl.ParseRef(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + dir := flag.Arg(1) + if !strings.ContainsRune(dir, filepath.Separator) { + dir = filepath.Join(image.DefaultStoreDir(), dir) + } + if err := ocidl.Download(ctx, r, dir, ocidl.Options{Parallel: *parallel, Logf: log.Printf}); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/oci-img-dl/ocidl/applelz4.go b/cmd/oci-img-dl/ocidl/applelz4.go new file mode 100644 index 0000000..30064d7 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/applelz4.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "encoding/binary" + "fmt" + "io" + "slices" + + "github.com/pierrec/lz4/v4" +) + +// Block magics of Apple's libcompression LZ4 stream format, as found in the +// disk layers of these images. +const ( + magicLZ4Block = "bv41" // u32le decoded size, u32le encoded size, LZ4 block + magicRawBlock = "bv4-" // u32le size, uncompressed bytes + magicEnd = "bv4$" // end of stream +) + +// lz4Window is how far back an LZ4 block may refer. +const lz4Window = 64 << 10 + +// decodeAppleLZ4 decodes an Apple LZ4 stream from r, calling emit with each +// decoded block in order. The slice passed to emit is only valid during the +// call. Blocks are decoded with the preceding 64 KiB of output as their +// dictionary, since a block may refer back into earlier output. +func decodeAppleLZ4(r io.Reader, emit func([]byte) error) error { + var ( + hdr [12]byte + src, dst []byte + dict []byte + ) + for { + if _, err := io.ReadFull(r, hdr[:4]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + switch string(hdr[:4]) { + case magicEnd: + return nil + case magicLZ4Block: + if _, err := io.ReadFull(r, hdr[4:12]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + en := int(binary.LittleEndian.Uint32(hdr[8:12])) + src = slices.Grow(src[:0], en)[:en] + if _, err := io.ReadFull(r, src); err != nil { + return fmt.Errorf("reading block: %w", err) + } + dst = slices.Grow(dst[:0], dn)[:dn] + n, err := lz4.UncompressBlockWithDict(src, dst, dict) + if err != nil { + return fmt.Errorf("decoding LZ4 block: %w", err) + } + if n != dn { + return fmt.Errorf("LZ4 block decoded to %d bytes, want %d", n, dn) + } + case magicRawBlock: + if _, err := io.ReadFull(r, hdr[4:8]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + dst = slices.Grow(dst[:0], dn)[:dn] + if _, err := io.ReadFull(r, dst); err != nil { + return fmt.Errorf("reading block: %w", err) + } + default: + return fmt.Errorf("unknown block magic %q", hdr[:4]) + } + if err := emit(dst); err != nil { + return err + } + dict = append(dict, dst...) + if len(dict) > lz4Window { + dict = append(dict[:0], dict[len(dict)-lz4Window:]...) + } + } +} diff --git a/cmd/oci-img-dl/ocidl/ocidl.go b/cmd/oci-img-dl/ocidl/ocidl.go new file mode 100644 index 0000000..75ddf43 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl.go @@ -0,0 +1,503 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package ocidl downloads a macOS VM image published to an OCI registry, as +// used by corp CI today (for example +// ghcr.io/cirruslabs/macos-tahoe-xcode:26.5), into a mantsana image +// directory (see package mantsana/image): +// +// manifest.json mantsana image manifest, written last +// disk.img raw disk, sparse (all-zero 4 KiB pages are left as holes) +// nvram.img the VM's auxiliary storage (NVRAM) +// oci-manifest.json the OCI manifest it was downloaded from +// +// It is for development and for the transition from Tart; mantsana's own +// image distribution will be different (see DESIGN.md). +// +// Disk layers are 512 MiB chunks of the raw disk, each compressed with +// Apple's LZ4 stream framing and annotated with its uncompressed size and +// SHA-256, which is verified. Chunks download in parallel, and an +// interrupted download resumes where it left off. +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/tailscale/tb/mantsana/image" + "tailscale.com/types/logger" +) + +// Layer media types and annotations of these VM images. +const ( + mediaTypeDisk = "application/vnd.cirruslabs.tart.disk.v2" + mediaTypeConfig = "application/vnd.cirruslabs.tart.config.v1" + mediaTypeNVRAM = "application/vnd.cirruslabs.tart.nvram.v1" + + annotUncompressedSize = "org.cirruslabs.tart.uncompressed-size" + annotUncompressedDigest = "org.cirruslabs.tart.uncompressed-content-digest" +) + +// progressFile records completed disk chunks, one index per line, after a +// first line naming the manifest digest they belong to. +const progressFile = ".oci-img-dl-progress" + +// ociManifestFile is where the OCI manifest is saved, for reference. +const ociManifestFile = "oci-manifest.json" + +const pageSize = 4096 + +// Options configures a download. +type Options struct { + // Parallel is how many disk chunks to download at once. Default 8. + Parallel int + // Logf receives progress messages. Default logger.Discard. + Logf logger.Logf +} + +// ErrComplete is returned (wrapped) by Download when the directory already +// holds a complete image. +var ErrComplete = errors.New("already a complete image") + +// chunk is one disk layer and where its contents go in disk.img. +type chunk struct { + index int + desc descriptor + off int64 + size int64 // uncompressed +} + +// Download downloads the image r into the image directory dir, creating it +// if needed. If dir holds an interrupted download of the same image, it +// resumes; if it holds a complete image, it returns an error wrapping +// ErrComplete. +func Download(ctx context.Context, r Ref, dir string, opts Options) error { + logf := opts.Logf + if logf == nil { + logf = logger.Discard + } + parallel := opts.Parallel + if parallel <= 0 { + parallel = 8 + } + c := newClient(r) + m, rawManifest, err := c.manifest(ctx, r) + if err != nil { + return err + } + sum := sha256.Sum256(rawManifest) + manifestDigest := "sha256:" + hex.EncodeToString(sum[:]) + + var ( + chunks []chunk + config, nvram *descriptor + total int64 + compressed int64 + ) + for i := range m.Layers { + l := &m.Layers[i] + switch l.MediaType { + case mediaTypeDisk: + n, err := strconv.ParseInt(l.Annotations[annotUncompressedSize], 10, 64) + if err != nil { + return fmt.Errorf("disk layer %d: bad %s: %v", len(chunks), annotUncompressedSize, err) + } + if l.Annotations[annotUncompressedDigest] == "" { + return fmt.Errorf("disk layer %d: no %s", len(chunks), annotUncompressedDigest) + } + if total%pageSize != 0 { + return fmt.Errorf("disk layer %d starts at unaligned offset %d", len(chunks), total) + } + chunks = append(chunks, chunk{index: len(chunks), desc: *l, off: total, size: n}) + total += n + compressed += l.Size + case mediaTypeConfig: + config = l + case mediaTypeNVRAM: + nvram = l + default: + return fmt.Errorf("unsupported layer media type %q", l.MediaType) + } + } + if len(chunks) == 0 || config == nil || nvram == nil { + return errors.New("manifest lacks a disk, config or NVRAM layer") + } + logf("%s (%s): %d disk chunks, %.1f GB disk, %.1f GB to download", + r, manifestDigest[:19], len(chunks), float64(total)/1e9, float64(compressed)/1e9) + + if _, err := os.Stat(filepath.Join(dir, image.ManifestFile)); err == nil { + return fmt.Errorf("%s is %w; remove it to download again", dir, ErrComplete) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + done, err := loadProgress(logf, dir, manifestDigest) + if err != nil { + return err + } + diskPath := filepath.Join(dir, image.DiskFile) + flags := os.O_RDWR | os.O_CREATE + if len(done) == 0 { + flags |= os.O_TRUNC // a fresh start, or a different image + } + disk, err := os.OpenFile(diskPath, flags, 0o644) + if err != nil { + return err + } + defer disk.Close() + if err := disk.Truncate(total); err != nil { + return err + } + prog, err := openProgress(dir, manifestDigest, len(done) == 0) + if err != nil { + return err + } + defer prog.Close() + + // Small layers first, so a config we can't use fails before the big + // download. + configJSON, err := fetchSmall(ctx, logf, c, *config) + if err != nil { + return err + } + im, err := imageManifest(configJSON, total, r, manifestDigest) + if err != nil { + return err + } + aux, err := fetchSmall(ctx, logf, c, *nvram) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, image.AuxFile), aux, 0o644); err != nil { + return err + } + + var todo []chunk + var todoBytes int64 + for _, ch := range chunks { + if !done[ch.index] { + todo = append(todo, ch) + todoBytes += ch.desc.Size + } + } + if len(done) > 0 { + logf("resuming: %d of %d chunks already done", len(chunks)-len(todo), len(chunks)) + } + + var ( + next atomic.Int64 + gotBytes atomic.Int64 + doneCount atomic.Int64 + progMu sync.Mutex + wg sync.WaitGroup + errOnce sync.Once + firstErr error + ) + parent := ctx + ctx, cancel := context.WithCancel(ctx) + defer cancel() + fail := func(err error) { + errOnce.Do(func() { firstErr = err; cancel() }) + } + t0 := time.Now() + go func() { + tick := time.NewTicker(10 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + got := gotBytes.Load() + el := time.Since(t0).Seconds() + rate := float64(got) / el + eta := time.Duration(float64(todoBytes-got)/max(rate, 1)) * time.Second + logf("%d/%d chunks, %.1f/%.1f GB, %.0f MB/s, ETA %v", + doneCount.Load(), len(todo), float64(got)/1e9, float64(todoBytes)/1e9, rate/1e6, eta.Round(time.Second)) + } + } + }() + for range min(parallel, len(todo)) { + wg.Add(1) + go func() { + defer wg.Done() + for { + i := int(next.Add(1) - 1) + if i >= len(todo) || ctx.Err() != nil { + return + } + ch := todo[i] + err := retry(ctx, logf, fmt.Sprintf("chunk %d", ch.index), func() error { + return fetchChunk(ctx, c, disk, ch, &gotBytes) + }) + if err != nil { + fail(fmt.Errorf("chunk %d: %w", ch.index, err)) + return + } + progMu.Lock() + _, err = fmt.Fprintf(prog, "%d\n", ch.index) + progMu.Unlock() + if err != nil { + fail(err) + return + } + doneCount.Add(1) + } + }() + } + wg.Wait() + if parent.Err() != nil { + return fmt.Errorf("interrupted after %d of %d chunks (rerun the same command to resume)", doneCount.Load(), len(todo)) + } + if firstErr != nil { + return fmt.Errorf("%w (rerun the same command to resume)", firstErr) + } + if err := disk.Sync(); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, ociManifestFile), rawManifest, 0o644); err != nil { + return err + } + // The image manifest goes last: its presence is what makes the + // directory an image. + if err := image.SaveManifest(filepath.Join(dir, image.ManifestFile), im); err != nil { + return err + } + prog.Close() + os.Remove(filepath.Join(dir, progressFile)) + logf("done in %v: image %q in %s", time.Since(t0).Round(time.Second), filepath.Base(dir), dir) + return nil +} + +// vmConfig is the VM configuration layer of these images. +type vmConfig struct { + OS string `json:"os"` + Arch string `json:"arch"` + HardwareModel []byte `json:"hardwareModel"` // VZMacHardwareModel.dataRepresentation + ECID []byte `json:"ecid"` // VZMacMachineIdentifier.dataRepresentation + CPUCount int `json:"cpuCount"` + MemorySize uint64 `json:"memorySize"` + DiskFormat string `json:"diskFormat"` + Display struct { + Width int `json:"width"` + Height int `json:"height"` + } `json:"display"` +} + +// imageManifest converts the VM configuration layer into a mantsana image +// manifest. +func imageManifest(configJSON []byte, diskBytes int64, r Ref, manifestDigest string) (*image.Manifest, error) { + var vc vmConfig + if err := json.Unmarshal(configJSON, &vc); err != nil { + return nil, fmt.Errorf("parsing VM config: %w", err) + } + if vc.OS != "darwin" || vc.Arch != "arm64" { + return nil, fmt.Errorf("VM is %s/%s; only darwin/arm64 is supported", vc.OS, vc.Arch) + } + if vc.DiskFormat != "" && vc.DiskFormat != "raw" { + return nil, fmt.Errorf("disk format %q; only raw is supported", vc.DiskFormat) + } + if len(vc.HardwareModel) == 0 || len(vc.ECID) == 0 { + return nil, errors.New("VM config lacks hardwareModel or ecid") + } + m := &image.Manifest{ + Version: 1, + OS: "macos", + Arch: "arm64", + HardwareModel: vc.HardwareModel, + MachineID: vc.ECID, + // Our fixed guest MAC rather than the image's, so every image looks + // the same to guestnet. + MAC: image.DefaultMAC, + CPUs: vc.CPUCount, + MemoryBytes: vc.MemorySize, + Display: image.DefaultDisplay, + MTU: image.DefaultMTU, + DiskBytes: diskBytes, + Created: time.Now().UTC().Truncate(time.Second), + Description: fmt.Sprintf("downloaded from %s (%s)", r, manifestDigest), + } + if vc.Display.Width > 0 && vc.Display.Height > 0 { + m.Display = image.Display{Width: vc.Display.Width, Height: vc.Display.Height, PPI: 72} + } + if err := m.Validate(); err != nil { + return nil, err + } + return m, nil +} + +// fetchChunk downloads one disk chunk, writes its non-zero pages to disk, +// and verifies its uncompressed digest. got counts compressed bytes read. +func fetchChunk(ctx context.Context, c *client, disk *os.File, ch chunk, got *atomic.Int64) error { + body, err := c.blob(ctx, ch.desc.Digest) + if err != nil { + return err + } + defer body.Close() + var counted int64 + defer func() { got.Add(-counted) }() // only completed chunks count toward progress + h := sha256.New() + off := ch.off + cr := &countingReader{r: body, n: got, local: &counted} + err = decodeAppleLZ4(cr, func(b []byte) error { + if off+int64(len(b)) > ch.off+ch.size { + return errors.New("chunk decodes to more than its uncompressed size") + } + h.Write(b) + if err := writeNonZero(disk, b, off); err != nil { + return err + } + off += int64(len(b)) + return nil + }) + if err != nil { + return err + } + if off != ch.off+ch.size { + return fmt.Errorf("chunk decoded to %d bytes, want %d", off-ch.off, ch.size) + } + want := ch.desc.Annotations[annotUncompressedDigest] + if gotDigest := "sha256:" + hex.EncodeToString(h.Sum(nil)); gotDigest != want { + return fmt.Errorf("uncompressed digest %s, want %s", gotDigest, want) + } + counted = 0 // keep this chunk's bytes in the total + return nil +} + +// writeNonZero writes b at off, skipping all-zero 4 KiB pages so they stay +// holes in the sparse file. off must be page aligned. +func writeNonZero(f *os.File, b []byte, off int64) error { + var zeros [pageSize]byte + start := -1 // start of the current run of non-zero pages + flush := func(end int) error { + if start < 0 { + return nil + } + _, err := f.WriteAt(b[start:end], off+int64(start)) + start = -1 + return err + } + for i := 0; i < len(b); i += pageSize { + p := b[i:min(i+pageSize, len(b))] + if bytes.Equal(p, zeros[:len(p)]) { + if err := flush(i); err != nil { + return err + } + continue + } + if start < 0 { + start = i + } + } + return flush(len(b)) +} + +// countingReader counts bytes read into two counters. +type countingReader struct { + r io.Reader + n *atomic.Int64 + local *int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n.Add(int64(n)) + *c.local += int64(n) + return n, err +} + +// retry calls f until it succeeds, up to 5 times with exponential backoff, +// logging failures as what. +func retry(ctx context.Context, logf logger.Logf, what string, f func() error) error { + var err error + for attempt := range 5 { + if err = f(); err == nil || ctx.Err() != nil { + return err + } + logf("%s, attempt %d: %v", what, attempt+1, err) + select { + case <-time.After(time.Second << attempt): + case <-ctx.Done(): + return ctx.Err() + } + } + return err +} + +// fetchSmall downloads a small layer, verifying its digest, with retries. +func fetchSmall(ctx context.Context, logf logger.Logf, c *client, d descriptor) (b []byte, err error) { + err = retry(ctx, logf, d.MediaType+" layer", func() error { + b, err = fetchSmallOnce(ctx, c, d) + return err + }) + return b, err +} + +func fetchSmallOnce(ctx context.Context, c *client, d descriptor) ([]byte, error) { + if d.Size > 1<<30 { + return nil, fmt.Errorf("%s layer is %d bytes; too big", d.MediaType, d.Size) + } + body, err := c.blob(ctx, d.Digest) + if err != nil { + return nil, err + } + defer body.Close() + b, err := io.ReadAll(io.LimitReader(body, d.Size+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != d.Digest { + return nil, fmt.Errorf("%s layer: digest %s, want %s", d.MediaType, got, d.Digest) + } + return b, nil +} + +// loadProgress returns the chunks already downloaded into dir for the +// manifest with the given digest. +func loadProgress(logf logger.Logf, dir, manifestDigest string) (map[int]bool, error) { + b, err := os.ReadFile(filepath.Join(dir, progressFile)) + if errors.Is(err, os.ErrNotExist) { + return map[int]bool{}, nil + } + if err != nil { + return nil, err + } + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + done := map[int]bool{} + if len(lines) == 0 || lines[0] != manifestDigest { + logf("existing download in %s is of a different image; starting over", dir) + return done, nil + } + for _, l := range lines[1:] { + if i, err := strconv.Atoi(l); err == nil { + done[i] = true + } + } + return done, nil +} + +// openProgress opens the progress file for appending, starting it afresh +// if fresh. +func openProgress(dir, manifestDigest string, fresh bool) (*os.File, error) { + path := filepath.Join(dir, progressFile) + if fresh { + if err := os.WriteFile(path, []byte(manifestDigest+"\n"), 0o644); err != nil { + return nil, err + } + } + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) +} diff --git a/cmd/oci-img-dl/ocidl/ocidl_test.go b/cmd/oci-img-dl/ocidl/ocidl_test.go new file mode 100644 index 0000000..28aba31 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl_test.go @@ -0,0 +1,377 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/pierrec/lz4/v4" + "github.com/tailscale/tb/mantsana/image" +) + +// encodeAppleLZ4 frames data as Apple LZ4 blocks of blockSize, compressing +// each block independently (or storing it raw if it doesn't compress). +func encodeAppleLZ4(t testing.TB, data []byte, blockSize int) []byte { + t.Helper() + var out bytes.Buffer + var ht [1 << 16]int + for len(data) > 0 { + blk := data[:min(blockSize, len(data))] + data = data[len(blk):] + dst := make([]byte, lz4.CompressBlockBound(len(blk))) + n, err := lz4.CompressBlock(blk, dst, ht[:]) + if err != nil { + t.Fatal(err) + } + if n == 0 || n >= len(blk) { + out.WriteString(magicRawBlock) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + out.Write(blk) + continue + } + out.WriteString(magicLZ4Block) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + binary.Write(&out, binary.LittleEndian, uint32(n)) + out.Write(dst[:n]) + } + out.WriteString(magicEnd) + return out.Bytes() +} + +func decodeAll(t testing.TB, stream []byte) ([]byte, error) { + t.Helper() + var out bytes.Buffer + err := decodeAppleLZ4(bytes.NewReader(stream), func(b []byte) error { + out.Write(b) + return nil + }) + return out.Bytes(), err +} + +func TestDecodeAppleLZ4(t *testing.T) { + data := make([]byte, 300<<10) + for i := range data { + data[i] = byte(i / 7) // compressible + } + copy(data[100<<10:], bytes.Repeat([]byte{0xff, 0x13, 0x77}, 1000)) + rnd := make([]byte, 5000) // incompressible, to get a raw block + for i := range rnd { + rnd[i] = byte(i*7919 + i*i*31) + } + data = append(data, rnd...) + got, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatal("round trip mismatch") + } + + if _, err := decodeAll(t, []byte("bv4X")); err == nil { + t.Error("unknown magic accepted") + } + if _, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)[:1000]); err == nil { + t.Error("truncated stream accepted") + } +} + +// TestDecodeAppleLZ4Dict checks that a block may copy from the previous +// block's output. +func TestDecodeAppleLZ4Dict(t *testing.T) { + first := []byte("0123456789abcdefghij") + // Second block: a match of 8 bytes at offset 10 (into the first block), + // then the required trailing literals. + blk := []byte{ + 0x04, // token: 0 literals, match length 4+4 + 10, 0, // offset 10 + 0x50, // token: 5 literals, no match + 'V', 'W', 'X', 'Y', 'Z', // literals + } + var s bytes.Buffer + s.WriteString(magicRawBlock) + binary.Write(&s, binary.LittleEndian, uint32(len(first))) + s.Write(first) + s.WriteString(magicLZ4Block) + binary.Write(&s, binary.LittleEndian, uint32(13)) + binary.Write(&s, binary.LittleEndian, uint32(len(blk))) + s.Write(blk) + s.WriteString(magicEnd) + got, err := decodeAll(t, s.Bytes()) + if err != nil { + t.Fatal(err) + } + want := string(first) + "abcdefgh" + "VWXYZ" + if string(got) != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestParseRef(t *testing.T) { + for _, tt := range []struct{ in, reg, repo, ref string }{ + {"cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "latest"}, + {"ghcr.io/cirruslabs/x@sha256:abcd", "ghcr.io", "cirruslabs/x", "sha256:abcd"}, + {"localhost:5000/foo/bar:v1", "localhost:5000", "foo/bar", "v1"}, + } { + r, err := ParseRef(tt.in) + if err != nil { + t.Errorf("ParseRef(%q): %v", tt.in, err) + continue + } + if r.Registry != tt.reg || r.Repo != tt.repo || r.TagOrDigest != tt.ref { + t.Errorf("ParseRef(%q) = %+v", tt.in, r) + } + } +} + +func sha(b []byte) string { + s := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(s[:]) +} + +// fakeRegistry serves one image, requiring a bearer token obtained through +// the WWW-Authenticate challenge. +type fakeRegistry struct { + blobs map[string][]byte + manifest []byte + failBlob atomic.Pointer[string] // blob digest to fail once + blobGets atomic.Int64 + corruptOne atomic.Bool +} + +func (f *fakeRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + if r.URL.Query().Get("scope") != "repository:test/vm:pull" { + http.Error(w, "bad scope", 400) + return + } + json.NewEncoder(w).Encode(map[string]string{"token": "tok123"}) + return + } + if r.Header.Get("Authorization") != "Bearer tok123" { + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="http://%s/token",service="fake",scope="repository:test/vm:pull"`, r.Host)) + http.Error(w, "unauthorized", 401) + return + } + switch { + case r.URL.Path == "/v2/test/vm/manifests/v1": + w.Header().Set("Content-Type", mediaTypeManifest) + w.Write(f.manifest) + case strings.HasPrefix(r.URL.Path, "/v2/test/vm/blobs/"): + d := strings.TrimPrefix(r.URL.Path, "/v2/test/vm/blobs/") + b, ok := f.blobs[d] + if !ok { + http.NotFound(w, r) + return + } + f.blobGets.Add(1) + if p := f.failBlob.Load(); p != nil && *p == d { + f.failBlob.Store(nil) + w.Write(b[:len(b)/2]) // truncated + return + } + if f.corruptOne.CompareAndSwap(true, false) && len(b) > 100 { + c := bytes.Clone(b) + c[len(c)-10] ^= 0xff + w.Write(c) + return + } + w.Write(b) + default: + http.NotFound(w, r) + } +} + +// newFakeImage builds a fake VM image with a disk of three chunks, one of +// them all zeros. +func newFakeImage(t *testing.T) (*fakeRegistry, []byte) { + const chunkSize = 256 << 10 + disk := make([]byte, 3*chunkSize-8192) // last chunk is short + for i := range chunkSize { + disk[i] = byte(i % 251) + } + // Chunk 1 stays zero; chunk 2 has data in its second page only. + copy(disk[2*chunkSize+pageSize:], bytes.Repeat([]byte("mantsana"), 512)) + + f := &fakeRegistry{blobs: map[string][]byte{}} + add := func(b []byte) string { + d := sha(b) + f.blobs[d] = b + return d + } + config := []byte(`{"os":"darwin","arch":"arm64","cpuCount":2,"memorySize":4294967296,` + + `"diskFormat":"raw","hardwareModel":"aHc=","ecid":"aWQ=","display":{"width":1024,"height":768}}`) + nvram := []byte("nvram bytes") + m := manifest{ + MediaType: mediaTypeManifest, + Config: descriptor{MediaType: "application/vnd.oci.image.config.v1+json", Digest: add([]byte("{}")), Size: 2}, + Layers: []descriptor{ + {MediaType: mediaTypeConfig, Digest: add(config), Size: int64(len(config))}, + }, + } + for off := 0; off < len(disk); off += chunkSize { + part := disk[off:min(off+chunkSize, len(disk))] + z := encodeAppleLZ4(t, part, 64<<10) + m.Layers = append(m.Layers, descriptor{ + MediaType: mediaTypeDisk, Digest: add(z), Size: int64(len(z)), + Annotations: map[string]string{ + annotUncompressedSize: strconv.Itoa(len(part)), + annotUncompressedDigest: sha(part), + }, + }) + } + m.Layers = append(m.Layers, descriptor{MediaType: mediaTypeNVRAM, Digest: add(nvram), Size: int64(len(nvram))}) + var err error + if f.manifest, err = json.Marshal(m); err != nil { + t.Fatal(err) + } + return f, disk +} + +func testRef(t *testing.T, srv *httptest.Server) Ref { + r, err := ParseRef(strings.TrimPrefix(srv.URL, "http://") + "/test/vm:v1") + if err != nil { + t.Fatal(err) + } + return r +} + +// withHTTP makes clients use plain HTTP for the duration of the test, for +// httptest servers. +func withHTTP(t *testing.T) { + old := scheme + scheme = "http" + t.Cleanup(func() { scheme = old }) +} + +func TestDownload(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + ctx := context.Background() + + // One truncated chunk response and one corrupted one are retried. + var chunk2 string + var m manifest + json.Unmarshal(f.manifest, &m) + for _, l := range m.Layers { + if l.MediaType == mediaTypeDisk { + chunk2 = l.Digest + } + } + f.failBlob.Store(&chunk2) + f.corruptOne.Store(true) + + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dir, "disk.img")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, disk) { + t.Fatal("disk.img differs from the image's disk") + } + if b, _ := os.ReadFile(filepath.Join(dir, image.AuxFile)); string(b) != "nvram bytes" { + t.Errorf("%s = %q", image.AuxFile, b) + } + if b, _ := os.ReadFile(filepath.Join(dir, ociManifestFile)); !bytes.Equal(b, f.manifest) { + t.Error("OCI manifest not saved") + } + if _, err := os.Stat(filepath.Join(dir, progressFile)); !os.IsNotExist(err) { + t.Error("progress file left behind") + } + // APFS allocates holes of up to about 16 MiB between written extents + // (observed on macOS 26.5), so this tiny disk is fully allocated there. + if blocks := allocatedBlocks(t, filepath.Join(dir, image.DiskFile)); runtime.GOOS != "darwin" && blocks >= 0 && blocks*512 >= int64(len(disk)) { + t.Errorf("disk.img is not sparse: %d bytes allocated for %d", blocks*512, len(disk)) + } + + // The directory is a mantsana image. + im, err := (&image.Store{Root: filepath.Dir(dir)}).Get(filepath.Base(dir)) + if err != nil { + t.Fatalf("loading downloaded image: %v", err) + } + mm := im.Manifest + if mm.OS != "macos" || mm.CPUs != 2 || mm.MemoryBytes != 4<<30 || string(mm.HardwareModel) != "hw" || + string(mm.MachineID) != "id" || mm.Display.Width != 1024 || mm.DiskBytes != int64(len(disk)) { + t.Errorf("image manifest = %+v", mm) + } + + // A complete image isn't overwritten. + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); !errors.Is(err, ErrComplete) { + t.Errorf("second download: %v", err) + } +} + +func TestBadConfig(t *testing.T) { + withHTTP(t) + f, _ := newFakeImage(t) + var m manifest + json.Unmarshal(f.manifest, &m) + for i, l := range m.Layers { + if l.MediaType == mediaTypeConfig { + linux := []byte(`{"os":"linux","arch":"arm64","cpuCount":2,"memorySize":1}`) + m.Layers[i].Digest, m.Layers[i].Size = sha(linux), int64(len(linux)) + f.blobs[sha(linux)] = linux + } + } + f.manifest, _ = json.Marshal(m) + srv := httptest.NewServer(f) + defer srv.Close() + before := f.blobGets.Load() + err := Download(context.Background(), testRef(t, srv), filepath.Join(t.TempDir(), "img"), Options{Logf: t.Logf}) + if err == nil || !strings.Contains(err.Error(), "only darwin/arm64") { + t.Fatalf("download of a linux VM: %v", err) + } + if n := f.blobGets.Load() - before; n != 1 { + t.Errorf("fetched %d blobs before rejecting the config, want 1", n) + } +} + +func TestResume(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + r := testRef(t, srv) + + // Pretend chunk 0 was already done, with its data in place. + os.MkdirAll(dir, 0o755) + os.WriteFile(filepath.Join(dir, progressFile), []byte(sha(f.manifest)+"\n0\n"), 0o644) + os.WriteFile(filepath.Join(dir, "disk.img"), disk[:256<<10], 0o644) + + before := f.blobGets.Load() + if err := Download(context.Background(), r, dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + // config, nvram, and two of the three chunks. + if n := f.blobGets.Load() - before; n != 4 { + t.Errorf("fetched %d blobs on resume, want 4", n) + } + got, _ := os.ReadFile(filepath.Join(dir, "disk.img")) + if !bytes.Equal(got, disk) { + t.Fatal("resumed disk.img differs") + } +} diff --git a/cmd/oci-img-dl/ocidl/registry.go b/cmd/oci-img-dl/ocidl/registry.go new file mode 100644 index 0000000..77374ff --- /dev/null +++ b/cmd/oci-img-dl/ocidl/registry.go @@ -0,0 +1,209 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" +) + +// Ref is a parsed image reference. +type Ref struct { + Registry string // "ghcr.io" + Repo string // "cirruslabs/macos-tahoe-xcode" + TagOrDigest string // "26.5" or "sha256:..." +} + +// ParseRef parses "[registry/]repo[:tag|@digest]". The registry defaults +// to ghcr.io and the tag to "latest". +func ParseRef(s string) (Ref, error) { + var r Ref + name := s + if i := strings.Index(s, "@"); i >= 0 { + name, r.TagOrDigest = s[:i], s[i+1:] + } else if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") { + name, r.TagOrDigest = s[:i], s[i+1:] + } else { + r.TagOrDigest = "latest" + } + first, rest, ok := strings.Cut(name, "/") + if ok && (strings.ContainsAny(first, ".:") || first == "localhost") { + r.Registry, r.Repo = first, rest + } else { + r.Registry, r.Repo = "ghcr.io", name + } + if r.Repo == "" || r.TagOrDigest == "" { + return Ref{}, fmt.Errorf("bad image reference %q", s) + } + return r, nil +} + +func (r Ref) String() string { + sep := ":" + if strings.HasPrefix(r.TagOrDigest, "sha256:") { + sep = "@" + } + return r.Registry + "/" + r.Repo + sep + r.TagOrDigest +} + +// client fetches from one repository, handling the registry's anonymous +// bearer token challenge. +type client struct { + hc *http.Client + base string // "https://ghcr.io/v2/cirruslabs/macos-tahoe-xcode" + + mu sync.Mutex + token string +} + +// scheme is the URL scheme for registries. Tests set it to "http". +var scheme = "https" + +func newClient(r Ref) *client { + return &client{hc: http.DefaultClient, base: scheme + "://" + r.Registry + "/v2/" + r.Repo} +} + +// get fetches path under the repository, authenticating if challenged. +// The caller must close the response body. +func (c *client) get(ctx context.Context, path, accept string) (*http.Response, error) { + for attempt := 0; ; attempt++ { + req, err := http.NewRequestWithContext(ctx, "GET", c.base+path, nil) + if err != nil { + return nil, err + } + if accept != "" { + req.Header.Set("Accept", accept) + } + c.mu.Lock() + tok := c.token + c.mu.Unlock() + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + res, err := c.hc.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode == http.StatusUnauthorized && attempt == 0 { + challenge := res.Header.Get("WWW-Authenticate") + res.Body.Close() + if err := c.authenticate(ctx, challenge); err != nil { + return nil, err + } + continue + } + if res.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(res.Body, 1<<10)) + res.Body.Close() + return nil, fmt.Errorf("GET %s: %s: %s", c.base+path, res.Status, strings.TrimSpace(string(b))) + } + return res, nil + } +} + +// authenticate gets an anonymous token for a +// `Bearer realm="...",service="...",scope="..."` challenge. +func (c *client) authenticate(ctx context.Context, challenge string) error { + scheme, params, _ := strings.Cut(challenge, " ") + if !strings.EqualFold(scheme, "Bearer") { + return fmt.Errorf("unsupported auth challenge %q", challenge) + } + p := map[string]string{} + for _, kv := range strings.Split(params, ",") { + k, v, _ := strings.Cut(strings.TrimSpace(kv), "=") + p[k] = strings.Trim(v, `"`) + } + if p["realm"] == "" { + return fmt.Errorf("auth challenge %q has no realm", challenge) + } + q := url.Values{} + for _, k := range []string{"service", "scope"} { + if p[k] != "" { + q.Set(k, p[k]) + } + } + req, err := http.NewRequestWithContext(ctx, "GET", p["realm"]+"?"+q.Encode(), nil) + if err != nil { + return err + } + res, err := c.hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("getting token: %s", res.Status) + } + var t struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(res.Body).Decode(&t); err != nil { + return fmt.Errorf("getting token: %w", err) + } + tok := cmp.Or(t.Token, t.AccessToken) + if tok == "" { + return errors.New("token response has no token") + } + c.mu.Lock() + c.token = tok + c.mu.Unlock() + return nil +} + +// descriptor is an OCI content descriptor. +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// manifest is an OCI image manifest. +type manifest struct { + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +const mediaTypeManifest = "application/vnd.oci.image.manifest.v1+json" + +// manifest fetches the manifest for r, returning it and its raw bytes. +func (c *client) manifest(ctx context.Context, r Ref) (*manifest, []byte, error) { + res, err := c.get(ctx, "/manifests/"+r.TagOrDigest, mediaTypeManifest) + if err != nil { + return nil, nil, err + } + defer res.Body.Close() + raw, err := io.ReadAll(io.LimitReader(res.Body, 16<<20)) + if err != nil { + return nil, nil, err + } + m := new(manifest) + if err := json.Unmarshal(raw, m); err != nil { + return nil, nil, fmt.Errorf("parsing manifest: %w", err) + } + if m.MediaType != "" && m.MediaType != mediaTypeManifest { + return nil, nil, fmt.Errorf("unsupported manifest media type %q", m.MediaType) + } + return m, raw, nil +} + +// blob opens the blob with the given digest. +func (c *client) blob(ctx context.Context, digest string) (io.ReadCloser, error) { + res, err := c.get(ctx, "/blobs/"+digest, "") + if err != nil { + return nil, err + } + return res.Body, nil +} diff --git a/cmd/oci-img-dl/ocidl/sparse_other_test.go b/cmd/oci-img-dl/ocidl/sparse_other_test.go new file mode 100644 index 0000000..5ccf42c --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_other_test.go @@ -0,0 +1,11 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !unix + +package ocidl + +import "testing" + +// allocatedBlocks returns -1: allocation isn't checked on this platform. +func allocatedBlocks(t *testing.T, path string) int64 { return -1 } diff --git a/cmd/oci-img-dl/ocidl/sparse_unix_test.go b/cmd/oci-img-dl/ocidl/sparse_unix_test.go new file mode 100644 index 0000000..0906220 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_unix_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package ocidl + +import ( + "os" + "syscall" + "testing" +) + +// allocatedBlocks returns the number of 512-byte blocks allocated to path. +func allocatedBlocks(t *testing.T, path string) int64 { + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return fi.Sys().(*syscall.Stat_t).Blocks +} diff --git a/go.mod b/go.mod index 2caf8f2..94343d3 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/prometheus/client_model v0.6.3 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 golang.org/x/time v0.16.0 gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 modernc.org/sqlite v1.51.0 @@ -73,7 +74,6 @@ require ( go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect - golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect modernc.org/libc v1.72.3 // indirect diff --git a/mantsana/image/clone_darwin.go b/mantsana/image/clone_darwin.go new file mode 100644 index 0000000..d7e21c4 --- /dev/null +++ b/mantsana/image/clone_darwin.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "errors" + "fmt" + + "golang.org/x/sys/unix" +) + +// CloneFile makes dst a copy-on-write clone of src. On APFS this is O(1) +// regardless of file size and consumes no space until written. dst must not +// exist. It falls back to a regular copy if the filesystem can't clone. +func CloneFile(src, dst string) error { + err := unix.Clonefile(src, dst, 0) + if err == nil { + return nil + } + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EXDEV) { + return copyFile(src, dst) + } + return fmt.Errorf("clonefile %s -> %s: %w", src, dst, err) +} diff --git a/mantsana/image/clone_other.go b/mantsana/image/clone_other.go new file mode 100644 index 0000000..2d603a1 --- /dev/null +++ b/mantsana/image/clone_other.go @@ -0,0 +1,10 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !darwin + +package image + +// CloneFile copies src to dst. Only macOS has the O(1) APFS clone; elsewhere +// this is a plain copy, which is fine for development and tests. +func CloneFile(src, dst string) error { return copyFile(src, dst) } diff --git a/mantsana/image/copy.go b/mantsana/image/copy.go new file mode 100644 index 0000000..b33803e --- /dev/null +++ b/mantsana/image/copy.go @@ -0,0 +1,30 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "io" + "os" +) + +// copyFile copies src to dst, which must not exist, preserving sparseness +// only as well as io.Copy does (not at all). It is the fallback for +// CloneFile. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + os.Remove(dst) + return err + } + return out.Close() +} diff --git a/mantsana/image/image.go b/mantsana/image/image.go new file mode 100644 index 0000000..1a33803 --- /dev/null +++ b/mantsana/image/image.go @@ -0,0 +1,407 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package image describes mantsana's on-disk VM images. +// +// An image is a directory of plain files: a manifest, a raw disk image, and +// the macOS auxiliary storage (NVRAM) blob. A warm subdirectory holds a +// host-local memory snapshot for fast restore; it is derived data, keyed to +// the host OS build, and rebuilt when stale. +// +// For now images are just files on local disk. Later the disk will be +// fetched lazily from a content-addressed chunk store; that plugs in below +// this package at the NBD BaseImage seam, so this layout does not change. +package image + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +// File names within an image directory and its warm subdirectory. +const ( + ManifestFile = "manifest.json" + DiskFile = "disk.img" + AuxFile = "nvram.img" // not "aux.img": AUX is a reserved file name on Windows + WarmDir = "warm" + StateFile = "state.vzvmsave" // in WarmDir only + WarmFile = "warm.json" // in WarmDir only +) + +// Display is a guest display geometry. It is part of the image because a +// memory snapshot bakes it into the saved configuration. +type Display struct { + Width int `json:"width"` + Height int `json:"height"` + PPI int `json:"ppi"` +} + +// DefaultDisplay is used when a manifest doesn't specify a display. +var DefaultDisplay = Display{Width: 1280, Height: 800, PPI: 72} + +// DefaultMTU is used when a manifest doesn't specify an MTU. +const DefaultMTU = 1500 + +// DefaultMAC is the guest MAC address images use unless they have a reason +// not to. It is guestnet's default guest MAC, so every image looks the same +// to guestnet; each guest's network is private, so there is no conflict. +var DefaultMAC = MACAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} + +// MACAddr is a net.HardwareAddr that encodes as text in JSON. +type MACAddr net.HardwareAddr + +// String returns the colon-separated form, or "" for an empty address. +func (m MACAddr) String() string { return net.HardwareAddr(m).String() } + +// MarshalText implements encoding.TextMarshaler. +func (m MACAddr) MarshalText() ([]byte, error) { return []byte(m.String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (m *MACAddr) UnmarshalText(b []byte) error { + if len(b) == 0 { + *m = nil + return nil + } + hw, err := net.ParseMAC(string(b)) + if err != nil { + return err + } + if len(hw) != 6 { + return fmt.Errorf("MAC %q is not 48 bits", b) + } + *m = MACAddr(hw) + return nil +} + +// Manifest describes an image. The hardware model, machine identifier, MAC, +// display and MTU are image properties because a memory snapshot depends on +// them. Two VMs from one image share them, which is fine since their +// networks are isolated. +type Manifest struct { + Version int `json:"version"` + OS string `json:"os"` // "macos" (later "linux") + Arch string `json:"arch"` // "arm64" + + // HardwareModel and MachineID are the VZMacHardwareModel and + // VZMacMachineIdentifier dataRepresentation blobs (base64 in JSON). + // Empty for non-macOS images. + HardwareModel []byte `json:"hardware_model,omitempty"` + MachineID []byte `json:"machine_id,omitempty"` + + MAC MACAddr `json:"mac,omitempty"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Display Display `json:"display"` + MTU uint32 `json:"mtu"` + DiskBytes int64 `json:"disk_bytes,omitempty"` + + Created time.Time `json:"created,omitzero"` + Description string `json:"description,omitempty"` +} + +// fillDefaults sets unset optional fields. +func (m *Manifest) fillDefaults() { + if m.Version == 0 { + m.Version = 1 + } + if m.Arch == "" { + m.Arch = "arm64" + } + if m.Display == (Display{}) { + m.Display = DefaultDisplay + } + if m.MTU == 0 { + m.MTU = DefaultMTU + } +} + +// Validate reports whether m is usable. +func (m *Manifest) Validate() error { + if m.Version != 1 { + return fmt.Errorf("unsupported manifest version %d", m.Version) + } + switch m.OS { + case "macos", "linux": + default: + return fmt.Errorf("unsupported os %q", m.OS) + } + if m.CPUs <= 0 || m.MemoryBytes == 0 { + return errors.New("cpus and memory_bytes must be set") + } + // Virtualization.framework rejects network MTUs outside this range. + if m.MTU < 1500 || m.MTU > 65535 { + return fmt.Errorf("mtu %d out of range 1500..65535", m.MTU) + } + return nil +} + +// LoadManifest reads and validates the manifest at path, filling defaults. +func LoadManifest(path string) (*Manifest, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + m := new(Manifest) + if err := json.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + m.fillDefaults() + if err := m.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return m, nil +} + +// SaveManifest atomically writes m to path. +func SaveManifest(path string, m *Manifest) error { + return writeJSON(path, m) +} + +// WarmInfo records what a warm snapshot was built against, so a stale one is +// never restored. +type WarmInfo struct { + HostOSBuild string `json:"host_os_build"` + MantsanaVersion string `json:"mantsana_version,omitempty"` + ManifestHash string `json:"manifest_hash"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Created time.Time `json:"created,omitzero"` + + // Stale is set when a restore from this snapshot failed. StaleReason + // says why. + Stale bool `json:"stale,omitempty"` + StaleReason string `json:"stale_reason,omitempty"` +} + +// Image is an image directory on disk. +type Image struct { + Name string + Dir string + Manifest Manifest + // Warm is the warm snapshot's info, or nil if the image has no complete + // warm snapshot. + Warm *WarmInfo +} + +// DiskPath returns the path to the pristine raw disk image. +func (im *Image) DiskPath() string { return filepath.Join(im.Dir, DiskFile) } + +// AuxPath returns the path to the pristine auxiliary storage blob. +func (im *Image) AuxPath() string { return filepath.Join(im.Dir, AuxFile) } + +// WarmPath returns the path of file within the warm subdirectory, or the +// subdirectory itself if file is empty. +func (im *Image) WarmPath(file string) string { + return filepath.Join(im.Dir, WarmDir, file) +} + +// ManifestHash returns a stable hash of the manifest file's contents, so a +// changed image invalidates its warm snapshot. +func (im *Image) ManifestHash() (string, error) { + b, err := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + if err != nil { + return "", err + } + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +// WarmUsable reports whether the warm snapshot can be restored on a host +// running hostOSBuild with the given CPU count and memory size. If not, why +// says why, for logs and the API. +func (im *Image) WarmUsable(hostOSBuild string, cpus int, memoryBytes uint64) (ok bool, why string) { + w := im.Warm + switch { + case w == nil: + return false, "no warm snapshot" + case w.Stale: + return false, "stale: " + w.StaleReason + case w.HostOSBuild != hostOSBuild: + return false, fmt.Sprintf("host OS changed (snapshot %s, host %s)", w.HostOSBuild, hostOSBuild) + case w.CPUs != cpus || w.MemoryBytes != memoryBytes: + return false, fmt.Sprintf("snapshot is %d CPUs/%d bytes, want %d/%d", w.CPUs, w.MemoryBytes, cpus, memoryBytes) + } + hash, err := im.ManifestHash() + if err != nil { + return false, err.Error() + } + if hash != w.ManifestHash { + return false, "image manifest changed since snapshot" + } + return true, "" +} + +// SaveWarmInfo writes wi as the warm snapshot's info, creating the warm +// directory if needed, and sets im.Warm. +func (im *Image) SaveWarmInfo(wi *WarmInfo) error { + if err := os.MkdirAll(im.WarmPath(""), 0o755); err != nil { + return err + } + if err := writeJSON(im.WarmPath(WarmFile), wi); err != nil { + return err + } + im.Warm = wi + return nil +} + +// MarkWarmStale records that the warm snapshot must not be used again, for +// example because restoring it failed. +func (im *Image) MarkWarmStale(reason string) error { + if im.Warm == nil { + return nil + } + wi := *im.Warm + wi.Stale = true + wi.StaleReason = reason + return im.SaveWarmInfo(&wi) +} + +// DeleteWarm removes the warm snapshot. +func (im *Image) DeleteWarm() error { + im.Warm = nil + return os.RemoveAll(im.WarmPath("")) +} + +// loadWarm loads the warm info. A missing warm directory is (nil, nil). A +// warm.json without all of its files is an error, since it means a warm-up +// was interrupted or files were removed by hand. +func (im *Image) loadWarm() (*WarmInfo, error) { + b, err := os.ReadFile(im.WarmPath(WarmFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + wi := new(WarmInfo) + if err := json.Unmarshal(b, wi); err != nil { + return nil, fmt.Errorf("%s: %w", im.WarmPath(WarmFile), err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + if _, err := os.Stat(im.WarmPath(f)); err != nil { + return nil, fmt.Errorf("incomplete warm snapshot: %w", err) + } + } + return wi, nil +} + +// NotFoundError is returned by [Store.Get] for a missing image. +type NotFoundError struct { + Name string +} + +func (e *NotFoundError) Error() string { return fmt.Sprintf("image %q not found", e.Name) } + +// Store is a directory of images, one subdirectory each. +type Store struct { + Root string +} + +// DefaultStoreDir returns the default image store directory for the current +// user: on macOS, ~/Library/Application Support/mantsana/images. It is not +// ~/Library/Caches, even though images can be downloaded again, because +// the system may purge caches, and warm snapshots are costly to rebuild. +func DefaultStoreDir() string { + dir, err := os.UserConfigDir() + if err != nil { + home, _ := os.UserHomeDir() + dir = home + } + return filepath.Join(dir, "mantsana", "images") +} + +// validName reports whether name is usable as a single path element. +func validName(name string) bool { + return name != "" && name != "." && name != ".." && + !strings.ContainsAny(name, `/\`) && !strings.HasPrefix(name, ".") +} + +// Get loads the named image. +func (s *Store) Get(name string) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + dir := filepath.Join(s.Root, name) + m, err := LoadManifest(filepath.Join(dir, ManifestFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, &NotFoundError{Name: name} + } + if err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + im := &Image{Name: name, Dir: dir, Manifest: *m} + if im.Warm, err = im.loadWarm(); err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + return im, nil +} + +// List returns the images in the store, sorted by name. Directories without +// a valid manifest are skipped. +func (s *Store) List() ([]*Image, error) { + ents, err := os.ReadDir(s.Root) + if err != nil { + return nil, err + } + var out []*Image + for _, e := range ents { + if !e.IsDir() || !validName(e.Name()) { + continue + } + if im, err := s.Get(e.Name()); err == nil { + out = append(out, im) + } + } + return out, nil +} + +// Create makes a new image directory containing only a manifest, filling +// manifest defaults. The caller then writes the disk and aux files (for +// example with a macOS installer run). It fails if the image exists. +func (s *Store) Create(name string, m Manifest) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + m.fillDefaults() + if m.Created.IsZero() { + m.Created = time.Now().UTC().Truncate(time.Second) + } + if err := m.Validate(); err != nil { + return nil, err + } + if err := os.MkdirAll(s.Root, 0o755); err != nil { + return nil, err + } + dir := filepath.Join(s.Root, name) + if err := os.Mkdir(dir, 0o755); err != nil { + return nil, err + } + if err := SaveManifest(filepath.Join(dir, ManifestFile), &m); err != nil { + os.RemoveAll(dir) + return nil, err + } + return &Image{Name: name, Dir: dir, Manifest: m}, nil +} + +// writeJSON atomically writes v as indented JSON to path. +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/mantsana/image/image_test.go b/mantsana/image/image_test.go new file mode 100644 index 0000000..f735cea --- /dev/null +++ b/mantsana/image/image_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "testing" +) + +func TestStoreRoundTrip(t *testing.T) { + s := &Store{Root: t.TempDir()} + mac, _ := net.ParseMAC("02:76:7a:00:00:02") + im, err := s.Create("macos-test", Manifest{ + OS: "macos", + Arch: "arm64", + HardwareModel: []byte("hw"), + MachineID: []byte("id"), + MAC: MACAddr(mac), + CPUs: 4, + MemoryBytes: 8 << 30, + }) + if err != nil { + t.Fatal(err) + } + if _, err := s.Create("macos-test", Manifest{}); err == nil { + t.Fatal("duplicate Create succeeded") + } + + got, err := s.Get("macos-test") + if err != nil { + t.Fatal(err) + } + if got.Manifest.MAC.String() != "02:76:7a:00:00:02" { + t.Errorf("MAC = %v", got.Manifest.MAC) + } + if got.Manifest.Display != DefaultDisplay { + t.Errorf("Display = %+v, want default", got.Manifest.Display) + } + if got.Manifest.MTU != 1500 { + t.Errorf("MTU = %d, want 1500", got.Manifest.MTU) + } + if got.Warm != nil { + t.Errorf("Warm = %+v, want nil", got.Warm) + } + + // The MAC is encoded as text. + b, _ := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + var raw map[string]any + json.Unmarshal(b, &raw) + if raw["mac"] != "02:76:7a:00:00:02" { + t.Errorf("mac in JSON = %v", raw["mac"]) + } + + list, err := s.List() + if err != nil || len(list) != 1 || list[0].Name != "macos-test" { + t.Fatalf("List = %v, %v", list, err) + } + + if _, err := s.Get("nope"); err == nil { + t.Fatal("Get missing succeeded") + } else { + var nf *NotFoundError + if !errors.As(err, &nf) { + t.Errorf("err = %v, want NotFoundError", err) + } + } + if _, err := s.Get("../etc"); err == nil { + t.Fatal("Get with path traversal succeeded") + } +} + +func TestWarm(t *testing.T) { + s := &Store{Root: t.TempDir()} + im, err := s.Create("img", Manifest{OS: "macos", CPUs: 4, MemoryBytes: 1 << 30}) + if err != nil { + t.Fatal(err) + } + if ok, why := im.WarmUsable("25A354", 4, 1<<30); ok || why != "no warm snapshot" { + t.Fatalf("WarmUsable = %v, %q", ok, why) + } + + // Warm info without the files is not usable and is not loaded. + hash, _ := im.ManifestHash() + wi := &WarmInfo{HostOSBuild: "25A354", ManifestHash: hash, CPUs: 4, MemoryBytes: 1 << 30} + if err := im.SaveWarmInfo(wi); err != nil { + t.Fatal(err) + } + if got, err := s.Get("img"); err == nil || got != nil { + t.Fatalf("Get with incomplete warm dir = %v, %v; want error", got, err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + os.WriteFile(im.WarmPath(f), []byte("x"), 0o644) + } + got, err := s.Get("img") + if err != nil { + t.Fatal(err) + } + if ok, why := got.WarmUsable("25A354", 4, 1<<30); !ok { + t.Errorf("WarmUsable = false: %s", why) + } + if ok, _ := got.WarmUsable("26A100", 4, 1<<30); ok { + t.Error("usable after host OS change") + } + if ok, _ := got.WarmUsable("25A354", 8, 1<<30); ok { + t.Error("usable with different CPU count") + } + if err := got.MarkWarmStale("restore failed"); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "stale: restore failed" { + t.Errorf("after stale: %v %q", ok, why) + } + + // Changing the manifest invalidates the snapshot. + got.Warm.Stale = false + got.SaveWarmInfo(got.Warm) + m := got.Manifest + m.Description = "changed" + SaveManifest(filepath.Join(got.Dir, ManifestFile), &m) + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "image manifest changed since snapshot" { + t.Errorf("after manifest change: %v %q", ok, why) + } + + if err := got.DeleteWarm(); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if got.Warm != nil { + t.Error("warm survived DeleteWarm") + } +} + +func TestCloneFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + os.WriteFile(src, []byte("hello"), 0o644) + dst := filepath.Join(dir, "dst") + if err := CloneFile(src, dst); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(dst) + if string(b) != "hello" { + t.Fatalf("dst = %q", b) + } + if err := CloneFile(src, dst); err == nil { + t.Fatal("clone over existing file succeeded") + } +} From a4baeb0cdf9b8a78d4eb2ed1237f345d3e6751ed Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Nov 2026 18:21:06 +0000 Subject: [PATCH 3/3] mantsana, imds: add a Virtualization.framework macOS VM backend for CI mantsana runs corp CI's macOS VMs with Virtualization.framework (via github.com/Code-Hex/vz/v3) instead of Tart. DESIGN.md has the design, the decisions so far and the behavior found on hardware. Each VM gets a private guestnet network, an IMDS server, and its disk over NBD from an in-process guestbd server that discards the guest's writes, so there are no per-VM disk copies. VMs restore from a warm memory snapshot in about 4 seconds instead of cold booting in about 8. - imds: the guest's instance metadata server (name, boot mode, user data, time, ready and log endpoints) at 169.254.169.254. - mantsana/guest: one VM's lifetime, independent of the hypervisor: network, IMDS, NBD disk, boot or restore with cold boot fallback, and warm-up, which boots, saves the memory state and persists the disk the state goes with. A restore that fails because the user's session is locked doesn't mark the snapshot stale. guesttest is a fake Backend so all of this is tested on Linux. - mantsana/vzvm: the darwin/arm64 Backend using Virtualization.framework. - cmd/mantsana: a development tool to warm images and self-test cold boot, warm-up and restore on a Mac, plus probes for framework behavior and a keep-awake LaunchAgent for Macs whose MDM enforces an idle screen lock. The corp ciguestlet will import mantsana/guest as a second VM backend alongside Tart, selected per host. Updates tailscale/corp#39994 Signed-off-by: Brad Fitzpatrick Change-Id: Ic05719ebb6ccdc12720ddcadf0e8defdee030681 --- DESIGN.md | 924 ++++++++++++++++++ cmd/mantsana/entitlementprobe.go | 78 ++ cmd/mantsana/guestssh.go | 97 ++ .../com.tailscale.mantsana.keepawake.plist | 45 + cmd/mantsana/mantsana.entitlements | 12 + cmd/mantsana/mantsana.go | 68 ++ cmd/mantsana/mantsana_other.go | 18 + cmd/mantsana/nbdlive.go | 124 +++ cmd/mantsana/nbdprobe.go | 157 +++ cmd/mantsana/restoreprobe.go | 167 ++++ cmd/mantsana/selftest.go | 225 +++++ cmd/mantsana/warm.go | 94 ++ go.mod | 5 +- go.sum | 4 + imds/imds.go | 186 ++++ imds/imds_test.go | 88 ++ mantsana/guest/backend.go | 99 ++ mantsana/guest/debug.go | 23 + mantsana/guest/guest.go | 663 +++++++++++++ mantsana/guest/guest_test.go | 296 ++++++ mantsana/guest/guesttest/guesttest.go | 188 ++++ mantsana/guest/warm.go | 139 +++ mantsana/vzvm/vzvm.go | 374 +++++++ 23 files changed, 4073 insertions(+), 1 deletion(-) create mode 100644 DESIGN.md create mode 100644 cmd/mantsana/entitlementprobe.go create mode 100644 cmd/mantsana/guestssh.go create mode 100644 cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist create mode 100644 cmd/mantsana/mantsana.entitlements create mode 100644 cmd/mantsana/mantsana.go create mode 100644 cmd/mantsana/mantsana_other.go create mode 100644 cmd/mantsana/nbdlive.go create mode 100644 cmd/mantsana/nbdprobe.go create mode 100644 cmd/mantsana/restoreprobe.go create mode 100644 cmd/mantsana/selftest.go create mode 100644 cmd/mantsana/warm.go create mode 100644 imds/imds.go create mode 100644 imds/imds_test.go create mode 100644 mantsana/guest/backend.go create mode 100644 mantsana/guest/debug.go create mode 100644 mantsana/guest/guest.go create mode 100644 mantsana/guest/guest_test.go create mode 100644 mantsana/guest/guesttest/guesttest.go create mode 100644 mantsana/guest/warm.go create mode 100644 mantsana/vzvm/vzvm.go diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..4bb5adc --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,924 @@ +# mantsana: headless macOS VMs for CI, replacing Tart + +Status: design + prototype verified on hardware. This document is the +source of truth for requirements and decisions; update it as they change. + +The name: it has "ts" in it, sounds like Spanish *manzana* (apple), means +"apple" in Western Huasteca Nahuatl, and in some Malagasy dialects describes +something quick and nimble. + +## 1. Goal + +`mantsana` runs ephemeral macOS guest VMs on Apple Silicon Macs for +Tailscale's CI system. It replaces [Tart](https://tart.run) and the `pf` +anchors in the corp macOS stack, and nothing else: corp's `cihostlet` and its +per-slot `ciguestlet` processes stay, and `ciguestlet` gains a mantsana VM +backend next to its Tart one (section 4.8). cimgr, the cihostlet API, the +guest state machine and the GitHub runner recipe are unchanged. + +mantsana is Go packages (and development tools) built on Apple's +Virtualization.framework via +[`github.com/Code-Hex/vz/v3`](https://github.com/Code-Hex/vz), which +`ciguestlet` links in-process. No Swift, no AppKit window, no App Store +signing or notarization: we `go build` on the host Mac and ad-hoc +`codesign` with the virtualization entitlement. + +It does exactly what our CI needs and not more: + +* Boot a macOS guest from a base image in seconds, normally by restoring a + saved memory snapshot rather than cold booting. +* Throw the guest away afterwards. All guest disk writes are discarded. +* Give each guest its own isolated network with a metadata service (IMDS), + intercepted DNS, and logged egress flows. +* Provide screenshots of the guest display for debugging, with no GUI on the + host (mechanism open; section 4.5). +* Let `ciguestlet` keep running the guest-side CI recipe (GitHub runner JIT + config, job-start hook, gomodfs NFS, gocache and git proxies) with its + services plugged into the guest's network instead of `pf` rules. + +Non-goals for now: OCI registries, GUI, running on Intel Macs, Linux/Windows +guests (planned later, see section 12), nested virtualization, hosting +anything but CI workloads. + +## 2. Requirements log + +Collected from the initial brief. Add new ones here with dates. + +2026-09-29, initial: + +* Go, using `github.com/Code-Hex/vz/v3`. Fix problems in vz upstream (or in a + fork) rather than working around them. +* Headless Go server. No macOS GUI. +* No signing/notarization. Compile on the host Mac, ad-hoc sign for the + `com.apple.security.virtualization` entitlement. +* Replaces the macOS path in corp `tailscale.io` CI (`ciguestlet`'s + `tart.go` and `cihostlet`'s `tart.go`). +* All VMs ephemeral. Boot fast (seconds to usable), typically from a restore + snapshot, then only fix up DHCP/ARP state. Guest writes are discarded via an + NBD block device backed by the discard-writes NBD server we already use on + Linux (`github.com/tailscale/tb/guestbd`). No ephemeral disk files. +* Disk images come from the network and will be lazily synced later. For now + the base image is plain files already on local disk. Content-addressed + chunks and incremental sync come later. +* Each VM in its own isolated network, so guests cannot talk to each other. + Two macOS guests max per host (framework limit); more slots later for Linux. +* An IMDS metadata IP providing services to the guest. +* Later: intercept and log all DNS, log all outgoing TCP/HTTP flows per guest + for accounting, intercept TCP to the IMDS IP for services. (The design does + these from day one because the network architecture makes them free.) +* Maybe share code with `tailscale/tailvisor`'s Go `netd`, but not its Swift. +* Internal VNC to render screenshots of each guest slot. +* macOS guests first; Linux guests later. + +2026-09-29, later: + +* Name is `mantsana`, not `vzd`. At least the Apple-specific parts are in + `./cmd/mantsana`. +* It may take over both the `cihostlet` and `ciguestlet` roles in one + process (decided: it does), but then it must act like the existing + `cihostlet` API so corp `cimgr` can create VMs on it unchanged. +* Move the cimgr/hostlet API types from corp into this repo. The corp + `ciguestlet` and `cihostlet` code will move here later too. +* `setec` is open source: depend on it directly and support it first class + for secrets (the GitHub App key for runner JIT configs). + +2026-09-30: + +* mantsana uses the Mac's own tailscaled (its tailnet address for the + cihostlet API listener, LocalAPI WhoIs for callers), as cihostlet does; + it is not a tsnet node. +* Framework facts come from Apple's documentation and our own experiments, + not from reading other hypervisors' source. +* `cmd/oci-img-dl` downloads registry images straight into mantsana images, + stored under `~/Library/Application Support/mantsana`. +* Keep corp's cihostlet and ciguestlet; replace only their Tart part. They + run as separate processes so a cihostlet deploy doesn't interrupt builds + running in guests; a one-process design would regress that, because a VM + (and its in-process network and disk servers) can't outlive the process + that created it. ciguestlet gets a mantsana backend and has 1:1 shared + fate with its VM. +* The mantsana backend coexists with the Tart one for a while, so Macs can + opt in one at a time: a cihostlet flag selects the backend and is passed + down to ciguestlet. + +2026-11-03: + +* Don't move the cimgr/hostlet API types from corp yet; corp shouldn't be + half migrated to this repo. + +## 3. What exists today (research summary) + +### 3.1 Corp CI (`tailscale.io`, `cmd/cihostlet` + `cmd/ciguestlet`) + +`cihostlet` runs one per Mac (launchd daemon `com.ci.hostlet`) and owns +slots. Per slot it launches one `ciguestlet` process (launchd +`com.ci.-`, `KeepAlive=false`) which supervises exactly one VM. +`ciguestlet` is host-side; nothing of ours runs inside the guest except a +baked-in `on-job-started.sh` hook. macOS defaults: `--guestlets=1 +--max-guestlets=2` (hard cap 2 on darwin). + +The macOS path in `ciguestlet/tart.go` today: + +``` +tart list --format json # stop+delete stale VMs with the slot prefix +tart clone +tart set --cpu N +tart set --memory MiB +tart run --no-audio --no-clipboard --root-disk-opts=sync=none,caching=cached + [--no-graphics] [--vnc-experimental] (env CI=... so VNC URL is printed) +tart ip # polled every 1s, forever +ssh admin@ (password admin) # polled every 250ms, forever + sudo route -n add -host 169.254.169.254 + sudo route -n add -host 169.254.169.253 + sudo networksetup -setdnsservers Ethernet 169.254.169.253 + mount -t nfs ... :/ /Users/admin/go/pkg/mod # gomodfs from the host + tar -xvf - (on-job-started.sh, marker files) + ACTIONS_RUNNER_HOOK_JOB_STARTED=... run.sh --jitconfig ... +tart stop ; tart delete +``` + +Networking on macOS is vmnet NAT shared by all Tart VMs on the host, which +forces: ephemeral ports for the per-VM guest server and DNS proxy, a +`guestOnlyListener` peer filter, per-slot `pf` anchors +(`rdr ... to 169.254.169.254 port 80 -> host:port`, DNS rdr, blocks for the +CGNAT range), and a leaked-anchor sweeper (tailscale/corp#45834). The +`route -n add` calls exist because 169.254/16 is on-link (APIPA) in the guest +and would never be routed to the gateway. + +IMDS surface the guest uses (must be preserved): + +| Guest-visible address | Service | +|---|---| +| `169.254.169.254:80` | guest server: `POST /` job-start hook (body: `env -0`) | +| `169.254.169.254:31364` | Go build cache proxy (`GOCACHEPROG`); disabled on darwin today | +| `169.254.169.253:53` | DNS (also every port 53 flow is redirected here) | +| `:2049` | gomodfs NFSv3, read-only | +| `:9418` | git proxy (opt-in) | + +Fast boot on Linux/QEMU today (Windows, FreeBSD guests) restores a migration +state file, then re-DHCPs the guest (Windows: `guest-exec ipconfig /renew`; +FreeBSD: hot-plug the NIC after `cont` so `devd` runs `dhclient`). Firecracker +picks a new guest MAC per boot so the host flushes its ARP entry. None of this +exists for macOS today: every macOS job is a full cold boot. + +The Linux NBD path: `guestbd.NewServer(guestbd.FileSource(img), +guestbd.WithSharedSnapshot(), guestbd.WithMaxMem(0))` served over TCP to +`/dev/nbdN` via a custom ioctl-based client. Writes go to an unlinked temp file +and vanish on disconnect. + +The `ciguestlet` states cihostlet watches (`statustype.GuestState`) must keep +being emitted by whatever replaces `tart.go`'s `guest.Run`; nothing above that +seam needs to change. + +The cimgr <-> cihostlet contract (plain HTTP over the tailnet, no +request-level auth beyond binding to the tailnet address): + +* cimgr to hostlet, on `:8692`: + `POST /api/vms` (`guestlet.Opts`, strict decode; 201 + `guestlet.Guestlet`; + 503 when draining or out of capacity, which makes cimgr try the next host), + `DELETE /api/vms/{name}`, `POST /api/vms/{name}/ttl` + (`hostlet.VMTTLRequest`/`Response`), + `GET /api/vms/{name}/boot-log?stream=true` (NDJSON `bootlog.Event`), + `POST /api/drain` (`hostlet.DrainRequest`). Also served: `GET /api/vms`, + `GET /api/vms/{name}`, `GET /api/drain`, and `POST /api/vms/{name}/ssh`, an + `Upgrade: ssh` hijack returning `Ssh-Username`/`Ssh-Password` headers, + authorized by `Guestlet.SSHBearerToken` or the creator's WhoIs login. +* Hostlet to cimgr: a websocket to `ws://ci-mgr/hostinfo-ws` carrying + `worktype.HostWSMessage{hello|ping|guest-update}` with a + `worktype.HostInfo` (name, OS, request endpoint, runner labels, max and + baseline guestlets, `Guestlets map[ciid.GuestletName]*guestlet.Guestlet`, + draining). +* cimgr treats a VM as ready from its `GuestState` (`github-runner-waiting`, + `github-runner-running-job`, `ready`, ...), and as having started its job + from `StartedJob`. Slots `1..--guestlets` are baseline runners the hostlet + recreates forever; the rest are allocated for API requests. Names are + `--`. +* The shared types (`ci/ciid`, `labels`, `cinet`, `guestlettype`, + `statustype`, `hostlet`, `guestlet`, `bootlog`, `worktype`, + `cimgr/cimgrapi`) are a closed, stdlib-only import graph. `vmimage` mixes + types with filesystem and exec code and needs splitting first. +* The GitHub runner JIT config is generated by ciguestlet itself + (`Actions.GenerateOrgJITConfig` with a GitHub App key from setec); cimgr + never sees it. + +### 3.2 Virtualization.framework (Apple's documentation) + +Framework behavior comes from Apple's +[Virtualization documentation](https://developer.apple.com/documentation/virtualization) +and from our own measurements on hardware (section 9). The facts we rely +on, as documented: + +* `VZFileHandleNetworkDeviceAttachment` (macOS 11) maps a network interface + to a *connected datagram socket* and carries data at the data link layer; + it needs no `com.apple.vm.networking` entitlement. Its + `maximumTransmissionUnit` (macOS 13) defaults to 1500 and must be within + 1500..65535, or the configuration is invalid. The socket's `SO_RCVBUF` + must be at least twice its `SO_SNDBUF`, and four times is recommended. +* `VZNetworkBlockDeviceStorageDeviceAttachment` (macOS 14) is an NBD client + for a server outside the framework, addressed by an NBD URI (per the NBD + project's URI spec, which includes `nbd+unix:///?socket=`). + It connects at `start()`, not when the attachment is created, reconnects + on recoverable errors throughout the VM's life, and requires the + `com.apple.security.network.client` entitlement. +* `saveMachineStateTo(url:)` (macOS 14) requires a paused VM and leaves it + paused. `restoreMachineStateFrom(url:)` requires a stopped VM whose + configuration is compatible with the saved state, and leaves it paused, + to be resumed. `validateSaveRestoreSupport()` reports whether a + configuration can be saved; Apple does not list which devices can't. + Apple does not document the state file's encryption or portability. +* `VZErrorVirtualMachineLimitExceeded`: starting a VM would exceed the + system's limit on simultaneously running VMs. For macOS guests the limit + is two, which corp's cihostlet also caps. +* `VZMacTrackpadConfiguration` (macOS 13) is used by guests running macOS 13 + and later; `VZMacKeyboardConfiguration` (macOS 14) carries Mac-specific + keys. Apple documents both in terms of `VZVirtualMachineView` and says + nothing about save/restore. +* The only guest display path Apple documents is `VZVirtualMachineView`, an + AppKit view. There is no documented headless framebuffer or screenshot + API. + +### 3.3 tailvisor `netd` (Go) + +A single-guest userspace L2-to-L4 gateway: raw Ethernet frames over a unix +datagram socket, gVisor netstack (`ipv4`, `tcp`, `icmp4`), promiscuous and +spoofing NIC, `tcp.NewForwarder` terminating every TCP flow and re-dialing it +through tsnet, hand-rolled ARP (gateway only), DHCP (fixed lease), DNS, ICMP. +Clean and reusable in structure, but: single-instance globals (UDP session +table, stats), tsnet-only egress, no IMDS, no IPv6, no tests, nested module in +a Swift-first repo. Decision: copy the architecture and selected code, don't +import it (section 4.3). + +### 3.4 `guestbd` (Go) + +Library NBD server: fixed newstyle handshake, `NBD_OPT_GO`, READ/WRITE/FLUSH +(no-op)/TRIM/DISC, base image via `BaseImage` interface (`io.ReaderAt` + +`Size`), per-connection or shared in-memory/temp-file COW snapshot with +SHA-256 page dedup, `Serve(net.Listener)` so it works over a unix socket. +Well tested including against the Linux kernel client. Decision: import it. +Its `BaseImage` seam is where lazy network fetching plugs in later. + +### 3.5 `Code-Hex/vz` (Go) + +Everything we need is bound: macOS platform/bootloader/installer, +save/restore (`SaveMachineStateToPath`, `RestoreMachineStateFromURL`, +`ValidateSaveRestoreSupport`, macOS 14+), `FileHandleNetworkDeviceAttachment` +(must be `SOCK_DGRAM`; raw Ethernet frames one per datagram; MTU settable +1500..65535 on macOS 13+), `NetworkBlockDeviceStorageDeviceAttachment` +(macOS 14+, NBD URI, reconnect timeout, `Connected()` / `DidEncounterError()` +channels), vsock, serial console over file handles, graphics device config, +Mac keyboard/trackpad. No framebuffer access. Per-VM dispatch queue, +so no main-thread requirement unless `StartGraphicApplication` is called +(we never call it). + +## 4. Architecture + +``` + cimgr ── cihostlet (root, LaunchDaemon; unchanged API, slots, TTLs, drain, HostInfo) + (tailnet) │ launchctl bootstrap user/ com.ci.- (one job per slot) + ▼ + ┌──────────────────── ciguestlet --vm-backend=mantsana (one process per VM, as the CI user) ───────┐ + │ existing recipe: runner JIT (setec), job-start hook, gomodfs NFS, gocache + git proxies, │ + │ GuestState stream and logs on its loopback API; plus a dial endpoint for cihostlet's SSH proxy │ + │ │ + │ ┌────────────┐ socketpair ┌──────────────┐ dial/listen host network │ + │ │ VZ VM │◄──SOCK_DGRAM─►│ guestnet │─────────────────► (default route) │ + │ │ (vz) │ raw ethernet │ gVisor stack │──► IMDS / service table: ciguestlet's guest │ + │ │ │ │ DHCP ARP DNS │ server, DNS, gocache, git, NFS handlers │ + │ │ virtio-blk │◄─nbd+unix────►│ NTP flows │──► DNS and flow logs │ + │ │ virtio-net │ └──────────────┘ │ + │ │ graphics │◄───────────── screenshots (mechanism open, section 4.5) │ + │ └────────────┘ guestbd NBD server (writes discarded) serving the image's (warm) disk.img │ + └──────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +### 4.1 Process model + +corp's split stays: `cihostlet` (one per Mac, root) owns slots, the +cimgr-facing API, HostInfo push, TTLs and drain; one `ciguestlet` process +per slot owns one VM's lifecycle, its network services, the GitHub runner +recipe and `GuestState` transitions. They are separate processes so that +deploying cihostlet doesn't interrupt builds running in guests. + +With the mantsana backend, ciguestlet creates the VM in-process +(`mantsana/guest` with the `mantsana/vzvm` backend) instead of shelling out +to `tart`. The VM, its guestnet netstack, its guestbd NBD server and its +IMDS all live in the ciguestlet process, so the VM has exactly the +ciguestlet's lifetime: a VM can't outlive the process that created it +(observed: killing the creating process ends the VM), and its network and +disk die with the process anyway. This is also crash containment: a +Virtualization.framework crash (corp has seen VM-side crashes, #43482) takes +down one slot. + +ciguestlet must run as the CI user in that user's launchd domain, not as a +LaunchDaemon, because saving and restoring machine state use per-user +security services (section 9, item 4). Measured on the test Mac: + +| How the per-VM process is launched | Save | Restore | +|---|---|---| +| root LaunchDaemon (system domain) | no (no per-user `secd`) | no | +| LaunchDaemon, system domain, `UserName=` (Tart-mode ciguestlet today) | no (no per-user `secd`) | no | +| root runs `launchctl bootstrap user/`, plist with `LimitLoadToSessionType=Background` | yes | yes, 4.2 s to sshd | +| root runs `launchctl bootstrap gui/` | yes | yes, 3.9 s to sshd | + +(Without `LimitLoadToSessionType=Background`, bootstrapping into `user/` +fails with "5: Input/output error", as root or as the user; agents +evidently default to the Aqua session type.) + +Per `launchctl(1)`, `user/` is the user domain, which "may exist +independently of a logged-in user" (its session type is Background), and +`gui/` is the user-login domain created when the user logs in at the +console (session type Aqua), existing only while that login does. The two +share a Mach bootstrap namespace but not services, which is presumably why +both can reach the user's security services while the system domain can't. + +So in mantsana mode cihostlet writes the guestlet plist without `UserName` +and with `LimitLoadToSessionType=Background`, and bootstraps it into +`user/` of the CI user. That keeps guestlet lifetimes independent of +the GUI login's, like today's LaunchDaemons, and the job is still separate +from cihostlet's own, so cihostlet restarts don't touch it. (`gui/` +also works if we ever need the window server, for example for screenshots, +section 4.5.) Cold boots work in any of these contexts, but restores also +need the user logged in and the session unlocked; the screen lock is +typically MDM policy, so CI Macs must be excluded from any idle-lock +policy. + +Host setup for a mantsana CI Mac: + +1. Boot straight into the CI user's session: automatic login + (`/etc/kcpassword`), which macOS only allows with FileVault off; + otherwise every reboot goes through `fdesetup authrestart`. +2. The session never locks, by policy: no MDM idle-lock profile, screen + saver password off. Display sleep is + then the only remaining trigger, and mantsana prevents it. +3. The ciguestlets are jobs in that session (above). On any Mac whose MDM + still enforces an idle lock, also install the keep-awake agent + `cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist`. It keeps + `caffeinate -dimsu` assertions held continuously (a new 600-second one + every 300 seconds, overlapping), so the session never sleeps and never + goes idle, which is what defeats an MDM idle lock (sleep assertions alone + don't). There must be no gap: the first version let launchd restart a + single `caffeinate -t 300`, which left 20 seconds without assertions + every 5 minutes, and the managed screen saver started (and locked the + screen 60 seconds later) in one of those gaps. As the CI user: + + ``` + # install + cp cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist ~/Library/LaunchAgents/ + launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.tailscale.mantsana.keepawake.plist + + # remove + launchctl bootout gui/$(id -u)/com.tailscale.mantsana.keepawake + rm ~/Library/LaunchAgents/com.tailscale.mantsana.keepawake.plist + ``` +4. If the session is locked anyway, a restore fails fast (about 100 ms, + surfaced as "permission denied"); `vzvm` reports that as a transient + `RestoreError`, and `mantsana/guest` cold boots without marking the warm + snapshot stale. + +### 4.2 Guest VM configuration (`mantsana/vzvm`, darwin && arm64 only) + +Built once per boot, identically for cold boot and restore, since the +restored state must match the configuration: + +| Device | Choice | Why | +|---|---|---| +| Boot loader | `MacOSBootLoader` | macOS guest | +| Platform | `MacPlatformConfiguration` with hardware model, machine identifier and aux storage from the image | required | +| CPU/memory | from the request, defaulting to the image's; a restored VM must match the warm snapshot's size | | +| Disk | one `VirtioBlockDeviceConfiguration`; attachment is `NetworkBlockDeviceStorageDeviceAttachment` to the in-process guestbd server over a unix socket, `nbd+unix:///?socket=`, sync mode none | discard writes, no per-VM disk files, lazy fetch later | +| Network | one `VirtioNetworkDeviceConfiguration` with fixed MAC from the image, `FileHandleNetworkDeviceAttachment` over one end of a `socketpair(AF_UNIX, SOCK_DGRAM)`; `SO_SNDBUF` 512 KiB and `SO_RCVBUF` 2 MiB on both ends (Apple requires `SO_RCVBUF` of at least twice `SO_SNDBUF` and recommends four times; absolute sizes to be tuned, section 9 item 7) | per-VM isolation, all packets visible to Go | +| Graphics | `MacGraphicsDeviceConfiguration` with one display, fixed pixel geometry from the image manifest (default 1280x800 @ 72 ppi) | for screenshots; part of the saved state's configuration | +| Input | `MacKeyboardConfiguration`, `MacTrackpadConfiguration` | Mac-specific input for macOS 13+ guests; in case screenshots grow into UI automation | +| vsock | `VirtioSocketDeviceConfiguration` | future guest agent; free | +| Serial | `VirtioConsoleDeviceSerialPortConfiguration` to a file handle pair, logged as the console log | debugging; macOS guests print little but it costs nothing | +| Entropy, sound, balloon, USB, shared dirs, clipboard | omitted | not needed; fewer devices to keep save/restore compatible; virtiofs on macOS is unreliable (corp uses NFS) | + +This exact device set passes `Validate()` and `ValidateSaveRestoreSupport()` +on hardware, with the disk over NBD and with or without the serial port, and +a state saved with it restores (section 9). Both validations run at every +boot and are logged; if save/restore validation fails the VM still boots +cold but can't be snapshotted. + +Lifecycle per slot: + +1. Allocate slot, socketpair, unix socket dir (short path, `sun_path` limit). +2. Start guestbd serving `disk.img` with `WithSharedSnapshot()` (the framework + NBD client reconnects on timeout; per-connection snapshots would lose the + guest's writes across a reconnect and corrupt the filesystem). +3. Clone the image's `nvram.img` (NVRAM, ~tens of MB) with `clonefile` into the + slot dir; the guest writes NVRAM variables. +4. Start guestnet on the other socketpair end. +5. Build config, validate, `NewVirtualMachine`. Keep the Go pointer reachable + for the VM's lifetime (vz finalizers release the ObjC object). +6. If the image has a warm state: `RestoreMachineStateFromURL` then + `Resume`. On restore error (host OS updated, mismatch): log, mark the warm + image stale, fall back to cold `Start`. +7. Watch `StateChangedNotify()` in one goroutine; publish events. +8. Stop: `RequestStop()` (graceful, guest shutdown) with a deadline, then + `Stop()`. Close everything; delete the slot dir. + +### 4.3 Guest networking (`guestnet`, portable Go) + +The macOS equivalent of "each VM in its own network namespace" is a private +userspace network stack per VM. There is no shared bridge, no vmnet, no `pf`. +A guest's frames only ever reach Go code owned by its slot. + +Design (adapted from tailvisor `netd` and containers' `gvisor-tap-vsock`): + +* One gVisor `stack.Stack` per VM: `ipv4`, `arp`, `tcp`, `udp`, `icmp`. NIC is + an `ethernet` link over a `channel` endpoint pumped by a `FrameLink` + (`ReadFrame`/`WriteFrame`), implemented over the unix datagram socket. For + tests, an in-memory link pair. +* The NIC is promiscuous and spoofing, so every IPv4 packet the guest sends, + to any destination, is delivered to our forwarders. +* Addresses on our NIC, so gVisor answers ARP and ICMP echo for them + natively: gateway `192.168.64.1/24`, IMDS `169.254.169.254`, DNS + `169.254.169.253`. Because the two link-local addresses are on the NIC, the + guest's on-link ARP for 169.254.x.x is answered directly and the + `route -n add` hack in the guest goes away. +* Guest gets a fixed lease: IP `192.168.64.2`, router `.1`, DNS + `169.254.169.253`, MTU option matching the link, long lease. Every slot uses + the same addresses; the L2 domains are separate so there is no conflict, + and it means a memory snapshot is valid in any slot without re-DHCP. + The gateway MAC is a constant, so the guest's ARP cache stays valid after + restore. DHCP renewals from a restored guest are ACKed for the same + address. That is the whole "twiddle DHCP/ARP" step. +* NTP: intercept every UDP/123 flow and answer with host time. A restored + guest's clock is stale by the snapshot age and `timed` fixes it on first + sync. The API also exposes a "guest ready" hook where ciguestlet can force + `sntp` over SSH if needed. +* TCP: `tcp.NewForwarder`. Each SYN is dispatched by (dst IP, dst port): + 1. service table hit (see below): hand the `gonet` conn to the handler. + 2. blocked destination (tailnet CGNAT `100.64.0.0/10`, other slots, the + host's own addresses unless allow-listed): RST. + 3. otherwise egress: dial from the host's network stack, splice, and record + a flow log entry (start, end, guest src port, dst, bytes in/out, SNI or + Host if cheaply visible). Flow logs are what "log all outgoing TCP/HTTP + flows" means; they are per slot and streamable from the API. +* UDP: forwarder with a per-slot session table keyed by 4-tuple, idle + timeout, same dispatch and logging rules. Port 53 to any destination goes + to the DNS service; port 123 to NTP. +* DNS: parse with `golang.org/x/net/dns/dnsmessage` for logging (qname, qtype, + rcode, upstream, latency), forward the raw query to configured upstreams + (default: the host's resolvers) over UDP with TCP fallback, and answer + configurable overrides (for example a MagicDNS suffix to the host's + `100.100.100.100` while still blocking the CGNAT range for everything else). + Every query is logged; this replaces `--debug-dns`. +* IPv6: dropped for now (corp blocks it today). +* Host to guest: `gonet.DialTCP` into the guest from the slot's stack. The API + exposes it as a connection upgrade (`POST /v0/vms/{name}/dial`), which is + how ciguestlet SSHes in. There is deliberately no host interface with a + route to the guest. + +Service table: the guest recipe registers guest-visible endpoints that mantsana +forwards to host-side targets, replacing the `pf rdr` rules: + +```json +"services": [ + {"guest": "169.254.169.254:80", "target": "unix:/var/run/ci/slot1-guest.sock"}, + {"guest": "169.254.169.254:31364","target": "tcp:127.0.0.1:31364"}, + {"guest": "192.168.64.1:2049", "target": "tcp:127.0.0.1:2049"}, + {"guest": "192.168.64.1:9418", "target": "tcp:127.0.0.1:9418"} +] +``` + +mantsana's built-in IMDS (section 4.4) serves anything on `169.254.169.254:80` that +the caller has not claimed. Because each slot has its own stack, the same +guest port maps to different host targets per slot with no ephemeral-port +games and no peer-IP filtering. + +Isolation properties: a guest can only send frames into its own stack; it can +reach (a) service-table targets, (b) egress destinations not denied by policy, +via connections mantsana originates. It cannot see another guest's frames, ARP +spoof anything real, or reach host listeners except those explicitly +published. The daemon's egress dial can additionally be constrained +(deny RFC1918, deny host addresses) by policy. + +Performance: one frame per `read`/`write` syscall on the datagram socket, no +batching on macOS. gVisor plus splice handles the CI workload (git, module +downloads over NFS from the host, HTTPS). Raise the link MTU (up to 65535) +if host-to-guest NFS throughput matters; it is baked into the snapshot via +DHCP option 26 so it is an image property. + +### 4.4 IMDS (`imds`) + +Plain `http.Handler` served to guest connections to `169.254.169.254:80` +(and reachable to the host process for tests). Built-in endpoints, all per +slot so the identity of the caller is implicit: + +* `GET /` index. +* `GET /v0/meta` JSON: VM name, slot, image, cpus, memory, boot mode + (cold/restored), boot time, host name. +* `GET /v0/user-data/`: opaque key/values supplied at create time (job + tokens, runner config, etc.), so nothing has to be pushed over SSH. +* `GET /v0/time`: host time, for guests that want it without NTP. +* `POST /v0/ready`: guest signals it is usable; surfaces as an event. +* `POST /v0/log`: append a line to the slot's event log. +* Everything else: 404, unless the caller claimed the port in the service + table, in which case mantsana never sees it. + +### 4.5 Screenshots + +Requirement: screenshots of each guest's display (the brief suggests an +internal VNC server), with no GUI on the host. The mechanism is open. The +only display path Apple documents is `VZVirtualMachineView`, an AppKit view, +and there is no documented headless framebuffer API. Candidates, to be +evaluated from Apple's documentation and experiment when we get to it +(milestone 5): + +* Host the documented `VZVirtualMachineView` in an offscreen window inside + ciguestlet (which runs in the user's Aqua session, section 4.1) and + capture it with documented AppKit/ScreenCaptureKit APIs. +* A VNC server in the guest (macOS Screen Sharing), reached through the + netstack like sshd, with a Go RFB client that grabs one frame. + +Either way the graphics device stays configured, since its geometry is part +of the saved state's configuration. ciguestlet's existing `/screenshot` +endpoint serves the result. + +### 4.6 Threading + +vz wraps every framework call in `dispatch_sync` on the VM's private queue. +No `runtime.LockOSThread`, no `NSApplication`, no main thread. The only +single-consumer resource is `StateChangedNotify()`; one goroutine per VM +drains it and fans out to the event log. KVO callbacks arrive on the dispatch +queue thread; they only touch a mutex and a channel. + +### 4.7 Images (`mantsana/image`) + +An image is a directory of plain files, in the image store directory, +by default `~/Library/Application Support/mantsana/images` (not +`~/Library/Caches`: the system may purge caches, and warm snapshots are +costly to rebuild): + +``` +// + manifest.json # see below + disk.img # raw, sparse; immutable once published + nvram.img # VZMacAuxiliaryStorage (NVRAM), pristine +``` + +`manifest.json`: + +```json +{ + "version": 1, + "os": "macos", + "arch": "arm64", + "hardware_model": "", + "machine_id": "", + "mac": "02:76:7a:00:00:02", + "cpus": 4, + "memory_bytes": 8589934592, + "display": {"width": 1280, "height": 800, "ppi": 72}, + "mtu": 1500, + "disk_bytes": 85899345920, + "created": "2026-09-29T00:00:00Z", + "description": "macOS 26.x + Xcode 26.5, admin/admin" +} +``` + +The MAC, machine identifier, display geometry and MTU are image properties +because a memory snapshot bakes them in. Two slots running the same image +share them; that is fine because their networks are separate and nothing +depends on machine identifier uniqueness. (Not yet verified: running two +VMs with the same machine identifier at once; section 9 item 10.) + +Warm snapshots are host-local derived data. Apple doesn't document the +state file's portability, but on hardware it is encrypted to a Secure +Enclave key of the logged-in user (section 9), so treat it as bound to the +host and user: + +``` +//warm/ + disk.img # clonefile of ../disk.img, then mutated by the warm boot + nvram.img # clonefile of ../nvram.img, then mutated + state.vzvmsave # saved memory state, consistent with the two files above + warm.json # host OS build, mantsana version, image manifest hash, cpus/mem +``` + +Warm-up flow (`POST /v0/images/{name}/warm`, also automatic on first use): +clone aux; boot cold with the *same* device set as production, disk over NBD +included; wait until the guest is +usable (TCP 22 reachable through the netstack, or `POST /v0/ready`); optionally +settle for a few seconds; `Pause`; `SaveMachineStateToPath`; `Stop`. Production +runs then serve `warm/disk.img` over NBD with discarded writes and restore +`warm/state.vzvmsave`. A warm image is invalidated when the host OS build or +the manifest hash changes, or when a restore fails; mantsana then cold boots and +re-warms in the background when a slot is free. + +Persisting the warm-up's disk: the framework only restores a state into a +configuration with the same kind of disk attachment it was saved with +(section 9), so the warm-up boot uses NBD exactly like production. After +`Stop`, mantsana clones `disk.img` to `warm/disk.img` and writes the guestbd +shared snapshot's dirty pages onto it (`Snapshot.WriteDirtyTo`). On the +first hardware run this was 81,317 pages (318 MiB) in 2.5 s. + +Cold-boot-only images (no warm dir) work too, just slower. + +The image store API is intentionally tiny (`list`, `get`, `warm`, +`delete-warm`); acquiring images is a separate concern. Later this becomes a +content-addressed chunk store with lazy fetch, plugged in at guestbd's +`BaseImage` seam and served to the framework's NBD client exactly as today. + +### 4.8 Corp integration: a mantsana backend in cihostlet and ciguestlet + +The mantsana backend coexists with the Tart one so Macs can opt in one at a +time. cihostlet gets a flag selecting the macOS VM backend (for example +`--vm-backend=tart|mantsana`, default `tart`), which it passes down to each +ciguestlet. Everything above that seam (cimgr, the cihostlet API and its +types, `GuestState`s, boot logs, TTLs, drain, baseline and API slots) is +unchanged in both modes. + +ciguestlet, in mantsana mode (replacing what `tart.go` does): + +1. Boot with `mantsana/guest` + `mantsana/vzvm` in-process: restore the + image's warm snapshot if usable, else cold boot; the disk is served over + NBD with writes discarded. No `tart clone`/`set`/`run`/`ip`/`stop`. +2. Wait for sshd with `guestnet.DialGuest(22)`, and SSH over that + connection. There is no `tart ip`: the guest address is fixed. The + in-guest `route -n add` and `networksetup` steps go away: IMDS and DNS + are on-link in the guest's netstack and DHCP hands out the DNS address. +3. Its existing services plug into guestnet's service table instead of + `pf rdr` rules (`imds_darwin.go`'s anchors go away): the guest server on + `169.254.169.254:80` (job-start hook `POST /`), its DNS proxy on + `169.254.169.253:53` (as guestnet's `Resolver`), the gocache proxy on + `169.254.169.254:31364`, gomodfs NFS on `:2049`, and the git + proxy on `:9418`. Egress policy (the tailnet CGNAT block) comes + from guestnet's deny list. +4. The rest of the recipe is unchanged: NFS mount, base files tarball, runner + JIT config and `run.sh --jitconfig`, the same `GuestState`s. +5. Its loopback API gains a dial endpoint (an HTTP upgrade to a TCP stream + into the guest, as `guestnet.DialGuest`), because there is no host route + to the guest. + +cihostlet, in mantsana mode: + +1. Launch each ciguestlet by bootstrapping its plist (no `UserName`, with + `LimitLoadToSessionType=Background`) into `user/` of the CI user, + instead of loading a LaunchDaemon with `UserName` (section 4.1). +2. Its SSH proxy (`POST /api/vms/{name}/ssh`) connects through the + guestlet's dial endpoint instead of dialing `PrivateIP:22`. +3. Replace `tart pull` of `--tart-image` with an image in the local image + store: download it (as `cmd/oci-img-dl` does, as a library) when the + configured reference changes, and build its warm snapshot (`guest.Warm`, + in a free slot's ciguestlet) before first use and after host OS updates. +4. Drop the `pf` anchor sweeping; keep `caffeinate`, and make sure the + session never locks (section 4.1). + +mantsana itself (this repo) provides the packages (`guestnet`, `imds`, +`mantsana/guest`, `mantsana/vzvm`, `mantsana/image`, the OCI image +download) and development tools (`cmd/mantsana selftest` and probes, +`cmd/oci-img-dl`). The corp ciguestlet and cihostlet code is expected to +move into this repo later, at which point the backend flag and Tart mode +can go. + +## 5. Repository layout + +``` +DESIGN.md +cmd/mantsana/ darwin&&arm64 dev tool: selftest (validate, cold boot, warm, restore) and debugging probes +cmd/mantsana/launchd/ keep-awake LaunchAgent for Macs whose MDM enforces an idle lock +cmd/oci-img-dl/ dev tool: download a VM image published to an OCI registry (as corp CI uses today) as a mantsana image +guestnet/ per-VM netstack: link, DHCP, DNS, NTP, TCP/UDP forwarding, service table, flow log +guestnet/guestnettest/ fake guest stack for tests +imds/ metadata http.Handler +mantsana/guest/ one VM: network, IMDS, NBD disk, boot/restore with cold fallback, warm-up (portable) +mantsana/guest/guesttest/ fake Backend for tests +mantsana/image/ image dir + manifest, warm-state bookkeeping +mantsana/vzvm/ darwin&&arm64: build config, boot/restore/save, NBD wiring, console +mantsana/screenshot/ guest display capture (mechanism open, section 4.5) +``` + +`guestnet` and `imds` are top level because Linux guests, and the corp +Linux hostlet when it moves here, will use them too. Everything except +`vzvm`, any darwin half of `screenshot`, and `cmd/mantsana` compiles and is +tested on Linux, where most development happens (`mantsana/guest` is +unix-only because guestbd is). + +## 6. Build and deploy + +Anything linking vz (the mantsana-mode ciguestlet, `cmd/mantsana`) needs cgo +and the macOS SDK, so it is built on macOS (macOS 14+ SDK for save/restore +and NBD), and ad hoc signed with the virtualization entitlement: + +``` +go build -o mantsana ./cmd/mantsana +codesign --force --entitlements cmd/mantsana/mantsana.entitlements -s - ./mantsana +``` + +corp's `deploy.sh` today cross-compiles ciguestlet for darwin/arm64 with cgo +off, which can't include vz. Plan: build and sign it in a single-use macOS +build VM (corp's tempvm service, running on this same infrastructure), then +copy the signed binary to the CI Macs as today; no signing on the targets. +Verified 2026-09-30: a binary built and ad hoc signed on one Mac (macOS 26.5, +SDK 26.4), copied byte-identically via Linux to a different Mac (Mac17,16, +macOS 27.0), passes `codesign -v --strict` there and creates and starts VMs; +the same binary re-signed there without the entitlement is refused ("The +process doesn't have the com.apple.security.virtualization entitlement"). +An ad hoc signature is a code-directory hash inside the binary, not bound to +a host, and copying with scp or tailcat adds no quarantine attribute. +`cmd/mantsana entitlementprobe` checks a binary on a Mac without an image. +Keep a manual escape hatch (build on any one Mac by hand) for when the +build VMs themselves are broken. + +`mantsana.entitlements` contains `com.apple.security.virtualization` (required), +plus `com.apple.security.network.client` and `.server` (harmless, and required +if the binary is ever sandboxed; the vz NBD docs mention `network.client`). +`com.apple.vm.networking` is not needed since we never use bridged vmnet. + +vz checks at runtime that it was built against an SDK new enough for the +APIs it calls (`ErrBuildTargetOSVersion`); a binary built with the 26.4 SDK +ran fine on macOS 27.0. + +## 7. Dependencies + +* `github.com/Code-Hex/vz/v3`. Fix problems there (via a fork with + `replace` until upstreamed) rather than work around them. +* `gvisor.dev/gvisor` netstack. +* `github.com/tailscale/tb/guestbd` NBD server. +* `github.com/insomniacslk/dhcp` for DHCPv4 message encoding (already used in + corp's `dhcp.go`). +* `golang.org/x/net/dns/dnsmessage` for DNS parsing. +* `github.com/tailscale/setec`, first class, for secrets (the GitHub App key + used to mint runner JIT configs). +* The host's tailscaled (via `tailscale.com/client/local`) for the tailnet + listener address and WhoIs, as cihostlet does today. Not tsnet. +* A GitHub API client for JIT configs and runner deregistration (corp uses + go-github). +* Possibly an RFB client for screenshots, depending on section 4.5 (corp + uses `github.com/mitchellh/go-vnc`). + +## 8. Decisions + +* mantsana replaces only Tart and `pf`: corp's cihostlet and ciguestlet stay, + and ciguestlet gets a mantsana backend (section 4.8), selected per host by + a cihostlet flag while Tart mode remains. (Rejected: one mantsana process + taking both roles. A VM can't outlive the process that created it, so + every deploy of that process would kill running builds.) +* One ciguestlet process per VM, with the VM, its netstack and its NBD server + in-process: 1:1 shared fate, and crash containment per slot. +* The cimgr/hostlet API types stay in corp for now (2026-11-03), so corp + isn't half migrated. Nothing here needs them: the corp ciguestlet + imports `mantsana/guest`, not the other way around. +* Userspace netstack per VM instead of vmnet/pf: it is the only way to get + real isolation, deterministic addressing, and packet-level visibility on + macOS without root-owned network state that leaks. +* NBD to an in-process guestbd rather than APFS clone per run: no disk space + churn, identical to the Linux path, and it is the seam for lazy image + fetching. Clone stays as the fallback if the framework rejects NBD for + save/restore. +* Fixed guest IP/MAC/gateway MAC shared by all slots so snapshots are + slot-independent and need no post-restore network fix-up beyond normal + DHCP renewals. +* No guest agent for now: SSH as `admin` over the API dial upgrade, exactly + like today. vsock is configured for a future agent. +* Copy tailvisor `netd`'s design, not its module. Import `guestbd`. +* Framework behavior comes from Apple's documentation and our own + experiments on hardware, not from reading other hypervisors' source. + +## 9. Open questions to settle on hardware + +Answers so far are from 2026-09-29 on an M1 Pro MacBook Pro (MacBookPro18,2), +macOS 26.5 (25F71), guest macOS Tahoe imported from a Tart VM, 4 CPUs, 8 GiB. + +1. Does `ValidateSaveRestoreSupport()` accept `NetworkBlockDeviceStorageDeviceAttachment` + and `FileHandleNetworkDeviceAttachment`? **Yes**, with and without the + virtio serial console. Cold boot over NBD with the userspace network + works: DHCP lease about 6 s after start, sshd reachable at 7.7 to 9.2 s. + (This needed guestbd to support `NBD_OPT_INFO`, which Apple's NBD + client sends before `NBD_OPT_GO`.) +2. Can a state saved with a disk-image attachment be restored with an NBD + attachment on the same virtio-blk device? **No** ("invalid argument"): + the attachment type must match. A state saved over NBD restores over + NBD, so warm-up boots over NBD too and persists the guest's writes onto + an APFS clone of the base disk afterwards (guestbd + `Server.SharedSnapshot` + `Snapshot.WriteDirtyTo`). Paths may differ + between save and restore; the aux file may be a clone. +3. How to take screenshots headlessly (section 4.5). +4. Session requirements (observed; Apple doesn't document them). The + framework's `VirtualMachineArchiveKeyManager` encrypts the saved state to + a Secure Enclave P-256 key of the user running the VM. + * As root in a LaunchDaemon (system domain) there is no per-user `secd`: + cold boot works, but saving fails with -25291 (`errSecNotAvailable`, + surfaced as "Internal Virtualization error"). + * As a regular user, saving works whether or not the screen is locked + (it only needs the public key), from the GUI (Aqua) session or a plain + SSH (Background) session. + * Restoring needs the private key, which the Secure Enclave only allows + while that user's session is **unlocked**. With the screen locked it + fails with -25308 (`errSecInteractionNotAllowed`, surfaced as + "permission denied"), in any session. Unlocked, restore works from + Aqua and from SSH alike. + + So mantsana runs as a logged-in regular user whose screen never locks: + no idle lock (MDM may enforce one, see 4.1), + display sleep off. corp's cihostlet already runs `caffeinate -dims` for + sleep. When restore fails this way mantsana cold boots, and should not + mark the snapshot stale, since it becomes usable again once unlocked. +5. Restore-to-usable latency: **about 4.0 s** from restore start to sshd + reachable (2.8 GiB state file for 8 GiB of guest memory; saving takes + 4 to 5.5 s, a whole warm-up about 26 s). Still open: whether `timed` corrects the clock fast + enough via intercepted NTP or we need an explicit `sntp` over SSH. +6. Whether macOS's DHCP client renews properly against our server after a + long snapshot age (lease should be long; test with expired leases too). +7. Throughput of the datagram link for NFS module cache reads; whether a + larger MTU is worth baking into images. +8. (Found on hardware) macOS guests ignore `RequestStop` (no shutdown within + 30 s). Since all writes are discarded, normal VMs are force-stopped + (about 170 ms). +9. (Found on hardware) The guest's DHCP client ARP-probes its offered + address (RFC 5227) and declines the lease if anything answers. A + promiscuous gVisor NIC answers ARP for every address, so guestnet only + passes ARP requests for its own addresses to gVisor. +10. Two VMs from the same image (same machine identifier and MAC, separate + networks) running at once. + +## 10. Testing + +* `guestnet`: a second gVisor stack plays the guest over an in-memory link + pair; tests cover DHCP lease, ARP for gateway and IMDS addresses, TCP egress + with flow log, service table dispatch, DNS interception and logging, NTP, + blocked destinations, host-to-guest dial. +* `mantsana/guest`: a fake backend (`guesttest`) whose machines are gVisor + guests; tests cover cold boot, IMDS from the guest, the NBD socket, + warm-up, restore, and restore failures (transient and not). +* `image`: manifest round trip, warm-state validity checks. +* `cmd/oci-img-dl`: LZ4 framing, and a fake registry with the bearer token + challenge, bad responses, sparseness and resume. +* Darwin packages: a `mantsana selftest` subcommand that boots an image cold, + checks save/restore validation, saves state, restores, + and reports timings. Run manually on the CI Macs. + +## 11. Milestones + +1. Prototype (done): `guestnet`, `imds`, `mantsana/image`, `mantsana/guest`, + `mantsana/vzvm`, `cmd/mantsana selftest`, `cmd/oci-img-dl`. +2. Hardware (done): cold boot, warm-up and restore on a Mac; section 9 items + 1, 2, 4 and 5 settled; launch context for save/restore settled + (section 4.1). +3. Corp: ciguestlet mantsana backend and cihostlet `--vm-backend` flag + (section 4.8), with Tart mode unchanged. +4. One Mac switched to `--vm-backend=mantsana`, taking real jobs from cimgr; + then the rest, one at a time. +5. Screenshots (section 4.5). +6. Image sync (content-addressed chunks, lazy NBD-backed fetch; section 13). +7. Linux guests (EFI boot loader, virtio-gpu or none, more than 2 slots). + +## 12. Linux guests (later) + +Same slot machinery. Differences: `EFIBootLoader` + variable store or +`LinuxBootLoader` (kernel+initrd), `GenericPlatformConfiguration`, no macOS VM +limit so slot count is CPU/RAM bound, virtio-gpu only if screenshots are +wanted, save/restore is not supported by the framework for Linux so fast boot +is cold boot from a small image (Firecracker-style), guest networking is +identical. The image manifest's `os` field selects the builder. + +## 13. Image storage measurements (input to milestone 7) + +2026-09-30: page-level similarity between public macOS Tahoe + Xcode VM +images (`ghcr.io/cirruslabs/macos-tahoe-xcode`). Each is a 140 GB raw disk, +about 80.5 GiB of it non-zero, published as 261 LZ4-compressed 512 MiB +chunks (69 GB). Pages are 4 KiB; "reused" means the page's content (SHA-256) +appears somewhere in the older image. Percentages are of the newer image's +non-zero pages. + +| old -> new | reused, same offset | reused, moved | dup within new | new content | +|---|---|---|---|---| +| 26.4.1 (2026-05-04) -> 26.5 previous build (2026-05-26) | 34.0% | 10.4% | 1.1% | 54.5% (43.7 GiB) | +| 26.5 previous build (2026-05-26) -> 26.5 rebuild (2026-09-16) | 0.03% | 66.5% | 1.2% | 32.3% (26.0 GiB) | +| 26.4.1 (2026-05-04) -> 26.5 rebuild (2026-09-16) | 0.03% | 22.2% | 1.4% | 76.5% (61.6 GiB) | + +(The "previous build" is the untagged `sha256:61f6e857...`, inferred to be +the earlier `26.5` from its date and download count.) + +Share of the newer image's non-zero data an aligned fixed-block +content-addressed store would have to fetch, given the older image: + +| old -> new | 4 KiB | 16 KiB | 64 KiB | 1 MiB | 16 MiB | +|---|---|---|---|---|---| +| 26.4.1 -> 26.5 previous | 54% | 63% | 66% | 68% | 72% | +| 26.5 previous -> 26.5 rebuild | 32% | 85% | 98% | 100% | 100% | +| 26.4.1 -> 26.5 rebuild | 76% | 93% | 99% | 100% | 100% | + +Reused pages that moved mostly moved as long contiguous runs (whole files at +a new 4 KiB-aligned offset): for the 26.5 rebuild, 469K old->new extents +cover all 14.0M reused pages, with 84% of them in extents of 1 MiB or more +and 48% in extents of 64 MiB or more (median extent is 1 page, a long tail +of small ones). + +Conclusions so far: + +* Rebuilds of "the same" image relocate nearly everything, so anything + keyed by disk offset (backing-file chains, offset-based block deltas, + fixed-block rsync) gets no reuse between rebuilds. +* Content addressing must be at 4 KiB granularity, the APFS block size; + fixed blocks of 16 KiB or more lose most of the reuse because relocated + files land at arbitrary 4 KiB alignments. Content-defined chunking over + the raw disk might recover larger chunks; not yet measured. +* Because moved content arrives in long runs, an image can be described + compactly as extents referring to earlier images' content (about 30x + fewer entries than per-page hashes), and fetches can be large even though + addressing is per page. +* About 1% of an image's non-zero pages duplicate other pages in the same + image; intra-image dedup is not worth much. diff --git a/cmd/mantsana/entitlementprobe.go b/cmd/mantsana/entitlementprobe.go new file mode 100644 index 0000000..71dfbd5 --- /dev/null +++ b/cmd/mantsana/entitlementprobe.go @@ -0,0 +1,78 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "errors" + "log" + "os" + "path/filepath" + "time" + + "github.com/Code-Hex/vz/v3" +) + +// cmdEntitlementProbe checks that this binary may use Virtualization.framework +// (that is, it carries the com.apple.security.virtualization entitlement), +// without needing an image: it validates, creates and starts a minimal EFI +// VM with no devices, then stops it. It is for checking that a binary signed +// on one Mac works on another. +func cmdEntitlementProbe(args []string) error { + if len(args) != 0 { + return errors.New("usage: entitlementprobe") + } + dir, err := os.MkdirTemp("", "mantsana-entitlementprobe") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + store, err := vz.NewEFIVariableStore(filepath.Join(dir, "efivars"), vz.WithCreatingEFIVariableStore()) + if err != nil { + return err + } + bl, err := vz.NewEFIBootLoader(vz.WithEFIVariableStore(store)) + if err != nil { + return err + } + vc, err := vz.NewVirtualMachineConfiguration(bl, 1, 512<<20) + if err != nil { + return err + } + platform, err := vz.NewGenericPlatformConfiguration() + if err != nil { + return err + } + vc.SetPlatformVirtualMachineConfiguration(platform) + if ok, err := vc.Validate(); !ok || err != nil { + return orErr(err, "invalid configuration") + } + log.Printf("configuration valid") + vm, err := vz.NewVirtualMachine(vc) + if err != nil { + return err + } + if err := vm.Start(); err != nil { + return err + } + log.Printf("VM started (state %v)", vm.State()) + time.Sleep(time.Second) + // With no boot device the firmware may already have stopped it. + if vm.CanStop() { + if err := vm.Stop(); err != nil { + return err + } + } + log.Printf("VM stopped (state %v); entitlement OK", vm.State()) + return nil +} + +func orErr(err error, msg string) error { + if err != nil { + return err + } + return errors.New(msg) +} diff --git a/cmd/mantsana/guestssh.go b/cmd/mantsana/guestssh.go new file mode 100644 index 0000000..7907f67 --- /dev/null +++ b/cmd/mantsana/guestssh.go @@ -0,0 +1,97 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "context" + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/mantsana/guest" + "golang.org/x/crypto/ssh" +) + +// guestSSH connects to the guest's sshd through its network, as corp's +// ciguestlet does (user admin, password admin; the CI images' convention). +func guestSSH(ctx context.Context, v *guest.VM) (*ssh.Client, error) { + c, err := v.Net().DialGuest(ctx, 22) + if err != nil { + return nil, err + } + addr := guestnet.DefaultGuestIP.String() + ":22" + cc, chans, reqs, err := ssh.NewClientConn(c, addr, &ssh.ClientConfig{ + User: "admin", + Auth: []ssh.AuthMethod{ssh.Password("admin")}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 10 * time.Second, + }) + if err != nil { + c.Close() + return nil, err + } + return ssh.NewClient(cc, chans, reqs), nil +} + +// sshOutput runs cmd on the guest and returns its combined output. +func sshOutput(sc *ssh.Client, cmd string) (string, error) { + sess, err := sc.NewSession() + if err != nil { + return "", err + } + defer sess.Close() + out, err := sess.CombinedOutput(cmd) + return strings.TrimSpace(string(out)), err +} + +// guestClockSkew returns how far the guest's clock is behind the host's. +func guestClockSkew(sc *ssh.Client) (time.Duration, error) { + // Nanoseconds; macOS date(1) has no %N, so use perl, which the images + // have. + before := time.Now() + out, err := sshOutput(sc, `perl -MTime::HiRes=time -e 'printf "%.6f\n", time'`) + if err != nil { + return 0, fmt.Errorf("reading guest time: %v: %s", err, out) + } + rtt := time.Since(before) + secs, err := strconv.ParseFloat(out, 64) + if err != nil { + return 0, fmt.Errorf("parsing guest time %q: %v", out, err) + } + guestNow := time.Unix(0, int64(secs*1e9)) + hostNow := before.Add(rtt / 2) + return hostNow.Sub(guestNow), nil +} + +// checkGuestClock reports the guest's clock skew, and if the guest was +// restored, fixes it with sntp against the gateway (guestnet answers NTP +// with host time) and reports it again. +func checkGuestClock(ctx context.Context, v *guest.VM) error { + sc, err := guestSSH(ctx, v) + if err != nil { + return fmt.Errorf("ssh: %w", err) + } + defer sc.Close() + skew, err := guestClockSkew(sc) + if err != nil { + return err + } + log.Printf("guest clock is %v behind the host", skew.Round(time.Millisecond)) + if !v.Restored() { + return nil + } + t0 := time.Now() + out, err := sshOutput(sc, "sudo -n sntp -sS "+guestnet.DefaultGatewayIP.String()) + log.Printf("sntp -sS (%v): err=%v %s", time.Since(t0).Round(time.Millisecond), err, out) + if skew, err = guestClockSkew(sc); err != nil { + return err + } + log.Printf("after sntp, guest clock is %v behind the host", skew.Round(time.Millisecond)) + return nil +} diff --git a/cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist b/cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist new file mode 100644 index 0000000..1936471 --- /dev/null +++ b/cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist @@ -0,0 +1,45 @@ + + + + + + Label + com.tailscale.mantsana.keepawake + ProgramArguments + + /bin/sh + -c + while :; do /usr/bin/caffeinate -dimsu -t 600 & sleep 300; done + + RunAtLoad + + KeepAlive + + ProcessType + Interactive + + diff --git a/cmd/mantsana/mantsana.entitlements b/cmd/mantsana/mantsana.entitlements new file mode 100644 index 0000000..eddc54d --- /dev/null +++ b/cmd/mantsana/mantsana.entitlements @@ -0,0 +1,12 @@ + + + + + com.apple.security.virtualization + + com.apple.security.network.client + + com.apple.security.network.server + + + diff --git a/cmd/mantsana/mantsana.go b/cmd/mantsana/mantsana.go new file mode 100644 index 0000000..e201d9e --- /dev/null +++ b/cmd/mantsana/mantsana.go @@ -0,0 +1,68 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +// The mantsana command runs ephemeral macOS CI guest VMs on Apple Silicon +// Macs. See DESIGN.md. +// +// Build it on the Mac and ad hoc sign it with the virtualization +// entitlement: +// +// go build ./cmd/mantsana && codesign --entitlements cmd/mantsana/mantsana.entitlements -s - ./mantsana +package main + +import ( + "flag" + "fmt" + "log" + "os" + + "github.com/tailscale/tb/mantsana/image" +) + +var imagesDir = flag.String("images", image.DefaultStoreDir(), "directory of VM images (get one with cmd/oci-img-dl)") + +var commands = map[string]func(args []string) error{ + "images": cmdImages, + "warm": cmdWarm, + "selftest": cmdSelftest, + "nbdprobe": cmdNBDProbe, + "restoreprobe": cmdRestoreProbe, + "entitlementprobe": cmdEntitlementProbe, +} + +func usage() { + fmt.Fprintf(os.Stderr, `usage: mantsana [flags] [args] + +Commands: + images list images and whether their warm snapshots are usable here + warm [flags] build an image's warm snapshot, so VMs restore in seconds + selftest [flags] validate, cold boot, warm up and restore an image, with timings + nbdprobe [flags] boot an image's disk through a logging NBD proxy (debugging) + restoreprobe [flags] try restoring an image's warm snapshot several ways (debugging) + entitlementprobe check this binary may use Virtualization.framework (no image needed) + +Flags: +`) + flag.PrintDefaults() +} + +func main() { + log.SetFlags(log.Ltime | log.Lmicroseconds) + flag.Usage = usage + flag.Parse() + if flag.NArg() == 0 { + usage() + os.Exit(2) + } + cmd, ok := commands[flag.Arg(0)] + if !ok { + fmt.Fprintf(os.Stderr, "mantsana: unknown command %q\n", flag.Arg(0)) + usage() + os.Exit(2) + } + if err := cmd(flag.Args()[1:]); err != nil { + log.Fatalf("%s: %v", flag.Arg(0), err) + } +} diff --git a/cmd/mantsana/mantsana_other.go b/cmd/mantsana/mantsana_other.go new file mode 100644 index 0000000..2d4373f --- /dev/null +++ b/cmd/mantsana/mantsana_other.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !(darwin && arm64) + +// The mantsana command runs ephemeral macOS CI guest VMs on Apple Silicon +// Macs. It only runs there. +package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Fprintln(os.Stderr, "mantsana only runs on Apple Silicon Macs") + os.Exit(1) +} diff --git a/cmd/mantsana/nbdlive.go b/cmd/mantsana/nbdlive.go new file mode 100644 index 0000000..9507ca1 --- /dev/null +++ b/cmd/mantsana/nbdlive.go @@ -0,0 +1,124 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "context" + "fmt" + "log" + "net" + "os" + "path/filepath" + "time" + + "github.com/tailscale/tb/guestbd" + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/image" + "github.com/tailscale/tb/mantsana/vzvm" +) + +// nbdLive cold boots the image's pristine disk over NBD, saves its state, +// and restores it over the same NBD server, whose shared snapshot still +// holds the first boot's writes. It checks whether a state saved with an +// NBD attachment restores with one. +func nbdLive(im *image.Image) error { + dir := "/tmp/mantsana-restoreprobe" + os.RemoveAll(dir) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + defer os.RemoveAll(dir) + sock := filepath.Join(dir, "nbd.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + return err + } + defer ln.Close() + srv := guestbd.NewServer(guestbd.FileSource(im.DiskPath()), guestbd.WithSharedSnapshot(), guestbd.WithMaxMem(0)) + go srv.Serve(ln) + defer srv.Close() + aux := filepath.Join(dir, image.AuxFile) + if err := image.CloneFile(im.AuxPath(), aux); err != nil { + return err + } + state := filepath.Join(dir, image.StateFile) + + boot := func(restore string) (guest.Machine, *guestnet.Net, func(), error) { + vmSide, link, err := guestnet.NewSocketPairLink() + if err != nil { + return nil, nil, nil, err + } + n, err := guestnet.New(guestnet.Config{Link: link, GuestMAC: net.HardwareAddr(im.Manifest.MAC), Logf: log.Printf}) + if err != nil { + vmSide.Close() + link.Close() + return nil, nil, nil, err + } + m, err := (&vzvm.Backend{}).Boot(context.Background(), &guest.BootSpec{ + Name: "nbd-live", + Manifest: &im.Manifest, + CPUs: im.Manifest.CPUs, + MemoryBytes: im.Manifest.MemoryBytes, + NBDURL: "nbd+unix:///?socket=" + sock, + AuxPath: aux, + Net: vmSide, + RestoreState: restore, + Logf: log.Printf, + }) + vmSide.Close() + if err != nil { + n.Close() + return nil, nil, nil, err + } + return m, n, func() { m.Stop(); <-m.Done(); n.Close() }, nil + } + waitSSH := func(n *guestnet.Net, t0 time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + for { + dctx, dcancel := context.WithTimeout(ctx, 2*time.Second) + c, err := n.DialGuest(dctx, 22) + dcancel() + if err == nil { + c.Close() + log.Printf("nbd-live: sshd reachable after %v", time.Since(t0).Round(time.Millisecond)) + return nil + } + if ctx.Err() != nil { + return err + } + time.Sleep(200 * time.Millisecond) + } + } + + t0 := time.Now() + m, n, stop, err := boot("") + if err != nil { + return fmt.Errorf("cold boot: %w", err) + } + if err := waitSSH(n, t0); err != nil { + stop() + return err + } + time.Sleep(10 * time.Second) + t1 := time.Now() + if err := m.SaveState(state); err != nil { + stop() + return fmt.Errorf("save: %w", err) + } + log.Printf("nbd-live: saved in %v", time.Since(t1).Round(time.Millisecond)) + stop() + + t2 := time.Now() + _, n, stop, err = boot(state) + if err != nil { + return err + } + defer stop() + log.Printf("nbd-live: restored and resumed in %v", time.Since(t2).Round(time.Millisecond)) + return waitSSH(n, t2) +} diff --git a/cmd/mantsana/nbdprobe.go b/cmd/mantsana/nbdprobe.go new file mode 100644 index 0000000..efcd1c9 --- /dev/null +++ b/cmd/mantsana/nbdprobe.go @@ -0,0 +1,157 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "context" + "encoding/hex" + "errors" + "flag" + "io" + "log" + "net" + "os" + "path/filepath" + "sync" + "time" + + "github.com/tailscale/tb/guestbd" + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/image" + "github.com/tailscale/tb/mantsana/vzvm" +) + +// cmdNBDProbe boots an image's disk over NBD through a proxy that logs +// every byte exchanged, to debug the framework's NBD client. +func cmdNBDProbe(args []string) error { + fs := flag.NewFlagSet("nbdprobe", flag.ExitOnError) + export := fs.String("export", "", "NBD export name to put in the URL") + dur := fs.Duration("duration", 20*time.Second, "how long to run the VM") + maxDump := fs.Int("max-dump", 512, "bytes of each direction to hex dump per connection") + fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: nbdprobe [flags] ") + } + im, err := (&image.Store{Root: *imagesDir}).Get(fs.Arg(0)) + if err != nil { + return err + } + dir := "/tmp/mantsana-nbdprobe" + os.RemoveAll(dir) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + defer os.RemoveAll(dir) + + // guestbd on an inner socket; the framework connects to the proxy. + innerSock := filepath.Join(dir, "inner.sock") + inner, err := net.Listen("unix", innerSock) + if err != nil { + return err + } + srv := guestbd.NewServer(guestbd.FileSource(im.DiskPath()), guestbd.WithSharedSnapshot(), guestbd.WithMaxMem(0)) + go srv.Serve(inner) + defer srv.Close() + + outerSock := filepath.Join(dir, "nbd.sock") + outer, err := net.Listen("unix", outerSock) + if err != nil { + return err + } + defer outer.Close() + go func() { + for n := 1; ; n++ { + c, err := outer.Accept() + if err != nil { + return + } + log.Printf("conn %d: accepted", n) + go traceProxy(n, c, innerSock, *maxDump) + } + }() + + aux := filepath.Join(dir, image.AuxFile) + if err := image.CloneFile(im.AuxPath(), aux); err != nil { + return err + } + vmSide, link, err := guestnet.NewSocketPairLink() + if err != nil { + return err + } + defer link.Close() + defer vmSide.Close() + + url := "nbd+unix:///" + *export + "?socket=" + outerSock + log.Printf("booting with %s", url) + b := &vzvm.Backend{} + m, err := b.Boot(context.Background(), &guest.BootSpec{ + Name: "nbdprobe", + Manifest: &im.Manifest, + CPUs: im.Manifest.CPUs, + MemoryBytes: im.Manifest.MemoryBytes, + NBDURL: url, + AuxPath: aux, + Net: vmSide, + Console: os.Stderr, + Logf: log.Printf, + }) + if err != nil { + return err + } + log.Printf("VM started; running for %v", *dur) + select { + case <-m.Done(): + log.Printf("VM stopped: %v", m.Err()) + case <-time.After(*dur): + log.Printf("stopping VM: %v", m.Stop()) + } + return nil +} + +// traceProxy relays c to the unix socket at target, hex dumping the first +// max bytes in each direction. +func traceProxy(n int, c net.Conn, target string, max int) { + defer c.Close() + t, err := net.Dial("unix", target) + if err != nil { + log.Printf("conn %d: dial guestbd: %v", n, err) + return + } + defer t.Close() + var wg sync.WaitGroup + relay := func(dir string, dst, src net.Conn) { + defer wg.Done() + d := &dumper{n: n, dir: dir, left: max} + nb, err := io.Copy(io.MultiWriter(dst, d), src) + log.Printf("conn %d: %s done after %d bytes: %v", n, dir, nb, err) + if uc, ok := dst.(*net.UnixConn); ok { + uc.CloseWrite() + } + } + wg.Add(2) + go relay("client->server", t, c) + go relay("server->client", c, t) + wg.Wait() +} + +type dumper struct { + n int + dir string + left int +} + +func (d *dumper) Write(p []byte) (int, error) { + if d.left > 0 { + q := p + if len(q) > d.left { + q = q[:d.left] + } + d.left -= len(q) + log.Printf("conn %d: %s %d bytes:\n%s", d.n, d.dir, len(p), hex.Dump(q)) + } + return len(p), nil +} diff --git a/cmd/mantsana/restoreprobe.go b/cmd/mantsana/restoreprobe.go new file mode 100644 index 0000000..fcf923a --- /dev/null +++ b/cmd/mantsana/restoreprobe.go @@ -0,0 +1,167 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/tailscale/tb/guestbd" + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/image" + "github.com/tailscale/tb/mantsana/vzvm" +) + +// cmdRestoreProbe tries restoring an image's warm snapshot with different +// disk attachments and file paths, to find what the framework requires. +func cmdRestoreProbe(args []string) error { + fs := flag.NewFlagSet("restoreprobe", flag.ExitOnError) + variants := fs.String("variants", "nbd,clone,samepath", "comma-separated variants: nbd (NBD of warm/disk.img), clone (disk-image attachment on a clone), samepath (clones at the paths used when saving), nbd-live (cold boot, save and restore all over one live NBD server)") + fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: restoreprobe [flags] ") + } + im, err := (&image.Store{Root: *imagesDir}).Get(fs.Arg(0)) + if err != nil { + return err + } + if im.Warm == nil { + return errors.New("image has no warm snapshot") + } + for _, v := range strings.Split(*variants, ",") { + t0 := time.Now() + var err error + if v == "nbd-live" { + err = nbdLive(im) + } else { + err = restoreVariant(im, v) + } + log.Printf("variant %-8s: %v after %v", v, okOrErr(err), time.Since(t0).Round(time.Millisecond)) + } + return nil +} + +func restoreVariant(im *image.Image, variant string) error { + dir := "/tmp/mantsana-restoreprobe" + os.RemoveAll(dir) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + defer os.RemoveAll(dir) + + spec := &guest.BootSpec{ + Name: "restoreprobe", + Manifest: &im.Manifest, + CPUs: im.Warm.CPUs, + MemoryBytes: im.Warm.MemoryBytes, + AuxPath: filepath.Join(dir, image.AuxFile), + RestoreState: im.WarmPath(image.StateFile), + Logf: log.Printf, + } + var cleanups []func() + defer func() { + for _, f := range cleanups { + f() + } + }() + clone := func(src, dst string) error { + os.Remove(dst) + if err := image.CloneFile(src, dst); err != nil { + return err + } + cleanups = append(cleanups, func() { os.Remove(dst) }) + return nil + } + switch variant { + case "nbd": + sock := filepath.Join(dir, "nbd.sock") + ln, err := net.Listen("unix", sock) + if err != nil { + return err + } + srv := guestbd.NewServer(guestbd.FileSource(im.WarmPath(image.DiskFile)), guestbd.WithSharedSnapshot(), guestbd.WithMaxMem(0)) + go srv.Serve(ln) + cleanups = append(cleanups, func() { ln.Close(); srv.Close() }) + spec.NBDURL = "nbd+unix:///?socket=" + sock + if err := clone(im.WarmPath(image.AuxFile), spec.AuxPath); err != nil { + return err + } + case "clone": + spec.DiskPath = filepath.Join(dir, image.DiskFile) + if err := clone(im.WarmPath(image.DiskFile), spec.DiskPath); err != nil { + return err + } + if err := clone(im.WarmPath(image.AuxFile), spec.AuxPath); err != nil { + return err + } + case "samepath": + // guest.Warm saves from .tmp and renames it into place. + tmp := filepath.Clean(im.WarmPath("")) + ".tmp" + if err := os.MkdirAll(tmp, 0o755); err != nil { + return err + } + cleanups = append(cleanups, func() { os.RemoveAll(tmp) }) + spec.DiskPath = filepath.Join(tmp, image.DiskFile) + spec.AuxPath = filepath.Join(tmp, image.AuxFile) + spec.RestoreState = filepath.Join(tmp, image.StateFile) + for _, f := range []string{image.DiskFile, image.AuxFile, image.StateFile} { + if err := clone(im.WarmPath(f), filepath.Join(tmp, f)); err != nil { + return err + } + } + default: + return fmt.Errorf("unknown variant %q", variant) + } + + vmSide, link, err := guestnet.NewSocketPairLink() + if err != nil { + return err + } + defer vmSide.Close() + n, err := guestnet.New(guestnet.Config{Link: link, GuestMAC: net.HardwareAddr(im.Manifest.MAC), Logf: log.Printf}) + if err != nil { + link.Close() + return err + } + defer n.Close() + spec.Net = vmSide + + t0 := time.Now() + m, err := (&vzvm.Backend{}).Boot(context.Background(), spec) + if err != nil { + return err + } + defer m.Stop() + log.Printf("variant %s: restored and resumed in %v; dialing sshd", variant, time.Since(t0).Round(time.Millisecond)) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + for { + dctx, dcancel := context.WithTimeout(ctx, 2*time.Second) + c, err := n.DialGuest(dctx, 22) + dcancel() + if err == nil { + c.Close() + log.Printf("variant %s: sshd reachable %v after restore started", variant, time.Since(t0).Round(time.Millisecond)) + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("restored, but sshd unreachable: %v", err) + case <-m.Done(): + return fmt.Errorf("VM stopped: %v", m.Err()) + case <-time.After(200 * time.Millisecond): + } + } +} diff --git a/cmd/mantsana/selftest.go b/cmd/mantsana/selftest.go new file mode 100644 index 0000000..c1fb1e0 --- /dev/null +++ b/cmd/mantsana/selftest.go @@ -0,0 +1,225 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/image" + "github.com/tailscale/tb/mantsana/vzvm" +) + +func cmdSelftest(args []string) error { + fs := flag.NewFlagSet("selftest", flag.ExitOnError) + validateOnly := fs.Bool("validate-only", false, "only validate configurations; don't boot") + noSerial := fs.Bool("no-serial", false, "omit the serial console device") + skipCold := fs.Bool("skip-cold", false, "skip the plain cold boot") + skipWarm := fs.Bool("skip-warm", false, "skip building the warm snapshot; restore the existing one") + readyTimeout := fs.Duration("ready-timeout", 5*time.Minute, "how long to wait for sshd in the guest") + settle := fs.Duration("settle", 10*time.Second, "how long to let the guest settle before saving warm state") + fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: selftest [flags] ") + } + s := &image.Store{Root: *imagesDir} + im, err := s.Get(fs.Arg(0)) + if err != nil { + return err + } + b := &vzvm.Backend{NoSerial: *noSerial} + log.Printf("host OS build %s; image %q: %d CPUs, %d MiB, display %+v", b.HostOSBuild(), im.Name, im.Manifest.CPUs, im.Manifest.MemoryBytes>>20, im.Manifest.Display) + + if err := validateMatrix(im); err != nil { + return err + } + if *validateOnly { + return nil + } + + // On a signal (including SIGHUP when the ssh session running us goes + // away), stop waiting so the VM is stopped and cleaned up. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + defer stop() + dir := "/tmp/mantsana-selftest" + cfg := guest.Config{ + Name: "selftest", + Slot: 1, + Image: im, + Dir: filepath.Join(dir, "s1"), + Backend: b, + Logf: log.Printf, + } + + if !*skipCold { + log.Printf("=== cold boot over NBD") + c := cfg + c.Boot = guest.BootCold + if err := bootAndProbe(ctx, c, *readyTimeout); err != nil { + return fmt.Errorf("cold boot: %w", err) + } + } + + if !*skipWarm { + log.Printf("=== warm-up (cold boot over NBD, then save state and persist the disk)") + t0 := time.Now() + wctx, cancel := context.WithTimeout(ctx, *readyTimeout+2*time.Minute) + wi, err := guest.Warm(wctx, guest.WarmConfig{Config: cfg, Settle: *settle}) + cancel() + if err != nil { + return fmt.Errorf("warm-up: %w", err) + } + log.Printf("warm snapshot built in %v: %+v", time.Since(t0).Round(time.Millisecond), wi) + if fi, err := os.Stat(im.WarmPath(image.StateFile)); err == nil { + log.Printf("state file is %d MiB", fi.Size()>>20) + } + } + + log.Printf("=== restore over NBD") + im, err = s.Get(im.Name) // reload warm info + if err != nil { + return err + } + c := cfg + c.Image = im + c.Boot = guest.BootRestore + if err := bootAndProbe(ctx, c, *readyTimeout); err != nil { + return fmt.Errorf("restore: %w", err) + } + log.Printf("=== selftest passed") + return nil +} + +// validateMatrix reports whether each disk attachment and console variant +// validates and supports save/restore, without booting. This is DESIGN.md +// section 9 question 1. +func validateMatrix(im *image.Image) error { + dir, err := os.MkdirTemp("", "mantsana-validate") + if err != nil { + return err + } + defer os.RemoveAll(dir) + aux := filepath.Join(dir, image.AuxFile) + if err := image.CloneFile(im.AuxPath(), aux); err != nil { + return err + } + vmSide, link, err := guestnet.NewSocketPairLink() + if err != nil { + return err + } + defer vmSide.Close() + defer link.Close() + + for _, noSerial := range []bool{false, true} { + for _, disk := range []string{"nbd", "disk-image"} { + spec := &guest.BootSpec{ + Name: "validate", + Manifest: &im.Manifest, + CPUs: im.Manifest.CPUs, + MemoryBytes: im.Manifest.MemoryBytes, + AuxPath: aux, + Net: vmSide, + Logf: log.Printf, + } + if disk == "nbd" { + spec.NBDURL = "nbd+unix:///?socket=" + filepath.Join(dir, "nbd.sock") + } else { + spec.DiskPath = im.DiskPath() + } + b := &vzvm.Backend{NoSerial: noSerial} + v, err := b.Validate(spec) + if err != nil { + log.Printf("validate disk=%s serial=%v: building config: %v", disk, !noSerial, err) + continue + } + log.Printf("validate disk=%-10s serial=%-5v: valid=%v save/restore=%v", disk, !noSerial, okOrErr(v.Valid), okOrErr(v.SaveRestore)) + } + } + return nil +} + +func okOrErr(err error) string { + if err == nil { + return "ok" + } + return err.Error() +} + +// bootAndProbe starts a VM, waits for sshd, reads its banner, reports +// timings, and stops it. +func bootAndProbe(ctx context.Context, cfg guest.Config, readyTimeout time.Duration) error { + t0 := time.Now() + v, err := guest.Start(ctx, cfg) + if err != nil { + return err + } + tBoot := time.Since(t0) + defer func() { + sctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + t := time.Now() + v.Stop(sctx, true) + log.Printf("stopped in %v (err=%v)", time.Since(t).Round(time.Millisecond), v.Err()) + }() + go logEvents(ctx, v, t0) + + rctx, cancel := context.WithTimeout(ctx, readyTimeout) + defer cancel() + if err := v.WaitReady(rctx, 22); err != nil { + return fmt.Errorf("waiting for sshd: %w (console: %q)", err, v.Console()) + } + tReady := time.Since(t0) + c, err := v.Net().DialGuest(rctx, 22) + if err != nil { + return err + } + c.SetDeadline(time.Now().Add(10 * time.Second)) + banner, err := bufio.NewReader(c).ReadString('\n') + c.Close() + if err != nil { + return fmt.Errorf("reading ssh banner: %w", err) + } + log.Printf("restored=%v boot=%v ssh-ready=%v banner=%q", + v.Restored(), tBoot.Round(time.Millisecond), tReady.Round(time.Millisecond), banner) + if err := checkGuestClock(rctx, v); err != nil { + log.Printf("checking guest clock: %v", err) + } + for _, f := range v.Net().Flows.Recent() { + if f.Closed() { + log.Printf(" flow %s %s -> %s kind=%s in=%d out=%d %s", f.Proto, f.Src, f.Dst, f.Kind, f.BytesIn, f.BytesOut, f.Err) + } + } + for _, d := range v.Net().DNS.Recent() { + log.Printf(" dns %s %s -> %s (%v)", d.Name, d.Type, d.RCode, d.Duration) + } + return nil +} + +func logEvents(ctx context.Context, v *guest.VM, t0 time.Time) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + <-v.Done() + cancel() + }() + for _, ev := range v.Events().Recent() { + log.Printf(" +%v event %s %s", ev.Time.Sub(t0).Round(time.Millisecond), ev.Kind, ev.Msg) + } + for ev := range v.Events().Subscribe(ctx) { + log.Printf(" +%v event %s %s", ev.Time.Sub(t0).Round(time.Millisecond), ev.Kind, ev.Msg) + } +} diff --git a/cmd/mantsana/warm.go b/cmd/mantsana/warm.go new file mode 100644 index 0000000..af8bce9 --- /dev/null +++ b/cmd/mantsana/warm.go @@ -0,0 +1,94 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "text/tabwriter" + "time" + + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/image" + "github.com/tailscale/tb/mantsana/vzvm" +) + +// cmdWarm builds an image's warm snapshot: it boots the image once, waits +// for the guest to be usable, and saves the machine state, so later VMs +// restore in seconds instead of cold booting. Snapshots are only valid on +// this host and its current OS build, so run it once per image per Mac and +// again after macOS updates. +func cmdWarm(args []string) error { + fs := flag.NewFlagSet("warm", flag.ExitOnError) + readyTimeout := fs.Duration("ready-timeout", 5*time.Minute, "how long to wait for sshd in the guest") + settle := fs.Duration("settle", 10*time.Second, "how long to let the guest settle before saving") + cpus := fs.Int("cpus", 0, "CPUs for the snapshot; 0 means the image's. VMs restoring it must use the same") + memGiB := fs.Int("mem-gib", 0, "memory in GiB for the snapshot; 0 means the image's. VMs restoring it must use the same") + fs.Parse(args) + if fs.NArg() != 1 { + return errors.New("usage: warm [flags] ") + } + im, err := (&image.Store{Root: *imagesDir}).Get(fs.Arg(0)) + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + defer stop() + ctx, cancel := context.WithTimeout(ctx, *readyTimeout+5*time.Minute) + defer cancel() + + t0 := time.Now() + wi, err := guest.Warm(ctx, guest.WarmConfig{ + Config: guest.Config{ + Name: "warm-" + im.Name, + Image: im, + CPUs: *cpus, + MemoryBytes: uint64(*memGiB) << 30, + Dir: fmt.Sprintf("/tmp/mantsana-warm-%d", os.Getpid()), + Backend: &vzvm.Backend{}, + Logf: log.Printf, + }, + Settle: *settle, + }) + if err != nil { + return fmt.Errorf("%w\n(saving and restoring need the user's own launchd domain and an unlocked session; see DESIGN.md section 4.1)", err) + } + log.Printf("warm snapshot of %q built in %v for host OS %s, %d CPUs, %d MiB", + im.Name, time.Since(t0).Round(time.Second), wi.HostOSBuild, wi.CPUs, wi.MemoryBytes>>20) + return nil +} + +// cmdImages lists the images in the store and whether each one's warm +// snapshot is usable on this host. +func cmdImages(args []string) error { + if len(args) != 0 { + return errors.New("usage: images") + } + ims, err := (&image.Store{Root: *imagesDir}).List() + if err != nil { + return err + } + host := (&vzvm.Backend{}).HostOSBuild() + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tCPUS\tMEMORY\tDISK\tWARM") + for _, im := range ims { + m := im.Manifest + warm := "no" + if ok, why := im.WarmUsable(host, m.CPUs, m.MemoryBytes); ok { + warm = "yes (" + im.Warm.Created.Local().Format(time.DateTime) + ")" + } else if im.Warm != nil { + warm = "unusable: " + why + } + fmt.Fprintf(tw, "%s\t%d\t%d GiB\t%d GB\t%s\n", im.Name, m.CPUs, m.MemoryBytes>>30, m.DiskBytes/1e9, warm) + } + return tw.Flush() +} diff --git a/go.mod b/go.mod index 94343d3..67a2233 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/tailscale/tb go 1.27.1 require ( + github.com/Code-Hex/vz/v3 v3.8.0 github.com/aws/aws-sdk-go-v2 v1.47.3 github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.36.5 @@ -18,6 +19,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.26 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 + golang.org/x/crypto v0.57.0 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 @@ -29,6 +31,7 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect + github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect github.com/akutz/memconn v0.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect @@ -72,8 +75,8 @@ require ( github.com/x448/float16 v0.8.4 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect - golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/mod v0.41.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect modernc.org/libc v1.72.3 // indirect diff --git a/go.sum b/go.sum index 180ac20..9a9a0bb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw= +github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY= +github.com/Code-Hex/vz/v3 v3.8.0 h1:wgOH3VY0le5fP3VPQFBf6wls8jSqzK5Haq/qIoO6pb4= +github.com/Code-Hex/vz/v3 v3.8.0/go.mod h1:eO4cMfeE9agtnGsC+zrcLFcPDBf1RlPbUcJJF0MIq9w= github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A= github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= diff --git a/imds/imds.go b/imds/imds.go new file mode 100644 index 0000000..6dd36af --- /dev/null +++ b/imds/imds.go @@ -0,0 +1,186 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package imds is the instance metadata service a guest reaches at +// http://169.254.169.254/. Each guest gets its own instance, so the caller's +// identity is implicit and no tokens are needed for reads. +// +// Endpoints: +// +// GET / index +// GET /v0/meta JSON [Meta] +// GET /v0/user-data JSON object of all user data +// GET /v0/user-data/{key} raw value +// GET /v0/time RFC 3339 host time +// POST /v0/ready guest says it is usable +// POST /v0/log append body as a log line +package imds + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +// Meta is what the guest learns about itself. +type Meta struct { + Name string `json:"name"` + Slot int `json:"slot"` + Image string `json:"image"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + BootMode string `json:"boot_mode"` // "cold" or "restored" + BootTime time.Time `json:"boot_time"` + Host string `json:"host"` + GuestIP string `json:"guest_ip"` + GatewayIP string `json:"gateway_ip"` +} + +// Handler serves the metadata API for one guest. +type Handler struct { + mu sync.Mutex + meta Meta + userData map[string]string + + ready chan struct{} + readyOnce sync.Once + + // OnLog receives lines the guest posts to /v0/log. Optional. + OnLog func(line string) + // Now returns the current time. Defaults to time.Now. + Now func() time.Time + + mux *http.ServeMux +} + +// New returns a Handler serving meta and userData. +func New(meta Meta, userData map[string]string) *Handler { + h := &Handler{ + meta: meta, + userData: make(map[string]string, len(userData)), + ready: make(chan struct{}), + Now: time.Now, + mux: http.NewServeMux(), + } + for k, v := range userData { + h.userData[k] = v + } + h.mux.HandleFunc("GET /{$}", h.serveIndex) + h.mux.HandleFunc("GET /v0/meta", h.serveMeta) + h.mux.HandleFunc("GET /v0/user-data", h.serveUserDataAll) + h.mux.HandleFunc("GET /v0/user-data/{key}", h.serveUserData) + h.mux.HandleFunc("GET /v0/time", h.serveTime) + h.mux.HandleFunc("POST /v0/ready", h.serveReady) + h.mux.HandleFunc("POST /v0/log", h.serveLog) + return h +} + +// ServeHTTP implements http.Handler. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "mantsana-imds") + h.mux.ServeHTTP(w, r) +} + +// Ready returns a channel closed when the guest posts to /v0/ready. +func (h *Handler) Ready() <-chan struct{} { return h.ready } + +// SetMeta updates the metadata (for example the boot mode once known). +func (h *Handler) SetMeta(f func(*Meta)) { + h.mu.Lock() + defer h.mu.Unlock() + f(&h.meta) +} + +// SetUserData sets or (with an empty value) removes a user-data key. +func (h *Handler) SetUserData(key, value string) { + h.mu.Lock() + defer h.mu.Unlock() + if value == "" { + delete(h.userData, key) + } else { + h.userData[key] = value + } +} + +func (h *Handler) serveIndex(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + io.WriteString(w, "mantsana instance metadata\n\nv0/meta\nv0/user-data\nv0/user-data/{key}\nv0/time\nv0/ready (POST)\nv0/log (POST)\n") +} + +func (h *Handler) serveMeta(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + m := h.meta + h.mu.Unlock() + writeJSON(w, m) +} + +func (h *Handler) serveUserDataAll(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + out := make(map[string]string, len(h.userData)) + for k, v := range h.userData { + out[k] = v + } + h.mu.Unlock() + writeJSON(w, out) +} + +func (h *Handler) serveUserData(w http.ResponseWriter, r *http.Request) { + key := r.PathValue("key") + h.mu.Lock() + v, ok := h.userData[key] + h.mu.Unlock() + if !ok { + http.Error(w, "no such key", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + io.WriteString(w, v) +} + +func (h *Handler) serveTime(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintln(w, h.Now().UTC().Format(time.RFC3339Nano)) +} + +func (h *Handler) serveReady(w http.ResponseWriter, r *http.Request) { + h.readyOnce.Do(func() { close(h.ready) }) + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) serveLog(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(io.LimitReader(r.Body, 64<<10)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if h.OnLog != nil { + for _, line := range strings.Split(strings.TrimRight(string(b), "\n"), "\n") { + h.OnLog(line) + } + } + w.WriteHeader(http.StatusNoContent) +} + +// Keys returns the user-data keys, sorted. +func (h *Handler) Keys() []string { + h.mu.Lock() + defer h.mu.Unlock() + keys := make([]string, 0, len(h.userData)) + for k := range h.userData { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(v) +} diff --git a/imds/imds_test.go b/imds/imds_test.go new file mode 100644 index 0000000..9cc9629 --- /dev/null +++ b/imds/imds_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package imds + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHandler(t *testing.T) { + var logged []string + h := New(Meta{Name: "vm1", Slot: 1, Image: "img", CPUs: 4}, map[string]string{"token": "abc"}) + h.OnLog = func(l string) { logged = append(logged, l) } + srv := httptest.NewServer(h) + defer srv.Close() + + get := func(path string) (int, string) { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) + } + + if code, body := get("/v0/meta"); code != 200 { + t.Fatalf("meta: %d %s", code, body) + } else { + var m Meta + json.Unmarshal([]byte(body), &m) + if m.Name != "vm1" || m.CPUs != 4 { + t.Errorf("meta = %+v", m) + } + } + if code, body := get("/v0/user-data/token"); code != 200 || body != "abc" { + t.Errorf("user-data/token: %d %q", code, body) + } + if code, _ := get("/v0/user-data/nope"); code != 404 { + t.Errorf("user-data/nope: %d", code) + } + if code, body := get("/v0/user-data"); code != 200 || !strings.Contains(body, `"token": "abc"`) { + t.Errorf("user-data: %d %q", code, body) + } + if code, body := get("/v0/time"); code != 200 || !strings.HasSuffix(strings.TrimSpace(body), "Z") { + t.Errorf("time: %d %q", code, body) + } + if code, _ := get("/v0/ready"); code != 405 { + t.Errorf("GET ready: %d, want 405", code) + } + + select { + case <-h.Ready(): + t.Fatal("ready before POST") + default: + } + resp, err := http.Post(srv.URL+"/v0/ready", "", nil) + if err != nil || resp.StatusCode != 204 { + t.Fatalf("POST ready: %v %v", resp, err) + } + select { + case <-h.Ready(): + default: + t.Fatal("not ready after POST") + } + + resp, err = http.Post(srv.URL+"/v0/log", "text/plain", strings.NewReader("line one\nline two\n")) + if err != nil || resp.StatusCode != 204 { + t.Fatalf("POST log: %v %v", resp, err) + } + if len(logged) != 2 || logged[1] != "line two" { + t.Errorf("logged = %q", logged) + } + + h.SetUserData("token", "") + if code, _ := get("/v0/user-data/token"); code != 404 { + t.Errorf("after removal: %d", code) + } + h.SetMeta(func(m *Meta) { m.BootMode = "restored" }) + if _, body := get("/v0/meta"); !strings.Contains(body, `"boot_mode": "restored"`) { + t.Errorf("meta after SetMeta: %s", body) + } +} diff --git a/mantsana/guest/backend.go b/mantsana/guest/backend.go new file mode 100644 index 0000000..80ce774 --- /dev/null +++ b/mantsana/guest/backend.go @@ -0,0 +1,99 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guest + +import ( + "context" + "errors" + "io" + "os" + + "github.com/tailscale/tb/mantsana/image" + "tailscale.com/types/logger" +) + +// Backend boots VMs. The real one is package vzvm (macOS only); tests use a +// fake. Everything else about a VM (its network, IMDS, NBD disk server, event +// log) is portable and owned by this package, so a backend only has to turn +// a BootSpec into a running machine. +type Backend interface { + // HostOSBuild returns the host OS build, such as "25A354". Warm + // snapshots are only valid on the build that saved them. + HostOSBuild() string + // VMLimit returns how many VMs may run at once (2 for macOS guests). + VMLimit() int + // Boot creates and starts (or restores and resumes) a VM. It returns + // once the VM is running. Errors wrapping ErrVMLimit or *RestoreError + // are reported specially. + Boot(ctx context.Context, spec *BootSpec) (Machine, error) +} + +// BootSpec is everything a backend needs to boot one VM. +type BootSpec struct { + Name string + Manifest *image.Manifest + CPUs int + MemoryBytes uint64 + + // Exactly one of NBDURL and DiskPath is set. NBDURL is an + // "nbd+unix:///?socket=/path" URL for the server's discard-writes NBD + // server; it is used for all normal VMs. DiskPath is a raw disk image + // to attach read-write, used when building a warm snapshot. + NBDURL string + DiskPath string + + // AuxPath is the macOS auxiliary storage (NVRAM) file. It is this VM's + // private copy and may be written. + AuxPath string + + // Net is the VM side of a SOCK_DGRAM socketpair carrying Ethernet + // frames. The backend must not close it; the caller closes it after + // Boot returns. + Net *os.File + + // RestoreState, if non-empty, is a saved state file to restore and + // resume instead of cold booting. + RestoreState string + + // Console receives the guest's serial console output. + Console io.Writer + + Logf logger.Logf +} + +// Machine is a running VM. +type Machine interface { + // Done is closed when the VM has stopped for any reason. + Done() <-chan struct{} + // Err returns why the VM stopped, or nil for a clean guest shutdown. It + // is only valid after Done is closed. + Err() error + // RequestStop asks the guest OS to shut down. + RequestStop() error + // Stop stops the VM immediately, like pulling the power. + Stop() error + // SaveState pauses the VM and saves its memory and device state to path. + // The VM stays paused; the caller then stops it. + SaveState(path string) error +} + +// ErrVMLimit is wrapped by Backend.Boot errors when the framework refuses +// to start another VM. +var ErrVMLimit = errors.New("virtualization framework VM limit exceeded") + +// RestoreError is returned by Backend.Boot when restoring RestoreState +// failed, so the caller can cold boot instead, and mark the snapshot stale +// unless the failure is Transient. +type RestoreError struct { + Err error + // Transient reports that the snapshot itself is probably fine and a later + // restore may succeed: on macOS, restoring needs the user's session to be + // unlocked (DESIGN.md section 9). + Transient bool +} + +func (e *RestoreError) Error() string { return "restore failed: " + e.Err.Error() } +func (e *RestoreError) Unwrap() error { return e.Err } diff --git a/mantsana/guest/debug.go b/mantsana/guest/debug.go new file mode 100644 index 0000000..0d5014a --- /dev/null +++ b/mantsana/guest/debug.go @@ -0,0 +1,23 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guest + +import ( + "os" + "runtime/pprof" +) + +// dumpGoroutines writes all goroutine stacks to a temporary file, for +// debugging a stuck teardown, and returns its name. +func dumpGoroutines() string { + f, err := os.CreateTemp("", "mantsana-goroutines-*.txt") + if err != nil { + return "(error: " + err.Error() + ")" + } + defer f.Close() + pprof.Lookup("goroutine").WriteTo(f, 2) + return f.Name() +} diff --git a/mantsana/guest/guest.go b/mantsana/guest/guest.go new file mode 100644 index 0000000..42856f8 --- /dev/null +++ b/mantsana/guest/guest.go @@ -0,0 +1,663 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +// Package guest runs one guest VM and everything the host provides it: its +// private network (guestnet), metadata service (imds), and disk, served over +// NBD with writes discarded (guestbd). Booting the machine itself is +// delegated to a [Backend], so everything here is portable and tested with a +// fake; the real backend is package vzvm on macOS. +// +// A VM restores from its image's warm snapshot when one is usable, and falls +// back to a cold boot (marking the snapshot stale) if restoring fails. +package guest + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "os" + "path/filepath" + "sync" + "time" + + "github.com/tailscale/tb/guestbd" + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/imds" + "github.com/tailscale/tb/mantsana/image" + "tailscale.com/types/logger" +) + +// BootMode selects how a VM starts. +type BootMode string + +const ( + // BootAuto restores the warm snapshot if it is usable, else cold boots. + BootAuto BootMode = "" + // BootCold always cold boots. + BootCold BootMode = "cold" + // BootRestore requires a usable warm snapshot. + BootRestore BootMode = "restore" +) + +// Config configures a VM. +type Config struct { + Name string + Slot int + Image *image.Image + + // CPUs and MemoryBytes default to the image's. + CPUs int + MemoryBytes uint64 + + Boot BootMode + + // Dir is a private directory for the VM's sockets and its copy of the + // auxiliary storage. Keep it short: it holds a unix socket, and + // sun_path is limited to 104 bytes on macOS. It is created if needed + // and removed when the VM is done. + Dir string + + Backend Backend + + // Services are extra guest-visible services, keyed by guest address. + // The address must be the gateway, IMDS or DNS address. A service on + // the IMDS address port 80 replaces the built-in metadata service. + Services map[netip.AddrPort]guestnet.Service + + // Deny, Dialer and Resolver are passed to guestnet. + Deny []netip.Prefix + Dialer guestnet.Dialer + Resolver guestnet.Resolver + + // UserData is served by the metadata service. + UserData map[string]string + // Hostname is the host's name, reported by the metadata service. + Hostname string + + // StopTimeout bounds a graceful stop. Default 30s. + StopTimeout time.Duration + + // Console, if non-nil, also receives the guest's serial console output + // (which VM.Console keeps the tail of regardless). + Console io.Writer + + Logf logger.Logf + Now func() time.Time +} + +// Event is a notable thing that happened to a VM. +type Event struct { + Time time.Time `json:"time"` + Kind EventKind `json:"kind"` + Msg string `json:"msg,omitempty"` +} + +// EventKind is the type of an [Event]. +type EventKind string + +const ( + // EventBoot reports the boot mode actually used: "cold" or "restored". + EventBoot EventKind = "boot" + // EventRestoreFailed reports that restoring the warm snapshot failed + // and the VM fell back to a cold boot. Msg is the error. + EventRestoreFailed EventKind = "restore-failed" + // EventNetworkUp reports that the guest has a DHCP lease (cold boot) + // or its network resumed (restore). + EventNetworkUp EventKind = "network-up" + // EventGuestReady reports that the guest posted to IMDS /v0/ready. + EventGuestReady EventKind = "guest-ready" + // EventLog is a line the guest posted to IMDS /v0/log. + EventLog EventKind = "log" + // EventNet is a notable network event (DHCP, blocked flows). + EventNet EventKind = "net" + // EventStopping reports that a stop was requested. + EventStopping EventKind = "stopping" + // EventExited reports that the machine stopped. Msg is the error, if + // any. + EventExited EventKind = "exited" +) + +// VM is a running guest. +type VM struct { + cfg Config + events *guestnet.Log[Event] + cons *ringBuffer + + // Set during start, then read-only. + net *guestnet.Net + imds *imds.Handler + imdsSrv *http.Server + nbd *guestbd.Server + nbdLn net.Listener + machine Machine + plan bootPlan // the plan that booted + + mu sync.Mutex + restored bool + stopping bool + err error + persist bool // persist the disk at teardown (see bootPlan.persistPath) + persistErr error // result of persisting + + done chan struct{} // closed when torn down + downOnce sync.Once +} + +// Start creates and boots a VM. It returns once the machine is running. +func Start(ctx context.Context, cfg Config) (*VM, error) { + cfg.withDefaults() + plans, err := cfg.plans() + if err != nil { + return nil, err + } + return start(ctx, cfg, plans) +} + +func (c *Config) withDefaults() { + if c.CPUs == 0 { + c.CPUs = c.Image.Manifest.CPUs + } + if c.MemoryBytes == 0 { + c.MemoryBytes = c.Image.Manifest.MemoryBytes + } + if c.StopTimeout == 0 { + c.StopTimeout = 30 * time.Second + } + if c.Logf == nil { + c.Logf = logger.Discard + } + if c.Now == nil { + c.Now = time.Now + } +} + +// bootPlan is one attempt at booting. +type bootPlan struct { + restore bool // restore statePath instead of cold booting + statePath string // saved state, if restore + diskPath string // raw disk, served over NBD with writes discarded + auxSrc string // aux file to clone to auxPath, or "" to use auxPath in place + auxPath string + + // persistPath, if set, is where the NBD disk's writes are persisted + // (as a clone of diskPath plus the dirty pages) at teardown, if the VM + // is armed to persist. Warm-up uses this. + persistPath string +} + +// plans returns the boot attempts for c, in order. +func (c *Config) plans() ([]bootPlan, error) { + im := c.Image + aux := filepath.Join(c.Dir, image.AuxFile) + cold := bootPlan{diskPath: im.DiskPath(), auxSrc: im.AuxPath(), auxPath: aux} + warm := bootPlan{ + restore: true, + statePath: im.WarmPath(image.StateFile), + diskPath: im.WarmPath(image.DiskFile), + auxSrc: im.WarmPath(image.AuxFile), + auxPath: aux, + } + ok, why := im.WarmUsable(c.Backend.HostOSBuild(), c.CPUs, c.MemoryBytes) + switch c.Boot { + case BootCold: + return []bootPlan{cold}, nil + case BootRestore: + if !ok { + return nil, fmt.Errorf("image %q has no usable warm snapshot: %s", im.Name, why) + } + return []bootPlan{warm}, nil + case BootAuto: + if ok { + return []bootPlan{warm, cold}, nil + } + c.Logf("guest %s: cold boot: %s", c.Name, why) + return []bootPlan{cold}, nil + } + return nil, fmt.Errorf("unknown boot mode %q", c.Boot) +} + +func start(ctx context.Context, cfg Config, plans []bootPlan) (_ *VM, retErr error) { + v := &VM{ + cfg: cfg, + events: guestnet.NewLog[Event](1024), + cons: &ringBuffer{max: 1 << 20}, + done: make(chan struct{}), + } + defer func() { + if retErr != nil { + v.teardown(retErr) + } + }() + if err := os.MkdirAll(cfg.Dir, 0o700); err != nil { + return nil, err + } + vmSide, link, err := guestnet.NewSocketPairLink() + if err != nil { + return nil, err + } + defer vmSide.Close() // the framework dups it + if err := v.startNet(link); err != nil { + link.Close() + return nil, err + } + + for i, p := range plans { + err = v.boot(ctx, p, vmSide) + if err == nil { + break + } + var re *RestoreError + if !errors.As(err, &re) || i == len(plans)-1 { + return nil, err + } + v.logf("%v; falling back to cold boot", err) + v.event(EventRestoreFailed, err.Error()) + if re.Transient { + continue // the snapshot is fine; don't mark it stale + } + if err := cfg.Image.MarkWarmStale(err.Error()); err != nil { + v.logf("marking warm snapshot stale: %v", err) + } + } + + mode := "cold" + if v.restored { + mode = "restored" + } + v.event(EventBoot, mode) + if v.restored { + v.event(EventNetworkUp, "restored") + } + v.imds.SetMeta(func(m *imds.Meta) { + m.BootMode = mode + m.BootTime = cfg.Now() + }) + go v.watch() + return v, nil +} + +func (v *VM) logf(format string, args ...any) { + v.cfg.Logf("guest %s: "+format, append([]any{v.cfg.Name}, args...)...) +} + +func (v *VM) event(kind EventKind, msg string) { + v.events.Add(Event{Time: v.cfg.Now(), Kind: kind, Msg: msg}) +} + +// startNet starts the guest network and metadata service on link. +func (v *VM) startNet(link guestnet.FrameLink) error { + cfg := v.cfg + ncfg := guestnet.Config{ + Link: link, + GuestMAC: net.HardwareAddr(cfg.Image.Manifest.MAC), + MTU: cfg.Image.Manifest.MTU, + Deny: cfg.Deny, + Dialer: cfg.Dialer, + Resolver: cfg.Resolver, + Services: map[netip.AddrPort]guestnet.Service{}, + Now: cfg.Now, + Logf: cfg.Logf, + } + for ap, svc := range cfg.Services { + switch ap.Addr() { + case guestnet.DefaultGatewayIP, guestnet.IMDSAddr, guestnet.DNSAddr: + default: + return fmt.Errorf("service address %v must be the gateway, IMDS or DNS address", ap) + } + ncfg.Services[ap] = svc + } + + v.imds = imds.New(imds.Meta{ + Name: cfg.Name, + Slot: cfg.Slot, + Image: cfg.Image.Name, + CPUs: cfg.CPUs, + MemoryBytes: cfg.MemoryBytes, + Host: cfg.Hostname, + GuestIP: guestnet.DefaultGuestIP.String(), + GatewayIP: guestnet.DefaultGatewayIP.String(), + }, cfg.UserData) + v.imds.Now = cfg.Now + v.imds.OnLog = func(line string) { v.event(EventLog, line) } + imdsAP := netip.AddrPortFrom(guestnet.IMDSAddr, 80) + if _, claimed := ncfg.Services[imdsAP]; !claimed { + ln := guestnet.NewListenerService("imds") + ncfg.Services[imdsAP] = ln + v.imdsSrv = &http.Server{Handler: v.imds, ReadHeaderTimeout: 10 * time.Second} + go v.imdsSrv.Serve(ln) + } + + var err error + if v.net, err = guestnet.New(ncfg); err != nil { + return err + } + go v.relayNet() + return nil +} + +// relayNet turns network and metadata signals into events until the VM is +// torn down. +func (v *VM) relayNet() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-v.done + cancel() + }() + netEvents := v.net.Events.Subscribe(ctx) + leased, ready := v.net.Leased(), v.imds.Ready() + for { + select { + case <-ctx.Done(): + return + case ev, ok := <-netEvents: + if !ok { + return + } + v.event(EventNet, ev.Kind+": "+ev.Msg) + case <-leased: + leased = nil + if !v.Restored() { + v.event(EventNetworkUp, "dhcp") + } + case <-ready: + ready = nil + v.event(EventGuestReady, "") + } + } +} + +// boot makes one boot attempt. Resources from a failed attempt are +// released before it returns. +func (v *VM) boot(ctx context.Context, p bootPlan, netFile *os.File) error { + cfg := v.cfg + spec := &BootSpec{ + Name: cfg.Name, + Manifest: &cfg.Image.Manifest, + CPUs: cfg.CPUs, + MemoryBytes: cfg.MemoryBytes, + AuxPath: p.auxPath, + Net: netFile, + Console: v.consoleWriter(), + Logf: cfg.Logf, + } + if p.restore { + spec.RestoreState = p.statePath + } + if p.auxSrc != "" { + os.Remove(p.auxPath) + if err := image.CloneFile(p.auxSrc, p.auxPath); err != nil { + return err + } + } + sock := filepath.Join(cfg.Dir, "nbd.sock") + os.Remove(sock) + ln, err := net.Listen("unix", sock) + if err != nil { + return err + } + srv := guestbd.NewServer(guestbd.FileSource(p.diskPath), guestbd.WithSharedSnapshot(), guestbd.WithMaxMem(0)) + go srv.Serve(ln) + v.nbd, v.nbdLn = srv, ln + spec.NBDURL = "nbd+unix:///?socket=" + sock + t0 := time.Now() + m, err := cfg.Backend.Boot(ctx, spec) + if err != nil { + v.closeNBD() + return err + } + v.logf("booted (restore=%v) in %v", p.restore, time.Since(t0).Round(time.Millisecond)) + v.mu.Lock() + v.machine = m + v.plan = p + v.restored = p.restore + v.mu.Unlock() + return nil +} + +// persistDisk writes the NBD disk's current contents to v.plan.persistPath: +// a clone of the base disk plus the pages the guest wrote. The machine must +// be stopped. +func (v *VM) persistDisk() error { + p := v.plan + if v.nbd == nil || p.persistPath == "" { + return errors.New("VM has no NBD disk to persist") + } + t0 := time.Now() + os.Remove(p.persistPath) + if err := image.CloneFile(p.diskPath, p.persistPath); err != nil { + return err + } + snap := v.nbd.SharedSnapshot() + if snap == nil { + // The disk was never connected, so never written. + return nil + } + f, err := os.OpenFile(p.persistPath, os.O_WRONLY, 0) + if err != nil { + return err + } + pages, err := snap.WriteDirtyTo(f) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return fmt.Errorf("persisting disk: %w", err) + } + v.logf("persisted %d written pages to %s in %v", pages, p.persistPath, time.Since(t0).Round(time.Millisecond)) + return nil +} + +func (v *VM) closeNBD() { + if v.nbdLn != nil { + v.nbdLn.Close() + v.nbd.Close() + v.nbd, v.nbdLn = nil, nil + } +} + +// watch waits for the machine to exit and tears down. +func (v *VM) watch() { + <-v.machine.Done() + err := v.machine.Err() + msg := "" + if err != nil { + msg = err.Error() + } + v.event(EventExited, msg) + v.teardown(err) +} + +// teardown releases everything. It is idempotent. +func (v *VM) teardown(err error) { + v.downOnce.Do(func() { + // Log any step that is slow, since a stuck teardown holds up Stop. + step := func(name string, f func()) { + t0 := time.Now() + slow := time.AfterFunc(5*time.Second, func() { + v.logf("teardown: still %s after 5s; goroutines in %s", name, dumpGoroutines()) + }) + f() + slow.Stop() + if d := time.Since(t0); d > time.Second { + v.logf("teardown: %s took %v", name, d.Round(time.Millisecond)) + } + } + if v.imdsSrv != nil { + step("closing IMDS server", func() { v.imdsSrv.Close() }) + } + if v.net != nil { + step("closing network", func() { v.net.Close() }) + } + v.mu.Lock() + persist := v.persist + v.mu.Unlock() + if persist { + perr := v.persistDisk() + v.mu.Lock() + v.persistErr = perr + v.mu.Unlock() + } + step("closing NBD server", v.closeNBD) + os.RemoveAll(v.cfg.Dir) + v.mu.Lock() + if !v.stopping { + v.err = err + } + v.mu.Unlock() + close(v.done) + }) +} + +// Stop stops the VM, first asking the guest to shut down unless force, +// and waits until it is torn down or ctx is done. +func (v *VM) Stop(ctx context.Context, force bool) { + v.mu.Lock() + m := v.machine + already := v.stopping + v.stopping = true + v.mu.Unlock() + if !already { + v.event(EventStopping, "") + } + if !force { + if err := m.RequestStop(); err != nil { + v.logf("RequestStop: %v", err) + } else { + t := time.NewTimer(v.cfg.StopTimeout) + select { + case <-m.Done(): + case <-t.C: + v.logf("guest did not shut down within %v; stopping", v.cfg.StopTimeout) + case <-ctx.Done(): + } + t.Stop() + } + } + select { + case <-m.Done(): + default: + t0 := time.Now() + if err := m.Stop(); err != nil { + v.logf("Stop: %v", err) + } + if d := time.Since(t0); d > time.Second { + v.logf("stopping the machine took %v", d.Round(time.Millisecond)) + } + } + t0 := time.Now() + select { + case <-m.Done(): + case <-ctx.Done(): + v.logf("machine not stopped after %v: %v", time.Since(t0).Round(time.Millisecond), ctx.Err()) + return + } + if d := time.Since(t0); d > time.Second { + v.logf("machine reported stopped %v after Stop", d.Round(time.Millisecond)) + } + select { + case <-v.done: + case <-ctx.Done(): + v.logf("teardown not done: %v", ctx.Err()) + } +} + +// Done returns a channel closed once the VM has stopped and its resources +// are released. +func (v *VM) Done() <-chan struct{} { return v.done } + +// Err returns why the VM stopped: nil after a clean guest shutdown or a +// requested Stop. It is valid after Done is closed. +func (v *VM) Err() error { + v.mu.Lock() + defer v.mu.Unlock() + return v.err +} + +// Restored reports whether the VM was restored from a warm snapshot rather +// than cold booted. +func (v *VM) Restored() bool { + v.mu.Lock() + defer v.mu.Unlock() + return v.restored +} + +// Net returns the guest's network. +func (v *VM) Net() *guestnet.Net { return v.net } + +// IMDS returns the guest's metadata service. +func (v *VM) IMDS() *imds.Handler { return v.imds } + +// Events returns the VM's event log. +func (v *VM) Events() *guestnet.Log[Event] { return v.events } + +// Console returns the most recent serial console output. +func (v *VM) Console() []byte { return v.cons.Bytes() } + +// consoleWriter returns where the backend writes serial console output. +func (v *VM) consoleWriter() io.Writer { + if v.cfg.Console == nil { + return v.cons + } + return io.MultiWriter(v.cons, v.cfg.Console) +} + +// WaitReady waits until the guest is usable: it has posted to IMDS +// /v0/ready, or a TCP connection to port on the guest succeeds. Port 0 +// means only the IMDS signal counts. +func (v *VM) WaitReady(ctx context.Context, port uint16) error { + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + if port != 0 { + dctx, cancel := context.WithTimeout(ctx, 2*time.Second) + c, err := v.net.DialGuest(dctx, port) + cancel() + if err == nil { + c.Close() + return nil + } + } + select { + case <-v.imds.Ready(): + return nil + case <-v.done: + return fmt.Errorf("guest exited before it was ready: %v", v.Err()) + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + } + } +} + +// ringBuffer keeps the last max bytes written to it. +type ringBuffer struct { + mu sync.Mutex + max int + b []byte +} + +func (r *ringBuffer) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.b = append(r.b, p...) + if over := len(r.b) - r.max; over > 0 { + r.b = append(r.b[:0], r.b[over:]...) + } + return len(p), nil +} + +func (r *ringBuffer) Bytes() []byte { + r.mu.Lock() + defer r.mu.Unlock() + return append([]byte(nil), r.b...) +} diff --git a/mantsana/guest/guest_test.go b/mantsana/guest/guest_test.go new file mode 100644 index 0000000..c3bc8d6 --- /dev/null +++ b/mantsana/guest/guest_test.go @@ -0,0 +1,296 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guest_test + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/netip" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/imds" + "github.com/tailscale/tb/mantsana/guest" + "github.com/tailscale/tb/mantsana/guest/guesttest" + "github.com/tailscale/tb/mantsana/image" +) + +func newTestImage(t *testing.T) *image.Image { + t.Helper() + s := &image.Store{Root: t.TempDir()} + im, err := s.Create("img", image.Manifest{ + OS: "macos", + CPUs: 4, + MemoryBytes: 8 << 30, + MAC: image.MACAddr(guestnet.DefaultGuestMAC), + }) + if err != nil { + t.Fatal(err) + } + for _, f := range []string{im.DiskPath(), im.AuxPath()} { + if err := os.WriteFile(f, make([]byte, 1<<20), 0o644); err != nil { + t.Fatal(err) + } + } + return im +} + +func testConfig(t *testing.T, b guest.Backend, im *image.Image) guest.Config { + // A short directory, since it holds a unix socket. + dir, err := os.MkdirTemp("", "mg") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + return guest.Config{ + Name: "test-1", + Slot: 1, + Image: im, + Dir: filepath.Join(dir, "s1"), + Backend: b, + Resolver: guestnet.StaticResolver{Addr: netip.MustParseAddr("192.0.2.1")}, + UserData: map[string]string{"k": "v"}, + Logf: t.Logf, + } +} + +func testContext(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + return ctx +} + +// waitEvent waits for an event of kind and returns it. +func waitEvent(t *testing.T, v *guest.VM, kind guest.EventKind) guest.Event { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + for _, ev := range v.Events().Recent() { + if ev.Kind == kind { + return ev + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("no %q event; have %+v", kind, v.Events().Recent()) + return guest.Event{} +} + +// lastMachine returns the most recently booted fake machine. +func lastMachine(t *testing.T, b *guesttest.Backend) *guesttest.Machine { + t.Helper() + ms := b.Machines() + if len(ms) == 0 { + t.Fatal("no machines booted") + } + return ms[len(ms)-1] +} + +// lastSpec returns the most recent Boot call's spec. +func lastSpec(t *testing.T, b *guesttest.Backend) guest.BootSpec { + t.Helper() + ss := b.Specs() + if len(ss) == 0 { + t.Fatal("no Boot calls") + } + return ss[len(ss)-1] +} + +// guestMeta fetches IMDS /v0/meta from inside the most recent guest. +func guestMeta(t *testing.T, ctx context.Context, b *guesttest.Backend) imds.Meta { + t.Helper() + m := lastMachine(t, b) + hc := &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return m.Guest.DialTCP(ctx, netip.AddrPortFrom(guestnet.IMDSAddr, 80)) + }, + }} + req, _ := http.NewRequestWithContext(ctx, "GET", "http://169.254.169.254/v0/meta", nil) + res, err := hc.Do(req) + if err != nil { + t.Fatalf("IMDS: %v", err) + } + defer res.Body.Close() + var meta imds.Meta + if err := json.NewDecoder(res.Body).Decode(&meta); err != nil { + t.Fatal(err) + } + return meta +} + +func TestColdBoot(t *testing.T) { + b := &guesttest.Backend{T: t} + im := newTestImage(t) + cfg := testConfig(t, b, im) + ctx := testContext(t) + v, err := guest.Start(ctx, cfg) + if err != nil { + t.Fatal(err) + } + if v.Restored() { + t.Error("Restored = true") + } + if ev := waitEvent(t, v, guest.EventBoot); ev.Msg != "cold" { + t.Errorf("boot event = %q, want cold", ev.Msg) + } + if err := v.WaitReady(ctx, 22); err != nil { + t.Fatal(err) + } + c, err := v.Net().DialGuest(ctx, 22) + if err != nil { + t.Fatal(err) + } + line, _ := bufio.NewReader(c).ReadString('\n') + c.Close() + if line != guesttest.SSHBanner { + t.Errorf("banner = %q", line) + } + + meta := guestMeta(t, ctx, b) + if meta.Name != "test-1" || meta.BootMode != "cold" || meta.CPUs != 4 { + t.Errorf("meta = %+v", meta) + } + + // The disk is served over NBD on a unix socket in the VM dir, and the + // aux file is a private copy there. + spec := lastSpec(t, b) + sock, ok := strings.CutPrefix(spec.NBDURL, "nbd+unix:///?socket=") + if !ok || filepath.Dir(sock) != cfg.Dir { + t.Fatalf("NBDURL = %q", spec.NBDURL) + } + nc, err := net.Dial("unix", sock) + if err != nil { + t.Fatal(err) + } + magic := make([]byte, 8) + nc.SetDeadline(time.Now().Add(5 * time.Second)) + if _, err := io.ReadFull(nc, magic); err != nil || string(magic) != "NBDMAGIC" { + t.Errorf("NBD handshake = %q, %v", magic, err) + } + nc.Close() + if spec.AuxPath != filepath.Join(cfg.Dir, image.AuxFile) { + t.Errorf("AuxPath = %q", spec.AuxPath) + } + if !strings.Contains(string(v.Console()), "fake console") { + t.Errorf("console = %q", v.Console()) + } + + v.Stop(ctx, false) + select { + case <-v.Done(): + default: + t.Fatal("not done after Stop") + } + if err := v.Err(); err != nil { + t.Errorf("Err = %v", err) + } + if _, err := os.Stat(cfg.Dir); !os.IsNotExist(err) { + t.Errorf("VM dir survived: %v", err) + } +} + +func TestWarmAndRestore(t *testing.T) { + b := &guesttest.Backend{T: t} + im := newTestImage(t) + ctx := testContext(t) + + cfg := testConfig(t, b, im) + if _, err := guest.Start(ctx, guest.Config{Image: im, Backend: b, Boot: guest.BootRestore, Dir: cfg.Dir}); err == nil { + t.Fatal("BootRestore without a warm snapshot succeeded") + } + + wi, err := guest.Warm(ctx, guest.WarmConfig{Config: cfg, Settle: time.Millisecond}) + if err != nil { + t.Fatalf("Warm: %v", err) + } + if wi.HostOSBuild != "25F71" || wi.CPUs != 4 { + t.Errorf("WarmInfo = %+v", wi) + } + if spec := lastSpec(t, b); spec.NBDURL == "" || spec.DiskPath != "" || spec.RestoreState != "" { + t.Errorf("warm-up spec = %+v; want NBD disk (as for restores), cold boot", spec) + } + for _, f := range []string{image.DiskFile, image.AuxFile, image.StateFile, image.WarmFile} { + if _, err := os.Stat(im.WarmPath(f)); err != nil { + t.Errorf("warm file: %v", err) + } + } + + v, err := guest.Start(ctx, cfg) + if err != nil { + t.Fatal(err) + } + if !v.Restored() { + t.Fatal("auto boot with a warm snapshot did not restore") + } + waitEvent(t, v, guest.EventNetworkUp) + if spec := lastSpec(t, b); spec.RestoreState != im.WarmPath(image.StateFile) || spec.NBDURL == "" { + t.Errorf("restore spec = %+v", spec) + } + if meta := guestMeta(t, ctx, b); meta.BootMode != "restored" { + t.Errorf("boot mode = %q", meta.BootMode) + } + v.Stop(ctx, true) + + // A transient restore failure (a locked session) falls back to cold + // boot and leaves the snapshot usable. + b.SetFailRestore(true, true) + v, err = guest.Start(ctx, cfg) + if err != nil { + t.Fatal(err) + } + if v.Restored() { + t.Error("restored despite failure") + } + waitEvent(t, v, guest.EventRestoreFailed) + v.Stop(ctx, true) + if ok, why := im.WarmUsable("25F71", 4, 8<<30); !ok { + t.Errorf("after transient restore failure: not usable: %s", why) + } + + // Any other failed restore falls back to cold boot and marks the + // snapshot stale. + b.SetFailRestore(true, false) + v, err = guest.Start(ctx, cfg) + if err != nil { + t.Fatal(err) + } + if v.Restored() { + t.Error("restored despite failure") + } + waitEvent(t, v, guest.EventRestoreFailed) + v.Stop(ctx, true) + if ok, why := im.WarmUsable("25F71", 4, 8<<30); ok || !strings.HasPrefix(why, "stale: ") { + t.Errorf("after failed restore: WarmUsable = %v, %q", ok, why) + } +} + +// TestDefaultMAC checks that images' default MAC is the guest MAC guestnet +// expects by default, which mantsana/image can't import guestnet to share. +func TestDefaultMAC(t *testing.T) { + if !bytes.Equal(image.DefaultMAC, guestnet.DefaultGuestMAC) { + t.Errorf("image.DefaultMAC %v != guestnet.DefaultGuestMAC %v", image.DefaultMAC, guestnet.DefaultGuestMAC) + } +} + +func TestBadService(t *testing.T) { + b := &guesttest.Backend{T: t} + cfg := testConfig(t, b, newTestImage(t)) + cfg.Services = map[netip.AddrPort]guestnet.Service{ + netip.MustParseAddrPort("8.8.8.8:53"): &guestnet.ForwardService{}, + } + if _, err := guest.Start(testContext(t), cfg); err == nil { + t.Fatal("service on a public address accepted") + } +} diff --git a/mantsana/guest/guesttest/guesttest.go b/mantsana/guest/guesttest/guesttest.go new file mode 100644 index 0000000..070bf0e --- /dev/null +++ b/mantsana/guest/guesttest/guesttest.go @@ -0,0 +1,188 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +// Package guesttest provides a fake guest.Backend for testing code built on +// package guest. Its machines are guestnettest guests attached to the VM end +// of the network socketpair, with the fixed guest address (as if holding a +// lease) and a fake sshd on port 22. +package guesttest + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "sync" + "syscall" + "testing" + + "github.com/tailscale/tb/guestnet" + "github.com/tailscale/tb/guestnet/guestnettest" + "github.com/tailscale/tb/mantsana/guest" +) + +// SSHBanner is what the fake sshd sends on each connection. +const SSHBanner = "SSH-2.0-fake\r\n" + +// Backend is a fake guest.Backend. +type Backend struct { + T testing.TB + + mu sync.Mutex + failRestore, transient bool // see SetFailRestore + specs []guest.BootSpec + machines []*Machine +} + +var _ guest.Backend = (*Backend)(nil) + +// SetFailRestore makes subsequent restores fail. A transient failure is +// like one due to the Mac's session being locked: the snapshot is fine. +func (b *Backend) SetFailRestore(fail, transient bool) { + b.mu.Lock() + defer b.mu.Unlock() + b.failRestore, b.transient = fail, transient +} + +// Specs returns the specs of every Boot call so far. +func (b *Backend) Specs() []guest.BootSpec { + b.mu.Lock() + defer b.mu.Unlock() + return append([]guest.BootSpec(nil), b.specs...) +} + +// Machines returns every machine booted so far. +func (b *Backend) Machines() []*Machine { + b.mu.Lock() + defer b.mu.Unlock() + return append([]*Machine(nil), b.machines...) +} + +// HostOSBuild implements guest.Backend. +func (b *Backend) HostOSBuild() string { return "25F71" } + +// VMLimit implements guest.Backend. +func (b *Backend) VMLimit() int { return 2 } + +// Boot implements guest.Backend. +func (b *Backend) Boot(ctx context.Context, spec *guest.BootSpec) (guest.Machine, error) { + b.mu.Lock() + b.specs = append(b.specs, *spec) + failRestore, transient := b.failRestore, b.transient + b.mu.Unlock() + if (spec.NBDURL == "") == (spec.DiskPath == "") { + return nil, fmt.Errorf("want exactly one of NBDURL and DiskPath; got %q, %q", spec.NBDURL, spec.DiskPath) + } + if _, err := os.Stat(spec.AuxPath); err != nil { + return nil, err + } + if spec.RestoreState != "" { + if failRestore { + return nil, &guest.RestoreError{Err: errors.New("fake restore failure"), Transient: transient} + } + if _, err := os.Stat(spec.RestoreState); err != nil { + return nil, &guest.RestoreError{Err: err} + } + } + // The caller closes spec.Net after Boot, as the framework dups it. + fd, err := dupFile(spec.Net) + if err != nil { + return nil, err + } + link := guestnet.NewFileLink(fd) + g := guestnettest.New(b.T, link, guestnettest.Config{ + IP: guestnet.DefaultGuestIP, + Subnet: guestnet.DefaultSubnet, + Gateway: guestnet.DefaultGatewayIP, + MAC: net.HardwareAddr(spec.Manifest.MAC), + }) + ln, err := g.ListenTCP(22) + if err != nil { + link.Close() + return nil, err + } + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + fmt.Fprint(c, SSHBanner) + c.Close() + } + }() + if spec.Console != nil { + fmt.Fprintf(spec.Console, "fake console for %s\n", spec.Name) + } + m := &Machine{Guest: g, link: link, ln: ln, done: make(chan struct{})} + b.mu.Lock() + b.machines = append(b.machines, m) + b.mu.Unlock() + return m, nil +} + +// Machine is a fake running VM. +type Machine struct { + // Guest is the machine's network stack, for dialing out of the guest. + Guest *guestnettest.Guest + + link *guestnet.FileLink + ln net.Listener + once sync.Once + done chan struct{} +} + +var _ guest.Machine = (*Machine)(nil) + +// Done implements guest.Machine. +func (m *Machine) Done() <-chan struct{} { return m.done } + +// Err implements guest.Machine. +func (m *Machine) Err() error { return nil } + +// RequestStop implements guest.Machine. The fake guest shuts down at once. +func (m *Machine) RequestStop() error { return m.Stop() } + +// Stop implements guest.Machine. +func (m *Machine) Stop() error { + m.once.Do(func() { + m.ln.Close() + m.link.Close() + close(m.done) + }) + return nil +} + +// SaveState implements guest.Machine. +func (m *Machine) SaveState(path string) error { + return os.WriteFile(path, []byte("fake state"), 0o644) +} + +// dupFile dups f as a non-blocking file, as the framework dups the network +// socket it is given. Non-blocking is shared with f, which is fine since +// the caller closes f after Boot. +func dupFile(f *os.File) (*os.File, error) { + sc, err := f.SyscallConn() + if err != nil { + return nil, err + } + var nf *os.File + var derr error + sc.Control(func(fd uintptr) { + nfd, err := syscall.Dup(int(fd)) + if err != nil { + derr = err + return + } + if err := syscall.SetNonblock(nfd, true); err != nil { + syscall.Close(nfd) + derr = err + return + } + nf = os.NewFile(uintptr(nfd), "dup") + }) + return nf, derr +} diff --git a/mantsana/guest/warm.go b/mantsana/guest/warm.go new file mode 100644 index 0000000..fbeb5d7 --- /dev/null +++ b/mantsana/guest/warm.go @@ -0,0 +1,139 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package guest + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/tailscale/tb/mantsana/image" +) + +// WarmConfig configures building a warm snapshot. +type WarmConfig struct { + // Config is the VM to boot. Boot and Services are ignored. + Config + + // ReadyPort is the guest TCP port whose reachability means the guest + // is usable, if it doesn't post to IMDS /v0/ready first. Default 22. + ReadyPort uint16 + // Settle is how long to wait after the guest is usable before saving, + // so background startup work finishes. Default 10s. + Settle time.Duration +} + +// Warm builds the image's warm snapshot: it cold boots a VM, waits until +// the guest is usable, saves the machine state, persists the disk's writes +// onto a clone of the image's disk, and publishes the disk, aux and state +// files as the image's warm directory, replacing any previous one. +// +// The VM is configured exactly like a normal VM, including its disk being +// served over NBD, because the framework only restores a state into a +// configuration with the same kind of disk attachment it was saved with. +func Warm(ctx context.Context, wc WarmConfig) (*image.WarmInfo, error) { + cfg := wc.Config + cfg.Boot = BootCold + cfg.Services = nil + cfg.withDefaults() + if wc.ReadyPort == 0 { + wc.ReadyPort = 22 + } + if wc.Settle == 0 { + wc.Settle = 10 * time.Second + } + im := cfg.Image + hash, err := im.ManifestHash() + if err != nil { + return nil, err + } + + final := im.WarmPath("") + tmp := filepath.Clean(final) + ".tmp" + if err := os.RemoveAll(tmp); err != nil { + return nil, err + } + if err := os.MkdirAll(tmp, 0o755); err != nil { + return nil, err + } + ok := false + defer func() { + if !ok { + os.RemoveAll(tmp) + } + }() + aux := filepath.Join(tmp, image.AuxFile) + if err := image.CloneFile(im.AuxPath(), aux); err != nil { + return nil, err + } + + t0 := time.Now() + v, err := start(ctx, cfg, []bootPlan{{ + diskPath: im.DiskPath(), + auxPath: aux, + persistPath: filepath.Join(tmp, image.DiskFile), + }}) + if err != nil { + return nil, err + } + // Stopping includes persisting the disk, which copies every page the + // guest wrote. + stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + defer v.Stop(stopCtx, true) + + if err := v.WaitReady(ctx, wc.ReadyPort); err != nil { + return nil, fmt.Errorf("waiting for guest: %w", err) + } + v.logf("warm-up guest ready after %v; settling for %v", time.Since(t0).Round(time.Millisecond), wc.Settle) + select { + case <-time.After(wc.Settle): + case <-ctx.Done(): + return nil, ctx.Err() + } + t1 := time.Now() + if err := v.machine.SaveState(filepath.Join(tmp, image.StateFile)); err != nil { + return nil, fmt.Errorf("saving state: %w", err) + } + v.logf("saved state in %v", time.Since(t1).Round(time.Millisecond)) + v.mu.Lock() + v.persist = true + v.mu.Unlock() + v.Stop(stopCtx, true) + select { + case <-v.Done(): + default: + return nil, errors.New("timed out stopping the warm-up VM and persisting its disk") + } + v.mu.Lock() + err = v.persistErr + v.mu.Unlock() + if err != nil { + return nil, err + } + + if err := os.RemoveAll(final); err != nil { + return nil, err + } + if err := os.Rename(tmp, final); err != nil { + return nil, err + } + ok = true + wi := &image.WarmInfo{ + HostOSBuild: cfg.Backend.HostOSBuild(), + ManifestHash: hash, + CPUs: cfg.CPUs, + MemoryBytes: cfg.MemoryBytes, + Created: cfg.Now().UTC(), + } + if err := im.SaveWarmInfo(wi); err != nil { + return nil, err + } + return wi, nil +} diff --git a/mantsana/vzvm/vzvm.go b/mantsana/vzvm/vzvm.go new file mode 100644 index 0000000..4abf889 --- /dev/null +++ b/mantsana/vzvm/vzvm.go @@ -0,0 +1,374 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build darwin && arm64 + +// Package vzvm is the Virtualization.framework backend for package guest: +// it turns a guest.BootSpec into a running macOS VM using +// github.com/Code-Hex/vz/v3. +// +// The device set is fixed and identical for cold boots, warm-up boots and +// restores, because a saved state only restores into a matching +// configuration. See DESIGN.md section 4.2. +package vzvm + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "sync" + "time" + + "github.com/Code-Hex/vz/v3" + "github.com/tailscale/tb/mantsana/guest" + "golang.org/x/sys/unix" +) + +// Backend is a guest.Backend using Virtualization.framework. +type Backend struct { + // NoSerial omits the virtio serial console, in case it is found to + // block save/restore. + NoSerial bool + // NBDTimeout is how long the framework's NBD client waits before + // reconnecting. Default 5s. + NBDTimeout time.Duration +} + +var _ guest.Backend = (*Backend)(nil) + +// HostOSBuild implements guest.Backend. +func (b *Backend) HostOSBuild() string { + s, _ := unix.Sysctl("kern.osversion") + return s +} + +// VMLimit implements guest.Backend. The framework allows two macOS VMs. +func (b *Backend) VMLimit() int { return 2 } + +// config is a built VM configuration and the host resources it references. +type config struct { + vc *vz.VirtualMachineConfiguration + consR *os.File // our end of the serial console output pipe + consW *os.File // the framework's end + devNull *os.File // the serial console's input +} + +func (c *config) close() { + for _, f := range []*os.File{c.consR, c.consW, c.devNull} { + if f != nil { + f.Close() + } + } +} + +func (b *Backend) buildConfig(spec *guest.BootSpec) (_ *config, retErr error) { + m := spec.Manifest + if m.OS != "macos" { + return nil, fmt.Errorf("vzvm: unsupported guest OS %q", m.OS) + } + c := new(config) + defer func() { + if retErr != nil { + c.close() + } + }() + + bl, err := vz.NewMacOSBootLoader() + if err != nil { + return nil, fmt.Errorf("boot loader: %w", err) + } + hw, err := vz.NewMacHardwareModelWithData(m.HardwareModel) + if err != nil { + return nil, fmt.Errorf("hardware model: %w", err) + } + if !hw.Supported() { + return nil, errors.New("image's hardware model is not supported on this host") + } + id, err := vz.NewMacMachineIdentifierWithData(m.MachineID) + if err != nil { + return nil, fmt.Errorf("machine identifier: %w", err) + } + aux, err := vz.NewMacAuxiliaryStorage(spec.AuxPath) + if err != nil { + return nil, fmt.Errorf("auxiliary storage %s: %w", spec.AuxPath, err) + } + platform, err := vz.NewMacPlatformConfiguration( + vz.WithMacHardwareModel(hw), + vz.WithMacMachineIdentifier(id), + vz.WithMacAuxiliaryStorage(aux), + ) + if err != nil { + return nil, fmt.Errorf("platform: %w", err) + } + + vc, err := vz.NewVirtualMachineConfiguration(bl, uint(spec.CPUs), spec.MemoryBytes) + if err != nil { + return nil, fmt.Errorf("configuration: %w", err) + } + vc.SetPlatformVirtualMachineConfiguration(platform) + + // Disk. + var att vz.StorageDeviceAttachment + switch { + case spec.NBDURL != "": + timeout := b.NBDTimeout + if timeout == 0 { + timeout = 5 * time.Second + } + att, err = vz.NewNetworkBlockDeviceStorageDeviceAttachment(spec.NBDURL, timeout, false, vz.DiskSynchronizationModeNone) + case spec.DiskPath != "": + att, err = vz.NewDiskImageStorageDeviceAttachmentWithCacheAndSync(spec.DiskPath, false, vz.DiskImageCachingModeCached, vz.DiskImageSynchronizationModeFsync) + default: + err = errors.New("no disk") + } + if err != nil { + return nil, fmt.Errorf("disk attachment: %w", err) + } + blk, err := vz.NewVirtioBlockDeviceConfiguration(att) + if err != nil { + return nil, fmt.Errorf("block device: %w", err) + } + vc.SetStorageDevicesVirtualMachineConfiguration([]vz.StorageDeviceConfiguration{blk}) + + // Network. + natt, err := vz.NewFileHandleNetworkDeviceAttachment(spec.Net) + if err != nil { + return nil, fmt.Errorf("network attachment: %w", err) + } + if mtu := int(m.MTU); mtu != 0 && mtu != 1500 { + if err := natt.SetMaximumTransmissionUnit(mtu); err != nil { + return nil, fmt.Errorf("network MTU %d: %w", mtu, err) + } + } + nic, err := vz.NewVirtioNetworkDeviceConfiguration(natt) + if err != nil { + return nil, fmt.Errorf("network device: %w", err) + } + if len(m.MAC) > 0 { + mac, err := vz.NewMACAddress(net.HardwareAddr(m.MAC)) + if err != nil { + return nil, fmt.Errorf("MAC: %w", err) + } + nic.SetMACAddress(mac) + } + vc.SetNetworkDevicesVirtualMachineConfiguration([]*vz.VirtioNetworkDeviceConfiguration{nic}) + + // Display, needed for screenshots even though there is no window. + gfx, err := vz.NewMacGraphicsDeviceConfiguration() + if err != nil { + return nil, fmt.Errorf("graphics: %w", err) + } + disp, err := vz.NewMacGraphicsDisplayConfiguration(int64(m.Display.Width), int64(m.Display.Height), int64(m.Display.PPI)) + if err != nil { + return nil, fmt.Errorf("display: %w", err) + } + gfx.SetDisplays(disp) + vc.SetGraphicsDevicesVirtualMachineConfiguration([]vz.GraphicsDeviceConfiguration{gfx}) + + // Mac keyboard and trackpad, which macOS 13+ guests use. This device set + // passes ValidateSaveRestoreSupport on hardware (DESIGN.md section 9). + kbd, err := vz.NewMacKeyboardConfiguration() + if err != nil { + return nil, fmt.Errorf("keyboard: %w", err) + } + vc.SetKeyboardsVirtualMachineConfiguration([]vz.KeyboardConfiguration{kbd}) + pad, err := vz.NewMacTrackpadConfiguration() + if err != nil { + return nil, fmt.Errorf("trackpad: %w", err) + } + vc.SetPointingDevicesVirtualMachineConfiguration([]vz.PointingDeviceConfiguration{pad}) + + // vsock, for a future guest agent. + vsock, err := vz.NewVirtioSocketDeviceConfiguration() + if err != nil { + return nil, fmt.Errorf("vsock: %w", err) + } + vc.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsock}) + + // Serial console. + if !b.NoSerial { + if c.devNull, err = os.Open(os.DevNull); err != nil { + return nil, err + } + if c.consR, c.consW, err = os.Pipe(); err != nil { + return nil, err + } + satt, err := vz.NewFileHandleSerialPortAttachment(c.devNull, c.consW) + if err != nil { + return nil, fmt.Errorf("serial attachment: %w", err) + } + serial, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(satt) + if err != nil { + return nil, fmt.Errorf("serial port: %w", err) + } + vc.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{serial}) + } + + c.vc = vc + return c, nil +} + +// Validation is the result of validating a configuration. +type Validation struct { + Valid error // nil if the configuration is valid + SaveRestore error // nil if it supports save and restore +} + +// Validate builds the configuration for spec and validates it without +// creating a VM. +func (b *Backend) Validate(spec *guest.BootSpec) (Validation, error) { + c, err := b.buildConfig(spec) + if err != nil { + return Validation{}, err + } + defer c.close() + var v Validation + if ok, err := c.vc.Validate(); !ok || err != nil { + v.Valid = orErr(err, "invalid configuration") + } + if ok, err := c.vc.ValidateSaveRestoreSupport(); !ok || err != nil { + v.SaveRestore = orErr(err, "save/restore not supported") + } + return v, nil +} + +func orErr(err error, msg string) error { + if err != nil { + return err + } + return errors.New(msg) +} + +// Boot implements guest.Backend. +func (b *Backend) Boot(ctx context.Context, spec *guest.BootSpec) (_ guest.Machine, retErr error) { + c, err := b.buildConfig(spec) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + c.close() + } + }() + if ok, err := c.vc.Validate(); !ok || err != nil { + return nil, fmt.Errorf("invalid VM configuration: %w", orErr(err, "invalid")) + } + if ok, err := c.vc.ValidateSaveRestoreSupport(); !ok || err != nil { + spec.Logf("vzvm %s: configuration does not support save/restore: %v", spec.Name, orErr(err, "unsupported")) + } + vm, err := vz.NewVirtualMachine(c.vc) + if err != nil { + return nil, fmt.Errorf("NewVirtualMachine: %w", err) + } + m := &machine{vm: vm, cfg: c, name: spec.Name, logf: spec.Logf, done: make(chan struct{})} + if c.consR != nil && spec.Console != nil { + go io.Copy(spec.Console, c.consR) + } + + if spec.RestoreState != "" { + if err := vm.RestoreMachineStateFromURL(spec.RestoreState); err != nil { + return nil, &guest.RestoreError{Err: err, Transient: isKeyUnavailable(err)} + } + if err := vm.Resume(); err != nil { + vm.Stop() + return nil, &guest.RestoreError{Err: fmt.Errorf("resume: %w", err)} + } + } else if err := vm.Start(); err != nil { + return nil, wrapStartErr(err) + } + go m.watch() + return m, nil +} + +// isKeyUnavailable reports whether a restore error is the framework failing +// to get at the saved state's key, which is what happens (observed; Apple +// doesn't document it) when the user's session is locked or the process +// isn't in the user's launchd domain. The saved state itself is fine then. +func isKeyUnavailable(err error) bool { + var nse *vz.NSError + if !errors.As(err, &nse) || nse.Code != int(vz.ErrorRestore) { + return false + } + return strings.Contains(nse.LocalizedDescription+nse.UserInfo, "permission denied") +} + +// wrapStartErr wraps the framework's VM-limit error in guest.ErrVMLimit. +func wrapStartErr(err error) error { + var nse *vz.NSError + if errors.As(err, &nse) && nse.Code == int(vz.ErrorVirtualMachineLimitExceeded) { + return fmt.Errorf("%w: %v", guest.ErrVMLimit, err) + } + return fmt.Errorf("start: %w", err) +} + +// machine is a running VM. It implements guest.Machine. +type machine struct { + vm *vz.VirtualMachine // kept reachable; vz finalizers release the ObjC object + cfg *config + name string + logf func(string, ...any) + + mu sync.Mutex + err error + done chan struct{} +} + +// watch drains the VM's state notifications (a single-consumer channel) +// until the VM stops. +func (m *machine) watch() { + for st := range m.vm.StateChangedNotify() { + switch st { + case vz.VirtualMachineStateStopped: + m.finish(nil) + return + case vz.VirtualMachineStateError: + m.finish(errors.New("virtual machine entered the error state")) + return + } + } + m.finish(errors.New("state notifications ended")) +} + +func (m *machine) finish(err error) { + m.mu.Lock() + m.err = err + m.mu.Unlock() + m.cfg.close() + close(m.done) +} + +func (m *machine) Done() <-chan struct{} { return m.done } + +func (m *machine) Err() error { + m.mu.Lock() + defer m.mu.Unlock() + return m.err +} + +func (m *machine) RequestStop() error { + if !m.vm.CanRequestStop() { + return errors.New("VM cannot request stop in its current state") + } + _, err := m.vm.RequestStop() + return err +} + +func (m *machine) Stop() error { + if !m.vm.CanStop() { + return nil + } + return m.vm.Stop() +} + +func (m *machine) SaveState(path string) error { + if m.vm.State() != vz.VirtualMachineStatePaused { + if err := m.vm.Pause(); err != nil { + return fmt.Errorf("pause: %w", err) + } + } + return m.vm.SaveMachineStateToPath(path) +}