mantsana, imds: add a Virtualization.framework macOS VM backend for CI

cmd/mantsana/entitlementprobe.go

@@ -0,0 +1,78 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build darwin && arm64
+
+package main
+
+import (
+	"errors"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Code-Hex/vz/v3"
+)
+
+// cmdEntitlementProbe checks that this binary may use Virtualization.framework
+// (that is, it carries the com.apple.security.virtualization entitlement),
+// without needing an image: it validates, creates and starts a minimal EFI
+// VM with no devices, then stops it. It is for checking that a binary signed
+// on one Mac works on another.
+func cmdEntitlementProbe(args []string) error {
+	if len(args) != 0 {
+		return errors.New("usage: entitlementprobe")
+	}
+	dir, err := os.MkdirTemp("", "mantsana-entitlementprobe")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(dir)
+
+	store, err := vz.NewEFIVariableStore(filepath.Join(dir, "efivars"), vz.WithCreatingEFIVariableStore())
+	if err != nil {
+		return err
+	}
+	bl, err := vz.NewEFIBootLoader(vz.WithEFIVariableStore(store))
+	if err != nil {
+		return err
+	}
+	vc, err := vz.NewVirtualMachineConfiguration(bl, 1, 512<<20)
+	if err != nil {
+		return err
+	}
+	platform, err := vz.NewGenericPlatformConfiguration()
+	if err != nil {
+		return err
+	}
+	vc.SetPlatformVirtualMachineConfiguration(platform)
+	if ok, err := vc.Validate(); !ok || err != nil {
+		return orErr(err, "invalid configuration")
+	}
+	log.Printf("configuration valid")
+	vm, err := vz.NewVirtualMachine(vc)
+	if err != nil {
+		return err
+	}
+	if err := vm.Start(); err != nil {
+		return err
+	}
+	log.Printf("VM started (state %v)", vm.State())
+	time.Sleep(time.Second)
+	// With no boot device the firmware may already have stopped it.
+	if vm.CanStop() {
+		if err := vm.Stop(); err != nil {
+			return err
+		}
+	}
+	log.Printf("VM stopped (state %v); entitlement OK", vm.State())
+	return nil
+}
+
+func orErr(err error, msg string) error {
+	if err != nil {
+		return err
+	}
+	return errors.New(msg)
+}

cmd/mantsana/guestssh.go

@@ -0,0 +1,97 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build darwin && arm64
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/tailscale/tb/guestnet"
+	"github.com/tailscale/tb/mantsana/guest"
+	"golang.org/x/crypto/ssh"
+)
+
+// guestSSH connects to the guest's sshd through its network, as corp's
+// ciguestlet does (user admin, password admin; the CI images' convention).
+func guestSSH(ctx context.Context, v *guest.VM) (*ssh.Client, error) {
+	c, err := v.Net().DialGuest(ctx, 22)
+	if err != nil {
+		return nil, err
+	}
+	addr := guestnet.DefaultGuestIP.String() + ":22"
+	cc, chans, reqs, err := ssh.NewClientConn(c, addr, &ssh.ClientConfig{
+		User:            "admin",
+		Auth:            []ssh.AuthMethod{ssh.Password("admin")},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         10 * time.Second,
+	})
+	if err != nil {
+		c.Close()
+		return nil, err
+	}
+	return ssh.NewClient(cc, chans, reqs), nil
+}
+
+// sshOutput runs cmd on the guest and returns its combined output.
+func sshOutput(sc *ssh.Client, cmd string) (string, error) {
+	sess, err := sc.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer sess.Close()
+	out, err := sess.CombinedOutput(cmd)
+	return strings.TrimSpace(string(out)), err
+}
+
+// guestClockSkew returns how far the guest's clock is behind the host's.
+func guestClockSkew(sc *ssh.Client) (time.Duration, error) {
+	// Nanoseconds; macOS date(1) has no %N, so use perl, which the images
+	// have.
+	before := time.Now()
+	out, err := sshOutput(sc, `perl -MTime::HiRes=time -e 'printf "%.6f\n", time'`)
+	if err != nil {
+		return 0, fmt.Errorf("reading guest time: %v: %s", err, out)
+	}
+	rtt := time.Since(before)
+	secs, err := strconv.ParseFloat(out, 64)
+	if err != nil {
+		return 0, fmt.Errorf("parsing guest time %q: %v", out, err)
+	}
+	guestNow := time.Unix(0, int64(secs*1e9))
+	hostNow := before.Add(rtt / 2)
+	return hostNow.Sub(guestNow), nil
+}
+
+// checkGuestClock reports the guest's clock skew, and if the guest was
+// restored, fixes it with sntp against the gateway (guestnet answers NTP
+// with host time) and reports it again.
+func checkGuestClock(ctx context.Context, v *guest.VM) error {
+	sc, err := guestSSH(ctx, v)
+	if err != nil {
+		return fmt.Errorf("ssh: %w", err)
+	}
+	defer sc.Close()
+	skew, err := guestClockSkew(sc)
+	if err != nil {
+		return err
+	}
+	log.Printf("guest clock is %v behind the host", skew.Round(time.Millisecond))
+	if !v.Restored() {
+		return nil
+	}
+	t0 := time.Now()
+	out, err := sshOutput(sc, "sudo -n sntp -sS "+guestnet.DefaultGatewayIP.String())
+	log.Printf("sntp -sS (%v): err=%v %s", time.Since(t0).Round(time.Millisecond), err, out)
+	if skew, err = guestClockSkew(sc); err != nil {
+		return err
+	}
+	log.Printf("after sntp, guest clock is %v behind the host", skew.Round(time.Millisecond))
+	return nil
+}

cmd/mantsana/launchd/com.tailscale.mantsana.keepawake.plist

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!--
+  com.tailscale.mantsana.keepawake keeps the CI user's login session from
+  going idle, so it never sleeps or locks. Restoring a VM's saved state needs
+  the session unlocked (see DESIGN.md sections 4.1 and 9), and on Macs whose
+  MDM enforces an idle screen lock, holding sleep assertions alone isn't
+  enough: this also declares the user active (caffeinate -u).
+
+  caffeinate -u holds its assertions only for -t seconds, and the idle
+  screen saver starts the moment they lapse (the real idle time is long
+  past the limit), so there must never be a gap between them: a new
+  600-second caffeinate starts every 300 seconds, overlapping the previous
+  one. (Letting launchd restart a single caffeinate left a 20-second gap,
+  enough for the screen saver and then the lock.) launchd restarts the loop
+  if it dies.
+
+  Install for the logged-in CI user:
+
+    cp com.tailscale.mantsana.keepawake.plist ~/Library/LaunchAgents/
+    launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.tailscale.mantsana.keepawake.plist
+
+  Remove:
+
+    launchctl bootout gui/$(id -u)/com.tailscale.mantsana.keepawake
+    rm ~/Library/LaunchAgents/com.tailscale.mantsana.keepawake.plist
+-->
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>com.tailscale.mantsana.keepawake</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>/bin/sh</string>
+		<string>-c</string>
+		<string>while :; do /usr/bin/caffeinate -dimsu -t 600 &amp; sleep 300; done</string>
+	</array>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>KeepAlive</key>
+	<true/>
+	<key>ProcessType</key>
+	<string>Interactive</string>
+</dict>
+</plist>

cmd/mantsana/mantsana.entitlements

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>com.apple.security.virtualization</key>
+	<true/>
+	<key>com.apple.security.network.client</key>
+	<true/>
+	<key>com.apple.security.network.server</key>
+	<true/>
+</dict>
+</plist>

cmd/mantsana/mantsana.go

@@ -0,0 +1,68 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build darwin && arm64
+
+// The mantsana command runs ephemeral macOS CI guest VMs on Apple Silicon
+// Macs. See DESIGN.md.
+//
+// Build it on the Mac and ad hoc sign it with the virtualization
+// entitlement:
+//
+//	go build ./cmd/mantsana && codesign --entitlements cmd/mantsana/mantsana.entitlements -s - ./mantsana
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/tailscale/tb/mantsana/image"
+)
+
+var imagesDir = flag.String("images", image.DefaultStoreDir(), "directory of VM images (get one with cmd/oci-img-dl)")
+
+var commands = map[string]func(args []string) error{
+	"images":           cmdImages,
+	"warm":             cmdWarm,
+	"selftest":         cmdSelftest,
+	"nbdprobe":         cmdNBDProbe,
+	"restoreprobe":     cmdRestoreProbe,
+	"entitlementprobe": cmdEntitlementProbe,
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr, `usage: mantsana [flags] <command> [args]
+
+Commands:
+  images                              list images and whether their warm snapshots are usable here
+  warm [flags] <image-name>           build an image's warm snapshot, so VMs restore in seconds
+  selftest [flags] <image-name>       validate, cold boot, warm up and restore an image, with timings
+  nbdprobe [flags] <image-name>       boot an image's disk through a logging NBD proxy (debugging)
+  restoreprobe [flags] <image-name>   try restoring an image's warm snapshot several ways (debugging)
+  entitlementprobe                    check this binary may use Virtualization.framework (no image needed)
+
+Flags:
+`)
+	flag.PrintDefaults()
+}
+
+func main() {
+	log.SetFlags(log.Ltime | log.Lmicroseconds)
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() == 0 {
+		usage()
+		os.Exit(2)
+	}
+	cmd, ok := commands[flag.Arg(0)]
+	if !ok {
+		fmt.Fprintf(os.Stderr, "mantsana: unknown command %q\n", flag.Arg(0))
+		usage()
+		os.Exit(2)
+	}
+	if err := cmd(flag.Args()[1:]); err != nil {
+		log.Fatalf("%s: %v", flag.Arg(0), err)
+	}
+}

cmd/mantsana/mantsana_other.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !(darwin && arm64)
+
+// The mantsana command runs ephemeral macOS CI guest VMs on Apple Silicon
+// Macs. It only runs there.
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	fmt.Fprintln(os.Stderr, "mantsana only runs on Apple Silicon Macs")
+	os.Exit(1)
+}