From cd91754d6b274df0f325cf0866c0e9b0edee6fd2 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 3 Nov 2026 18:17:20 +0000 Subject: [PATCH] mantsana/image, cmd/oci-img-dl: add VM images and an OCI image downloader These are the on-disk VM images for mantsana, a macOS CI VM backend replacing Tart (see the DESIGN.md that follows), and a way to get them for now. An image is a directory of plain files: a manifest (hardware model, machine identifier, CPUs, memory, display, MTU), a raw sparse disk, and the macOS auxiliary storage (NVRAM). A warm subdirectory holds a saved VM memory snapshot and the disk and NVRAM it goes with, for restoring in seconds instead of cold booting. Snapshots are only usable on the host OS build and at the VM size they were saved with, so WarmUsable checks, and a failed restore can mark one stale. cmd/oci-img-dl (package ocidl) downloads the macOS VM images corp CI uses today, published to OCI registries such as ghcr.io/cirruslabs/macos-tahoe-xcode, straight into such an image. Their disks are 512 MiB chunks in Apple's LZ4 stream framing, each annotated with its uncompressed SHA-256, which is verified. Chunks download in parallel, all-zero pages are left as holes, and interrupted downloads resume. It's for development and the transition; mantsana's own image distribution will be different. Updates tailscale/corp#39994 Signed-off-by: Brad Fitzpatrick Change-Id: I54ff615054bbe0ff0e1fecfa920ffa2c947bd6e7 --- cmd/oci-img-dl/main.go | 52 +++ cmd/oci-img-dl/ocidl/applelz4.go | 81 ++++ cmd/oci-img-dl/ocidl/ocidl.go | 503 ++++++++++++++++++++++ cmd/oci-img-dl/ocidl/ocidl_test.go | 377 ++++++++++++++++ cmd/oci-img-dl/ocidl/registry.go | 209 +++++++++ cmd/oci-img-dl/ocidl/sparse_other_test.go | 11 + cmd/oci-img-dl/ocidl/sparse_unix_test.go | 21 + go.mod | 2 +- mantsana/image/clone_darwin.go | 25 ++ mantsana/image/clone_other.go | 10 + mantsana/image/copy.go | 30 ++ mantsana/image/image.go | 407 +++++++++++++++++ mantsana/image/image_test.go | 155 +++++++ 13 files changed, 1882 insertions(+), 1 deletion(-) create mode 100644 cmd/oci-img-dl/main.go create mode 100644 cmd/oci-img-dl/ocidl/applelz4.go create mode 100644 cmd/oci-img-dl/ocidl/ocidl.go create mode 100644 cmd/oci-img-dl/ocidl/ocidl_test.go create mode 100644 cmd/oci-img-dl/ocidl/registry.go create mode 100644 cmd/oci-img-dl/ocidl/sparse_other_test.go create mode 100644 cmd/oci-img-dl/ocidl/sparse_unix_test.go create mode 100644 mantsana/image/clone_darwin.go create mode 100644 mantsana/image/clone_other.go create mode 100644 mantsana/image/copy.go create mode 100644 mantsana/image/image.go create mode 100644 mantsana/image/image_test.go diff --git a/cmd/oci-img-dl/main.go b/cmd/oci-img-dl/main.go new file mode 100644 index 0000000..742649d --- /dev/null +++ b/cmd/oci-img-dl/main.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The oci-img-dl command downloads a macOS VM image published to an OCI +// registry (for example ghcr.io/cirruslabs/macos-tahoe-xcode:26.5) into a +// mantsana image directory. See package ocidl. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "github.com/tailscale/tb/cmd/oci-img-dl/ocidl" + "github.com/tailscale/tb/mantsana/image" +) + +var parallel = flag.Int("j", 8, "number of chunks to download at once") + +func main() { + log.SetFlags(log.Ltime) + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: oci-img-dl [-j N] <[registry/]repo[:tag|@digest]> \n\n") + fmt.Fprintf(os.Stderr, "A bare image name is created in %s.\n\n", image.DefaultStoreDir()) + fmt.Fprintf(os.Stderr, "example: oci-img-dl ghcr.io/cirruslabs/macos-tahoe-xcode:26.5 tahoe-xcode-26.5\n\n") + flag.PrintDefaults() + } + flag.Parse() + if flag.NArg() != 2 { + flag.Usage() + os.Exit(2) + } + r, err := ocidl.ParseRef(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + dir := flag.Arg(1) + if !strings.ContainsRune(dir, filepath.Separator) { + dir = filepath.Join(image.DefaultStoreDir(), dir) + } + if err := ocidl.Download(ctx, r, dir, ocidl.Options{Parallel: *parallel, Logf: log.Printf}); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/oci-img-dl/ocidl/applelz4.go b/cmd/oci-img-dl/ocidl/applelz4.go new file mode 100644 index 0000000..30064d7 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/applelz4.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "encoding/binary" + "fmt" + "io" + "slices" + + "github.com/pierrec/lz4/v4" +) + +// Block magics of Apple's libcompression LZ4 stream format, as found in the +// disk layers of these images. +const ( + magicLZ4Block = "bv41" // u32le decoded size, u32le encoded size, LZ4 block + magicRawBlock = "bv4-" // u32le size, uncompressed bytes + magicEnd = "bv4$" // end of stream +) + +// lz4Window is how far back an LZ4 block may refer. +const lz4Window = 64 << 10 + +// decodeAppleLZ4 decodes an Apple LZ4 stream from r, calling emit with each +// decoded block in order. The slice passed to emit is only valid during the +// call. Blocks are decoded with the preceding 64 KiB of output as their +// dictionary, since a block may refer back into earlier output. +func decodeAppleLZ4(r io.Reader, emit func([]byte) error) error { + var ( + hdr [12]byte + src, dst []byte + dict []byte + ) + for { + if _, err := io.ReadFull(r, hdr[:4]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + switch string(hdr[:4]) { + case magicEnd: + return nil + case magicLZ4Block: + if _, err := io.ReadFull(r, hdr[4:12]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + en := int(binary.LittleEndian.Uint32(hdr[8:12])) + src = slices.Grow(src[:0], en)[:en] + if _, err := io.ReadFull(r, src); err != nil { + return fmt.Errorf("reading block: %w", err) + } + dst = slices.Grow(dst[:0], dn)[:dn] + n, err := lz4.UncompressBlockWithDict(src, dst, dict) + if err != nil { + return fmt.Errorf("decoding LZ4 block: %w", err) + } + if n != dn { + return fmt.Errorf("LZ4 block decoded to %d bytes, want %d", n, dn) + } + case magicRawBlock: + if _, err := io.ReadFull(r, hdr[4:8]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + dst = slices.Grow(dst[:0], dn)[:dn] + if _, err := io.ReadFull(r, dst); err != nil { + return fmt.Errorf("reading block: %w", err) + } + default: + return fmt.Errorf("unknown block magic %q", hdr[:4]) + } + if err := emit(dst); err != nil { + return err + } + dict = append(dict, dst...) + if len(dict) > lz4Window { + dict = append(dict[:0], dict[len(dict)-lz4Window:]...) + } + } +} diff --git a/cmd/oci-img-dl/ocidl/ocidl.go b/cmd/oci-img-dl/ocidl/ocidl.go new file mode 100644 index 0000000..75ddf43 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl.go @@ -0,0 +1,503 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package ocidl downloads a macOS VM image published to an OCI registry, as +// used by corp CI today (for example +// ghcr.io/cirruslabs/macos-tahoe-xcode:26.5), into a mantsana image +// directory (see package mantsana/image): +// +// manifest.json mantsana image manifest, written last +// disk.img raw disk, sparse (all-zero 4 KiB pages are left as holes) +// nvram.img the VM's auxiliary storage (NVRAM) +// oci-manifest.json the OCI manifest it was downloaded from +// +// It is for development and for the transition from Tart; mantsana's own +// image distribution will be different (see DESIGN.md). +// +// Disk layers are 512 MiB chunks of the raw disk, each compressed with +// Apple's LZ4 stream framing and annotated with its uncompressed size and +// SHA-256, which is verified. Chunks download in parallel, and an +// interrupted download resumes where it left off. +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/tailscale/tb/mantsana/image" + "tailscale.com/types/logger" +) + +// Layer media types and annotations of these VM images. +const ( + mediaTypeDisk = "application/vnd.cirruslabs.tart.disk.v2" + mediaTypeConfig = "application/vnd.cirruslabs.tart.config.v1" + mediaTypeNVRAM = "application/vnd.cirruslabs.tart.nvram.v1" + + annotUncompressedSize = "org.cirruslabs.tart.uncompressed-size" + annotUncompressedDigest = "org.cirruslabs.tart.uncompressed-content-digest" +) + +// progressFile records completed disk chunks, one index per line, after a +// first line naming the manifest digest they belong to. +const progressFile = ".oci-img-dl-progress" + +// ociManifestFile is where the OCI manifest is saved, for reference. +const ociManifestFile = "oci-manifest.json" + +const pageSize = 4096 + +// Options configures a download. +type Options struct { + // Parallel is how many disk chunks to download at once. Default 8. + Parallel int + // Logf receives progress messages. Default logger.Discard. + Logf logger.Logf +} + +// ErrComplete is returned (wrapped) by Download when the directory already +// holds a complete image. +var ErrComplete = errors.New("already a complete image") + +// chunk is one disk layer and where its contents go in disk.img. +type chunk struct { + index int + desc descriptor + off int64 + size int64 // uncompressed +} + +// Download downloads the image r into the image directory dir, creating it +// if needed. If dir holds an interrupted download of the same image, it +// resumes; if it holds a complete image, it returns an error wrapping +// ErrComplete. +func Download(ctx context.Context, r Ref, dir string, opts Options) error { + logf := opts.Logf + if logf == nil { + logf = logger.Discard + } + parallel := opts.Parallel + if parallel <= 0 { + parallel = 8 + } + c := newClient(r) + m, rawManifest, err := c.manifest(ctx, r) + if err != nil { + return err + } + sum := sha256.Sum256(rawManifest) + manifestDigest := "sha256:" + hex.EncodeToString(sum[:]) + + var ( + chunks []chunk + config, nvram *descriptor + total int64 + compressed int64 + ) + for i := range m.Layers { + l := &m.Layers[i] + switch l.MediaType { + case mediaTypeDisk: + n, err := strconv.ParseInt(l.Annotations[annotUncompressedSize], 10, 64) + if err != nil { + return fmt.Errorf("disk layer %d: bad %s: %v", len(chunks), annotUncompressedSize, err) + } + if l.Annotations[annotUncompressedDigest] == "" { + return fmt.Errorf("disk layer %d: no %s", len(chunks), annotUncompressedDigest) + } + if total%pageSize != 0 { + return fmt.Errorf("disk layer %d starts at unaligned offset %d", len(chunks), total) + } + chunks = append(chunks, chunk{index: len(chunks), desc: *l, off: total, size: n}) + total += n + compressed += l.Size + case mediaTypeConfig: + config = l + case mediaTypeNVRAM: + nvram = l + default: + return fmt.Errorf("unsupported layer media type %q", l.MediaType) + } + } + if len(chunks) == 0 || config == nil || nvram == nil { + return errors.New("manifest lacks a disk, config or NVRAM layer") + } + logf("%s (%s): %d disk chunks, %.1f GB disk, %.1f GB to download", + r, manifestDigest[:19], len(chunks), float64(total)/1e9, float64(compressed)/1e9) + + if _, err := os.Stat(filepath.Join(dir, image.ManifestFile)); err == nil { + return fmt.Errorf("%s is %w; remove it to download again", dir, ErrComplete) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + done, err := loadProgress(logf, dir, manifestDigest) + if err != nil { + return err + } + diskPath := filepath.Join(dir, image.DiskFile) + flags := os.O_RDWR | os.O_CREATE + if len(done) == 0 { + flags |= os.O_TRUNC // a fresh start, or a different image + } + disk, err := os.OpenFile(diskPath, flags, 0o644) + if err != nil { + return err + } + defer disk.Close() + if err := disk.Truncate(total); err != nil { + return err + } + prog, err := openProgress(dir, manifestDigest, len(done) == 0) + if err != nil { + return err + } + defer prog.Close() + + // Small layers first, so a config we can't use fails before the big + // download. + configJSON, err := fetchSmall(ctx, logf, c, *config) + if err != nil { + return err + } + im, err := imageManifest(configJSON, total, r, manifestDigest) + if err != nil { + return err + } + aux, err := fetchSmall(ctx, logf, c, *nvram) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, image.AuxFile), aux, 0o644); err != nil { + return err + } + + var todo []chunk + var todoBytes int64 + for _, ch := range chunks { + if !done[ch.index] { + todo = append(todo, ch) + todoBytes += ch.desc.Size + } + } + if len(done) > 0 { + logf("resuming: %d of %d chunks already done", len(chunks)-len(todo), len(chunks)) + } + + var ( + next atomic.Int64 + gotBytes atomic.Int64 + doneCount atomic.Int64 + progMu sync.Mutex + wg sync.WaitGroup + errOnce sync.Once + firstErr error + ) + parent := ctx + ctx, cancel := context.WithCancel(ctx) + defer cancel() + fail := func(err error) { + errOnce.Do(func() { firstErr = err; cancel() }) + } + t0 := time.Now() + go func() { + tick := time.NewTicker(10 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + got := gotBytes.Load() + el := time.Since(t0).Seconds() + rate := float64(got) / el + eta := time.Duration(float64(todoBytes-got)/max(rate, 1)) * time.Second + logf("%d/%d chunks, %.1f/%.1f GB, %.0f MB/s, ETA %v", + doneCount.Load(), len(todo), float64(got)/1e9, float64(todoBytes)/1e9, rate/1e6, eta.Round(time.Second)) + } + } + }() + for range min(parallel, len(todo)) { + wg.Add(1) + go func() { + defer wg.Done() + for { + i := int(next.Add(1) - 1) + if i >= len(todo) || ctx.Err() != nil { + return + } + ch := todo[i] + err := retry(ctx, logf, fmt.Sprintf("chunk %d", ch.index), func() error { + return fetchChunk(ctx, c, disk, ch, &gotBytes) + }) + if err != nil { + fail(fmt.Errorf("chunk %d: %w", ch.index, err)) + return + } + progMu.Lock() + _, err = fmt.Fprintf(prog, "%d\n", ch.index) + progMu.Unlock() + if err != nil { + fail(err) + return + } + doneCount.Add(1) + } + }() + } + wg.Wait() + if parent.Err() != nil { + return fmt.Errorf("interrupted after %d of %d chunks (rerun the same command to resume)", doneCount.Load(), len(todo)) + } + if firstErr != nil { + return fmt.Errorf("%w (rerun the same command to resume)", firstErr) + } + if err := disk.Sync(); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, ociManifestFile), rawManifest, 0o644); err != nil { + return err + } + // The image manifest goes last: its presence is what makes the + // directory an image. + if err := image.SaveManifest(filepath.Join(dir, image.ManifestFile), im); err != nil { + return err + } + prog.Close() + os.Remove(filepath.Join(dir, progressFile)) + logf("done in %v: image %q in %s", time.Since(t0).Round(time.Second), filepath.Base(dir), dir) + return nil +} + +// vmConfig is the VM configuration layer of these images. +type vmConfig struct { + OS string `json:"os"` + Arch string `json:"arch"` + HardwareModel []byte `json:"hardwareModel"` // VZMacHardwareModel.dataRepresentation + ECID []byte `json:"ecid"` // VZMacMachineIdentifier.dataRepresentation + CPUCount int `json:"cpuCount"` + MemorySize uint64 `json:"memorySize"` + DiskFormat string `json:"diskFormat"` + Display struct { + Width int `json:"width"` + Height int `json:"height"` + } `json:"display"` +} + +// imageManifest converts the VM configuration layer into a mantsana image +// manifest. +func imageManifest(configJSON []byte, diskBytes int64, r Ref, manifestDigest string) (*image.Manifest, error) { + var vc vmConfig + if err := json.Unmarshal(configJSON, &vc); err != nil { + return nil, fmt.Errorf("parsing VM config: %w", err) + } + if vc.OS != "darwin" || vc.Arch != "arm64" { + return nil, fmt.Errorf("VM is %s/%s; only darwin/arm64 is supported", vc.OS, vc.Arch) + } + if vc.DiskFormat != "" && vc.DiskFormat != "raw" { + return nil, fmt.Errorf("disk format %q; only raw is supported", vc.DiskFormat) + } + if len(vc.HardwareModel) == 0 || len(vc.ECID) == 0 { + return nil, errors.New("VM config lacks hardwareModel or ecid") + } + m := &image.Manifest{ + Version: 1, + OS: "macos", + Arch: "arm64", + HardwareModel: vc.HardwareModel, + MachineID: vc.ECID, + // Our fixed guest MAC rather than the image's, so every image looks + // the same to guestnet. + MAC: image.DefaultMAC, + CPUs: vc.CPUCount, + MemoryBytes: vc.MemorySize, + Display: image.DefaultDisplay, + MTU: image.DefaultMTU, + DiskBytes: diskBytes, + Created: time.Now().UTC().Truncate(time.Second), + Description: fmt.Sprintf("downloaded from %s (%s)", r, manifestDigest), + } + if vc.Display.Width > 0 && vc.Display.Height > 0 { + m.Display = image.Display{Width: vc.Display.Width, Height: vc.Display.Height, PPI: 72} + } + if err := m.Validate(); err != nil { + return nil, err + } + return m, nil +} + +// fetchChunk downloads one disk chunk, writes its non-zero pages to disk, +// and verifies its uncompressed digest. got counts compressed bytes read. +func fetchChunk(ctx context.Context, c *client, disk *os.File, ch chunk, got *atomic.Int64) error { + body, err := c.blob(ctx, ch.desc.Digest) + if err != nil { + return err + } + defer body.Close() + var counted int64 + defer func() { got.Add(-counted) }() // only completed chunks count toward progress + h := sha256.New() + off := ch.off + cr := &countingReader{r: body, n: got, local: &counted} + err = decodeAppleLZ4(cr, func(b []byte) error { + if off+int64(len(b)) > ch.off+ch.size { + return errors.New("chunk decodes to more than its uncompressed size") + } + h.Write(b) + if err := writeNonZero(disk, b, off); err != nil { + return err + } + off += int64(len(b)) + return nil + }) + if err != nil { + return err + } + if off != ch.off+ch.size { + return fmt.Errorf("chunk decoded to %d bytes, want %d", off-ch.off, ch.size) + } + want := ch.desc.Annotations[annotUncompressedDigest] + if gotDigest := "sha256:" + hex.EncodeToString(h.Sum(nil)); gotDigest != want { + return fmt.Errorf("uncompressed digest %s, want %s", gotDigest, want) + } + counted = 0 // keep this chunk's bytes in the total + return nil +} + +// writeNonZero writes b at off, skipping all-zero 4 KiB pages so they stay +// holes in the sparse file. off must be page aligned. +func writeNonZero(f *os.File, b []byte, off int64) error { + var zeros [pageSize]byte + start := -1 // start of the current run of non-zero pages + flush := func(end int) error { + if start < 0 { + return nil + } + _, err := f.WriteAt(b[start:end], off+int64(start)) + start = -1 + return err + } + for i := 0; i < len(b); i += pageSize { + p := b[i:min(i+pageSize, len(b))] + if bytes.Equal(p, zeros[:len(p)]) { + if err := flush(i); err != nil { + return err + } + continue + } + if start < 0 { + start = i + } + } + return flush(len(b)) +} + +// countingReader counts bytes read into two counters. +type countingReader struct { + r io.Reader + n *atomic.Int64 + local *int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n.Add(int64(n)) + *c.local += int64(n) + return n, err +} + +// retry calls f until it succeeds, up to 5 times with exponential backoff, +// logging failures as what. +func retry(ctx context.Context, logf logger.Logf, what string, f func() error) error { + var err error + for attempt := range 5 { + if err = f(); err == nil || ctx.Err() != nil { + return err + } + logf("%s, attempt %d: %v", what, attempt+1, err) + select { + case <-time.After(time.Second << attempt): + case <-ctx.Done(): + return ctx.Err() + } + } + return err +} + +// fetchSmall downloads a small layer, verifying its digest, with retries. +func fetchSmall(ctx context.Context, logf logger.Logf, c *client, d descriptor) (b []byte, err error) { + err = retry(ctx, logf, d.MediaType+" layer", func() error { + b, err = fetchSmallOnce(ctx, c, d) + return err + }) + return b, err +} + +func fetchSmallOnce(ctx context.Context, c *client, d descriptor) ([]byte, error) { + if d.Size > 1<<30 { + return nil, fmt.Errorf("%s layer is %d bytes; too big", d.MediaType, d.Size) + } + body, err := c.blob(ctx, d.Digest) + if err != nil { + return nil, err + } + defer body.Close() + b, err := io.ReadAll(io.LimitReader(body, d.Size+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != d.Digest { + return nil, fmt.Errorf("%s layer: digest %s, want %s", d.MediaType, got, d.Digest) + } + return b, nil +} + +// loadProgress returns the chunks already downloaded into dir for the +// manifest with the given digest. +func loadProgress(logf logger.Logf, dir, manifestDigest string) (map[int]bool, error) { + b, err := os.ReadFile(filepath.Join(dir, progressFile)) + if errors.Is(err, os.ErrNotExist) { + return map[int]bool{}, nil + } + if err != nil { + return nil, err + } + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + done := map[int]bool{} + if len(lines) == 0 || lines[0] != manifestDigest { + logf("existing download in %s is of a different image; starting over", dir) + return done, nil + } + for _, l := range lines[1:] { + if i, err := strconv.Atoi(l); err == nil { + done[i] = true + } + } + return done, nil +} + +// openProgress opens the progress file for appending, starting it afresh +// if fresh. +func openProgress(dir, manifestDigest string, fresh bool) (*os.File, error) { + path := filepath.Join(dir, progressFile) + if fresh { + if err := os.WriteFile(path, []byte(manifestDigest+"\n"), 0o644); err != nil { + return nil, err + } + } + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) +} diff --git a/cmd/oci-img-dl/ocidl/ocidl_test.go b/cmd/oci-img-dl/ocidl/ocidl_test.go new file mode 100644 index 0000000..28aba31 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl_test.go @@ -0,0 +1,377 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/pierrec/lz4/v4" + "github.com/tailscale/tb/mantsana/image" +) + +// encodeAppleLZ4 frames data as Apple LZ4 blocks of blockSize, compressing +// each block independently (or storing it raw if it doesn't compress). +func encodeAppleLZ4(t testing.TB, data []byte, blockSize int) []byte { + t.Helper() + var out bytes.Buffer + var ht [1 << 16]int + for len(data) > 0 { + blk := data[:min(blockSize, len(data))] + data = data[len(blk):] + dst := make([]byte, lz4.CompressBlockBound(len(blk))) + n, err := lz4.CompressBlock(blk, dst, ht[:]) + if err != nil { + t.Fatal(err) + } + if n == 0 || n >= len(blk) { + out.WriteString(magicRawBlock) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + out.Write(blk) + continue + } + out.WriteString(magicLZ4Block) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + binary.Write(&out, binary.LittleEndian, uint32(n)) + out.Write(dst[:n]) + } + out.WriteString(magicEnd) + return out.Bytes() +} + +func decodeAll(t testing.TB, stream []byte) ([]byte, error) { + t.Helper() + var out bytes.Buffer + err := decodeAppleLZ4(bytes.NewReader(stream), func(b []byte) error { + out.Write(b) + return nil + }) + return out.Bytes(), err +} + +func TestDecodeAppleLZ4(t *testing.T) { + data := make([]byte, 300<<10) + for i := range data { + data[i] = byte(i / 7) // compressible + } + copy(data[100<<10:], bytes.Repeat([]byte{0xff, 0x13, 0x77}, 1000)) + rnd := make([]byte, 5000) // incompressible, to get a raw block + for i := range rnd { + rnd[i] = byte(i*7919 + i*i*31) + } + data = append(data, rnd...) + got, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatal("round trip mismatch") + } + + if _, err := decodeAll(t, []byte("bv4X")); err == nil { + t.Error("unknown magic accepted") + } + if _, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)[:1000]); err == nil { + t.Error("truncated stream accepted") + } +} + +// TestDecodeAppleLZ4Dict checks that a block may copy from the previous +// block's output. +func TestDecodeAppleLZ4Dict(t *testing.T) { + first := []byte("0123456789abcdefghij") + // Second block: a match of 8 bytes at offset 10 (into the first block), + // then the required trailing literals. + blk := []byte{ + 0x04, // token: 0 literals, match length 4+4 + 10, 0, // offset 10 + 0x50, // token: 5 literals, no match + 'V', 'W', 'X', 'Y', 'Z', // literals + } + var s bytes.Buffer + s.WriteString(magicRawBlock) + binary.Write(&s, binary.LittleEndian, uint32(len(first))) + s.Write(first) + s.WriteString(magicLZ4Block) + binary.Write(&s, binary.LittleEndian, uint32(13)) + binary.Write(&s, binary.LittleEndian, uint32(len(blk))) + s.Write(blk) + s.WriteString(magicEnd) + got, err := decodeAll(t, s.Bytes()) + if err != nil { + t.Fatal(err) + } + want := string(first) + "abcdefgh" + "VWXYZ" + if string(got) != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestParseRef(t *testing.T) { + for _, tt := range []struct{ in, reg, repo, ref string }{ + {"cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "latest"}, + {"ghcr.io/cirruslabs/x@sha256:abcd", "ghcr.io", "cirruslabs/x", "sha256:abcd"}, + {"localhost:5000/foo/bar:v1", "localhost:5000", "foo/bar", "v1"}, + } { + r, err := ParseRef(tt.in) + if err != nil { + t.Errorf("ParseRef(%q): %v", tt.in, err) + continue + } + if r.Registry != tt.reg || r.Repo != tt.repo || r.TagOrDigest != tt.ref { + t.Errorf("ParseRef(%q) = %+v", tt.in, r) + } + } +} + +func sha(b []byte) string { + s := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(s[:]) +} + +// fakeRegistry serves one image, requiring a bearer token obtained through +// the WWW-Authenticate challenge. +type fakeRegistry struct { + blobs map[string][]byte + manifest []byte + failBlob atomic.Pointer[string] // blob digest to fail once + blobGets atomic.Int64 + corruptOne atomic.Bool +} + +func (f *fakeRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + if r.URL.Query().Get("scope") != "repository:test/vm:pull" { + http.Error(w, "bad scope", 400) + return + } + json.NewEncoder(w).Encode(map[string]string{"token": "tok123"}) + return + } + if r.Header.Get("Authorization") != "Bearer tok123" { + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="http://%s/token",service="fake",scope="repository:test/vm:pull"`, r.Host)) + http.Error(w, "unauthorized", 401) + return + } + switch { + case r.URL.Path == "/v2/test/vm/manifests/v1": + w.Header().Set("Content-Type", mediaTypeManifest) + w.Write(f.manifest) + case strings.HasPrefix(r.URL.Path, "/v2/test/vm/blobs/"): + d := strings.TrimPrefix(r.URL.Path, "/v2/test/vm/blobs/") + b, ok := f.blobs[d] + if !ok { + http.NotFound(w, r) + return + } + f.blobGets.Add(1) + if p := f.failBlob.Load(); p != nil && *p == d { + f.failBlob.Store(nil) + w.Write(b[:len(b)/2]) // truncated + return + } + if f.corruptOne.CompareAndSwap(true, false) && len(b) > 100 { + c := bytes.Clone(b) + c[len(c)-10] ^= 0xff + w.Write(c) + return + } + w.Write(b) + default: + http.NotFound(w, r) + } +} + +// newFakeImage builds a fake VM image with a disk of three chunks, one of +// them all zeros. +func newFakeImage(t *testing.T) (*fakeRegistry, []byte) { + const chunkSize = 256 << 10 + disk := make([]byte, 3*chunkSize-8192) // last chunk is short + for i := range chunkSize { + disk[i] = byte(i % 251) + } + // Chunk 1 stays zero; chunk 2 has data in its second page only. + copy(disk[2*chunkSize+pageSize:], bytes.Repeat([]byte("mantsana"), 512)) + + f := &fakeRegistry{blobs: map[string][]byte{}} + add := func(b []byte) string { + d := sha(b) + f.blobs[d] = b + return d + } + config := []byte(`{"os":"darwin","arch":"arm64","cpuCount":2,"memorySize":4294967296,` + + `"diskFormat":"raw","hardwareModel":"aHc=","ecid":"aWQ=","display":{"width":1024,"height":768}}`) + nvram := []byte("nvram bytes") + m := manifest{ + MediaType: mediaTypeManifest, + Config: descriptor{MediaType: "application/vnd.oci.image.config.v1+json", Digest: add([]byte("{}")), Size: 2}, + Layers: []descriptor{ + {MediaType: mediaTypeConfig, Digest: add(config), Size: int64(len(config))}, + }, + } + for off := 0; off < len(disk); off += chunkSize { + part := disk[off:min(off+chunkSize, len(disk))] + z := encodeAppleLZ4(t, part, 64<<10) + m.Layers = append(m.Layers, descriptor{ + MediaType: mediaTypeDisk, Digest: add(z), Size: int64(len(z)), + Annotations: map[string]string{ + annotUncompressedSize: strconv.Itoa(len(part)), + annotUncompressedDigest: sha(part), + }, + }) + } + m.Layers = append(m.Layers, descriptor{MediaType: mediaTypeNVRAM, Digest: add(nvram), Size: int64(len(nvram))}) + var err error + if f.manifest, err = json.Marshal(m); err != nil { + t.Fatal(err) + } + return f, disk +} + +func testRef(t *testing.T, srv *httptest.Server) Ref { + r, err := ParseRef(strings.TrimPrefix(srv.URL, "http://") + "/test/vm:v1") + if err != nil { + t.Fatal(err) + } + return r +} + +// withHTTP makes clients use plain HTTP for the duration of the test, for +// httptest servers. +func withHTTP(t *testing.T) { + old := scheme + scheme = "http" + t.Cleanup(func() { scheme = old }) +} + +func TestDownload(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + ctx := context.Background() + + // One truncated chunk response and one corrupted one are retried. + var chunk2 string + var m manifest + json.Unmarshal(f.manifest, &m) + for _, l := range m.Layers { + if l.MediaType == mediaTypeDisk { + chunk2 = l.Digest + } + } + f.failBlob.Store(&chunk2) + f.corruptOne.Store(true) + + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dir, "disk.img")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, disk) { + t.Fatal("disk.img differs from the image's disk") + } + if b, _ := os.ReadFile(filepath.Join(dir, image.AuxFile)); string(b) != "nvram bytes" { + t.Errorf("%s = %q", image.AuxFile, b) + } + if b, _ := os.ReadFile(filepath.Join(dir, ociManifestFile)); !bytes.Equal(b, f.manifest) { + t.Error("OCI manifest not saved") + } + if _, err := os.Stat(filepath.Join(dir, progressFile)); !os.IsNotExist(err) { + t.Error("progress file left behind") + } + // APFS allocates holes of up to about 16 MiB between written extents + // (observed on macOS 26.5), so this tiny disk is fully allocated there. + if blocks := allocatedBlocks(t, filepath.Join(dir, image.DiskFile)); runtime.GOOS != "darwin" && blocks >= 0 && blocks*512 >= int64(len(disk)) { + t.Errorf("disk.img is not sparse: %d bytes allocated for %d", blocks*512, len(disk)) + } + + // The directory is a mantsana image. + im, err := (&image.Store{Root: filepath.Dir(dir)}).Get(filepath.Base(dir)) + if err != nil { + t.Fatalf("loading downloaded image: %v", err) + } + mm := im.Manifest + if mm.OS != "macos" || mm.CPUs != 2 || mm.MemoryBytes != 4<<30 || string(mm.HardwareModel) != "hw" || + string(mm.MachineID) != "id" || mm.Display.Width != 1024 || mm.DiskBytes != int64(len(disk)) { + t.Errorf("image manifest = %+v", mm) + } + + // A complete image isn't overwritten. + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); !errors.Is(err, ErrComplete) { + t.Errorf("second download: %v", err) + } +} + +func TestBadConfig(t *testing.T) { + withHTTP(t) + f, _ := newFakeImage(t) + var m manifest + json.Unmarshal(f.manifest, &m) + for i, l := range m.Layers { + if l.MediaType == mediaTypeConfig { + linux := []byte(`{"os":"linux","arch":"arm64","cpuCount":2,"memorySize":1}`) + m.Layers[i].Digest, m.Layers[i].Size = sha(linux), int64(len(linux)) + f.blobs[sha(linux)] = linux + } + } + f.manifest, _ = json.Marshal(m) + srv := httptest.NewServer(f) + defer srv.Close() + before := f.blobGets.Load() + err := Download(context.Background(), testRef(t, srv), filepath.Join(t.TempDir(), "img"), Options{Logf: t.Logf}) + if err == nil || !strings.Contains(err.Error(), "only darwin/arm64") { + t.Fatalf("download of a linux VM: %v", err) + } + if n := f.blobGets.Load() - before; n != 1 { + t.Errorf("fetched %d blobs before rejecting the config, want 1", n) + } +} + +func TestResume(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + r := testRef(t, srv) + + // Pretend chunk 0 was already done, with its data in place. + os.MkdirAll(dir, 0o755) + os.WriteFile(filepath.Join(dir, progressFile), []byte(sha(f.manifest)+"\n0\n"), 0o644) + os.WriteFile(filepath.Join(dir, "disk.img"), disk[:256<<10], 0o644) + + before := f.blobGets.Load() + if err := Download(context.Background(), r, dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + // config, nvram, and two of the three chunks. + if n := f.blobGets.Load() - before; n != 4 { + t.Errorf("fetched %d blobs on resume, want 4", n) + } + got, _ := os.ReadFile(filepath.Join(dir, "disk.img")) + if !bytes.Equal(got, disk) { + t.Fatal("resumed disk.img differs") + } +} diff --git a/cmd/oci-img-dl/ocidl/registry.go b/cmd/oci-img-dl/ocidl/registry.go new file mode 100644 index 0000000..77374ff --- /dev/null +++ b/cmd/oci-img-dl/ocidl/registry.go @@ -0,0 +1,209 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" +) + +// Ref is a parsed image reference. +type Ref struct { + Registry string // "ghcr.io" + Repo string // "cirruslabs/macos-tahoe-xcode" + TagOrDigest string // "26.5" or "sha256:..." +} + +// ParseRef parses "[registry/]repo[:tag|@digest]". The registry defaults +// to ghcr.io and the tag to "latest". +func ParseRef(s string) (Ref, error) { + var r Ref + name := s + if i := strings.Index(s, "@"); i >= 0 { + name, r.TagOrDigest = s[:i], s[i+1:] + } else if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") { + name, r.TagOrDigest = s[:i], s[i+1:] + } else { + r.TagOrDigest = "latest" + } + first, rest, ok := strings.Cut(name, "/") + if ok && (strings.ContainsAny(first, ".:") || first == "localhost") { + r.Registry, r.Repo = first, rest + } else { + r.Registry, r.Repo = "ghcr.io", name + } + if r.Repo == "" || r.TagOrDigest == "" { + return Ref{}, fmt.Errorf("bad image reference %q", s) + } + return r, nil +} + +func (r Ref) String() string { + sep := ":" + if strings.HasPrefix(r.TagOrDigest, "sha256:") { + sep = "@" + } + return r.Registry + "/" + r.Repo + sep + r.TagOrDigest +} + +// client fetches from one repository, handling the registry's anonymous +// bearer token challenge. +type client struct { + hc *http.Client + base string // "https://ghcr.io/v2/cirruslabs/macos-tahoe-xcode" + + mu sync.Mutex + token string +} + +// scheme is the URL scheme for registries. Tests set it to "http". +var scheme = "https" + +func newClient(r Ref) *client { + return &client{hc: http.DefaultClient, base: scheme + "://" + r.Registry + "/v2/" + r.Repo} +} + +// get fetches path under the repository, authenticating if challenged. +// The caller must close the response body. +func (c *client) get(ctx context.Context, path, accept string) (*http.Response, error) { + for attempt := 0; ; attempt++ { + req, err := http.NewRequestWithContext(ctx, "GET", c.base+path, nil) + if err != nil { + return nil, err + } + if accept != "" { + req.Header.Set("Accept", accept) + } + c.mu.Lock() + tok := c.token + c.mu.Unlock() + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + res, err := c.hc.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode == http.StatusUnauthorized && attempt == 0 { + challenge := res.Header.Get("WWW-Authenticate") + res.Body.Close() + if err := c.authenticate(ctx, challenge); err != nil { + return nil, err + } + continue + } + if res.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(res.Body, 1<<10)) + res.Body.Close() + return nil, fmt.Errorf("GET %s: %s: %s", c.base+path, res.Status, strings.TrimSpace(string(b))) + } + return res, nil + } +} + +// authenticate gets an anonymous token for a +// `Bearer realm="...",service="...",scope="..."` challenge. +func (c *client) authenticate(ctx context.Context, challenge string) error { + scheme, params, _ := strings.Cut(challenge, " ") + if !strings.EqualFold(scheme, "Bearer") { + return fmt.Errorf("unsupported auth challenge %q", challenge) + } + p := map[string]string{} + for _, kv := range strings.Split(params, ",") { + k, v, _ := strings.Cut(strings.TrimSpace(kv), "=") + p[k] = strings.Trim(v, `"`) + } + if p["realm"] == "" { + return fmt.Errorf("auth challenge %q has no realm", challenge) + } + q := url.Values{} + for _, k := range []string{"service", "scope"} { + if p[k] != "" { + q.Set(k, p[k]) + } + } + req, err := http.NewRequestWithContext(ctx, "GET", p["realm"]+"?"+q.Encode(), nil) + if err != nil { + return err + } + res, err := c.hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("getting token: %s", res.Status) + } + var t struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(res.Body).Decode(&t); err != nil { + return fmt.Errorf("getting token: %w", err) + } + tok := cmp.Or(t.Token, t.AccessToken) + if tok == "" { + return errors.New("token response has no token") + } + c.mu.Lock() + c.token = tok + c.mu.Unlock() + return nil +} + +// descriptor is an OCI content descriptor. +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// manifest is an OCI image manifest. +type manifest struct { + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +const mediaTypeManifest = "application/vnd.oci.image.manifest.v1+json" + +// manifest fetches the manifest for r, returning it and its raw bytes. +func (c *client) manifest(ctx context.Context, r Ref) (*manifest, []byte, error) { + res, err := c.get(ctx, "/manifests/"+r.TagOrDigest, mediaTypeManifest) + if err != nil { + return nil, nil, err + } + defer res.Body.Close() + raw, err := io.ReadAll(io.LimitReader(res.Body, 16<<20)) + if err != nil { + return nil, nil, err + } + m := new(manifest) + if err := json.Unmarshal(raw, m); err != nil { + return nil, nil, fmt.Errorf("parsing manifest: %w", err) + } + if m.MediaType != "" && m.MediaType != mediaTypeManifest { + return nil, nil, fmt.Errorf("unsupported manifest media type %q", m.MediaType) + } + return m, raw, nil +} + +// blob opens the blob with the given digest. +func (c *client) blob(ctx context.Context, digest string) (io.ReadCloser, error) { + res, err := c.get(ctx, "/blobs/"+digest, "") + if err != nil { + return nil, err + } + return res.Body, nil +} diff --git a/cmd/oci-img-dl/ocidl/sparse_other_test.go b/cmd/oci-img-dl/ocidl/sparse_other_test.go new file mode 100644 index 0000000..5ccf42c --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_other_test.go @@ -0,0 +1,11 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !unix + +package ocidl + +import "testing" + +// allocatedBlocks returns -1: allocation isn't checked on this platform. +func allocatedBlocks(t *testing.T, path string) int64 { return -1 } diff --git a/cmd/oci-img-dl/ocidl/sparse_unix_test.go b/cmd/oci-img-dl/ocidl/sparse_unix_test.go new file mode 100644 index 0000000..0906220 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_unix_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package ocidl + +import ( + "os" + "syscall" + "testing" +) + +// allocatedBlocks returns the number of 512-byte blocks allocated to path. +func allocatedBlocks(t *testing.T, path string) int64 { + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return fi.Sys().(*syscall.Stat_t).Blocks +} diff --git a/go.mod b/go.mod index c781e5a..a9d2079 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 golang.org/x/time v0.16.0 modernc.org/sqlite v1.51.0 tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef @@ -69,7 +70,6 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect golang.org/x/net v0.59.0 // indirect - golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect modernc.org/libc v1.72.3 // indirect diff --git a/mantsana/image/clone_darwin.go b/mantsana/image/clone_darwin.go new file mode 100644 index 0000000..d7e21c4 --- /dev/null +++ b/mantsana/image/clone_darwin.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "errors" + "fmt" + + "golang.org/x/sys/unix" +) + +// CloneFile makes dst a copy-on-write clone of src. On APFS this is O(1) +// regardless of file size and consumes no space until written. dst must not +// exist. It falls back to a regular copy if the filesystem can't clone. +func CloneFile(src, dst string) error { + err := unix.Clonefile(src, dst, 0) + if err == nil { + return nil + } + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EXDEV) { + return copyFile(src, dst) + } + return fmt.Errorf("clonefile %s -> %s: %w", src, dst, err) +} diff --git a/mantsana/image/clone_other.go b/mantsana/image/clone_other.go new file mode 100644 index 0000000..2d603a1 --- /dev/null +++ b/mantsana/image/clone_other.go @@ -0,0 +1,10 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !darwin + +package image + +// CloneFile copies src to dst. Only macOS has the O(1) APFS clone; elsewhere +// this is a plain copy, which is fine for development and tests. +func CloneFile(src, dst string) error { return copyFile(src, dst) } diff --git a/mantsana/image/copy.go b/mantsana/image/copy.go new file mode 100644 index 0000000..b33803e --- /dev/null +++ b/mantsana/image/copy.go @@ -0,0 +1,30 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "io" + "os" +) + +// copyFile copies src to dst, which must not exist, preserving sparseness +// only as well as io.Copy does (not at all). It is the fallback for +// CloneFile. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + os.Remove(dst) + return err + } + return out.Close() +} diff --git a/mantsana/image/image.go b/mantsana/image/image.go new file mode 100644 index 0000000..1a33803 --- /dev/null +++ b/mantsana/image/image.go @@ -0,0 +1,407 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package image describes mantsana's on-disk VM images. +// +// An image is a directory of plain files: a manifest, a raw disk image, and +// the macOS auxiliary storage (NVRAM) blob. A warm subdirectory holds a +// host-local memory snapshot for fast restore; it is derived data, keyed to +// the host OS build, and rebuilt when stale. +// +// For now images are just files on local disk. Later the disk will be +// fetched lazily from a content-addressed chunk store; that plugs in below +// this package at the NBD BaseImage seam, so this layout does not change. +package image + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +// File names within an image directory and its warm subdirectory. +const ( + ManifestFile = "manifest.json" + DiskFile = "disk.img" + AuxFile = "nvram.img" // not "aux.img": AUX is a reserved file name on Windows + WarmDir = "warm" + StateFile = "state.vzvmsave" // in WarmDir only + WarmFile = "warm.json" // in WarmDir only +) + +// Display is a guest display geometry. It is part of the image because a +// memory snapshot bakes it into the saved configuration. +type Display struct { + Width int `json:"width"` + Height int `json:"height"` + PPI int `json:"ppi"` +} + +// DefaultDisplay is used when a manifest doesn't specify a display. +var DefaultDisplay = Display{Width: 1280, Height: 800, PPI: 72} + +// DefaultMTU is used when a manifest doesn't specify an MTU. +const DefaultMTU = 1500 + +// DefaultMAC is the guest MAC address images use unless they have a reason +// not to. It is guestnet's default guest MAC, so every image looks the same +// to guestnet; each guest's network is private, so there is no conflict. +var DefaultMAC = MACAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} + +// MACAddr is a net.HardwareAddr that encodes as text in JSON. +type MACAddr net.HardwareAddr + +// String returns the colon-separated form, or "" for an empty address. +func (m MACAddr) String() string { return net.HardwareAddr(m).String() } + +// MarshalText implements encoding.TextMarshaler. +func (m MACAddr) MarshalText() ([]byte, error) { return []byte(m.String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (m *MACAddr) UnmarshalText(b []byte) error { + if len(b) == 0 { + *m = nil + return nil + } + hw, err := net.ParseMAC(string(b)) + if err != nil { + return err + } + if len(hw) != 6 { + return fmt.Errorf("MAC %q is not 48 bits", b) + } + *m = MACAddr(hw) + return nil +} + +// Manifest describes an image. The hardware model, machine identifier, MAC, +// display and MTU are image properties because a memory snapshot depends on +// them. Two VMs from one image share them, which is fine since their +// networks are isolated. +type Manifest struct { + Version int `json:"version"` + OS string `json:"os"` // "macos" (later "linux") + Arch string `json:"arch"` // "arm64" + + // HardwareModel and MachineID are the VZMacHardwareModel and + // VZMacMachineIdentifier dataRepresentation blobs (base64 in JSON). + // Empty for non-macOS images. + HardwareModel []byte `json:"hardware_model,omitempty"` + MachineID []byte `json:"machine_id,omitempty"` + + MAC MACAddr `json:"mac,omitempty"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Display Display `json:"display"` + MTU uint32 `json:"mtu"` + DiskBytes int64 `json:"disk_bytes,omitempty"` + + Created time.Time `json:"created,omitzero"` + Description string `json:"description,omitempty"` +} + +// fillDefaults sets unset optional fields. +func (m *Manifest) fillDefaults() { + if m.Version == 0 { + m.Version = 1 + } + if m.Arch == "" { + m.Arch = "arm64" + } + if m.Display == (Display{}) { + m.Display = DefaultDisplay + } + if m.MTU == 0 { + m.MTU = DefaultMTU + } +} + +// Validate reports whether m is usable. +func (m *Manifest) Validate() error { + if m.Version != 1 { + return fmt.Errorf("unsupported manifest version %d", m.Version) + } + switch m.OS { + case "macos", "linux": + default: + return fmt.Errorf("unsupported os %q", m.OS) + } + if m.CPUs <= 0 || m.MemoryBytes == 0 { + return errors.New("cpus and memory_bytes must be set") + } + // Virtualization.framework rejects network MTUs outside this range. + if m.MTU < 1500 || m.MTU > 65535 { + return fmt.Errorf("mtu %d out of range 1500..65535", m.MTU) + } + return nil +} + +// LoadManifest reads and validates the manifest at path, filling defaults. +func LoadManifest(path string) (*Manifest, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + m := new(Manifest) + if err := json.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + m.fillDefaults() + if err := m.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return m, nil +} + +// SaveManifest atomically writes m to path. +func SaveManifest(path string, m *Manifest) error { + return writeJSON(path, m) +} + +// WarmInfo records what a warm snapshot was built against, so a stale one is +// never restored. +type WarmInfo struct { + HostOSBuild string `json:"host_os_build"` + MantsanaVersion string `json:"mantsana_version,omitempty"` + ManifestHash string `json:"manifest_hash"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Created time.Time `json:"created,omitzero"` + + // Stale is set when a restore from this snapshot failed. StaleReason + // says why. + Stale bool `json:"stale,omitempty"` + StaleReason string `json:"stale_reason,omitempty"` +} + +// Image is an image directory on disk. +type Image struct { + Name string + Dir string + Manifest Manifest + // Warm is the warm snapshot's info, or nil if the image has no complete + // warm snapshot. + Warm *WarmInfo +} + +// DiskPath returns the path to the pristine raw disk image. +func (im *Image) DiskPath() string { return filepath.Join(im.Dir, DiskFile) } + +// AuxPath returns the path to the pristine auxiliary storage blob. +func (im *Image) AuxPath() string { return filepath.Join(im.Dir, AuxFile) } + +// WarmPath returns the path of file within the warm subdirectory, or the +// subdirectory itself if file is empty. +func (im *Image) WarmPath(file string) string { + return filepath.Join(im.Dir, WarmDir, file) +} + +// ManifestHash returns a stable hash of the manifest file's contents, so a +// changed image invalidates its warm snapshot. +func (im *Image) ManifestHash() (string, error) { + b, err := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + if err != nil { + return "", err + } + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +// WarmUsable reports whether the warm snapshot can be restored on a host +// running hostOSBuild with the given CPU count and memory size. If not, why +// says why, for logs and the API. +func (im *Image) WarmUsable(hostOSBuild string, cpus int, memoryBytes uint64) (ok bool, why string) { + w := im.Warm + switch { + case w == nil: + return false, "no warm snapshot" + case w.Stale: + return false, "stale: " + w.StaleReason + case w.HostOSBuild != hostOSBuild: + return false, fmt.Sprintf("host OS changed (snapshot %s, host %s)", w.HostOSBuild, hostOSBuild) + case w.CPUs != cpus || w.MemoryBytes != memoryBytes: + return false, fmt.Sprintf("snapshot is %d CPUs/%d bytes, want %d/%d", w.CPUs, w.MemoryBytes, cpus, memoryBytes) + } + hash, err := im.ManifestHash() + if err != nil { + return false, err.Error() + } + if hash != w.ManifestHash { + return false, "image manifest changed since snapshot" + } + return true, "" +} + +// SaveWarmInfo writes wi as the warm snapshot's info, creating the warm +// directory if needed, and sets im.Warm. +func (im *Image) SaveWarmInfo(wi *WarmInfo) error { + if err := os.MkdirAll(im.WarmPath(""), 0o755); err != nil { + return err + } + if err := writeJSON(im.WarmPath(WarmFile), wi); err != nil { + return err + } + im.Warm = wi + return nil +} + +// MarkWarmStale records that the warm snapshot must not be used again, for +// example because restoring it failed. +func (im *Image) MarkWarmStale(reason string) error { + if im.Warm == nil { + return nil + } + wi := *im.Warm + wi.Stale = true + wi.StaleReason = reason + return im.SaveWarmInfo(&wi) +} + +// DeleteWarm removes the warm snapshot. +func (im *Image) DeleteWarm() error { + im.Warm = nil + return os.RemoveAll(im.WarmPath("")) +} + +// loadWarm loads the warm info. A missing warm directory is (nil, nil). A +// warm.json without all of its files is an error, since it means a warm-up +// was interrupted or files were removed by hand. +func (im *Image) loadWarm() (*WarmInfo, error) { + b, err := os.ReadFile(im.WarmPath(WarmFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + wi := new(WarmInfo) + if err := json.Unmarshal(b, wi); err != nil { + return nil, fmt.Errorf("%s: %w", im.WarmPath(WarmFile), err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + if _, err := os.Stat(im.WarmPath(f)); err != nil { + return nil, fmt.Errorf("incomplete warm snapshot: %w", err) + } + } + return wi, nil +} + +// NotFoundError is returned by [Store.Get] for a missing image. +type NotFoundError struct { + Name string +} + +func (e *NotFoundError) Error() string { return fmt.Sprintf("image %q not found", e.Name) } + +// Store is a directory of images, one subdirectory each. +type Store struct { + Root string +} + +// DefaultStoreDir returns the default image store directory for the current +// user: on macOS, ~/Library/Application Support/mantsana/images. It is not +// ~/Library/Caches, even though images can be downloaded again, because +// the system may purge caches, and warm snapshots are costly to rebuild. +func DefaultStoreDir() string { + dir, err := os.UserConfigDir() + if err != nil { + home, _ := os.UserHomeDir() + dir = home + } + return filepath.Join(dir, "mantsana", "images") +} + +// validName reports whether name is usable as a single path element. +func validName(name string) bool { + return name != "" && name != "." && name != ".." && + !strings.ContainsAny(name, `/\`) && !strings.HasPrefix(name, ".") +} + +// Get loads the named image. +func (s *Store) Get(name string) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + dir := filepath.Join(s.Root, name) + m, err := LoadManifest(filepath.Join(dir, ManifestFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, &NotFoundError{Name: name} + } + if err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + im := &Image{Name: name, Dir: dir, Manifest: *m} + if im.Warm, err = im.loadWarm(); err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + return im, nil +} + +// List returns the images in the store, sorted by name. Directories without +// a valid manifest are skipped. +func (s *Store) List() ([]*Image, error) { + ents, err := os.ReadDir(s.Root) + if err != nil { + return nil, err + } + var out []*Image + for _, e := range ents { + if !e.IsDir() || !validName(e.Name()) { + continue + } + if im, err := s.Get(e.Name()); err == nil { + out = append(out, im) + } + } + return out, nil +} + +// Create makes a new image directory containing only a manifest, filling +// manifest defaults. The caller then writes the disk and aux files (for +// example with a macOS installer run). It fails if the image exists. +func (s *Store) Create(name string, m Manifest) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + m.fillDefaults() + if m.Created.IsZero() { + m.Created = time.Now().UTC().Truncate(time.Second) + } + if err := m.Validate(); err != nil { + return nil, err + } + if err := os.MkdirAll(s.Root, 0o755); err != nil { + return nil, err + } + dir := filepath.Join(s.Root, name) + if err := os.Mkdir(dir, 0o755); err != nil { + return nil, err + } + if err := SaveManifest(filepath.Join(dir, ManifestFile), &m); err != nil { + os.RemoveAll(dir) + return nil, err + } + return &Image{Name: name, Dir: dir, Manifest: m}, nil +} + +// writeJSON atomically writes v as indented JSON to path. +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/mantsana/image/image_test.go b/mantsana/image/image_test.go new file mode 100644 index 0000000..f735cea --- /dev/null +++ b/mantsana/image/image_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "testing" +) + +func TestStoreRoundTrip(t *testing.T) { + s := &Store{Root: t.TempDir()} + mac, _ := net.ParseMAC("02:76:7a:00:00:02") + im, err := s.Create("macos-test", Manifest{ + OS: "macos", + Arch: "arm64", + HardwareModel: []byte("hw"), + MachineID: []byte("id"), + MAC: MACAddr(mac), + CPUs: 4, + MemoryBytes: 8 << 30, + }) + if err != nil { + t.Fatal(err) + } + if _, err := s.Create("macos-test", Manifest{}); err == nil { + t.Fatal("duplicate Create succeeded") + } + + got, err := s.Get("macos-test") + if err != nil { + t.Fatal(err) + } + if got.Manifest.MAC.String() != "02:76:7a:00:00:02" { + t.Errorf("MAC = %v", got.Manifest.MAC) + } + if got.Manifest.Display != DefaultDisplay { + t.Errorf("Display = %+v, want default", got.Manifest.Display) + } + if got.Manifest.MTU != 1500 { + t.Errorf("MTU = %d, want 1500", got.Manifest.MTU) + } + if got.Warm != nil { + t.Errorf("Warm = %+v, want nil", got.Warm) + } + + // The MAC is encoded as text. + b, _ := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + var raw map[string]any + json.Unmarshal(b, &raw) + if raw["mac"] != "02:76:7a:00:00:02" { + t.Errorf("mac in JSON = %v", raw["mac"]) + } + + list, err := s.List() + if err != nil || len(list) != 1 || list[0].Name != "macos-test" { + t.Fatalf("List = %v, %v", list, err) + } + + if _, err := s.Get("nope"); err == nil { + t.Fatal("Get missing succeeded") + } else { + var nf *NotFoundError + if !errors.As(err, &nf) { + t.Errorf("err = %v, want NotFoundError", err) + } + } + if _, err := s.Get("../etc"); err == nil { + t.Fatal("Get with path traversal succeeded") + } +} + +func TestWarm(t *testing.T) { + s := &Store{Root: t.TempDir()} + im, err := s.Create("img", Manifest{OS: "macos", CPUs: 4, MemoryBytes: 1 << 30}) + if err != nil { + t.Fatal(err) + } + if ok, why := im.WarmUsable("25A354", 4, 1<<30); ok || why != "no warm snapshot" { + t.Fatalf("WarmUsable = %v, %q", ok, why) + } + + // Warm info without the files is not usable and is not loaded. + hash, _ := im.ManifestHash() + wi := &WarmInfo{HostOSBuild: "25A354", ManifestHash: hash, CPUs: 4, MemoryBytes: 1 << 30} + if err := im.SaveWarmInfo(wi); err != nil { + t.Fatal(err) + } + if got, err := s.Get("img"); err == nil || got != nil { + t.Fatalf("Get with incomplete warm dir = %v, %v; want error", got, err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + os.WriteFile(im.WarmPath(f), []byte("x"), 0o644) + } + got, err := s.Get("img") + if err != nil { + t.Fatal(err) + } + if ok, why := got.WarmUsable("25A354", 4, 1<<30); !ok { + t.Errorf("WarmUsable = false: %s", why) + } + if ok, _ := got.WarmUsable("26A100", 4, 1<<30); ok { + t.Error("usable after host OS change") + } + if ok, _ := got.WarmUsable("25A354", 8, 1<<30); ok { + t.Error("usable with different CPU count") + } + if err := got.MarkWarmStale("restore failed"); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "stale: restore failed" { + t.Errorf("after stale: %v %q", ok, why) + } + + // Changing the manifest invalidates the snapshot. + got.Warm.Stale = false + got.SaveWarmInfo(got.Warm) + m := got.Manifest + m.Description = "changed" + SaveManifest(filepath.Join(got.Dir, ManifestFile), &m) + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "image manifest changed since snapshot" { + t.Errorf("after manifest change: %v %q", ok, why) + } + + if err := got.DeleteWarm(); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if got.Warm != nil { + t.Error("warm survived DeleteWarm") + } +} + +func TestCloneFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + os.WriteFile(src, []byte("hello"), 0o644) + dst := filepath.Join(dir, "dst") + if err := CloneFile(src, dst); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(dst) + if string(b) != "hello" { + t.Fatalf("dst = %q", b) + } + if err := CloneFile(src, dst); err == nil { + t.Fatal("clone over existing file succeeded") + } +}