diff --git a/cmd/oci-img-dl/main.go b/cmd/oci-img-dl/main.go new file mode 100644 index 0000000..742649d --- /dev/null +++ b/cmd/oci-img-dl/main.go @@ -0,0 +1,52 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The oci-img-dl command downloads a macOS VM image published to an OCI +// registry (for example ghcr.io/cirruslabs/macos-tahoe-xcode:26.5) into a +// mantsana image directory. See package ocidl. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "github.com/tailscale/tb/cmd/oci-img-dl/ocidl" + "github.com/tailscale/tb/mantsana/image" +) + +var parallel = flag.Int("j", 8, "number of chunks to download at once") + +func main() { + log.SetFlags(log.Ltime) + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: oci-img-dl [-j N] <[registry/]repo[:tag|@digest]> \n\n") + fmt.Fprintf(os.Stderr, "A bare image name is created in %s.\n\n", image.DefaultStoreDir()) + fmt.Fprintf(os.Stderr, "example: oci-img-dl ghcr.io/cirruslabs/macos-tahoe-xcode:26.5 tahoe-xcode-26.5\n\n") + flag.PrintDefaults() + } + flag.Parse() + if flag.NArg() != 2 { + flag.Usage() + os.Exit(2) + } + r, err := ocidl.ParseRef(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + dir := flag.Arg(1) + if !strings.ContainsRune(dir, filepath.Separator) { + dir = filepath.Join(image.DefaultStoreDir(), dir) + } + if err := ocidl.Download(ctx, r, dir, ocidl.Options{Parallel: *parallel, Logf: log.Printf}); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/oci-img-dl/ocidl/applelz4.go b/cmd/oci-img-dl/ocidl/applelz4.go new file mode 100644 index 0000000..30064d7 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/applelz4.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "encoding/binary" + "fmt" + "io" + "slices" + + "github.com/pierrec/lz4/v4" +) + +// Block magics of Apple's libcompression LZ4 stream format, as found in the +// disk layers of these images. +const ( + magicLZ4Block = "bv41" // u32le decoded size, u32le encoded size, LZ4 block + magicRawBlock = "bv4-" // u32le size, uncompressed bytes + magicEnd = "bv4$" // end of stream +) + +// lz4Window is how far back an LZ4 block may refer. +const lz4Window = 64 << 10 + +// decodeAppleLZ4 decodes an Apple LZ4 stream from r, calling emit with each +// decoded block in order. The slice passed to emit is only valid during the +// call. Blocks are decoded with the preceding 64 KiB of output as their +// dictionary, since a block may refer back into earlier output. +func decodeAppleLZ4(r io.Reader, emit func([]byte) error) error { + var ( + hdr [12]byte + src, dst []byte + dict []byte + ) + for { + if _, err := io.ReadFull(r, hdr[:4]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + switch string(hdr[:4]) { + case magicEnd: + return nil + case magicLZ4Block: + if _, err := io.ReadFull(r, hdr[4:12]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + en := int(binary.LittleEndian.Uint32(hdr[8:12])) + src = slices.Grow(src[:0], en)[:en] + if _, err := io.ReadFull(r, src); err != nil { + return fmt.Errorf("reading block: %w", err) + } + dst = slices.Grow(dst[:0], dn)[:dn] + n, err := lz4.UncompressBlockWithDict(src, dst, dict) + if err != nil { + return fmt.Errorf("decoding LZ4 block: %w", err) + } + if n != dn { + return fmt.Errorf("LZ4 block decoded to %d bytes, want %d", n, dn) + } + case magicRawBlock: + if _, err := io.ReadFull(r, hdr[4:8]); err != nil { + return fmt.Errorf("reading block header: %w", err) + } + dn := int(binary.LittleEndian.Uint32(hdr[4:8])) + dst = slices.Grow(dst[:0], dn)[:dn] + if _, err := io.ReadFull(r, dst); err != nil { + return fmt.Errorf("reading block: %w", err) + } + default: + return fmt.Errorf("unknown block magic %q", hdr[:4]) + } + if err := emit(dst); err != nil { + return err + } + dict = append(dict, dst...) + if len(dict) > lz4Window { + dict = append(dict[:0], dict[len(dict)-lz4Window:]...) + } + } +} diff --git a/cmd/oci-img-dl/ocidl/ocidl.go b/cmd/oci-img-dl/ocidl/ocidl.go new file mode 100644 index 0000000..75ddf43 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl.go @@ -0,0 +1,503 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package ocidl downloads a macOS VM image published to an OCI registry, as +// used by corp CI today (for example +// ghcr.io/cirruslabs/macos-tahoe-xcode:26.5), into a mantsana image +// directory (see package mantsana/image): +// +// manifest.json mantsana image manifest, written last +// disk.img raw disk, sparse (all-zero 4 KiB pages are left as holes) +// nvram.img the VM's auxiliary storage (NVRAM) +// oci-manifest.json the OCI manifest it was downloaded from +// +// It is for development and for the transition from Tart; mantsana's own +// image distribution will be different (see DESIGN.md). +// +// Disk layers are 512 MiB chunks of the raw disk, each compressed with +// Apple's LZ4 stream framing and annotated with its uncompressed size and +// SHA-256, which is verified. Chunks download in parallel, and an +// interrupted download resumes where it left off. +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/tailscale/tb/mantsana/image" + "tailscale.com/types/logger" +) + +// Layer media types and annotations of these VM images. +const ( + mediaTypeDisk = "application/vnd.cirruslabs.tart.disk.v2" + mediaTypeConfig = "application/vnd.cirruslabs.tart.config.v1" + mediaTypeNVRAM = "application/vnd.cirruslabs.tart.nvram.v1" + + annotUncompressedSize = "org.cirruslabs.tart.uncompressed-size" + annotUncompressedDigest = "org.cirruslabs.tart.uncompressed-content-digest" +) + +// progressFile records completed disk chunks, one index per line, after a +// first line naming the manifest digest they belong to. +const progressFile = ".oci-img-dl-progress" + +// ociManifestFile is where the OCI manifest is saved, for reference. +const ociManifestFile = "oci-manifest.json" + +const pageSize = 4096 + +// Options configures a download. +type Options struct { + // Parallel is how many disk chunks to download at once. Default 8. + Parallel int + // Logf receives progress messages. Default logger.Discard. + Logf logger.Logf +} + +// ErrComplete is returned (wrapped) by Download when the directory already +// holds a complete image. +var ErrComplete = errors.New("already a complete image") + +// chunk is one disk layer and where its contents go in disk.img. +type chunk struct { + index int + desc descriptor + off int64 + size int64 // uncompressed +} + +// Download downloads the image r into the image directory dir, creating it +// if needed. If dir holds an interrupted download of the same image, it +// resumes; if it holds a complete image, it returns an error wrapping +// ErrComplete. +func Download(ctx context.Context, r Ref, dir string, opts Options) error { + logf := opts.Logf + if logf == nil { + logf = logger.Discard + } + parallel := opts.Parallel + if parallel <= 0 { + parallel = 8 + } + c := newClient(r) + m, rawManifest, err := c.manifest(ctx, r) + if err != nil { + return err + } + sum := sha256.Sum256(rawManifest) + manifestDigest := "sha256:" + hex.EncodeToString(sum[:]) + + var ( + chunks []chunk + config, nvram *descriptor + total int64 + compressed int64 + ) + for i := range m.Layers { + l := &m.Layers[i] + switch l.MediaType { + case mediaTypeDisk: + n, err := strconv.ParseInt(l.Annotations[annotUncompressedSize], 10, 64) + if err != nil { + return fmt.Errorf("disk layer %d: bad %s: %v", len(chunks), annotUncompressedSize, err) + } + if l.Annotations[annotUncompressedDigest] == "" { + return fmt.Errorf("disk layer %d: no %s", len(chunks), annotUncompressedDigest) + } + if total%pageSize != 0 { + return fmt.Errorf("disk layer %d starts at unaligned offset %d", len(chunks), total) + } + chunks = append(chunks, chunk{index: len(chunks), desc: *l, off: total, size: n}) + total += n + compressed += l.Size + case mediaTypeConfig: + config = l + case mediaTypeNVRAM: + nvram = l + default: + return fmt.Errorf("unsupported layer media type %q", l.MediaType) + } + } + if len(chunks) == 0 || config == nil || nvram == nil { + return errors.New("manifest lacks a disk, config or NVRAM layer") + } + logf("%s (%s): %d disk chunks, %.1f GB disk, %.1f GB to download", + r, manifestDigest[:19], len(chunks), float64(total)/1e9, float64(compressed)/1e9) + + if _, err := os.Stat(filepath.Join(dir, image.ManifestFile)); err == nil { + return fmt.Errorf("%s is %w; remove it to download again", dir, ErrComplete) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + done, err := loadProgress(logf, dir, manifestDigest) + if err != nil { + return err + } + diskPath := filepath.Join(dir, image.DiskFile) + flags := os.O_RDWR | os.O_CREATE + if len(done) == 0 { + flags |= os.O_TRUNC // a fresh start, or a different image + } + disk, err := os.OpenFile(diskPath, flags, 0o644) + if err != nil { + return err + } + defer disk.Close() + if err := disk.Truncate(total); err != nil { + return err + } + prog, err := openProgress(dir, manifestDigest, len(done) == 0) + if err != nil { + return err + } + defer prog.Close() + + // Small layers first, so a config we can't use fails before the big + // download. + configJSON, err := fetchSmall(ctx, logf, c, *config) + if err != nil { + return err + } + im, err := imageManifest(configJSON, total, r, manifestDigest) + if err != nil { + return err + } + aux, err := fetchSmall(ctx, logf, c, *nvram) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, image.AuxFile), aux, 0o644); err != nil { + return err + } + + var todo []chunk + var todoBytes int64 + for _, ch := range chunks { + if !done[ch.index] { + todo = append(todo, ch) + todoBytes += ch.desc.Size + } + } + if len(done) > 0 { + logf("resuming: %d of %d chunks already done", len(chunks)-len(todo), len(chunks)) + } + + var ( + next atomic.Int64 + gotBytes atomic.Int64 + doneCount atomic.Int64 + progMu sync.Mutex + wg sync.WaitGroup + errOnce sync.Once + firstErr error + ) + parent := ctx + ctx, cancel := context.WithCancel(ctx) + defer cancel() + fail := func(err error) { + errOnce.Do(func() { firstErr = err; cancel() }) + } + t0 := time.Now() + go func() { + tick := time.NewTicker(10 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + got := gotBytes.Load() + el := time.Since(t0).Seconds() + rate := float64(got) / el + eta := time.Duration(float64(todoBytes-got)/max(rate, 1)) * time.Second + logf("%d/%d chunks, %.1f/%.1f GB, %.0f MB/s, ETA %v", + doneCount.Load(), len(todo), float64(got)/1e9, float64(todoBytes)/1e9, rate/1e6, eta.Round(time.Second)) + } + } + }() + for range min(parallel, len(todo)) { + wg.Add(1) + go func() { + defer wg.Done() + for { + i := int(next.Add(1) - 1) + if i >= len(todo) || ctx.Err() != nil { + return + } + ch := todo[i] + err := retry(ctx, logf, fmt.Sprintf("chunk %d", ch.index), func() error { + return fetchChunk(ctx, c, disk, ch, &gotBytes) + }) + if err != nil { + fail(fmt.Errorf("chunk %d: %w", ch.index, err)) + return + } + progMu.Lock() + _, err = fmt.Fprintf(prog, "%d\n", ch.index) + progMu.Unlock() + if err != nil { + fail(err) + return + } + doneCount.Add(1) + } + }() + } + wg.Wait() + if parent.Err() != nil { + return fmt.Errorf("interrupted after %d of %d chunks (rerun the same command to resume)", doneCount.Load(), len(todo)) + } + if firstErr != nil { + return fmt.Errorf("%w (rerun the same command to resume)", firstErr) + } + if err := disk.Sync(); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, ociManifestFile), rawManifest, 0o644); err != nil { + return err + } + // The image manifest goes last: its presence is what makes the + // directory an image. + if err := image.SaveManifest(filepath.Join(dir, image.ManifestFile), im); err != nil { + return err + } + prog.Close() + os.Remove(filepath.Join(dir, progressFile)) + logf("done in %v: image %q in %s", time.Since(t0).Round(time.Second), filepath.Base(dir), dir) + return nil +} + +// vmConfig is the VM configuration layer of these images. +type vmConfig struct { + OS string `json:"os"` + Arch string `json:"arch"` + HardwareModel []byte `json:"hardwareModel"` // VZMacHardwareModel.dataRepresentation + ECID []byte `json:"ecid"` // VZMacMachineIdentifier.dataRepresentation + CPUCount int `json:"cpuCount"` + MemorySize uint64 `json:"memorySize"` + DiskFormat string `json:"diskFormat"` + Display struct { + Width int `json:"width"` + Height int `json:"height"` + } `json:"display"` +} + +// imageManifest converts the VM configuration layer into a mantsana image +// manifest. +func imageManifest(configJSON []byte, diskBytes int64, r Ref, manifestDigest string) (*image.Manifest, error) { + var vc vmConfig + if err := json.Unmarshal(configJSON, &vc); err != nil { + return nil, fmt.Errorf("parsing VM config: %w", err) + } + if vc.OS != "darwin" || vc.Arch != "arm64" { + return nil, fmt.Errorf("VM is %s/%s; only darwin/arm64 is supported", vc.OS, vc.Arch) + } + if vc.DiskFormat != "" && vc.DiskFormat != "raw" { + return nil, fmt.Errorf("disk format %q; only raw is supported", vc.DiskFormat) + } + if len(vc.HardwareModel) == 0 || len(vc.ECID) == 0 { + return nil, errors.New("VM config lacks hardwareModel or ecid") + } + m := &image.Manifest{ + Version: 1, + OS: "macos", + Arch: "arm64", + HardwareModel: vc.HardwareModel, + MachineID: vc.ECID, + // Our fixed guest MAC rather than the image's, so every image looks + // the same to guestnet. + MAC: image.DefaultMAC, + CPUs: vc.CPUCount, + MemoryBytes: vc.MemorySize, + Display: image.DefaultDisplay, + MTU: image.DefaultMTU, + DiskBytes: diskBytes, + Created: time.Now().UTC().Truncate(time.Second), + Description: fmt.Sprintf("downloaded from %s (%s)", r, manifestDigest), + } + if vc.Display.Width > 0 && vc.Display.Height > 0 { + m.Display = image.Display{Width: vc.Display.Width, Height: vc.Display.Height, PPI: 72} + } + if err := m.Validate(); err != nil { + return nil, err + } + return m, nil +} + +// fetchChunk downloads one disk chunk, writes its non-zero pages to disk, +// and verifies its uncompressed digest. got counts compressed bytes read. +func fetchChunk(ctx context.Context, c *client, disk *os.File, ch chunk, got *atomic.Int64) error { + body, err := c.blob(ctx, ch.desc.Digest) + if err != nil { + return err + } + defer body.Close() + var counted int64 + defer func() { got.Add(-counted) }() // only completed chunks count toward progress + h := sha256.New() + off := ch.off + cr := &countingReader{r: body, n: got, local: &counted} + err = decodeAppleLZ4(cr, func(b []byte) error { + if off+int64(len(b)) > ch.off+ch.size { + return errors.New("chunk decodes to more than its uncompressed size") + } + h.Write(b) + if err := writeNonZero(disk, b, off); err != nil { + return err + } + off += int64(len(b)) + return nil + }) + if err != nil { + return err + } + if off != ch.off+ch.size { + return fmt.Errorf("chunk decoded to %d bytes, want %d", off-ch.off, ch.size) + } + want := ch.desc.Annotations[annotUncompressedDigest] + if gotDigest := "sha256:" + hex.EncodeToString(h.Sum(nil)); gotDigest != want { + return fmt.Errorf("uncompressed digest %s, want %s", gotDigest, want) + } + counted = 0 // keep this chunk's bytes in the total + return nil +} + +// writeNonZero writes b at off, skipping all-zero 4 KiB pages so they stay +// holes in the sparse file. off must be page aligned. +func writeNonZero(f *os.File, b []byte, off int64) error { + var zeros [pageSize]byte + start := -1 // start of the current run of non-zero pages + flush := func(end int) error { + if start < 0 { + return nil + } + _, err := f.WriteAt(b[start:end], off+int64(start)) + start = -1 + return err + } + for i := 0; i < len(b); i += pageSize { + p := b[i:min(i+pageSize, len(b))] + if bytes.Equal(p, zeros[:len(p)]) { + if err := flush(i); err != nil { + return err + } + continue + } + if start < 0 { + start = i + } + } + return flush(len(b)) +} + +// countingReader counts bytes read into two counters. +type countingReader struct { + r io.Reader + n *atomic.Int64 + local *int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n.Add(int64(n)) + *c.local += int64(n) + return n, err +} + +// retry calls f until it succeeds, up to 5 times with exponential backoff, +// logging failures as what. +func retry(ctx context.Context, logf logger.Logf, what string, f func() error) error { + var err error + for attempt := range 5 { + if err = f(); err == nil || ctx.Err() != nil { + return err + } + logf("%s, attempt %d: %v", what, attempt+1, err) + select { + case <-time.After(time.Second << attempt): + case <-ctx.Done(): + return ctx.Err() + } + } + return err +} + +// fetchSmall downloads a small layer, verifying its digest, with retries. +func fetchSmall(ctx context.Context, logf logger.Logf, c *client, d descriptor) (b []byte, err error) { + err = retry(ctx, logf, d.MediaType+" layer", func() error { + b, err = fetchSmallOnce(ctx, c, d) + return err + }) + return b, err +} + +func fetchSmallOnce(ctx context.Context, c *client, d descriptor) ([]byte, error) { + if d.Size > 1<<30 { + return nil, fmt.Errorf("%s layer is %d bytes; too big", d.MediaType, d.Size) + } + body, err := c.blob(ctx, d.Digest) + if err != nil { + return nil, err + } + defer body.Close() + b, err := io.ReadAll(io.LimitReader(body, d.Size+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != d.Digest { + return nil, fmt.Errorf("%s layer: digest %s, want %s", d.MediaType, got, d.Digest) + } + return b, nil +} + +// loadProgress returns the chunks already downloaded into dir for the +// manifest with the given digest. +func loadProgress(logf logger.Logf, dir, manifestDigest string) (map[int]bool, error) { + b, err := os.ReadFile(filepath.Join(dir, progressFile)) + if errors.Is(err, os.ErrNotExist) { + return map[int]bool{}, nil + } + if err != nil { + return nil, err + } + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + done := map[int]bool{} + if len(lines) == 0 || lines[0] != manifestDigest { + logf("existing download in %s is of a different image; starting over", dir) + return done, nil + } + for _, l := range lines[1:] { + if i, err := strconv.Atoi(l); err == nil { + done[i] = true + } + } + return done, nil +} + +// openProgress opens the progress file for appending, starting it afresh +// if fresh. +func openProgress(dir, manifestDigest string, fresh bool) (*os.File, error) { + path := filepath.Join(dir, progressFile) + if fresh { + if err := os.WriteFile(path, []byte(manifestDigest+"\n"), 0o644); err != nil { + return nil, err + } + } + return os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) +} diff --git a/cmd/oci-img-dl/ocidl/ocidl_test.go b/cmd/oci-img-dl/ocidl/ocidl_test.go new file mode 100644 index 0000000..28aba31 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/ocidl_test.go @@ -0,0 +1,377 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/pierrec/lz4/v4" + "github.com/tailscale/tb/mantsana/image" +) + +// encodeAppleLZ4 frames data as Apple LZ4 blocks of blockSize, compressing +// each block independently (or storing it raw if it doesn't compress). +func encodeAppleLZ4(t testing.TB, data []byte, blockSize int) []byte { + t.Helper() + var out bytes.Buffer + var ht [1 << 16]int + for len(data) > 0 { + blk := data[:min(blockSize, len(data))] + data = data[len(blk):] + dst := make([]byte, lz4.CompressBlockBound(len(blk))) + n, err := lz4.CompressBlock(blk, dst, ht[:]) + if err != nil { + t.Fatal(err) + } + if n == 0 || n >= len(blk) { + out.WriteString(magicRawBlock) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + out.Write(blk) + continue + } + out.WriteString(magicLZ4Block) + binary.Write(&out, binary.LittleEndian, uint32(len(blk))) + binary.Write(&out, binary.LittleEndian, uint32(n)) + out.Write(dst[:n]) + } + out.WriteString(magicEnd) + return out.Bytes() +} + +func decodeAll(t testing.TB, stream []byte) ([]byte, error) { + t.Helper() + var out bytes.Buffer + err := decodeAppleLZ4(bytes.NewReader(stream), func(b []byte) error { + out.Write(b) + return nil + }) + return out.Bytes(), err +} + +func TestDecodeAppleLZ4(t *testing.T) { + data := make([]byte, 300<<10) + for i := range data { + data[i] = byte(i / 7) // compressible + } + copy(data[100<<10:], bytes.Repeat([]byte{0xff, 0x13, 0x77}, 1000)) + rnd := make([]byte, 5000) // incompressible, to get a raw block + for i := range rnd { + rnd[i] = byte(i*7919 + i*i*31) + } + data = append(data, rnd...) + got, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, data) { + t.Fatal("round trip mismatch") + } + + if _, err := decodeAll(t, []byte("bv4X")); err == nil { + t.Error("unknown magic accepted") + } + if _, err := decodeAll(t, encodeAppleLZ4(t, data, 64<<10)[:1000]); err == nil { + t.Error("truncated stream accepted") + } +} + +// TestDecodeAppleLZ4Dict checks that a block may copy from the previous +// block's output. +func TestDecodeAppleLZ4Dict(t *testing.T) { + first := []byte("0123456789abcdefghij") + // Second block: a match of 8 bytes at offset 10 (into the first block), + // then the required trailing literals. + blk := []byte{ + 0x04, // token: 0 literals, match length 4+4 + 10, 0, // offset 10 + 0x50, // token: 5 literals, no match + 'V', 'W', 'X', 'Y', 'Z', // literals + } + var s bytes.Buffer + s.WriteString(magicRawBlock) + binary.Write(&s, binary.LittleEndian, uint32(len(first))) + s.Write(first) + s.WriteString(magicLZ4Block) + binary.Write(&s, binary.LittleEndian, uint32(13)) + binary.Write(&s, binary.LittleEndian, uint32(len(blk))) + s.Write(blk) + s.WriteString(magicEnd) + got, err := decodeAll(t, s.Bytes()) + if err != nil { + t.Fatal(err) + } + want := string(first) + "abcdefgh" + "VWXYZ" + if string(got) != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestParseRef(t *testing.T) { + for _, tt := range []struct{ in, reg, repo, ref string }{ + {"cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode:26.5", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "26.5"}, + {"ghcr.io/cirruslabs/macos-tahoe-xcode", "ghcr.io", "cirruslabs/macos-tahoe-xcode", "latest"}, + {"ghcr.io/cirruslabs/x@sha256:abcd", "ghcr.io", "cirruslabs/x", "sha256:abcd"}, + {"localhost:5000/foo/bar:v1", "localhost:5000", "foo/bar", "v1"}, + } { + r, err := ParseRef(tt.in) + if err != nil { + t.Errorf("ParseRef(%q): %v", tt.in, err) + continue + } + if r.Registry != tt.reg || r.Repo != tt.repo || r.TagOrDigest != tt.ref { + t.Errorf("ParseRef(%q) = %+v", tt.in, r) + } + } +} + +func sha(b []byte) string { + s := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(s[:]) +} + +// fakeRegistry serves one image, requiring a bearer token obtained through +// the WWW-Authenticate challenge. +type fakeRegistry struct { + blobs map[string][]byte + manifest []byte + failBlob atomic.Pointer[string] // blob digest to fail once + blobGets atomic.Int64 + corruptOne atomic.Bool +} + +func (f *fakeRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + if r.URL.Query().Get("scope") != "repository:test/vm:pull" { + http.Error(w, "bad scope", 400) + return + } + json.NewEncoder(w).Encode(map[string]string{"token": "tok123"}) + return + } + if r.Header.Get("Authorization") != "Bearer tok123" { + w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="http://%s/token",service="fake",scope="repository:test/vm:pull"`, r.Host)) + http.Error(w, "unauthorized", 401) + return + } + switch { + case r.URL.Path == "/v2/test/vm/manifests/v1": + w.Header().Set("Content-Type", mediaTypeManifest) + w.Write(f.manifest) + case strings.HasPrefix(r.URL.Path, "/v2/test/vm/blobs/"): + d := strings.TrimPrefix(r.URL.Path, "/v2/test/vm/blobs/") + b, ok := f.blobs[d] + if !ok { + http.NotFound(w, r) + return + } + f.blobGets.Add(1) + if p := f.failBlob.Load(); p != nil && *p == d { + f.failBlob.Store(nil) + w.Write(b[:len(b)/2]) // truncated + return + } + if f.corruptOne.CompareAndSwap(true, false) && len(b) > 100 { + c := bytes.Clone(b) + c[len(c)-10] ^= 0xff + w.Write(c) + return + } + w.Write(b) + default: + http.NotFound(w, r) + } +} + +// newFakeImage builds a fake VM image with a disk of three chunks, one of +// them all zeros. +func newFakeImage(t *testing.T) (*fakeRegistry, []byte) { + const chunkSize = 256 << 10 + disk := make([]byte, 3*chunkSize-8192) // last chunk is short + for i := range chunkSize { + disk[i] = byte(i % 251) + } + // Chunk 1 stays zero; chunk 2 has data in its second page only. + copy(disk[2*chunkSize+pageSize:], bytes.Repeat([]byte("mantsana"), 512)) + + f := &fakeRegistry{blobs: map[string][]byte{}} + add := func(b []byte) string { + d := sha(b) + f.blobs[d] = b + return d + } + config := []byte(`{"os":"darwin","arch":"arm64","cpuCount":2,"memorySize":4294967296,` + + `"diskFormat":"raw","hardwareModel":"aHc=","ecid":"aWQ=","display":{"width":1024,"height":768}}`) + nvram := []byte("nvram bytes") + m := manifest{ + MediaType: mediaTypeManifest, + Config: descriptor{MediaType: "application/vnd.oci.image.config.v1+json", Digest: add([]byte("{}")), Size: 2}, + Layers: []descriptor{ + {MediaType: mediaTypeConfig, Digest: add(config), Size: int64(len(config))}, + }, + } + for off := 0; off < len(disk); off += chunkSize { + part := disk[off:min(off+chunkSize, len(disk))] + z := encodeAppleLZ4(t, part, 64<<10) + m.Layers = append(m.Layers, descriptor{ + MediaType: mediaTypeDisk, Digest: add(z), Size: int64(len(z)), + Annotations: map[string]string{ + annotUncompressedSize: strconv.Itoa(len(part)), + annotUncompressedDigest: sha(part), + }, + }) + } + m.Layers = append(m.Layers, descriptor{MediaType: mediaTypeNVRAM, Digest: add(nvram), Size: int64(len(nvram))}) + var err error + if f.manifest, err = json.Marshal(m); err != nil { + t.Fatal(err) + } + return f, disk +} + +func testRef(t *testing.T, srv *httptest.Server) Ref { + r, err := ParseRef(strings.TrimPrefix(srv.URL, "http://") + "/test/vm:v1") + if err != nil { + t.Fatal(err) + } + return r +} + +// withHTTP makes clients use plain HTTP for the duration of the test, for +// httptest servers. +func withHTTP(t *testing.T) { + old := scheme + scheme = "http" + t.Cleanup(func() { scheme = old }) +} + +func TestDownload(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + ctx := context.Background() + + // One truncated chunk response and one corrupted one are retried. + var chunk2 string + var m manifest + json.Unmarshal(f.manifest, &m) + for _, l := range m.Layers { + if l.MediaType == mediaTypeDisk { + chunk2 = l.Digest + } + } + f.failBlob.Store(&chunk2) + f.corruptOne.Store(true) + + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filepath.Join(dir, "disk.img")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, disk) { + t.Fatal("disk.img differs from the image's disk") + } + if b, _ := os.ReadFile(filepath.Join(dir, image.AuxFile)); string(b) != "nvram bytes" { + t.Errorf("%s = %q", image.AuxFile, b) + } + if b, _ := os.ReadFile(filepath.Join(dir, ociManifestFile)); !bytes.Equal(b, f.manifest) { + t.Error("OCI manifest not saved") + } + if _, err := os.Stat(filepath.Join(dir, progressFile)); !os.IsNotExist(err) { + t.Error("progress file left behind") + } + // APFS allocates holes of up to about 16 MiB between written extents + // (observed on macOS 26.5), so this tiny disk is fully allocated there. + if blocks := allocatedBlocks(t, filepath.Join(dir, image.DiskFile)); runtime.GOOS != "darwin" && blocks >= 0 && blocks*512 >= int64(len(disk)) { + t.Errorf("disk.img is not sparse: %d bytes allocated for %d", blocks*512, len(disk)) + } + + // The directory is a mantsana image. + im, err := (&image.Store{Root: filepath.Dir(dir)}).Get(filepath.Base(dir)) + if err != nil { + t.Fatalf("loading downloaded image: %v", err) + } + mm := im.Manifest + if mm.OS != "macos" || mm.CPUs != 2 || mm.MemoryBytes != 4<<30 || string(mm.HardwareModel) != "hw" || + string(mm.MachineID) != "id" || mm.Display.Width != 1024 || mm.DiskBytes != int64(len(disk)) { + t.Errorf("image manifest = %+v", mm) + } + + // A complete image isn't overwritten. + if err := Download(ctx, testRef(t, srv), dir, Options{Logf: t.Logf}); !errors.Is(err, ErrComplete) { + t.Errorf("second download: %v", err) + } +} + +func TestBadConfig(t *testing.T) { + withHTTP(t) + f, _ := newFakeImage(t) + var m manifest + json.Unmarshal(f.manifest, &m) + for i, l := range m.Layers { + if l.MediaType == mediaTypeConfig { + linux := []byte(`{"os":"linux","arch":"arm64","cpuCount":2,"memorySize":1}`) + m.Layers[i].Digest, m.Layers[i].Size = sha(linux), int64(len(linux)) + f.blobs[sha(linux)] = linux + } + } + f.manifest, _ = json.Marshal(m) + srv := httptest.NewServer(f) + defer srv.Close() + before := f.blobGets.Load() + err := Download(context.Background(), testRef(t, srv), filepath.Join(t.TempDir(), "img"), Options{Logf: t.Logf}) + if err == nil || !strings.Contains(err.Error(), "only darwin/arm64") { + t.Fatalf("download of a linux VM: %v", err) + } + if n := f.blobGets.Load() - before; n != 1 { + t.Errorf("fetched %d blobs before rejecting the config, want 1", n) + } +} + +func TestResume(t *testing.T) { + withHTTP(t) + f, disk := newFakeImage(t) + srv := httptest.NewServer(f) + defer srv.Close() + dir := filepath.Join(t.TempDir(), "img") + r := testRef(t, srv) + + // Pretend chunk 0 was already done, with its data in place. + os.MkdirAll(dir, 0o755) + os.WriteFile(filepath.Join(dir, progressFile), []byte(sha(f.manifest)+"\n0\n"), 0o644) + os.WriteFile(filepath.Join(dir, "disk.img"), disk[:256<<10], 0o644) + + before := f.blobGets.Load() + if err := Download(context.Background(), r, dir, Options{Logf: t.Logf}); err != nil { + t.Fatal(err) + } + // config, nvram, and two of the three chunks. + if n := f.blobGets.Load() - before; n != 4 { + t.Errorf("fetched %d blobs on resume, want 4", n) + } + got, _ := os.ReadFile(filepath.Join(dir, "disk.img")) + if !bytes.Equal(got, disk) { + t.Fatal("resumed disk.img differs") + } +} diff --git a/cmd/oci-img-dl/ocidl/registry.go b/cmd/oci-img-dl/ocidl/registry.go new file mode 100644 index 0000000..77374ff --- /dev/null +++ b/cmd/oci-img-dl/ocidl/registry.go @@ -0,0 +1,209 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ocidl + +import ( + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" +) + +// Ref is a parsed image reference. +type Ref struct { + Registry string // "ghcr.io" + Repo string // "cirruslabs/macos-tahoe-xcode" + TagOrDigest string // "26.5" or "sha256:..." +} + +// ParseRef parses "[registry/]repo[:tag|@digest]". The registry defaults +// to ghcr.io and the tag to "latest". +func ParseRef(s string) (Ref, error) { + var r Ref + name := s + if i := strings.Index(s, "@"); i >= 0 { + name, r.TagOrDigest = s[:i], s[i+1:] + } else if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") { + name, r.TagOrDigest = s[:i], s[i+1:] + } else { + r.TagOrDigest = "latest" + } + first, rest, ok := strings.Cut(name, "/") + if ok && (strings.ContainsAny(first, ".:") || first == "localhost") { + r.Registry, r.Repo = first, rest + } else { + r.Registry, r.Repo = "ghcr.io", name + } + if r.Repo == "" || r.TagOrDigest == "" { + return Ref{}, fmt.Errorf("bad image reference %q", s) + } + return r, nil +} + +func (r Ref) String() string { + sep := ":" + if strings.HasPrefix(r.TagOrDigest, "sha256:") { + sep = "@" + } + return r.Registry + "/" + r.Repo + sep + r.TagOrDigest +} + +// client fetches from one repository, handling the registry's anonymous +// bearer token challenge. +type client struct { + hc *http.Client + base string // "https://ghcr.io/v2/cirruslabs/macos-tahoe-xcode" + + mu sync.Mutex + token string +} + +// scheme is the URL scheme for registries. Tests set it to "http". +var scheme = "https" + +func newClient(r Ref) *client { + return &client{hc: http.DefaultClient, base: scheme + "://" + r.Registry + "/v2/" + r.Repo} +} + +// get fetches path under the repository, authenticating if challenged. +// The caller must close the response body. +func (c *client) get(ctx context.Context, path, accept string) (*http.Response, error) { + for attempt := 0; ; attempt++ { + req, err := http.NewRequestWithContext(ctx, "GET", c.base+path, nil) + if err != nil { + return nil, err + } + if accept != "" { + req.Header.Set("Accept", accept) + } + c.mu.Lock() + tok := c.token + c.mu.Unlock() + if tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + res, err := c.hc.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode == http.StatusUnauthorized && attempt == 0 { + challenge := res.Header.Get("WWW-Authenticate") + res.Body.Close() + if err := c.authenticate(ctx, challenge); err != nil { + return nil, err + } + continue + } + if res.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(res.Body, 1<<10)) + res.Body.Close() + return nil, fmt.Errorf("GET %s: %s: %s", c.base+path, res.Status, strings.TrimSpace(string(b))) + } + return res, nil + } +} + +// authenticate gets an anonymous token for a +// `Bearer realm="...",service="...",scope="..."` challenge. +func (c *client) authenticate(ctx context.Context, challenge string) error { + scheme, params, _ := strings.Cut(challenge, " ") + if !strings.EqualFold(scheme, "Bearer") { + return fmt.Errorf("unsupported auth challenge %q", challenge) + } + p := map[string]string{} + for _, kv := range strings.Split(params, ",") { + k, v, _ := strings.Cut(strings.TrimSpace(kv), "=") + p[k] = strings.Trim(v, `"`) + } + if p["realm"] == "" { + return fmt.Errorf("auth challenge %q has no realm", challenge) + } + q := url.Values{} + for _, k := range []string{"service", "scope"} { + if p[k] != "" { + q.Set(k, p[k]) + } + } + req, err := http.NewRequestWithContext(ctx, "GET", p["realm"]+"?"+q.Encode(), nil) + if err != nil { + return err + } + res, err := c.hc.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("getting token: %s", res.Status) + } + var t struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(res.Body).Decode(&t); err != nil { + return fmt.Errorf("getting token: %w", err) + } + tok := cmp.Or(t.Token, t.AccessToken) + if tok == "" { + return errors.New("token response has no token") + } + c.mu.Lock() + c.token = tok + c.mu.Unlock() + return nil +} + +// descriptor is an OCI content descriptor. +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// manifest is an OCI image manifest. +type manifest struct { + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +const mediaTypeManifest = "application/vnd.oci.image.manifest.v1+json" + +// manifest fetches the manifest for r, returning it and its raw bytes. +func (c *client) manifest(ctx context.Context, r Ref) (*manifest, []byte, error) { + res, err := c.get(ctx, "/manifests/"+r.TagOrDigest, mediaTypeManifest) + if err != nil { + return nil, nil, err + } + defer res.Body.Close() + raw, err := io.ReadAll(io.LimitReader(res.Body, 16<<20)) + if err != nil { + return nil, nil, err + } + m := new(manifest) + if err := json.Unmarshal(raw, m); err != nil { + return nil, nil, fmt.Errorf("parsing manifest: %w", err) + } + if m.MediaType != "" && m.MediaType != mediaTypeManifest { + return nil, nil, fmt.Errorf("unsupported manifest media type %q", m.MediaType) + } + return m, raw, nil +} + +// blob opens the blob with the given digest. +func (c *client) blob(ctx context.Context, digest string) (io.ReadCloser, error) { + res, err := c.get(ctx, "/blobs/"+digest, "") + if err != nil { + return nil, err + } + return res.Body, nil +} diff --git a/cmd/oci-img-dl/ocidl/sparse_other_test.go b/cmd/oci-img-dl/ocidl/sparse_other_test.go new file mode 100644 index 0000000..5ccf42c --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_other_test.go @@ -0,0 +1,11 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !unix + +package ocidl + +import "testing" + +// allocatedBlocks returns -1: allocation isn't checked on this platform. +func allocatedBlocks(t *testing.T, path string) int64 { return -1 } diff --git a/cmd/oci-img-dl/ocidl/sparse_unix_test.go b/cmd/oci-img-dl/ocidl/sparse_unix_test.go new file mode 100644 index 0000000..0906220 --- /dev/null +++ b/cmd/oci-img-dl/ocidl/sparse_unix_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build unix + +package ocidl + +import ( + "os" + "syscall" + "testing" +) + +// allocatedBlocks returns the number of 512-byte blocks allocated to path. +func allocatedBlocks(t *testing.T, path string) int64 { + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return fi.Sys().(*syscall.Stat_t).Blocks +} diff --git a/go.mod b/go.mod index c781e5a..a9d2079 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 golang.org/x/time v0.16.0 modernc.org/sqlite v1.51.0 tailscale.com v1.103.0-pre.0.20260920011920-7d96cf5a62ef @@ -69,7 +70,6 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect golang.org/x/net v0.59.0 // indirect - golang.org/x/sys v0.48.0 // indirect golang.zx2c4.com/wireguard/windows v1.0.1 // indirect google.golang.org/protobuf v1.36.12 // indirect modernc.org/libc v1.72.3 // indirect diff --git a/mantsana/image/clone_darwin.go b/mantsana/image/clone_darwin.go new file mode 100644 index 0000000..d7e21c4 --- /dev/null +++ b/mantsana/image/clone_darwin.go @@ -0,0 +1,25 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "errors" + "fmt" + + "golang.org/x/sys/unix" +) + +// CloneFile makes dst a copy-on-write clone of src. On APFS this is O(1) +// regardless of file size and consumes no space until written. dst must not +// exist. It falls back to a regular copy if the filesystem can't clone. +func CloneFile(src, dst string) error { + err := unix.Clonefile(src, dst, 0) + if err == nil { + return nil + } + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EXDEV) { + return copyFile(src, dst) + } + return fmt.Errorf("clonefile %s -> %s: %w", src, dst, err) +} diff --git a/mantsana/image/clone_other.go b/mantsana/image/clone_other.go new file mode 100644 index 0000000..2d603a1 --- /dev/null +++ b/mantsana/image/clone_other.go @@ -0,0 +1,10 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !darwin + +package image + +// CloneFile copies src to dst. Only macOS has the O(1) APFS clone; elsewhere +// this is a plain copy, which is fine for development and tests. +func CloneFile(src, dst string) error { return copyFile(src, dst) } diff --git a/mantsana/image/copy.go b/mantsana/image/copy.go new file mode 100644 index 0000000..b33803e --- /dev/null +++ b/mantsana/image/copy.go @@ -0,0 +1,30 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "io" + "os" +) + +// copyFile copies src to dst, which must not exist, preserving sparseness +// only as well as io.Copy does (not at all). It is the fallback for +// CloneFile. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + os.Remove(dst) + return err + } + return out.Close() +} diff --git a/mantsana/image/image.go b/mantsana/image/image.go new file mode 100644 index 0000000..1a33803 --- /dev/null +++ b/mantsana/image/image.go @@ -0,0 +1,407 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package image describes mantsana's on-disk VM images. +// +// An image is a directory of plain files: a manifest, a raw disk image, and +// the macOS auxiliary storage (NVRAM) blob. A warm subdirectory holds a +// host-local memory snapshot for fast restore; it is derived data, keyed to +// the host OS build, and rebuilt when stale. +// +// For now images are just files on local disk. Later the disk will be +// fetched lazily from a content-addressed chunk store; that plugs in below +// this package at the NBD BaseImage seam, so this layout does not change. +package image + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +// File names within an image directory and its warm subdirectory. +const ( + ManifestFile = "manifest.json" + DiskFile = "disk.img" + AuxFile = "nvram.img" // not "aux.img": AUX is a reserved file name on Windows + WarmDir = "warm" + StateFile = "state.vzvmsave" // in WarmDir only + WarmFile = "warm.json" // in WarmDir only +) + +// Display is a guest display geometry. It is part of the image because a +// memory snapshot bakes it into the saved configuration. +type Display struct { + Width int `json:"width"` + Height int `json:"height"` + PPI int `json:"ppi"` +} + +// DefaultDisplay is used when a manifest doesn't specify a display. +var DefaultDisplay = Display{Width: 1280, Height: 800, PPI: 72} + +// DefaultMTU is used when a manifest doesn't specify an MTU. +const DefaultMTU = 1500 + +// DefaultMAC is the guest MAC address images use unless they have a reason +// not to. It is guestnet's default guest MAC, so every image looks the same +// to guestnet; each guest's network is private, so there is no conflict. +var DefaultMAC = MACAddr{0x02, 0x76, 0x7a, 0x00, 0x00, 0x02} + +// MACAddr is a net.HardwareAddr that encodes as text in JSON. +type MACAddr net.HardwareAddr + +// String returns the colon-separated form, or "" for an empty address. +func (m MACAddr) String() string { return net.HardwareAddr(m).String() } + +// MarshalText implements encoding.TextMarshaler. +func (m MACAddr) MarshalText() ([]byte, error) { return []byte(m.String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (m *MACAddr) UnmarshalText(b []byte) error { + if len(b) == 0 { + *m = nil + return nil + } + hw, err := net.ParseMAC(string(b)) + if err != nil { + return err + } + if len(hw) != 6 { + return fmt.Errorf("MAC %q is not 48 bits", b) + } + *m = MACAddr(hw) + return nil +} + +// Manifest describes an image. The hardware model, machine identifier, MAC, +// display and MTU are image properties because a memory snapshot depends on +// them. Two VMs from one image share them, which is fine since their +// networks are isolated. +type Manifest struct { + Version int `json:"version"` + OS string `json:"os"` // "macos" (later "linux") + Arch string `json:"arch"` // "arm64" + + // HardwareModel and MachineID are the VZMacHardwareModel and + // VZMacMachineIdentifier dataRepresentation blobs (base64 in JSON). + // Empty for non-macOS images. + HardwareModel []byte `json:"hardware_model,omitempty"` + MachineID []byte `json:"machine_id,omitempty"` + + MAC MACAddr `json:"mac,omitempty"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Display Display `json:"display"` + MTU uint32 `json:"mtu"` + DiskBytes int64 `json:"disk_bytes,omitempty"` + + Created time.Time `json:"created,omitzero"` + Description string `json:"description,omitempty"` +} + +// fillDefaults sets unset optional fields. +func (m *Manifest) fillDefaults() { + if m.Version == 0 { + m.Version = 1 + } + if m.Arch == "" { + m.Arch = "arm64" + } + if m.Display == (Display{}) { + m.Display = DefaultDisplay + } + if m.MTU == 0 { + m.MTU = DefaultMTU + } +} + +// Validate reports whether m is usable. +func (m *Manifest) Validate() error { + if m.Version != 1 { + return fmt.Errorf("unsupported manifest version %d", m.Version) + } + switch m.OS { + case "macos", "linux": + default: + return fmt.Errorf("unsupported os %q", m.OS) + } + if m.CPUs <= 0 || m.MemoryBytes == 0 { + return errors.New("cpus and memory_bytes must be set") + } + // Virtualization.framework rejects network MTUs outside this range. + if m.MTU < 1500 || m.MTU > 65535 { + return fmt.Errorf("mtu %d out of range 1500..65535", m.MTU) + } + return nil +} + +// LoadManifest reads and validates the manifest at path, filling defaults. +func LoadManifest(path string) (*Manifest, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + m := new(Manifest) + if err := json.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + m.fillDefaults() + if err := m.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return m, nil +} + +// SaveManifest atomically writes m to path. +func SaveManifest(path string, m *Manifest) error { + return writeJSON(path, m) +} + +// WarmInfo records what a warm snapshot was built against, so a stale one is +// never restored. +type WarmInfo struct { + HostOSBuild string `json:"host_os_build"` + MantsanaVersion string `json:"mantsana_version,omitempty"` + ManifestHash string `json:"manifest_hash"` + CPUs int `json:"cpus"` + MemoryBytes uint64 `json:"memory_bytes"` + Created time.Time `json:"created,omitzero"` + + // Stale is set when a restore from this snapshot failed. StaleReason + // says why. + Stale bool `json:"stale,omitempty"` + StaleReason string `json:"stale_reason,omitempty"` +} + +// Image is an image directory on disk. +type Image struct { + Name string + Dir string + Manifest Manifest + // Warm is the warm snapshot's info, or nil if the image has no complete + // warm snapshot. + Warm *WarmInfo +} + +// DiskPath returns the path to the pristine raw disk image. +func (im *Image) DiskPath() string { return filepath.Join(im.Dir, DiskFile) } + +// AuxPath returns the path to the pristine auxiliary storage blob. +func (im *Image) AuxPath() string { return filepath.Join(im.Dir, AuxFile) } + +// WarmPath returns the path of file within the warm subdirectory, or the +// subdirectory itself if file is empty. +func (im *Image) WarmPath(file string) string { + return filepath.Join(im.Dir, WarmDir, file) +} + +// ManifestHash returns a stable hash of the manifest file's contents, so a +// changed image invalidates its warm snapshot. +func (im *Image) ManifestHash() (string, error) { + b, err := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + if err != nil { + return "", err + } + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]), nil +} + +// WarmUsable reports whether the warm snapshot can be restored on a host +// running hostOSBuild with the given CPU count and memory size. If not, why +// says why, for logs and the API. +func (im *Image) WarmUsable(hostOSBuild string, cpus int, memoryBytes uint64) (ok bool, why string) { + w := im.Warm + switch { + case w == nil: + return false, "no warm snapshot" + case w.Stale: + return false, "stale: " + w.StaleReason + case w.HostOSBuild != hostOSBuild: + return false, fmt.Sprintf("host OS changed (snapshot %s, host %s)", w.HostOSBuild, hostOSBuild) + case w.CPUs != cpus || w.MemoryBytes != memoryBytes: + return false, fmt.Sprintf("snapshot is %d CPUs/%d bytes, want %d/%d", w.CPUs, w.MemoryBytes, cpus, memoryBytes) + } + hash, err := im.ManifestHash() + if err != nil { + return false, err.Error() + } + if hash != w.ManifestHash { + return false, "image manifest changed since snapshot" + } + return true, "" +} + +// SaveWarmInfo writes wi as the warm snapshot's info, creating the warm +// directory if needed, and sets im.Warm. +func (im *Image) SaveWarmInfo(wi *WarmInfo) error { + if err := os.MkdirAll(im.WarmPath(""), 0o755); err != nil { + return err + } + if err := writeJSON(im.WarmPath(WarmFile), wi); err != nil { + return err + } + im.Warm = wi + return nil +} + +// MarkWarmStale records that the warm snapshot must not be used again, for +// example because restoring it failed. +func (im *Image) MarkWarmStale(reason string) error { + if im.Warm == nil { + return nil + } + wi := *im.Warm + wi.Stale = true + wi.StaleReason = reason + return im.SaveWarmInfo(&wi) +} + +// DeleteWarm removes the warm snapshot. +func (im *Image) DeleteWarm() error { + im.Warm = nil + return os.RemoveAll(im.WarmPath("")) +} + +// loadWarm loads the warm info. A missing warm directory is (nil, nil). A +// warm.json without all of its files is an error, since it means a warm-up +// was interrupted or files were removed by hand. +func (im *Image) loadWarm() (*WarmInfo, error) { + b, err := os.ReadFile(im.WarmPath(WarmFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + wi := new(WarmInfo) + if err := json.Unmarshal(b, wi); err != nil { + return nil, fmt.Errorf("%s: %w", im.WarmPath(WarmFile), err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + if _, err := os.Stat(im.WarmPath(f)); err != nil { + return nil, fmt.Errorf("incomplete warm snapshot: %w", err) + } + } + return wi, nil +} + +// NotFoundError is returned by [Store.Get] for a missing image. +type NotFoundError struct { + Name string +} + +func (e *NotFoundError) Error() string { return fmt.Sprintf("image %q not found", e.Name) } + +// Store is a directory of images, one subdirectory each. +type Store struct { + Root string +} + +// DefaultStoreDir returns the default image store directory for the current +// user: on macOS, ~/Library/Application Support/mantsana/images. It is not +// ~/Library/Caches, even though images can be downloaded again, because +// the system may purge caches, and warm snapshots are costly to rebuild. +func DefaultStoreDir() string { + dir, err := os.UserConfigDir() + if err != nil { + home, _ := os.UserHomeDir() + dir = home + } + return filepath.Join(dir, "mantsana", "images") +} + +// validName reports whether name is usable as a single path element. +func validName(name string) bool { + return name != "" && name != "." && name != ".." && + !strings.ContainsAny(name, `/\`) && !strings.HasPrefix(name, ".") +} + +// Get loads the named image. +func (s *Store) Get(name string) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + dir := filepath.Join(s.Root, name) + m, err := LoadManifest(filepath.Join(dir, ManifestFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, &NotFoundError{Name: name} + } + if err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + im := &Image{Name: name, Dir: dir, Manifest: *m} + if im.Warm, err = im.loadWarm(); err != nil { + return nil, fmt.Errorf("image %q: %w", name, err) + } + return im, nil +} + +// List returns the images in the store, sorted by name. Directories without +// a valid manifest are skipped. +func (s *Store) List() ([]*Image, error) { + ents, err := os.ReadDir(s.Root) + if err != nil { + return nil, err + } + var out []*Image + for _, e := range ents { + if !e.IsDir() || !validName(e.Name()) { + continue + } + if im, err := s.Get(e.Name()); err == nil { + out = append(out, im) + } + } + return out, nil +} + +// Create makes a new image directory containing only a manifest, filling +// manifest defaults. The caller then writes the disk and aux files (for +// example with a macOS installer run). It fails if the image exists. +func (s *Store) Create(name string, m Manifest) (*Image, error) { + if !validName(name) { + return nil, fmt.Errorf("invalid image name %q", name) + } + m.fillDefaults() + if m.Created.IsZero() { + m.Created = time.Now().UTC().Truncate(time.Second) + } + if err := m.Validate(); err != nil { + return nil, err + } + if err := os.MkdirAll(s.Root, 0o755); err != nil { + return nil, err + } + dir := filepath.Join(s.Root, name) + if err := os.Mkdir(dir, 0o755); err != nil { + return nil, err + } + if err := SaveManifest(filepath.Join(dir, ManifestFile), &m); err != nil { + os.RemoveAll(dir) + return nil, err + } + return &Image{Name: name, Dir: dir, Manifest: m}, nil +} + +// writeJSON atomically writes v as indented JSON to path. +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} diff --git a/mantsana/image/image_test.go b/mantsana/image/image_test.go new file mode 100644 index 0000000..f735cea --- /dev/null +++ b/mantsana/image/image_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package image + +import ( + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "testing" +) + +func TestStoreRoundTrip(t *testing.T) { + s := &Store{Root: t.TempDir()} + mac, _ := net.ParseMAC("02:76:7a:00:00:02") + im, err := s.Create("macos-test", Manifest{ + OS: "macos", + Arch: "arm64", + HardwareModel: []byte("hw"), + MachineID: []byte("id"), + MAC: MACAddr(mac), + CPUs: 4, + MemoryBytes: 8 << 30, + }) + if err != nil { + t.Fatal(err) + } + if _, err := s.Create("macos-test", Manifest{}); err == nil { + t.Fatal("duplicate Create succeeded") + } + + got, err := s.Get("macos-test") + if err != nil { + t.Fatal(err) + } + if got.Manifest.MAC.String() != "02:76:7a:00:00:02" { + t.Errorf("MAC = %v", got.Manifest.MAC) + } + if got.Manifest.Display != DefaultDisplay { + t.Errorf("Display = %+v, want default", got.Manifest.Display) + } + if got.Manifest.MTU != 1500 { + t.Errorf("MTU = %d, want 1500", got.Manifest.MTU) + } + if got.Warm != nil { + t.Errorf("Warm = %+v, want nil", got.Warm) + } + + // The MAC is encoded as text. + b, _ := os.ReadFile(filepath.Join(im.Dir, ManifestFile)) + var raw map[string]any + json.Unmarshal(b, &raw) + if raw["mac"] != "02:76:7a:00:00:02" { + t.Errorf("mac in JSON = %v", raw["mac"]) + } + + list, err := s.List() + if err != nil || len(list) != 1 || list[0].Name != "macos-test" { + t.Fatalf("List = %v, %v", list, err) + } + + if _, err := s.Get("nope"); err == nil { + t.Fatal("Get missing succeeded") + } else { + var nf *NotFoundError + if !errors.As(err, &nf) { + t.Errorf("err = %v, want NotFoundError", err) + } + } + if _, err := s.Get("../etc"); err == nil { + t.Fatal("Get with path traversal succeeded") + } +} + +func TestWarm(t *testing.T) { + s := &Store{Root: t.TempDir()} + im, err := s.Create("img", Manifest{OS: "macos", CPUs: 4, MemoryBytes: 1 << 30}) + if err != nil { + t.Fatal(err) + } + if ok, why := im.WarmUsable("25A354", 4, 1<<30); ok || why != "no warm snapshot" { + t.Fatalf("WarmUsable = %v, %q", ok, why) + } + + // Warm info without the files is not usable and is not loaded. + hash, _ := im.ManifestHash() + wi := &WarmInfo{HostOSBuild: "25A354", ManifestHash: hash, CPUs: 4, MemoryBytes: 1 << 30} + if err := im.SaveWarmInfo(wi); err != nil { + t.Fatal(err) + } + if got, err := s.Get("img"); err == nil || got != nil { + t.Fatalf("Get with incomplete warm dir = %v, %v; want error", got, err) + } + for _, f := range []string{DiskFile, AuxFile, StateFile} { + os.WriteFile(im.WarmPath(f), []byte("x"), 0o644) + } + got, err := s.Get("img") + if err != nil { + t.Fatal(err) + } + if ok, why := got.WarmUsable("25A354", 4, 1<<30); !ok { + t.Errorf("WarmUsable = false: %s", why) + } + if ok, _ := got.WarmUsable("26A100", 4, 1<<30); ok { + t.Error("usable after host OS change") + } + if ok, _ := got.WarmUsable("25A354", 8, 1<<30); ok { + t.Error("usable with different CPU count") + } + if err := got.MarkWarmStale("restore failed"); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "stale: restore failed" { + t.Errorf("after stale: %v %q", ok, why) + } + + // Changing the manifest invalidates the snapshot. + got.Warm.Stale = false + got.SaveWarmInfo(got.Warm) + m := got.Manifest + m.Description = "changed" + SaveManifest(filepath.Join(got.Dir, ManifestFile), &m) + got, _ = s.Get("img") + if ok, why := got.WarmUsable("25A354", 4, 1<<30); ok || why != "image manifest changed since snapshot" { + t.Errorf("after manifest change: %v %q", ok, why) + } + + if err := got.DeleteWarm(); err != nil { + t.Fatal(err) + } + got, _ = s.Get("img") + if got.Warm != nil { + t.Error("warm survived DeleteWarm") + } +} + +func TestCloneFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + os.WriteFile(src, []byte("hello"), 0o644) + dst := filepath.Join(dir, "dst") + if err := CloneFile(src, dst); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(dst) + if string(b) != "hello" { + t.Fatalf("dst = %q", b) + } + if err := CloneFile(src, dst); err == nil { + t.Fatal("clone over existing file succeeded") + } +}