From f8152a346e6bdf7bc014e05ba676be0c2d1925ad Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 13 Sep 2026 08:56:23 -0400 Subject: [PATCH] Slice stored .conda zip members instead of copying Real .conda packages store their .tar.zst members uncompressed, so DataOffset locates the payload inside the outer bytes and a slice returns it with no io.ReadAll copy or buffer growth. Non-Store or out-of-range members fall back to Open+ReadAll. The stored path skips archive/zip's CRC32; the inner zstd frame check still catches corruption. --- conda.go | 47 +++++++++++++++++------- conda_test.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 14 deletions(-) diff --git a/conda.go b/conda.go index c4786f4..00da255 100644 --- a/conda.go +++ b/conda.go @@ -38,7 +38,7 @@ func openConda(raw []byte) (*tarReader, error) { if !strings.HasPrefix(f.Name, "pkg-") && !strings.HasPrefix(f.Name, "info-") { continue } - entries, size, err := readCondaMember(f, len(files)) + entries, size, err := readCondaMember(raw, f, len(files)) if err != nil { return nil, err } @@ -62,19 +62,10 @@ func openConda(raw []byte) (*tarReader, error) { return &tarReader{raw: raw, files: files, index: index}, nil } -func readCondaMember(f *zip.File, initialEntryCount int) ([]tarFileEntry, int64, error) { - rc, err := f.Open() - if err != nil { - return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err) - } - defer func() { _ = rc.Close() }() - - data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1)) +func readCondaMember(raw []byte, f *zip.File, initialEntryCount int) ([]tarFileEntry, int64, error) { + data, err := condaMemberBytes(raw, f) if err != nil { - return nil, 0, fmt.Errorf("reading %s: %w", f.Name, err) - } - if int64(len(data)) > maxDecompressedSize { - return nil, 0, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize) + return nil, 0, err } tr, err := openTarWithInitialEntryCount(data, "zstd", initialEntryCount) @@ -87,3 +78,33 @@ func readCondaMember(f *zip.File, initialEntryCount int) ([]tarFileEntry, int64, } return tr.files, size, nil } + +// condaMemberBytes returns the raw bytes of a .conda zip member. Real +// .conda packages store members uncompressed (zip.Store), so DataOffset +// locates the payload inside raw and a slice returns it with no copy or +// buffer growth. This bypasses archive/zip's CRC32 check on the member; +// the inner zstd frame check catches corruption anyway. Non-Store members +// or out-of-range headers fall through to the original Open+ReadAll path. +func condaMemberBytes(raw []byte, f *zip.File) ([]byte, error) { + if f.Method == zip.Store { + size := f.CompressedSize64 + if off, err := f.DataOffset(); err == nil && + off >= 0 && size <= uint64(len(raw)) && off <= int64(len(raw))-int64(size) { + return raw[off : off+int64(size)], nil + } + } + + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("opening %s: %w", f.Name, err) + } + defer func() { _ = rc.Close() }() + data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1)) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", f.Name, err) + } + if int64(len(data)) > maxDecompressedSize { + return nil, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize) + } + return data, nil +} diff --git a/conda_test.go b/conda_test.go index f284514..d5efd0e 100644 --- a/conda_test.go +++ b/conda_test.go @@ -6,13 +6,14 @@ import ( "bytes" "errors" "io" + "math/rand" "strings" "testing" "github.com/klauspost/compress/zstd" ) -func writeTarZst(t *testing.T, files map[string]string) []byte { +func writeTarZst(t testing.TB, files map[string]string) []byte { t.Helper() var buf bytes.Buffer enc, err := zstd.NewWriter(&buf) @@ -238,6 +239,101 @@ func TestOpenCondaStopsAtCombinedEntryLimit(t *testing.T) { } } +func TestCondaDeflatedMemberFallback(t *testing.T) { + // Real .conda packages use zip.Store, but a hand-built or repacked + // one might not. The reader must still open it via f.Open(). + pkg := writeTarZst(t, map[string]string{"lib/x.so": "content"}) + info := writeTarZst(t, map[string]string{"info/index.json": `{"name":"x"}`}) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, m := range []struct { + name string + data []byte + }{ + {"pkg-x-1.tar.zst", pkg}, + {"info-x-1.tar.zst", info}, + } { + w, err := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Deflate}) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(m.data); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + reader, err := openConda(buf.Bytes()) + if err != nil { + t.Fatalf("openConda deflated: %v", err) + } + defer func() { _ = reader.Close() }() + + rc, err := reader.Extract("lib/x.so") + if err != nil { + t.Fatalf("Extract: %v", err) + } + content, _ := io.ReadAll(rc) + _ = rc.Close() + if string(content) != "content" { + t.Errorf("content = %q", string(content)) + } +} + +// createBenchConda builds a .conda whose pkg member is filled with +// random bytes so the stored .tar.zst is close to memberSize and the +// zero-copy path has real work to avoid. +func createBenchConda(b *testing.B, memberSize int) []byte { + b.Helper() + rnd := rand.New(rand.NewSource(1)) //nolint:gosec + blob := make([]byte, memberSize) + rnd.Read(blob) + + var pkg bytes.Buffer + enc, err := zstd.NewWriter(&pkg) + if err != nil { + b.Fatal(err) + } + tw := tar.NewWriter(enc) + _ = tw.WriteHeader(&tar.Header{Name: "lib/blob", Size: int64(len(blob)), Mode: 0o644}) + _, _ = tw.Write(blob) + _ = tw.Close() + _ = enc.Close() + + info := writeTarZst(b, map[string]string{"info/index.json": `{"name":"bench"}`}) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, m := range []struct { + name string + data []byte + }{ + {"pkg-bench-1.tar.zst", pkg.Bytes()}, + {"info-bench-1.tar.zst", info}, + } { + w, _ := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Store}) + _, _ = w.Write(m.data) + } + _ = zw.Close() + return buf.Bytes() +} + +func BenchmarkOpenConda(b *testing.B) { + raw := createBenchConda(b, 8<<20) + b.SetBytes(int64(len(raw))) + b.ReportAllocs() + for b.Loop() { + r, err := openConda(raw) + if err != nil { + b.Fatal(err) + } + _ = r.Close() + } +} + func TestOpenDoesNotInferConda(t *testing.T) { reader, err := OpenBytes("artifact", createTestConda(t)) if err != nil {