Slice stored .conda zip members instead of copying

conda.go

@@ -38,7 +38,7 @@ func openConda(raw []byte) (*tarReader, error) {
 		if !strings.HasPrefix(f.Name, "pkg-") && !strings.HasPrefix(f.Name, "info-") {
 			continue
 		}
-		entries, size, err := readCondaMember(f, len(files))
+		entries, size, err := readCondaMember(raw, f, len(files))
 		if err != nil {
 			return nil, err
 		}
@@ -62,19 +62,10 @@ func openConda(raw []byte) (*tarReader, error) {
 	return &tarReader{raw: raw, files: files, index: index}, nil
 }
 
-func readCondaMember(f *zip.File, initialEntryCount int) ([]tarFileEntry, int64, error) {
-	rc, err := f.Open()
-	if err != nil {
-		return nil, 0, fmt.Errorf("opening %s: %w", f.Name, err)
-	}
-	defer func() { _ = rc.Close() }()
-
-	data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1))
+func readCondaMember(raw []byte, f *zip.File, initialEntryCount int) ([]tarFileEntry, int64, error) {
+	data, err := condaMemberBytes(raw, f)
 	if err != nil {
-		return nil, 0, fmt.Errorf("reading %s: %w", f.Name, err)
-	}
-	if int64(len(data)) > maxDecompressedSize {
-		return nil, 0, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize)
+		return nil, 0, err
 	}
 
 	tr, err := openTarWithInitialEntryCount(data, "zstd", initialEntryCount)
@@ -87,3 +78,33 @@ func readCondaMember(f *zip.File, initialEntryCount int) ([]tarFileEntry, int64,
 	}
 	return tr.files, size, nil
 }
+
+// condaMemberBytes returns the raw bytes of a .conda zip member. Real
+// .conda packages store members uncompressed (zip.Store), so DataOffset
+// locates the payload inside raw and a slice returns it with no copy or
+// buffer growth. This bypasses archive/zip's CRC32 check on the member;
+// the inner zstd frame check catches corruption anyway. Non-Store members
+// or out-of-range headers fall through to the original Open+ReadAll path.
+func condaMemberBytes(raw []byte, f *zip.File) ([]byte, error) {
+	if f.Method == zip.Store {
+		size := f.CompressedSize64
+		if off, err := f.DataOffset(); err == nil &&
+			off >= 0 && size <= uint64(len(raw)) && off <= int64(len(raw))-int64(size) {
+			return raw[off : off+int64(size)], nil
+		}
+	}
+
+	rc, err := f.Open()
+	if err != nil {
+		return nil, fmt.Errorf("opening %s: %w", f.Name, err)
+	}
+	defer func() { _ = rc.Close() }()
+	data, err := io.ReadAll(io.LimitReader(rc, maxDecompressedSize+1))
+	if err != nil {
+		return nil, fmt.Errorf("reading %s: %w", f.Name, err)
+	}
+	if int64(len(data)) > maxDecompressedSize {
+		return nil, fmt.Errorf("%w: %s exceeds %d bytes", ErrDecompressLimit, f.Name, maxDecompressedSize)
+	}
+	return data, nil
+}

conda_test.go

@@ -6,13 +6,14 @@ import (
 	"bytes"
 	"errors"
 	"io"
+	"math/rand"
 	"strings"
 	"testing"
 
 	"github.com/klauspost/compress/zstd"
 )
 
-func writeTarZst(t *testing.T, files map[string]string) []byte {
+func writeTarZst(t testing.TB, files map[string]string) []byte {
 	t.Helper()
 	var buf bytes.Buffer
 	enc, err := zstd.NewWriter(&buf)
@@ -238,6 +239,101 @@ func TestOpenCondaStopsAtCombinedEntryLimit(t *testing.T) {
 	}
 }
 
+func TestCondaDeflatedMemberFallback(t *testing.T) {
+	// Real .conda packages use zip.Store, but a hand-built or repacked
+	// one might not. The reader must still open it via f.Open().
+	pkg := writeTarZst(t, map[string]string{"lib/x.so": "content"})
+	info := writeTarZst(t, map[string]string{"info/index.json": `{"name":"x"}`})
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, m := range []struct {
+		name string
+		data []byte
+	}{
+		{"pkg-x-1.tar.zst", pkg},
+		{"info-x-1.tar.zst", info},
+	} {
+		w, err := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Deflate})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := w.Write(m.data); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := openConda(buf.Bytes())
+	if err != nil {
+		t.Fatalf("openConda deflated: %v", err)
+	}
+	defer func() { _ = reader.Close() }()
+
+	rc, err := reader.Extract("lib/x.so")
+	if err != nil {
+		t.Fatalf("Extract: %v", err)
+	}
+	content, _ := io.ReadAll(rc)
+	_ = rc.Close()
+	if string(content) != "content" {
+		t.Errorf("content = %q", string(content))
+	}
+}
+
+// createBenchConda builds a .conda whose pkg member is filled with
+// random bytes so the stored .tar.zst is close to memberSize and the
+// zero-copy path has real work to avoid.
+func createBenchConda(b *testing.B, memberSize int) []byte {
+	b.Helper()
+	rnd := rand.New(rand.NewSource(1)) //nolint:gosec
+	blob := make([]byte, memberSize)
+	rnd.Read(blob)
+
+	var pkg bytes.Buffer
+	enc, err := zstd.NewWriter(&pkg)
+	if err != nil {
+		b.Fatal(err)
+	}
+	tw := tar.NewWriter(enc)
+	_ = tw.WriteHeader(&tar.Header{Name: "lib/blob", Size: int64(len(blob)), Mode: 0o644})
+	_, _ = tw.Write(blob)
+	_ = tw.Close()
+	_ = enc.Close()
+
+	info := writeTarZst(b, map[string]string{"info/index.json": `{"name":"bench"}`})
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, m := range []struct {
+		name string
+		data []byte
+	}{
+		{"pkg-bench-1.tar.zst", pkg.Bytes()},
+		{"info-bench-1.tar.zst", info},
+	} {
+		w, _ := zw.CreateHeader(&zip.FileHeader{Name: m.name, Method: zip.Store})
+		_, _ = w.Write(m.data)
+	}
+	_ = zw.Close()
+	return buf.Bytes()
+}
+
+func BenchmarkOpenConda(b *testing.B) {
+	raw := createBenchConda(b, 8<<20)
+	b.SetBytes(int64(len(raw)))
+	b.ReportAllocs()
+	for b.Loop() {
+		r, err := openConda(raw)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_ = r.Close()
+	}
+}
+
 func TestOpenDoesNotInferConda(t *testing.T) {
 	reader, err := OpenBytes("artifact", createTestConda(t))
 	if err != nil {