Reduce allocations in Composer, NuGet, Go and Python parsers

internal/composer/composer.go

@@ -111,6 +111,9 @@ func (p *composerLockParser) Parse(filename string, content []byte) (*core.Resul
 	}
 
 	var deps []core.Dependency
+	if count := len(lock.Packages) + len(lock.PackagesDev); count > 0 {
+		deps = make([]core.Dependency, 0, count)
+	}
 
 	for _, pkg := range lock.Packages {
 		integrity := ""

internal/golang/golang.go

@@ -42,13 +42,13 @@ var (
 )
 
 func (p *goModParser) Parse(filename string, content []byte) (*core.Result, error) {
-	lines := strings.Split(string(content), "\n")
-	tools := collectToolPaths(lines)
-	replaced := collectReplacedModules(lines)
-	deps, declarations := collectRequireDeps(lines, tools, replaced)
+	text := string(content)
+	tools := collectToolPaths(text)
+	replaced := collectReplacedModules(text)
+	deps, declarations := collectRequireDeps(text, tools, replaced)
 
 	var modulePath string
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		trimmed := strings.TrimSpace(line)
 		if strings.HasPrefix(trimmed, "module ") || strings.HasPrefix(trimmed, "module\t") {
 			modulePath = strings.TrimSpace(strings.Trim(strings.TrimSpace(trimmed[len("module"):]), `"`))
@@ -61,11 +61,11 @@ func (p *goModParser) Parse(filename string, content []byte) (*core.Result, erro
 
 // collectToolPaths scans go.mod lines for tool directives (both single-line and block form)
 // and returns a set of tool import paths.
-func collectToolPaths(lines []string) map[string]bool {
+func collectToolPaths(text string) map[string]bool {
 	tools := make(map[string]bool)
 	inToolBlock := false
 
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		trimmed := strings.TrimSpace(line)
 
 		if trimmed == "" || strings.HasPrefix(trimmed, "//") {
@@ -101,13 +101,13 @@ func collectToolPaths(lines []string) map[string]bool {
 
 // collectRequireDeps scans go.mod lines for require directives (both single-line and block form)
 // and returns dependencies, marking tool-related modules as development scope.
-func collectRequireDeps(lines []string, tools map[string]bool, replaced map[moduleVersion]bool) ([]core.Dependency, []core.Declaration) {
+func collectRequireDeps(text string, tools map[string]bool, replaced map[moduleVersion]bool) ([]core.Dependency, []core.Declaration) {
 	var deps []core.Dependency
 	var declarations []core.Declaration
 	inRequireBlock := false
 	locations := make(map[string]int)
 
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		trimmed := strings.TrimSpace(line)
 
 		if trimmed == "" || strings.HasPrefix(trimmed, "//") {
@@ -169,10 +169,10 @@ func appendGoDeclaration(
 
 // collectReplacedModules returns module paths named on the left side of a
 // replace directive, in either single-line or block form.
-func collectReplacedModules(lines []string) map[moduleVersion]bool {
+func collectReplacedModules(text string) map[moduleVersion]bool {
 	replaced := make(map[moduleVersion]bool)
 	inReplaceBlock := false
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		trimmed := strings.TrimSpace(line)
 		if trimmed == "" || strings.HasPrefix(trimmed, "//") {
 			continue
@@ -251,9 +251,9 @@ type goSumKey struct {
 func (p *goSumParser) Parse(filename string, content []byte) (*core.Result, error) {
 	var deps []core.Dependency
 	seen := make(map[goSumKey]bool)
-	lines := strings.Split(string(content), "\n")
+	text := string(content)
 
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
 			continue
@@ -311,11 +311,11 @@ func (p *goGraphParser) Parse(filename string, content []byte) (*core.Result, er
 	var deps []core.Dependency
 	seen := make(map[string]bool)
 	directDeps := make(map[string]bool)
-	lines := strings.Split(string(content), "\n")
+	text := string(content)
 
 	// First pass: identify direct dependencies (those required by the main module)
 	// The main module appears without a version in the first column
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
 			continue
@@ -340,7 +340,7 @@ func (p *goGraphParser) Parse(filename string, content []byte) (*core.Result, er
 	}
 
 	// Second pass: collect all dependencies
-	for _, line := range lines {
+	for line := range strings.SplitSeq(text, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
 			continue

internal/nuget/nuget.go

@@ -498,7 +498,14 @@ func (p *packagesLockParser) Parse(filename string, content []byte) (*core.Resul
 		return nil, &core.ParseError{Filename: filename, Err: err}
 	}
 
+	capacity := 0
+	for _, framework := range lock.Dependencies {
+		capacity = max(capacity, len(framework))
+	}
 	var deps []core.Dependency
+	if capacity > 0 {
+		deps = make([]core.Dependency, 0, capacity)
+	}
 	seen := make(map[string]bool)
 
 	for _, framework := range lock.Dependencies {
@@ -533,11 +540,12 @@ var (
 
 func (p *paketLockParser) Parse(filename string, content []byte) (*core.Result, error) {
 	var deps []core.Dependency
-	lines := strings.Split(string(content), "\n")
+	lines := strings.SplitSeq(string(content), "\n")
 	seen := make(map[string]bool)
 	inNuget := false
 
-	for _, line := range lines {
+	for line := range lines {
+		line = strings.TrimSuffix(line, "\r")
 		// Check for NUGET section
 		if line == "NUGET" {
 			inNuget = true
@@ -594,7 +602,20 @@ func (p *projectAssetsParser) Parse(filename string, content []byte) (*core.Resu
 		return nil, &core.ParseError{Filename: filename, Err: err}
 	}
 
+	capacity := 0
+	for _, framework := range assets.Targets {
+		count := 0
+		for key, pkg := range framework {
+			if pkg.Type == "package" && strings.Contains(key, "/") {
+				count++
+			}
+		}
+		capacity = max(capacity, count)
+	}
 	var deps []core.Dependency
+	if capacity > 0 {
+		deps = make([]core.Dependency, 0, capacity)
+	}
 	seen := make(map[string]bool)
 
 	for _, framework := range assets.Targets {
@@ -697,7 +718,16 @@ func parseLibraries(filename string, content []byte) ([]core.Dependency, error)
 		return nil, &core.ParseError{Filename: filename, Err: err}
 	}
 
+	count := 0
+	for key, lib := range raw.Libraries {
+		if lib.Type != "project" && strings.Contains(key, "/") {
+			count++
+		}
+	}
 	var deps []core.Dependency
+	if count > 0 {
+		deps = make([]core.Dependency, 0, count)
+	}
 
 	for key, lib := range raw.Libraries {
 		if lib.Type == "project" {

internal/pypi/pypi.go

@@ -73,9 +73,9 @@ func (p *requirementsTxtParser) Parse(filename string, content []byte) (*core.Re
 	var deps []core.Dependency
 	var declarations []core.Declaration
 	locations := make(map[string]int)
-	lines := strings.Split(string(content), "\n")
+	lines := strings.SplitSeq(string(content), "\n")
 
-	for _, line := range lines {
+	for line := range lines {
 		// Remove comments
 		if idx := strings.Index(line, "#"); idx >= 0 {
 			line = line[:idx]
@@ -771,9 +771,9 @@ type pipResolvedDepsParser struct{}
 
 func (p *pipResolvedDepsParser) Parse(filename string, content []byte) (*core.Result, error) {
 	var deps []core.Dependency
-	lines := strings.Split(string(content), "\n")
+	lines := strings.SplitSeq(string(content), "\n")
 
-	for _, line := range lines {
+	for line := range lines {
 		line = strings.TrimSpace(line)
 		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "-") {
 			continue

line_iteration_test.go

@@ -0,0 +1,90 @@
+package manifests
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestLineParserEndings(t *testing.T) {
+	for _, path := range []string{"golang/go.mod", "golang/go.sum", "golang/go.graph", "pypi/requirements.txt", "pypi/pip-resolved-dependencies.txt", "nuget/paket.lock"} {
+		t.Run(path, func(t *testing.T) {
+			content, err := os.ReadFile("testdata/" + path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			name := path[strings.LastIndexByte(path, '/')+1:]
+			lf := bytes.TrimRight(bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), "\n")
+			want, err := Parse(name, lf)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(want.Dependencies) == 0 {
+				t.Fatal("fixture has no dependencies")
+			}
+			variants := [][]byte{append(bytes.Clone(lf), '\n'), append(bytes.Clone(lf), '\n', '\n'), bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n"))}
+			for _, data := range variants {
+				got, err := Parse(name, data)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !reflect.DeepEqual(got, want) {
+					t.Fatalf("line endings changed %s output", name)
+				}
+			}
+		})
+	}
+}
+
+func lineBenchmarkInput(name string, count int) []byte {
+	var s strings.Builder
+	switch name {
+	case "go.mod":
+		s.WriteString("module example.com/demo\n\ngo 1.26\nrequire (\n")
+	case "paket.lock":
+		s.WriteString("NUGET\n  remote: https://api.nuget.org/v3/index.json\n  specs:\n")
+	}
+	for i := range count {
+		switch name {
+		case "go.mod":
+			fmt.Fprintf(&s, "example.com/package%d v1.2.3 // indirect\n", i)
+		case "go.sum":
+			fmt.Fprintf(&s, "example.com/package%d v1.2.3 h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n", i)
+		case "go.graph":
+			fmt.Fprintf(&s, "example.com/demo example.com/package%d@v1.2.3\n", i)
+		case "requirements.txt", "pip-resolved-dependencies.txt":
+			fmt.Fprintf(&s, "package%d==1.2.3\n", i)
+		case "paket.lock":
+			fmt.Fprintf(&s, "    Package%d (1.2.3)\n", i)
+		}
+	}
+	if name == "go.mod" {
+		s.WriteString(")\n")
+	}
+	return []byte(s.String())
+}
+func BenchmarkLineParsers(b *testing.B) {
+	for _, name := range []string{"go.mod", "go.sum", "go.graph", "requirements.txt", "pip-resolved-dependencies.txt", "paket.lock"} {
+		b.Run(name, func(b *testing.B) {
+			for _, count := range []int{100, 10000} {
+				b.Run(fmt.Sprint(count), func(b *testing.B) {
+					content := lineBenchmarkInput(name, count)
+					b.ReportAllocs()
+					b.ResetTimer()
+					for b.Loop() {
+						result, err := Parse(name, content)
+						if err != nil {
+							b.Fatal(err)
+						}
+						if len(result.Dependencies) != count {
+							b.Fatalf("got %d dependencies, want %d", len(result.Dependencies), count)
+						}
+					}
+				})
+			}
+		})
+	}
+}