diff --git a/internal/composer/composer.go b/internal/composer/composer.go index 9079ba0..7fc3368 100644 --- a/internal/composer/composer.go +++ b/internal/composer/composer.go @@ -111,6 +111,9 @@ func (p *composerLockParser) Parse(filename string, content []byte) (*core.Resul } var deps []core.Dependency + if count := len(lock.Packages) + len(lock.PackagesDev); count > 0 { + deps = make([]core.Dependency, 0, count) + } for _, pkg := range lock.Packages { integrity := "" diff --git a/internal/golang/golang.go b/internal/golang/golang.go index 26e8526..d712eb8 100644 --- a/internal/golang/golang.go +++ b/internal/golang/golang.go @@ -42,13 +42,13 @@ var ( ) func (p *goModParser) Parse(filename string, content []byte) (*core.Result, error) { - lines := strings.Split(string(content), "\n") - tools := collectToolPaths(lines) - replaced := collectReplacedModules(lines) - deps, declarations := collectRequireDeps(lines, tools, replaced) + text := string(content) + tools := collectToolPaths(text) + replaced := collectReplacedModules(text) + deps, declarations := collectRequireDeps(text, tools, replaced) var modulePath string - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "module ") || strings.HasPrefix(trimmed, "module\t") { modulePath = strings.TrimSpace(strings.Trim(strings.TrimSpace(trimmed[len("module"):]), `"`)) @@ -61,11 +61,11 @@ func (p *goModParser) Parse(filename string, content []byte) (*core.Result, erro // collectToolPaths scans go.mod lines for tool directives (both single-line and block form) // and returns a set of tool import paths. -func collectToolPaths(lines []string) map[string]bool { +func collectToolPaths(text string) map[string]bool { tools := make(map[string]bool) inToolBlock := false - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { trimmed := strings.TrimSpace(line) if trimmed == "" || strings.HasPrefix(trimmed, "//") { @@ -101,13 +101,13 @@ func collectToolPaths(lines []string) map[string]bool { // collectRequireDeps scans go.mod lines for require directives (both single-line and block form) // and returns dependencies, marking tool-related modules as development scope. -func collectRequireDeps(lines []string, tools map[string]bool, replaced map[moduleVersion]bool) ([]core.Dependency, []core.Declaration) { +func collectRequireDeps(text string, tools map[string]bool, replaced map[moduleVersion]bool) ([]core.Dependency, []core.Declaration) { var deps []core.Dependency var declarations []core.Declaration inRequireBlock := false locations := make(map[string]int) - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { trimmed := strings.TrimSpace(line) if trimmed == "" || strings.HasPrefix(trimmed, "//") { @@ -169,10 +169,10 @@ func appendGoDeclaration( // collectReplacedModules returns module paths named on the left side of a // replace directive, in either single-line or block form. -func collectReplacedModules(lines []string) map[moduleVersion]bool { +func collectReplacedModules(text string) map[moduleVersion]bool { replaced := make(map[moduleVersion]bool) inReplaceBlock := false - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { trimmed := strings.TrimSpace(line) if trimmed == "" || strings.HasPrefix(trimmed, "//") { continue @@ -251,9 +251,9 @@ type goSumKey struct { func (p *goSumParser) Parse(filename string, content []byte) (*core.Result, error) { var deps []core.Dependency seen := make(map[goSumKey]bool) - lines := strings.Split(string(content), "\n") + text := string(content) - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { line = strings.TrimSpace(line) if line == "" { continue @@ -311,11 +311,11 @@ func (p *goGraphParser) Parse(filename string, content []byte) (*core.Result, er var deps []core.Dependency seen := make(map[string]bool) directDeps := make(map[string]bool) - lines := strings.Split(string(content), "\n") + text := string(content) // First pass: identify direct dependencies (those required by the main module) // The main module appears without a version in the first column - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { line = strings.TrimSpace(line) if line == "" { continue @@ -340,7 +340,7 @@ func (p *goGraphParser) Parse(filename string, content []byte) (*core.Result, er } // Second pass: collect all dependencies - for _, line := range lines { + for line := range strings.SplitSeq(text, "\n") { line = strings.TrimSpace(line) if line == "" { continue diff --git a/internal/nuget/nuget.go b/internal/nuget/nuget.go index 8b75547..0d82d56 100644 --- a/internal/nuget/nuget.go +++ b/internal/nuget/nuget.go @@ -498,7 +498,14 @@ func (p *packagesLockParser) Parse(filename string, content []byte) (*core.Resul return nil, &core.ParseError{Filename: filename, Err: err} } + capacity := 0 + for _, framework := range lock.Dependencies { + capacity = max(capacity, len(framework)) + } var deps []core.Dependency + if capacity > 0 { + deps = make([]core.Dependency, 0, capacity) + } seen := make(map[string]bool) for _, framework := range lock.Dependencies { @@ -533,11 +540,12 @@ var ( func (p *paketLockParser) Parse(filename string, content []byte) (*core.Result, error) { var deps []core.Dependency - lines := strings.Split(string(content), "\n") + lines := strings.SplitSeq(string(content), "\n") seen := make(map[string]bool) inNuget := false - for _, line := range lines { + for line := range lines { + line = strings.TrimSuffix(line, "\r") // Check for NUGET section if line == "NUGET" { inNuget = true @@ -594,7 +602,20 @@ func (p *projectAssetsParser) Parse(filename string, content []byte) (*core.Resu return nil, &core.ParseError{Filename: filename, Err: err} } + capacity := 0 + for _, framework := range assets.Targets { + count := 0 + for key, pkg := range framework { + if pkg.Type == "package" && strings.Contains(key, "/") { + count++ + } + } + capacity = max(capacity, count) + } var deps []core.Dependency + if capacity > 0 { + deps = make([]core.Dependency, 0, capacity) + } seen := make(map[string]bool) for _, framework := range assets.Targets { @@ -697,7 +718,16 @@ func parseLibraries(filename string, content []byte) ([]core.Dependency, error) return nil, &core.ParseError{Filename: filename, Err: err} } + count := 0 + for key, lib := range raw.Libraries { + if lib.Type != "project" && strings.Contains(key, "/") { + count++ + } + } var deps []core.Dependency + if count > 0 { + deps = make([]core.Dependency, 0, count) + } for key, lib := range raw.Libraries { if lib.Type == "project" { diff --git a/internal/pypi/pypi.go b/internal/pypi/pypi.go index 7bfd815..171599b 100644 --- a/internal/pypi/pypi.go +++ b/internal/pypi/pypi.go @@ -73,9 +73,9 @@ func (p *requirementsTxtParser) Parse(filename string, content []byte) (*core.Re var deps []core.Dependency var declarations []core.Declaration locations := make(map[string]int) - lines := strings.Split(string(content), "\n") + lines := strings.SplitSeq(string(content), "\n") - for _, line := range lines { + for line := range lines { // Remove comments if idx := strings.Index(line, "#"); idx >= 0 { line = line[:idx] @@ -771,9 +771,9 @@ type pipResolvedDepsParser struct{} func (p *pipResolvedDepsParser) Parse(filename string, content []byte) (*core.Result, error) { var deps []core.Dependency - lines := strings.Split(string(content), "\n") + lines := strings.SplitSeq(string(content), "\n") - for _, line := range lines { + for line := range lines { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "-") { continue diff --git a/line_iteration_test.go b/line_iteration_test.go new file mode 100644 index 0000000..7bdd693 --- /dev/null +++ b/line_iteration_test.go @@ -0,0 +1,90 @@ +package manifests + +import ( + "bytes" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +func TestLineParserEndings(t *testing.T) { + for _, path := range []string{"golang/go.mod", "golang/go.sum", "golang/go.graph", "pypi/requirements.txt", "pypi/pip-resolved-dependencies.txt", "nuget/paket.lock"} { + t.Run(path, func(t *testing.T) { + content, err := os.ReadFile("testdata/" + path) + if err != nil { + t.Fatal(err) + } + name := path[strings.LastIndexByte(path, '/')+1:] + lf := bytes.TrimRight(bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), "\n") + want, err := Parse(name, lf) + if err != nil { + t.Fatal(err) + } + if len(want.Dependencies) == 0 { + t.Fatal("fixture has no dependencies") + } + variants := [][]byte{append(bytes.Clone(lf), '\n'), append(bytes.Clone(lf), '\n', '\n'), bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n"))} + for _, data := range variants { + got, err := Parse(name, data) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("line endings changed %s output", name) + } + } + }) + } +} + +func lineBenchmarkInput(name string, count int) []byte { + var s strings.Builder + switch name { + case "go.mod": + s.WriteString("module example.com/demo\n\ngo 1.26\nrequire (\n") + case "paket.lock": + s.WriteString("NUGET\n remote: https://api.nuget.org/v3/index.json\n specs:\n") + } + for i := range count { + switch name { + case "go.mod": + fmt.Fprintf(&s, "example.com/package%d v1.2.3 // indirect\n", i) + case "go.sum": + fmt.Fprintf(&s, "example.com/package%d v1.2.3 h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\n", i) + case "go.graph": + fmt.Fprintf(&s, "example.com/demo example.com/package%d@v1.2.3\n", i) + case "requirements.txt", "pip-resolved-dependencies.txt": + fmt.Fprintf(&s, "package%d==1.2.3\n", i) + case "paket.lock": + fmt.Fprintf(&s, " Package%d (1.2.3)\n", i) + } + } + if name == "go.mod" { + s.WriteString(")\n") + } + return []byte(s.String()) +} +func BenchmarkLineParsers(b *testing.B) { + for _, name := range []string{"go.mod", "go.sum", "go.graph", "requirements.txt", "pip-resolved-dependencies.txt", "paket.lock"} { + b.Run(name, func(b *testing.B) { + for _, count := range []int{100, 10000} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + content := lineBenchmarkInput(name, count) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + result, err := Parse(name, content) + if err != nil { + b.Fatal(err) + } + if len(result.Dependencies) != count { + b.Fatalf("got %d dependencies, want %d", len(result.Dependencies), count) + } + } + }) + } + }) + } +} diff --git a/lock_capacity_test.go b/lock_capacity_test.go new file mode 100644 index 0000000..76f3915 --- /dev/null +++ b/lock_capacity_test.go @@ -0,0 +1,168 @@ +package manifests + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "testing" +) + +func TestLockDependencyCapacity(t *testing.T) { + tests := []struct { + name, content string + want []Dependency + }{ + {"composer.lock", `{"packages":[{"name":"vendor/runtime","version":"1.2.3","dist":{"shasum":"abc","sha256":"def"}}],"packages-dev":[{"name":"vendor/dev","version":"2.0.0","dist":{"shasum":"abc"}}]}`, []Dependency{ + {Name: "vendor/runtime", Version: "1.2.3", Scope: Runtime, Integrity: "sha256-def", PURL: "pkg:composer/vendor/runtime@1.2.3"}, + {Name: "vendor/dev", Version: "2.0.0", Scope: Development, Integrity: "sha1-abc", PURL: "pkg:composer/vendor/dev@2.0.0"}, + }}, + {"packages.lock.json", `{"version":1,"dependencies":{"net8.0":{"Alpha":{"type":"Direct","resolved":"1.2.3","contentHash":"abc"}},"net9.0":{"Alpha":{"type":"Direct","resolved":"1.2.3","contentHash":"abc"},"Beta":{"type":"Transitive","resolved":"2.0.0"}},"net10.0":{"Gamma":{"type":"Transitive","resolved":"3.0.0"}}}}`, []Dependency{ + {Name: "Alpha", Version: "1.2.3", Scope: Runtime, Integrity: "sha512-abc", Direct: true, PURL: "pkg:nuget/Alpha@1.2.3"}, + {Name: "Beta", Version: "2.0.0", Scope: Runtime, PURL: "pkg:nuget/Beta@2.0.0"}, + {Name: "Gamma", Version: "3.0.0", Scope: Runtime, PURL: "pkg:nuget/Gamma@3.0.0"}, + }}, + {"project.assets.json", `{"targets":{"net8.0":{"Alpha/1.2.3":{"type":"package"},"App/1.0":{"type":"project"},"invalid":{"type":"package"}},"net9.0":{"Alpha/1.2.3":{"type":"package"},"Beta/2.0.0":{"type":"package"}},"net10.0":{"Gamma/3.0.0":{"type":"package"}}},"libraries":{"Alpha/1.2.3":{"sha512":"abc"}}}`, []Dependency{ + {Name: "Alpha", Version: "1.2.3", Scope: Runtime, Integrity: "sha512-abc", PURL: "pkg:nuget/Alpha@1.2.3"}, + {Name: "Beta", Version: "2.0.0", Scope: Runtime, PURL: "pkg:nuget/Beta@2.0.0"}, + {Name: "Gamma", Version: "3.0.0", Scope: Runtime, PURL: "pkg:nuget/Gamma@3.0.0"}, + }}, + } + for _, name := range []string{"example.deps.json", "Project.lock.json"} { + tests = append(tests, struct { + name, content string + want []Dependency + }{name, `{"libraries":{"Alpha/1.2.3":{"type":"package","sha512":"abc"},"Beta/2.0.0":{"type":"package"},"App/1.0":{"type":"project"},"invalid":{"type":"package"}}}`, []Dependency{ + {Name: "Alpha", Version: "1.2.3", Scope: Runtime, Integrity: "sha512-abc", PURL: "pkg:nuget/Alpha@1.2.3"}, + {Name: "Beta", Version: "2.0.0", Scope: Runtime, PURL: "pkg:nuget/Beta@2.0.0"}, + }}) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := Parse(tt.name, []byte(tt.content)) + if err != nil { + t.Fatal(err) + } + sortDeps := func(deps []Dependency) { + slices.SortFunc(deps, func(a, b Dependency) int { + if a.Name < b.Name { + return -1 + } + if a.Name > b.Name { + return 1 + } + return 0 + }) + } + sortDeps(result.Dependencies) + sortDeps(tt.want) + if !reflect.DeepEqual(result.Dependencies, tt.want) { + t.Fatalf("got %#v, want %#v", result.Dependencies, tt.want) + } + for _, empty := range []string{`{}`, `{"packages":[],"packages-dev":[],"dependencies":{"net8.0":{}},"targets":{"net8.0":{"App/1.0":{"type":"project"},"invalid":{"type":"package"}}},"libraries":{"App/1.0":{"type":"project"},"invalid":{"type":"package"}}}`} { + result, err = Parse(tt.name, []byte(empty)) + if err != nil { + t.Fatal(err) + } + if result.Dependencies != nil { + t.Fatalf("empty dependencies = %#v", result.Dependencies) + } + } + }) + } +} + +func lockCapacityInput(name string, count, frameworks int) []byte { + packages := make([]map[string]any, 0, count) + libraries := make(map[string]any, count) + dependencies := make(map[string]any, count) + targets := make(map[string]any, count) + for i := range count { + pkg := fmt.Sprintf("Package%d", i) + packages = append(packages, map[string]any{"name": "vendor/" + pkg, "version": "1.2.3"}) + libraries[pkg+"/1.2.3"] = map[string]any{"type": "package", "sha512": "abc"} + targets[pkg+"/1.2.3"] = map[string]any{"type": "package"} + dependencies[pkg] = map[string]any{"type": "Direct", "resolved": "1.2.3", "contentHash": "abc"} + } + var input any + switch name { + case "composer.lock": + input = map[string]any{"packages": packages[:count/2], "packages-dev": packages[count/2:]} + case "packages.lock.json", "project.assets.json": + groups := make(map[string]any, frameworks) + for i := range frameworks { + if name == "packages.lock.json" { + groups[fmt.Sprintf("net%d.0", 8+i)] = dependencies + } else { + groups[fmt.Sprintf("net%d.0", 8+i)] = targets + } + } + if name == "packages.lock.json" { + input = map[string]any{"version": 1, "dependencies": groups} + } else { + input = map[string]any{"targets": groups, "libraries": libraries} + } + default: + input = map[string]any{"libraries": libraries} + } + content, err := json.Marshal(input) + if err != nil { + panic(err) + } + return content +} + +func BenchmarkLockCapacity(b *testing.B) { + for _, name := range []string{"composer.lock", "packages.lock.json", "project.assets.json", "example.deps.json", "Project.lock.json"} { + for _, count := range []int{2, 100, 10000} { + groups := []int{1} + if name == "packages.lock.json" || name == "project.assets.json" { + groups = append(groups, 3) + } + for _, frameworks := range groups { + b.Run(fmt.Sprintf("%s/%d/%d", name, count, frameworks), func(b *testing.B) { + content := lockCapacityInput(name, count, frameworks) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + result, err := Parse(name, content) + if err != nil { + b.Fatal(err) + } + if len(result.Dependencies) != count { + b.Fatalf("got %d dependencies, want %d", len(result.Dependencies), count) + } + } + }) + } + } + } +} + +func BenchmarkLockCapacityFixtures(b *testing.B) { + for _, path := range []string{"composer/composer.lock", "nuget/packages.lock.json", "nuget/nuget_project.assets.json", "nuget/example.deps.json", "nuget/Project.lock.json"} { + b.Run(path, func(b *testing.B) { + content, err := os.ReadFile("testdata/" + path) + if err != nil { + b.Fatal(err) + } + name := filepath.Base(path) + if name == "nuget_project.assets.json" { + name = "project.assets.json" + } + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + result, err := Parse(name, content) + if err != nil { + b.Fatal(err) + } + if len(result.Dependencies) == 0 { + b.Fatal("no dependencies") + } + } + }) + } +}