Skip to content

Decode npm lockfile packages as JSON #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
andrew wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Decode npm lockfile packages as JSON #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 171 additions & 70 deletions internal/npm/npm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package npm
import (
"bytes"
"encoding/json"
"errors"
"iter"
"net/url"
"strings"
Expand Down Expand Up @@ -162,26 +163,32 @@ type packageLockRoot struct {
PeerDependencies map[string]json.RawMessage `json:"peerDependencies"`
}

var packageLockPackagesKey = []byte(`"packages"`)

var errPackageLock = errors.New("malformed JSON")

func (p *npmPackageLockParser) Parse(filename string, content []byte) (*core.Result, error) {
// Quick check for lockfile version to determine parsing strategy
// v3 (lockfileVersion >= 2 with packages) uses line-based parsing
// v1 uses JSON parsing for nested dependencies
const headerPeekSize = 200
const packagesPeekSize = 600
header := string(content[:min(headerPeekSize, len(content))])

// v2+ with packages section uses line-based v3 parsing
if strings.Contains(header, `"lockfileVersion": 3`) ||
(strings.Contains(header, `"lockfileVersion": 2`) && strings.Contains(string(content[:min(packagesPeekSize, len(content))]), `"packages"`)) {
return &core.Result{Dependencies: parsePackageLockV3Lines(content)}, nil
}

// v1 format uses JSON (nested dependencies make line parsing complex)
var lock packageLockJSON
if err := json.Unmarshal(content, &lock); err != nil {
// Without a packages section the lockfile is v1: nested dependencies only.
if !bytes.Contains(content, packageLockPackagesKey) {
var lock packageLockJSON
if err := json.Unmarshal(content, &lock); err != nil {
return nil, &core.ParseError{Filename: filename, Err: err}
}
return &core.Result{Dependencies: parsePackageLockV1(lock.Dependencies)}, nil
}

// npm writes one key per line, which the line scanner reads without
// decoding the document.
if deps := parsePackageLockV3Lines(content); len(deps) > 0 {
return &core.Result{Dependencies: deps}, nil
}

// Any other formatting, compact JSON included.
deps, err := decodePackageLock(content)
if err != nil {
return nil, &core.ParseError{Filename: filename, Err: err}
}
return &core.Result{Dependencies: parsePackageLockV1(lock.Dependencies)}, nil
return &core.Result{Dependencies: deps}, nil
}

func parsePackageLockV1(deps map[string]packageLockDep) []core.Dependency {
Expand Down Expand Up @@ -225,54 +232,48 @@ func appendPackageLockV1(result []core.Dependency, deps map[string]packageLockDe
return result
}

// v3PackageEntry holds the state accumulated while parsing a single package
// entry in the v3 lockfile format.
type v3PackageEntry struct {
path string
version string
integrity string
resolved string
dev bool
optional bool
devOptional bool
link bool
// packageLockEntry is one entry of the v2/v3 "packages" object, filled either
// by the line scanner or by the JSON decoder. Path holds the install path the
// entry is keyed by.
type packageLockEntry struct {
Path string `json:"-"`
Version string `json:"version"`
Integrity string `json:"integrity"`
Resolved string `json:"resolved"`
Dev bool `json:"dev"`
Optional bool `json:"optional"`
DevOptional bool `json:"devOptional"`
Link bool `json:"link"`
}

func (e *v3PackageEntry) reset(path string) {
e.path = path
e.version = ""
e.integrity = ""
e.resolved = ""
e.dev = false
e.optional = false
e.devOptional = false
e.link = false
func (e *packageLockEntry) reset(path string) {
*e = packageLockEntry{Path: path}
}

func (e *v3PackageEntry) hasContent() bool {
return e.path != "" && (e.version != "" || e.link)
func (e *packageLockEntry) hasContent() bool {
return e.Path != "" && (e.Version != "" || e.Link)
}

func (e *v3PackageEntry) toDependency(directDependencies map[string]bool) (core.Dependency, bool) {
name := extractPackageName(e.path)
if name == "" {
func (e *packageLockEntry) toDependency(directDependencies map[string]bool) (core.Dependency, bool) {
name := extractPackageName(e.Path)
if name == "" || (e.Version == "" && !e.Link) {
return core.Dependency{}, false
}
scope := core.Runtime
if e.dev || e.devOptional {
if e.Dev || e.DevOptional {
scope = core.Development
} else if e.optional {
} else if e.Optional {
scope = core.Optional
}
topLevel := !strings.Contains(strings.TrimPrefix(e.path, "node_modules/"), "node_modules/")
topLevel := !strings.Contains(strings.TrimPrefix(e.Path, "node_modules/"), "node_modules/")
direct := topLevel && directDependencies[name]
return core.Dependency{
Name: name,
Version: e.version,
Version: e.Version,
Scope: scope,
Integrity: e.integrity,
Integrity: e.Integrity,
Direct: direct,
RegistryURL: e.resolved,
RegistryURL: e.Resolved,
}, true
}

Expand All @@ -290,7 +291,7 @@ func parsePackageLockDirectDependencies(content []byte) map[string]bool {
if key == "packages" {
return decodePackageLockRootDependencies(decoder)
}
if !discardJSONValue(decoder) {
if !skipJSONValue(decoder) {
return nil
}
}
Expand All @@ -309,28 +310,32 @@ func decodePackageLockRootDependencies(decoder *json.Decoder) map[string]bool {
return nil
}
if path == "" {
return decodePackageLockRoot(decoder)
declared, err := decodePackageLockRoot(decoder)
if err != nil {
return nil
}
return declared
}
if !discardJSONValue(decoder) {
if !skipJSONValue(decoder) {
return nil
}
}

return nil
}

func decodePackageLockRoot(decoder *json.Decoder) map[string]bool {
func decodePackageLockRoot(decoder *json.Decoder) (map[string]bool, error) {
var root packageLockRoot
if err := decoder.Decode(&root); err != nil {
return nil
return nil, err
}

directDependencies := make(map[string]bool)
collectPackageLockDependencyNames(directDependencies, root.Dependencies)
collectPackageLockDependencyNames(directDependencies, root.DevDependencies)
collectPackageLockDependencyNames(directDependencies, root.OptionalDependencies)
collectPackageLockDependencyNames(directDependencies, root.PeerDependencies)
return directDependencies
return directDependencies, nil
}

func decodeJSONObjectOpening(decoder *json.Decoder) bool {
Expand All @@ -347,11 +352,6 @@ func decodeJSONKey(decoder *json.Decoder) (string, bool) {
return key, ok
}

func discardJSONValue(decoder *json.Decoder) bool {
var discarded json.RawMessage
return decoder.Decode(&discarded) == nil
}

func collectPackageLockDependencyNames(directDependencies map[string]bool, dependencies map[string]json.RawMessage) {
for name := range dependencies {
directDependencies[name] = true
Expand All @@ -360,28 +360,28 @@ func collectPackageLockDependencyNames(directDependencies map[string]bool, depen

// updateFromLine reads a trimmed line and updates the entry's fields.
// Returns true if the line was consumed.
func (e *v3PackageEntry) updateFromLine(trimmed string) bool {
func (e *packageLockEntry) updateFromLine(trimmed string) bool {
switch {
case strings.HasPrefix(trimmed, `"version"`):
if v := extractJSONStringValue(trimmed); v != "" {
e.version = v
e.Version = v
}
case strings.HasPrefix(trimmed, `"integrity"`):
if v := extractJSONStringValue(trimmed); v != "" {
e.integrity = v
e.Integrity = v
}
case strings.HasPrefix(trimmed, `"resolved"`):
if v := extractJSONStringValue(trimmed); v != "" {
e.resolved = v
e.Resolved = v
}
case strings.HasPrefix(trimmed, `"dev": true`):
e.dev = true
e.Dev = true
case strings.HasPrefix(trimmed, `"optional": true`):
e.optional = true
e.Optional = true
case strings.HasPrefix(trimmed, `"devOptional": true`):
e.devOptional = true
e.DevOptional = true
case strings.HasPrefix(trimmed, `"link": true`):
e.link = true
e.Link = true
default:
return false
}
Expand Down Expand Up @@ -436,10 +436,10 @@ func parsePackageLockV3Lines(content []byte) []core.Dependency {
return deps
}

func packageLockV3Entries(content string) iter.Seq[v3PackageEntry] {
return func(yield func(v3PackageEntry) bool) {
func packageLockV3Entries(content string) iter.Seq[packageLockEntry] {
return func(yield func(packageLockEntry) bool) {
inPackages := false
var entry v3PackageEntry
var entry packageLockEntry

for line := range strings.SplitSeq(content, "\n") {
trimmed := strings.TrimSpace(line)
Expand Down Expand Up @@ -493,6 +493,107 @@ func extractJSONStringValue(line string) string {
return rest[start+1 : start+1+end]
}

// decodePackageLock decodes the document, for lockfiles the line scanner cannot
// read. v2 and v3 list every installed package under "packages"; v1 only has
// the nested "dependencies" tree.
func decodePackageLock(content []byte) ([]core.Dependency, error) {
decoder := json.NewDecoder(bytes.NewReader(content))
if !decodeJSONObjectOpening(decoder) {
return nil, errPackageLock
}

var tree map[string]packageLockDep
for decoder.More() {
key, ok := decodeJSONKey(decoder)
if !ok {
return nil, errPackageLock
}
switch key {
case "packages":
if !decodeJSONObjectOpening(decoder) {
return nil, errPackageLock
}
return decodePackageLockEntries(decoder)
case "dependencies":
if err := decoder.Decode(&tree); err != nil {
return nil, err
}
default:
if !skipJSONValue(decoder) {
return nil, errPackageLock
}
}
}

return parsePackageLockV1(tree), nil
}

// decodePackageLockEntries reads the entries of an open "packages" object in
// document order. The root entry is converted last because it can appear after
// the packages it declares.
func decodePackageLockEntries(decoder *json.Decoder) ([]core.Dependency, error) {
var entries []packageLockEntry
var directDependencies map[string]bool

for decoder.More() {
path, ok := decodeJSONKey(decoder)
if !ok {
return nil, errPackageLock
}
if path == "" {
declared, err := decodePackageLockRoot(decoder)
if err != nil {
return nil, err
}
directDependencies = declared
continue
}
entries = append(entries, packageLockEntry{Path: path})
if err := decoder.Decode(&entries[len(entries)-1]); err != nil {
return nil, err
}
}

deps := make([]core.Dependency, 0, len(entries))
for i := range entries {
if dep, ok := entries[i].toDependency(directDependencies); ok {
deps = append(deps, dep)
}
}
if len(deps) == 0 {
return nil, nil
}
return deps, nil
}

// skipJSONValue reads past the next value without copying it.
func skipJSONValue(decoder *json.Decoder) bool {
token, err := decoder.Token()
if err != nil {
return false
}
depth := 0
switch token {
case json.Delim('{'), json.Delim('['):
depth = 1
default:
return true
}
for depth > 0 {
token, err = decoder.Token()
if err != nil {
return false
}
switch token {
case json.Delim('{'), json.Delim('['):
depth++
case json.Delim('}'), json.Delim(']'):
depth--
}
}
return true
}

// extractPackageName extracts the package name from a node_modules path.
func extractPackageName(path string) string {
// Remove leading node_modules/
Expand Down
32 changes: 32 additions & 0 deletions internal/npm/npm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package npm

import (
"os"
"reflect"
"testing"

"github.com/git-pkgs/manifests/internal/core"
Expand Down Expand Up @@ -1207,3 +1208,34 @@ func TestYarnV4Lock(t *testing.T) {
t.Error("workspace package should be excluded")
}
}

// TestPackageLockPathsAgree keeps the line scanner and the JSON decoder from
// drifting apart on the lockfiles npm itself writes.
func TestPackageLockPathsAgree(t *testing.T) {
paths := []string{
"../../testdata/npm/npm-lockfile-version-1/package-lock.json",
"../../testdata/npm/npm-lockfile-version-2/package-lock.json",
"../../testdata/npm/npm-lockfile-version-3/package-lock.json",
"../../testdata/npm/npm-local-file/package-lock.json",
"../../testdata/misc/multiple_versions/package-lock.json",
}
for _, path := range paths {
t.Run(path, func(t *testing.T) {
content, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
scanned := parsePackageLockV3Lines(content)
if len(scanned) == 0 {
t.Fatal("line scanner read no dependencies")
}
decoded, err := decodePackageLock(content)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(scanned, decoded) {
t.Fatalf("line scanner gave %+v, decoder gave %+v", scanned, decoded)
}
})
}
}
Loading
Loading