diff --git a/internal/sync/local/adapter_safety_test.go b/internal/sync/local/adapter_safety_test.go new file mode 100644 index 00000000..31e710d7 --- /dev/null +++ b/internal/sync/local/adapter_safety_test.go @@ -0,0 +1,503 @@ +package local + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +func TestReferenceDescriptorValidationFailures(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + for name, content := range map[string]string{ + "invalid YAML": "formatVersion: [", + "unknown field": "formatVersion: 1\nfile: prompt.md\nformat: markdown\nunknown: true\n", + "multiple docs": "formatVersion: 1\nfile: prompt.md\nformat: markdown\n---\nextra: true\n", + "unsupported format": "formatVersion: 99\nfile: prompt.md\nformat: markdown\n", + "missing file": "formatVersion: 1\nformat: markdown\n", + } { + t.Run(name, func(t *testing.T) { + fsys := fstest.MapFS{"value.ref.yaml": {Data: []byte(content)}} + _, err := ReadDescriptor(fsys, "value.ref.yaml", id) + require.Error(t, err) + }) + } + _, err := ReadDescriptor(fstest.MapFS{}, "missing.ref.yaml", id) + require.Error(t, err) +} + +func TestReferencePathValidationFailures(t *testing.T) { + for name, reference := range map[string]Reference{ + "missing file": {Format: "markdown"}, + "missing format": {File: "prompt.md"}, + "absolute": {File: "/prompt.md", Format: "markdown"}, + "managed": {File: ".launchdarkly/prompt.md", Format: "markdown"}, + "unclean": {File: "dir/../prompt.md", Format: "markdown"}, + } { + t.Run(name, func(t *testing.T) { + require.Error(t, validateReference(reference)) + }) + } + + root := t.TempDir() + writeTestFile(t, root, "directory/file.md", "body") + _, err := resolveReferencePath(root, Reference{File: "missing.md", Format: "markdown"}) + require.Error(t, err) + _, err = resolveReferencePath(root, Reference{File: "directory", Format: "markdown"}) + require.Error(t, err) + + outside := filepath.Join(t.TempDir(), "outside.md") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + link := filepath.Join(root, "link.md") + if err := os.Symlink(outside, link); err == nil { + _, err = resolveReferencePath(root, Reference{File: "link.md", Format: "markdown"}) + require.Error(t, err) + } + + _, err = readReferenceFromFS(fstest.MapFS{}, Reference{}) + require.Error(t, err) + _, err = readReferenceFromFS(fstest.MapFS{}, Reference{File: "prompt.md", Format: "markdown"}) + require.Error(t, err) + data, err := readReferenceFromFS( + fstest.MapFS{"prompt.md": {Data: []byte("Help")}}, + Reference{File: "prompt.md", Format: "markdown"}, + ) + require.NoError(t, err) + assert.Equal(t, "Help", string(data)) +} + +func TestNewSourceReferenceValidatesRepositoryBoundaries(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "work"), 0o755)) + writeTestFile(t, root, "work/prompt.md", "Help") + + _, err := NewSourceReference(root, filepath.Join(root, "work"), "", "markdown", "", nil) + require.Error(t, err) + _, err = NewSourceReference(root, filepath.Join(root, "work"), "missing.md", "markdown", "", nil) + require.Error(t, err) + + outside := filepath.Join(t.TempDir(), "outside.md") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + _, err = NewSourceReference(root, root, outside, "markdown", "", nil) + require.Error(t, err) + + reference, err := NewSourceReference(root, filepath.Join(root, "work"), "prompt.md", "markdown", "", nil) + require.NoError(t, err) + assert.Equal(t, "work/prompt.md", reference.File) +} + +func TestSourceDefinitionAndCompileSafety(t *testing.T) { + assert.Empty(t, (SourceDefinition{}).ID()) + variationID := syncdomain.VariationID("project", "config", "default") + variation := syncdomain.SyncedResource{Kind: variationID.Kind, ProjectKey: variationID.ProjectKey, LookupKey: variationID.LookupKey} + assert.Equal(t, variationID, (SourceDefinition{Variation: &variation}).ID()) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + attachment := syncdomain.SyncedAttachment{ + ProjectKey: "project", + Attachment: syncdomain.Attachment{Kind: syncdomain.AttachmentTool, Tool: &syncdomain.Tool{Key: "tool"}}, + } + assert.Equal(t, toolID, (SourceDefinition{Attachment: &attachment}).ID()) + + root := t.TempDir() + writeTestFile(t, root, "value.json", `{"description":"Tool"}`) + definition, err := CompileSource(root, toolID, Reference{ + File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}, + }) + require.NoError(t, err) + require.NotNil(t, definition.Attachment) + + _, err = CompileSource(root, syncdomain.ResourceID{Kind: "unknown"}, Reference{ + File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}, + }) + require.Error(t, err) + + generated := syncreference.SourceBundle{Primary: syncreference.Document{ + Path: "generated.py", Data: []byte("# Code generated. DO NOT EDIT.\n"), + }} + _, err = compileVariationSource(variationID, Reference{File: "generated.py", Format: "python"}, generated) + require.Error(t, err) + _, err = compileAttachmentSource(toolID, Reference{File: "generated.py", Format: "python"}, generated) + require.Error(t, err) + _, err = compileAttachmentSource( + syncdomain.ResourceID{Kind: "unknown", ProjectKey: "project", LookupKey: "value"}, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}}, + syncreference.SourceBundle{Primary: syncreference.Document{Path: "value.json", Data: []byte(`{"description":"value"}`)}}, + ) + require.Error(t, err) +} + +func TestSourceBundleHandlesMissingAndFailedDependencies(t *testing.T) { + reference := Reference{File: "app/main.py", Format: "python"} + content := []byte("from .values import PROMPT\n") + bundle, err := sourceBundle(reference, content, func(Reference) ([]byte, error) { + return nil, fs.ErrNotExist + }) + require.NoError(t, err) + assert.Empty(t, bundle.Dependencies) + + expected := errors.New("read failed") + _, err = sourceBundle(reference, content, func(Reference) ([]byte, error) { + return nil, expected + }) + require.ErrorIs(t, err, expected) + + _, err = sourceBundle(Reference{File: "value", Format: "missing"}, nil, func(Reference) ([]byte, error) { + return nil, nil + }) + require.Error(t, err) + + _, err = sourceBundle(reference, content, func(dependency Reference) ([]byte, error) { + if strings.HasSuffix(dependency.File, "values.py") { + return make([]byte, 8<<20), nil + } + return nil, fs.ErrNotExist + }) + require.ErrorContains(t, err, "8 MiB") +} + +func TestSourceEditHelpersEnforceAtomicSafety(t *testing.T) { + problem := publicationProblem("value.py", "unsafe", "fix it", assert.AnError) + typed, ok := syncdomain.ProblemFrom(problem) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorPublicationUnsupported, typed.Code) + + original := []byte("abcdef") + updated, err := applySourceEdits("value.txt", original, []syncreference.TextEdit{ + {File: "value.txt", Start: 4, End: 6, Expected: []byte("ef"), Replacement: []byte("EF")}, + {File: "value.txt", Start: 0, End: 2, Expected: []byte("ab"), Replacement: []byte("AB")}, + }) + require.NoError(t, err) + assert.Equal(t, "ABcdEF", string(updated)) + + for name, edits := range map[string][]syncreference.TextEdit{ + "invalid range": {{File: "value.txt", Start: -1}}, + "wrong file": {{File: "other.txt", Start: 0, End: 1, Expected: []byte("a")}}, + "stale bytes": {{File: "value.txt", Start: 0, End: 1, Expected: []byte("x")}}, + "overlap": { + {File: "value.txt", Start: 0, End: 3, Expected: []byte("abc")}, + {File: "value.txt", Start: 2, End: 4, Expected: []byte("cd")}, + }, + } { + t.Run(name, func(t *testing.T) { + _, err := applySourceEdits("value.txt", original, edits) + require.Error(t, err) + }) + } + + id := syncdomain.VariationID("project", "config", "default") + prompt := "Help" + projection := syncdomain.VariationProjection{Instructions: &prompt} + assert.True(t, sameProjection(id, projection, projection)) + other := "Other" + assert.False(t, sameProjection(id, projection, syncdomain.VariationProjection{Instructions: &other})) + + assert.True(t, generatedSource("vendor/value.go", nil)) + assert.True(t, generatedSource("value.min.js", nil)) + assert.True(t, generatedSource("value.go", []byte("// Code generated by tool. DO NOT EDIT."))) + assert.False(t, generatedSource("value.go", []byte("package value"))) + + bundle := syncreference.SourceBundle{ + Primary: syncreference.Document{Path: "main.py"}, + Dependencies: map[string]syncreference.Document{"dependency.py": {Path: "dependency.py"}}, + } + _, ok = sourceBundleDocument(bundle, "main.py") + assert.True(t, ok) + _, ok = sourceBundleDocument(bundle, "dependency.py") + assert.True(t, ok) + _, ok = sourceBundleDocument(bundle, "missing.py") + assert.False(t, ok) +} + +func TestCheckSourcesDoesNotWrite(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Old prompt\n") + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{File: "prompt.md", Format: "markdown"} + writeDescriptorForTest(t, root, id, reference) + prompt := "New prompt" + + err := NewStore(root).CheckSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(root, "prompt.md")) + require.NoError(t, err) + assert.Equal(t, "Old prompt\n", string(content)) +} + +func TestIndexedSourceErrorsDoNotWrite(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + prompt := "New prompt" + for name, selectPath := range map[string]string{ + "malformed": "items[-1]", + "out of range": "items[2]", + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + original := []byte("{\n \"items\": [{\"prompt\": \"Old prompt\"}]\n}\n") + writeTestFile(t, root, "values.json", string(original)) + reference := Reference{ + File: "values.json", Format: "json", Select: selectPath, + Map: map[string]string{"prompt": "prompt"}, + } + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + + require.Error(t, err) + actual, readErr := os.ReadFile(filepath.Join(root, "values.json")) + require.NoError(t, readErr) + assert.Equal(t, original, actual) + }) + } +} + +func TestIndexedSourceChangeAfterReviewDoesNotWrite(t *testing.T) { + root := t.TempDir() + original := []byte(`{"items":[{"name":"first","prompt":"Keep"},{"name":"second","prompt":"Old"}]}` + "\n") + writeTestFile(t, root, "values.json", string(original)) + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{ + File: "values.json", Format: "json", Select: "items[1]", + Map: map[string]string{"prompt": "prompt"}, + } + prompt := "New" + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + ExpectedIndexedSelections: []syncdomain.IndexedSelection{{ + Path: "select#0", CollectionLength: 2, + Fingerprint: "sha256:" + strings.Repeat("0", 64), + }}, + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorIndexedSelectionChanged, problem.Code) + actual, readErr := os.ReadFile(filepath.Join(root, "values.json")) + require.NoError(t, readErr) + assert.Equal(t, original, actual) +} + +func TestDescriptorStoreRejectsConflictsAndInvalidTargets(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + invalidID := syncdomain.ResourceID{Kind: "unknown", ProjectKey: "project", LookupKey: "value"} + require.Error(t, store.ValidateDescriptorTarget(invalidID)) + _, err := store.WriteDescriptor(invalidID, Reference{File: "prompt.md", Format: "markdown"}) + require.Error(t, err) + + id := syncdomain.VariationID("project", "config", "default") + writeTestFile(t, root, ".launchdarkly/project/configs/config/default.prompt.md", "managed") + require.Error(t, store.ValidateDescriptorTarget(id)) + + _, err = RenderDescriptor(id, Reference{File: "prompt.md", Format: "missing"}) + require.Error(t, err) + + require.NoError(t, os.Remove(filepath.Join(root, ".launchdarkly/project/configs/config/default.prompt.md"))) + writeTestFile(t, root, "prompt.md", "Help") + _, err = store.WriteDescriptor(id, Reference{File: "prompt.md", Format: "missing"}) + require.Error(t, err) + + _, err = NewSourceReference(filepath.Join(root, "missing-root"), root, filepath.Join(root, "prompt.md"), "markdown", "", nil) + require.Error(t, err) + _, err = resolveReferencePath(filepath.Join(root, "missing-root"), Reference{File: "prompt.md", Format: "markdown"}) + require.Error(t, err) +} + +func TestSourceEditPreparationFailureBranches(t *testing.T) { + missingStore := NewStore(filepath.Join(t.TempDir(), "missing")) + _, err := missingStore.ReplaceSources(nil) + require.Error(t, err) + + root := t.TempDir() + store := NewStore(root) + paths, err := store.ReplaceSources(nil) + require.NoError(t, err) + assert.Empty(t, paths) + + writeTestFile(t, root, "bad.py", "def broken(") + id := syncdomain.VariationID("project", "config", "default") + reference := Reference{ + File: "bad.py", Format: "python", Select: "$", + Map: map[string]string{"prompt": "$"}, + } + prompt := "Help" + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.Error(t, err) + + writeTestFile(t, root, "agents.py", `from agents import Agent +agent = Agent(tools=["search"]) +`) + toolReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="agent"]`, + Map: map[string]string{"prompt": "instructions", "tools": "tools"}, + } + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: toolReference, + Projection: syncdomain.VariationProjection{Instructions: &prompt}, + }}) + require.Error(t, err) + + longGenerated := append(make([]byte, 600), []byte("code generated do not edit")...) + assert.False(t, generatedSource("source.go", longGenerated)) + + badRoot := t.TempDir() + writeTestFile(t, badRoot, ".launchdarkly/project/configs/config/default.ref.yaml", "invalid: [") + err = NewStore(badRoot).CheckSources(nil) + require.Error(t, err) + + writeTestFile(t, root, "agent.py", `from agents import Agent +agent = Agent(instructions="Help", tools=["search"]) +`) + agentReference := Reference{ + File: "agent.py", Format: "python", Select: `agents[id="agent"]`, + Map: map[string]string{"prompt": "instructions", "tools": "tools"}, + } + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: agentReference, + Projection: syncdomain.ToolProjection{}, + }}) + require.Error(t, err) + + changedTools := []string{"search", "read"} + _, err = store.ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: agentReference, + Projection: syncdomain.VariationProjection{Instructions: &prompt, Tools: &changedTools}, + }}) + require.Error(t, err) +} + +func TestStageSourceDocumentRejectsUnsafeFiles(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + writeTestFile(t, root, "generated.go", "// Code generated by tool. DO NOT EDIT.\n") + _, err := store.stageSourceDocument( + "generated.go", + []byte("// Code generated by tool. DO NOT EDIT.\n"), + []byte("updated"), + ) + require.Error(t, err) + + _, err = store.stageSourceDocument("missing.go", nil, []byte("updated")) + require.Error(t, err) + + writeTestFile(t, root, "original.go", "package original\n") + hardlink := filepath.Join(root, "hardlink.go") + if err := os.Link(filepath.Join(root, "original.go"), hardlink); err == nil { + _, err = store.stageSourceDocument("original.go", []byte("package original\n"), []byte("package changed\n")) + require.Error(t, err) + } + + outside := filepath.Join(t.TempDir(), "outside.go") + require.NoError(t, os.WriteFile(outside, []byte("outside"), 0o644)) + if err := os.Symlink(outside, filepath.Join(root, "link.go")); err == nil { + _, err = store.stageSourceDocument("link.go", []byte("outside"), []byte("changed")) + require.Error(t, err) + } +} + +func TestSourceBundleDependencyLimit(t *testing.T) { + var source strings.Builder + for index := 0; index < 33; index++ { + source.WriteString("import value") + source.WriteString(string(rune('A' + index%26))) + source.WriteString(" from './dependency") + source.WriteString(strings.Repeat("x", index+1)) + source.WriteString("';\n") + } + _, err := sourceBundle( + Reference{File: "main.ts", Format: "typescript"}, + []byte(source.String()), + func(Reference) ([]byte, error) { return nil, fs.ErrNotExist }, + ) + require.ErrorContains(t, err, "more than 32 dependencies") +} + +func TestCompileHelpersReturnReadAndParseErrors(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + fsys := fstest.MapFS{"value.ref.yaml": {Data: []byte("invalid: [")}} + _, err := compileReferencedVariation(fsys, "value.ref.yaml", id, func(Reference) ([]byte, error) { + return nil, nil + }) + require.Error(t, err) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: prompt.md +format: markdown +`)}} + expectedRead := errors.New("primary read failed") + _, err = compileReferencedVariation(fsys, "value.ref.yaml", id, func(Reference) ([]byte, error) { + return nil, expectedRead + }) + require.ErrorIs(t, err, expectedRead) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: app/main.py +format: python +select: 'agents[id="agent"]' +map: + prompt: instructions +`)}} + calls := 0 + dependencyError := errors.New("dependency read failed") + _, err = compileReferencedVariation(fsys, "value.ref.yaml", id, func(Reference) ([]byte, error) { + calls++ + if calls == 1 { + return []byte("from .prompt import TEXT\n"), nil + } + return nil, dependencyError + }) + require.ErrorIs(t, err, dependencyError) + + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + _, err = compileReferencedAttachment( + fstest.MapFS{"bad.ref.yaml": {Data: []byte("invalid: [")}}, + "bad.ref.yaml", + toolID, + func(Reference) ([]byte, error) { return nil, nil }, + ) + require.Error(t, err) + + fsys = fstest.MapFS{"value.ref.yaml": {Data: []byte(`formatVersion: 1 +file: tool.json +format: json +select: $ +map: + description: description +`)}} + expected := errors.New("read failed") + _, err = compileReferencedAttachment(fsys, "value.ref.yaml", toolID, func(Reference) ([]byte, error) { + return nil, expected + }) + require.ErrorIs(t, err, expected) + + _, err = compileVariationSource( + id, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"prompt": "prompt"}}, + syncreference.SourceBundle{Primary: syncreference.Document{Path: "value.json", Data: []byte(`{"prompt":`)}}, + ) + require.Error(t, err) + _, err = compileAttachmentSource( + toolID, + Reference{File: "value.json", Format: "json", Select: "$", Map: map[string]string{"description": "description"}}, + syncreference.SourceBundle{Primary: syncreference.Document{Path: "value.json", Data: []byte(`{"description":`)}}, + ) + require.Error(t, err) +} diff --git a/internal/sync/local/attachment.go b/internal/sync/local/attachment.go index 7521ecd8..f8bf9e2d 100644 --- a/internal/sync/local/attachment.go +++ b/internal/sync/local/attachment.go @@ -51,16 +51,36 @@ func CompileAttachments(repositoryRoot string, include func(syncdomain.ResourceI fsys := workspaceFS{root: root} var attachments []syncdomain.SyncedAttachment + definitions := make(map[syncdomain.ResourceID]string) for _, file := range files { id, ok := ParseManagedPath(file) + if !ok { + id, ok = ParseReferencePath(file) + } if !ok || id.Kind == syncdomain.KindVariation || !include(id) { continue } - attachment, err := readAttachment(fsys, id.ProjectKey, syncdomain.AttachmentKind(id.Kind), id.LookupKey) + if existing, duplicate := definitions[id]; duplicate { + return nil, ParseError{Path: file, Err: definitionConflict( + id, fmt.Sprintf("%s is also defined by %s", id.Label(), existing), + )} + } + definitions[id] = file + + var attachment syncdomain.SyncedAttachment + if strings.HasSuffix(file, referenceFileSuffix) { + attachment, err = compileReferencedAttachment(fsys, file, id, func(reference Reference) ([]byte, error) { + return readReferenceFromFS(fsys, reference) + }) + } else { + var value syncdomain.Attachment + value, err = readAttachment(fsys, id.ProjectKey, syncdomain.AttachmentKind(id.Kind), id.LookupKey) + attachment = syncdomain.SyncedAttachment{ProjectKey: id.ProjectKey, Attachment: value} + } if err != nil { return nil, ParseError{Path: file, Err: err} } - attachments = append(attachments, syncdomain.SyncedAttachment{ProjectKey: id.ProjectKey, Attachment: attachment}) + attachments = append(attachments, attachment) } return attachments, nil } diff --git a/internal/sync/local/compile.go b/internal/sync/local/compile.go index 6c92875d..e27f469e 100644 --- a/internal/sync/local/compile.go +++ b/internal/sync/local/compile.go @@ -28,7 +28,9 @@ func (e ParseError) Unwrap() error { return e.Err } // Compile reads every variation in a file system that contains a managed // directory. Tests use it with an in-memory file system. func Compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { - return compile(fsys) + return compile(fsys, func(reference Reference) ([]byte, error) { + return readReferenceFromFS(fsys, reference) + }) } // CompileWorkspace reads every variation in a repository. It rejects a @@ -38,7 +40,9 @@ func CompileWorkspace(repositoryRoot string) ([]syncdomain.SyncedResource, error if err != nil { return nil, fmt.Errorf("resolve repository root: %w", err) } - return compile(workspaceFS{root: root}) + return compile(workspaceFS{root: root}, func(reference Reference) ([]byte, error) { + return ReadReference(root, reference) + }) } // Compile reads every variation in the repository of the store. @@ -68,7 +72,7 @@ func (fsys workspaceFS) Open(name string) (fs.File, error) { } // compile reads the variation files of each project, in identity order. -func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { +func compile(fsys fs.FS, readReference func(Reference) ([]byte, error)) ([]syncdomain.SyncedResource, error) { entries, err := fs.ReadDir(fsys, syncdomain.RootDir) if errors.Is(err, fs.ErrNotExist) { return nil, ErrNoDirectory @@ -84,7 +88,7 @@ func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { if !entry.IsDir() { continue } - projectVariations, err := compileProject(fsys, entry.Name()) + projectVariations, err := compileProject(fsys, entry.Name(), readReference) if err != nil { return nil, err } @@ -101,13 +105,32 @@ func compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { func compileProject( fsys fs.FS, projectKey string, + readReference func(Reference) ([]byte, error), ) ([]syncdomain.SyncedResource, error) { configsRoot := path.Join(syncdomain.RootDir, projectKey, configsDir) readAttachment := func(kind syncdomain.AttachmentKind, key string) (syncdomain.Attachment, error) { + id := syncdomain.ResourceID{Kind: syncdomain.Kind(kind), ProjectKey: projectKey, LookupKey: key} + descriptorPath, err := ReferencePath(id) + if err != nil { + return syncdomain.Attachment{}, err + } + if _, err := fs.Stat(fsys, descriptorPath); err == nil { + managedPath, _ := ResourcePath(id) + if _, managedErr := fs.Stat(fsys, managedPath); managedErr == nil { + return syncdomain.Attachment{}, definitionConflict( + id, id.Label()+" has a managed file and a descriptor", + ) + } + attachment, err := compileReferencedAttachment(fsys, descriptorPath, id, readReference) + return attachment.Attachment, err + } else if !errors.Is(err, fs.ErrNotExist) { + return syncdomain.Attachment{}, err + } return readAttachment(fsys, projectKey, kind, key) } var variations []syncdomain.SyncedResource + definitions := make(map[syncdomain.ResourceID]string) err := fs.WalkDir(fsys, configsRoot, func(name string, entry fs.DirEntry, err error) error { switch { case name == configsRoot && errors.Is(err, fs.ErrNotExist): @@ -118,20 +141,35 @@ func compileProject( case entry.IsDir(): return nil } - if id, ok := ParseManagedPath(name); !ok || id.Kind != syncdomain.KindVariation { + id, managed := ParseManagedPath(name) + if !managed { + id, managed = ParseReferencePath(name) + } + if !managed || id.Kind != syncdomain.KindVariation { return nil } - - data, err := fs.ReadFile(fsys, name) - if err != nil { - return err + if existing, duplicate := definitions[id]; duplicate { + return ParseError{Path: name, Err: definitionConflict( + id, fmt.Sprintf("%s is also defined by %s", id.Label(), existing), + )} + } + definitions[id] = name + + var variation syncdomain.SyncedResource + if strings.HasSuffix(name, referenceFileSuffix) { + variation, err = compileReferencedVariation(fsys, name, id, readReference) + } else { + var data []byte + data, err = fs.ReadFile(fsys, name) + if err == nil { + variation, err = parseVariation(localFile{ + ProjectKey: projectKey, + RelPath: strings.TrimPrefix(name, configsRoot+"/"), + Data: data, + ReadAttachment: readAttachment, + }) + } } - variation, err := parseVariation(localFile{ - ProjectKey: projectKey, - RelPath: strings.TrimPrefix(name, configsRoot+"/"), - Data: data, - ReadAttachment: readAttachment, - }) if err != nil { return ParseError{Path: name, Err: err} } diff --git a/internal/sync/local/reference.go b/internal/sync/local/reference.go index 7d69d065..4316f868 100644 --- a/internal/sync/local/reference.go +++ b/internal/sync/local/reference.go @@ -231,6 +231,17 @@ func resolveReferencePath(repositoryRoot string, reference Reference) (string, e return target, nil } +func readReferenceFromFS(fsys fs.FS, reference Reference) ([]byte, error) { + if err := validateReference(reference); err != nil { + return nil, err + } + data, err := fs.ReadFile(fsys, reference.File) + if err != nil { + return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) + } + return data, nil +} + // ReadReference reads a linked file. It resolves symbolic links and makes sure // that the file is a regular file in the repository. func ReadReference(repositoryRoot string, reference Reference) ([]byte, error) { diff --git a/internal/sync/local/source_edit.go b/internal/sync/local/source_edit.go new file mode 100644 index 00000000..dd66dd50 --- /dev/null +++ b/internal/sync/local/source_edit.go @@ -0,0 +1,367 @@ +package local + +import ( + "bytes" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +// SourceReplacement is one descriptor-backed resource update. +type SourceReplacement struct { + ID syncdomain.ResourceID + Mode syncdomain.VariationMode + Reference Reference + Projection syncdomain.Projection + // ExpectedFingerprint is the mapped source state from the reviewed plan. + ExpectedFingerprint string + // ExpectedProjection supplies reviewed values for unreadable fields. + ExpectedProjection syncdomain.Projection + // ExpectedIndexedSelections identifies the indexed items from the reviewed plan. + ExpectedIndexedSelections []syncdomain.IndexedSelection +} + +func publicationProblem(file, message, suggestion string, cause error) error { + return syncdomain.WrapProblem( + syncdomain.NewProblem( + syncdomain.ErrorPublicationUnsupported, + message, + suggestion, + ).WithFile(file, 0, 0), + cause, + ) +} + +// ReplaceSources applies checked edits to each selected repository source. +func (store Store) ReplaceSources(replacements []SourceReplacement) ([]string, error) { + staged, err := store.prepareSourceReplacements(replacements) + if err != nil { + return nil, err + } + if err := replaceFiles(staged); err != nil { + return nil, err + } + paths := make([]string, len(staged)) + for index := range staged { + paths[index] = staged[index].displayPath + } + return paths, nil +} + +// CheckSources verifies grouped source edits without writing files. +func (store Store) CheckSources(replacements []SourceReplacement) error { + _, err := store.prepareSourceReplacements(replacements) + return err +} + +func (store Store) prepareSourceReplacements(replacements []SourceReplacement) ([]stagedFile, error) { + groups := make(map[string][]SourceReplacement) + for _, replacement := range replacements { + groups[replacement.Reference.File] = append(groups[replacement.Reference.File], replacement) + } + bindings, err := store.sourceBindings() + if err != nil { + return nil, err + } + + var staged []stagedFile + files := slices.Collect(maps.Keys(groups)) + slices.Sort(files) + for _, file := range files { + next, err := store.prepareSourceFiles(file, groups[file], bindings[file]) + if err != nil { + return nil, err + } + staged = append(staged, next...) + } + return staged, nil +} + +type sourceBinding struct { + id syncdomain.ResourceID + mode syncdomain.VariationMode + reference Reference +} + +func (store Store) sourceBindings() (map[string][]sourceBinding, error) { + files, err := SourceFiles(store.repositoryRoot) + if err != nil { + return nil, err + } + root, err := filepath.EvalSymlinks(store.repositoryRoot) + if err != nil { + return nil, fmt.Errorf("resolve repository root: %w", err) + } + fsys := workspaceFS{root: root} + result := make(map[string][]sourceBinding) + for _, file := range files { + id, ok := ParseReferencePath(file) + if !ok { + continue + } + reference, err := ReadDescriptor(fsys, file, id) + if err != nil { + return nil, ParseError{Path: file, Err: err} + } + mode := syncdomain.VariationMode("") + if id.Kind == syncdomain.KindVariation { + mode = syncdomain.VariationModeAgent + } + result[reference.File] = append(result[reference.File], sourceBinding{id: id, mode: mode, reference: reference}) + } + return result, nil +} + +func (store Store) prepareSourceFiles( + file string, + replacements []SourceReplacement, + bindings []sourceBinding, +) ([]stagedFile, error) { + bundle, err := ReadSourceBundle(store.repositoryRoot, replacements[0].Reference) + if err != nil { + return nil, err + } + + var edits []syncreference.TextEdit + targets := make(map[syncdomain.ResourceID]struct{}, len(replacements)) + for _, replacement := range replacements { + targets[replacement.ID] = struct{}{} + current, indexedSelections, err := syncreference.ParseSourceWithSelections( + bundle, + replacement.ID, + replacement.Mode, + replacement.Reference, + ) + if err != nil { + return nil, publicationProblem(file, "source cannot be updated safely", "Change the source to a supported static form.", err) + } + if replacement.ExpectedIndexedSelections != nil && + !slices.Equal(replacement.ExpectedIndexedSelections, indexedSelections) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorIndexedSelectionChanged, + "an indexed source selection changed after review", + "Run sync again. Use a keyed selector when the item has a stable key.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + if replacement.ExpectedProjection != nil { + current, err = syncdomain.FillReadOnlyFields(current, replacement.ExpectedProjection) + if err != nil { + return nil, err + } + } + fingerprint, err := syncdomain.FingerprintProjection(replacement.ID, current) + if err != nil { + return nil, err + } + if replacement.ExpectedFingerprint != "" && fingerprint != replacement.ExpectedFingerprint { + return nil, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, + "mapped source fields changed after review", + "Run sync again.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + result, err := syncreference.UpdateSource( + bundle, + replacement.ID, + replacement.Mode, + replacement.Reference, + replacement.Projection, + ) + if err != nil { + return nil, err + } + for field, update := range result.Fields { + if update.Capability != syncreference.CapabilityDirect { + if update.Capability == syncreference.CapabilityReadOnly { + return nil, syncdomain.NewProblem( + syncdomain.ErrorFieldReadOnly, + fmt.Sprintf("field %q is read-only: %s", field, update.Reason), + "Change the field to a directly writable form.", + ).WithResource(replacement.ID).WithFile(file, 0, 0) + } + return nil, publicationProblem( + file, + fmt.Sprintf("field %q of %s is %s: %s", field, replacement.ID.Label(), update.Capability, update.Reason), + "Change the field to a directly writable form.", + nil, + ) + } + edits = append(edits, update.Edits...) + } + } + editsByFile := map[string][]syncreference.TextEdit{} + for _, edit := range edits { + editsByFile[edit.File] = append(editsByFile[edit.File], edit) + } + updatedBundle := bundle + updatedBundle.Dependencies = maps.Clone(bundle.Dependencies) + updatedDocuments := map[string]syncreference.Document{} + for editFile, fileEdits := range editsByFile { + document, ok := sourceBundleDocument(bundle, editFile) + if !ok { + return nil, fmt.Errorf("edit targets unknown dependency %q", editFile) + } + updated, err := applySourceEdits(editFile, document.Data, fileEdits) + if err != nil { + return nil, err + } + document.Data = updated + updatedDocuments[editFile] = document + if editFile == bundle.Primary.Path { + updatedBundle.Primary = document + } else { + updatedBundle.Dependencies[editFile] = document + } + } + + for _, replacement := range replacements { + actual, err := syncreference.ParseSource( + updatedBundle, + replacement.ID, + replacement.Mode, + replacement.Reference, + ) + if err != nil { + return nil, fmt.Errorf("verify updated %s: %w", replacement.ID.Label(), err) + } + if !syncdomain.ProjectionIncludes(actual, replacement.Projection) { + return nil, fmt.Errorf("updated projection does not match %s", replacement.ID.Label()) + } + } + for _, binding := range bindings { + if _, changed := targets[binding.id]; changed { + continue + } + before, err := syncreference.ParseSource( + bundle, + binding.id, binding.mode, binding.reference, + ) + if err != nil { + return nil, err + } + after, err := syncreference.ParseSource( + updatedBundle, + binding.id, binding.mode, binding.reference, + ) + if err != nil || !sameProjection(binding.id, before, after) { + return nil, fmt.Errorf("non-target projection %s changed", binding.id.Label()) + } + } + var staged []stagedFile + for changedFile, updated := range updatedDocuments { + original, _ := sourceBundleDocument(bundle, changedFile) + next, err := store.stageSourceDocument(changedFile, original.Data, updated.Data) + if err != nil { + return nil, err + } + if !bytes.Equal(next.original, next.replacement) { + staged = append(staged, next) + } + } + return staged, nil +} + +func sourceBundleDocument(bundle syncreference.SourceBundle, file string) (syncreference.Document, bool) { + if file == bundle.Primary.Path { + return bundle.Primary, true + } + document, ok := bundle.Dependencies[file] + return document, ok +} + +func (store Store) stageSourceDocument(file string, original, updated []byte) (stagedFile, error) { + root, err := filepath.EvalSymlinks(store.repositoryRoot) + if err != nil { + return stagedFile{}, fmt.Errorf("resolve repository root: %w", err) + } + reference := Reference{File: file, Format: "dependency"} + unresolved := filepath.Join(root, filepath.FromSlash(file)) + if err := rejectSymlinkedPath(root, unresolved); err != nil { + return stagedFile{}, publicationProblem(file, "source path is not safe to replace", "Use a regular repository file.", err) + } + path, err := resolveReferencePath(store.repositoryRoot, reference) + if err != nil { + return stagedFile{}, err + } + info, err := os.Lstat(path) + if err != nil { + return stagedFile{}, fmt.Errorf("inspect source file %q: %w", file, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return stagedFile{}, publicationProblem(file, "source is a symbolic link", "Use a regular repository file.", nil) + } + if stat, ok := info.Sys().(*syscall.Stat_t); ok && stat.Nlink > 1 { + return stagedFile{}, publicationProblem(file, "source has multiple hard links", "Use a file with one link.", nil) + } + if generatedSource(file, original) { + return stagedFile{}, publicationProblem(file, "source is generated, vendored, or minified", "Link the authored source file.", nil) + } + return stagedFile{ + displayPath: file, path: path, original: original, originalExists: true, + replacement: updated, mode: info.Mode().Perm(), + }, nil +} + +func applySourceEdits(file string, original []byte, edits []syncreference.TextEdit) ([]byte, error) { + for _, edit := range edits { + if edit.File != file || edit.Start < 0 || edit.End < edit.Start || edit.End > len(original) { + return nil, fmt.Errorf("invalid edit for %q", file) + } + if !bytes.Equal(original[edit.Start:edit.End], edit.Expected) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorStateChanged, + "source bytes changed after review", + "Run sync again.", + ).WithFile(file, 0, 0) + } + } + slices.SortFunc(edits, func(left, right syncreference.TextEdit) int { + if left.Start != right.Start { + return right.Start - left.Start + } + return right.End - left.End + }) + for index := 1; index < len(edits); index++ { + if edits[index-1].Start < edits[index].End { + return nil, syncdomain.NewProblem( + syncdomain.ErrorOverlappingEdits, + "source updates overlap", + "Use mappings that own separate source values.", + ).WithFile(file, 0, 0) + } + } + updated := bytes.Clone(original) + for _, edit := range edits { + next := make([]byte, 0, len(updated)-edit.End+edit.Start+len(edit.Replacement)) + next = append(next, updated[:edit.Start]...) + next = append(next, edit.Replacement...) + next = append(next, updated[edit.End:]...) + updated = next + } + return updated, nil +} + +func sameProjection(id syncdomain.ResourceID, left, right syncdomain.Projection) bool { + leftFingerprint, leftErr := syncdomain.FingerprintProjection(id, left) + rightFingerprint, rightErr := syncdomain.FingerprintProjection(id, right) + return leftErr == nil && rightErr == nil && leftFingerprint == rightFingerprint +} + +func generatedSource(file string, data []byte) bool { + path := filepath.ToSlash(file) + if strings.Contains("/"+path+"/", "/vendor/") || strings.HasSuffix(path, ".min.js") { + return true + } + prefix := strings.ToLower(string(data)) + if len(prefix) > 512 { + prefix = prefix[:512] + } + return strings.Contains(prefix, "code generated") && strings.Contains(prefix, "do not edit") +} diff --git a/internal/sync/local/source_reference.go b/internal/sync/local/source_reference.go new file mode 100644 index 00000000..9a763458 --- /dev/null +++ b/internal/sync/local/source_reference.go @@ -0,0 +1,225 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "maps" + "slices" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +// SourceDefinition is one compiled descriptor-backed resource. +type SourceDefinition struct { + Variation *syncdomain.SyncedResource + Attachment *syncdomain.SyncedAttachment +} + +// ID returns the resource identity of a source definition. +func (definition SourceDefinition) ID() syncdomain.ResourceID { + if definition.Variation != nil { + return definition.Variation.ID() + } + if definition.Attachment != nil { + return definition.Attachment.ID() + } + return syncdomain.ResourceID{} +} + +// CompileSource compiles a proposed reference without writing its descriptor. +func CompileSource( + repositoryRoot string, + id syncdomain.ResourceID, + reference Reference, +) (SourceDefinition, error) { + bundle, err := ReadSourceBundle(repositoryRoot, reference) + if err != nil { + return SourceDefinition{}, err + } + switch id.Kind { + case syncdomain.KindVariation: + variation, err := compileVariationSource(id, reference, bundle) + return SourceDefinition{Variation: &variation}, err + case syncdomain.KindTool, syncdomain.KindSkill: + attachment, err := compileAttachmentSource(id, reference, bundle) + return SourceDefinition{Attachment: &attachment}, err + default: + return SourceDefinition{}, fmt.Errorf("unsupported resource kind %q", id.Kind) + } +} + +// ReadSourceBundle reads a primary source and its static dependencies. +func ReadSourceBundle(repositoryRoot string, reference Reference) (syncreference.SourceBundle, error) { + content, err := ReadReference(repositoryRoot, reference) + if err != nil { + return syncreference.SourceBundle{}, err + } + return sourceBundle(reference, content, func(dependency Reference) ([]byte, error) { + return ReadReference(repositoryRoot, dependency) + }) +} + +func sourceBundle( + reference Reference, + content []byte, + read func(Reference) ([]byte, error), +) (syncreference.SourceBundle, error) { + bundle := syncreference.SourceBundle{ + Primary: syncreference.Document{Path: reference.File, Data: content}, + Dependencies: map[string]syncreference.Document{}, + } + paths, err := syncreference.DependencyPaths(reference.Format, bundle.Primary) + if err != nil { + return syncreference.SourceBundle{}, err + } + if len(paths) > 32 { + return syncreference.SourceBundle{}, fmt.Errorf("source has more than 32 dependencies") + } + total := len(content) + for _, dependencyPath := range paths { + dependency := reference + dependency.File = dependencyPath + data, err := read(dependency) + problem, isProblem := syncdomain.ProblemFrom(err) + if errors.Is(err, fs.ErrNotExist) || + isProblem && problem.Code == syncdomain.ErrorLinkedSourceMissing { + continue + } + if err != nil { + return syncreference.SourceBundle{}, err + } + total += len(data) + if total > 8<<20 { + return syncreference.SourceBundle{}, fmt.Errorf("source dependencies exceed 8 MiB") + } + bundle.Dependencies[dependencyPath] = syncreference.Document{ + Path: dependencyPath, Data: data, + } + } + return bundle, nil +} + +func compileReferencedVariation( + fsys fs.FS, + descriptorPath string, + id syncdomain.ResourceID, + readSource func(Reference) ([]byte, error), +) (syncdomain.SyncedResource, error) { + reference, err := ReadDescriptor(fsys, descriptorPath, id) + if err != nil { + return syncdomain.SyncedResource{}, err + } + content, err := readSource(reference) + if err != nil { + return syncdomain.SyncedResource{}, err + } + bundle, err := sourceBundle(reference, content, readSource) + if err != nil { + return syncdomain.SyncedResource{}, err + } + return compileVariationSource(id, reference, bundle) +} + +func compileVariationSource( + id syncdomain.ResourceID, + reference Reference, + bundle syncreference.SourceBundle, +) (syncdomain.SyncedResource, error) { + if generatedSource(reference.File, bundle.Primary.Data) { + return syncdomain.SyncedResource{}, publicationProblem( + reference.File, "source is generated, vendored, or minified", "Link the authored source file.", nil, + ) + } + projection, indexedSelections, err := syncreference.ParseSourceWithSelections( + bundle, + id, + syncdomain.VariationModeAgent, + reference, + ) + if err != nil { + return syncdomain.SyncedResource{}, err + } + variationProjection, ok := projection.(syncdomain.VariationProjection) + if !ok { + return syncdomain.SyncedResource{}, fmt.Errorf("source returned %T for a variation", projection) + } + _, variationKey, _ := id.VariationKeys() + variation := syncdomain.ApplyVariationProjection(syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, + Key: variationKey, + Name: syncdomain.DisplayName(variationKey), + }, variationProjection) + return syncdomain.SyncedResource{ + Kind: id.Kind, ProjectKey: id.ProjectKey, LookupKey: id.LookupKey, + Source: &reference, Dependencies: sortedDependencyPaths(bundle), + IndexedSelections: indexedSelections, Projection: projection, Variation: variation, + }, nil +} + +func compileReferencedAttachment( + fsys fs.FS, + descriptorPath string, + id syncdomain.ResourceID, + readSource func(Reference) ([]byte, error), +) (syncdomain.SyncedAttachment, error) { + reference, err := ReadDescriptor(fsys, descriptorPath, id) + if err != nil { + return syncdomain.SyncedAttachment{}, err + } + content, err := readSource(reference) + if err != nil { + return syncdomain.SyncedAttachment{}, err + } + bundle, err := sourceBundle(reference, content, readSource) + if err != nil { + return syncdomain.SyncedAttachment{}, err + } + return compileAttachmentSource(id, reference, bundle) +} + +func compileAttachmentSource( + id syncdomain.ResourceID, + reference Reference, + bundle syncreference.SourceBundle, +) (syncdomain.SyncedAttachment, error) { + if generatedSource(reference.File, bundle.Primary.Data) { + return syncdomain.SyncedAttachment{}, publicationProblem( + reference.File, "source is generated, vendored, or minified", "Link the authored source file.", nil, + ) + } + projection, indexedSelections, err := syncreference.ParseSourceWithSelections( + bundle, + id, + "", + reference, + ) + if err != nil { + return syncdomain.SyncedAttachment{}, err + } + + attachment := syncdomain.Attachment{Kind: syncdomain.AttachmentKind(id.Kind)} + switch id.Kind { + case syncdomain.KindTool: + attachment.Tool = &syncdomain.Tool{Key: id.LookupKey} + case syncdomain.KindSkill: + attachment.Skill = &syncdomain.Skill{Key: id.LookupKey} + default: + return syncdomain.SyncedAttachment{}, fmt.Errorf("unsupported attachment kind %q", id.Kind) + } + attachment, err = syncdomain.ApplyAttachmentProjection(attachment, projection) + if err != nil { + return syncdomain.SyncedAttachment{}, err + } + return syncdomain.SyncedAttachment{ + ProjectKey: id.ProjectKey, Attachment: attachment, Source: &reference, + Dependencies: sortedDependencyPaths(bundle), IndexedSelections: indexedSelections, Projection: projection, + }, nil +} + +func sortedDependencyPaths(bundle syncreference.SourceBundle) []string { + paths := slices.Collect(maps.Keys(bundle.Dependencies)) + slices.Sort(paths) + return paths +} diff --git a/internal/sync/local/source_reference_test.go b/internal/sync/local/source_reference_test.go index b111a940..3eb24bb1 100644 --- a/internal/sync/local/source_reference_test.go +++ b/internal/sync/local/source_reference_test.go @@ -1,6 +1,7 @@ package local import ( + "errors" "os" "path/filepath" "testing" @@ -11,10 +12,82 @@ import ( syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) +func TestCompileWorkspaceReadsVariationDescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "agents.py", `from agents import Agent +support = Agent(instructions="Help with orders.") +`) + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", `formatVersion: 1 +file: agents.py +format: python +select: 'agents[id="support"]' +map: + prompt: instructions +`) + + resources, err := CompileWorkspace(root) + require.NoError(t, err) + require.Len(t, resources, 1) + assert.Equal(t, syncdomain.VariationID("project", "support", "default"), resources[0].ID()) + assert.Equal(t, "Help with orders.", resources[0].Variation.Instructions) + require.NotNil(t, resources[0].Source) + assert.Equal(t, "agents.py", resources[0].Source.File) +} + +func TestCompileSourceWithoutWritingDescriptor(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help with orders.\n") + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{File: "prompt.md", Format: "markdown"} + + definition, err := CompileSource(root, id, reference) + + require.NoError(t, err) + assert.Equal(t, id, definition.ID()) + require.NotNil(t, definition.Variation) + assert.Equal(t, "Help with orders.", definition.Variation.Variation.Instructions) + require.NoFileExists(t, filepath.Join(root, ".launchdarkly", "project", "configs", "support", "default.ref.yaml")) +} + +func TestImportedSourceEditsDependencyAtomically(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "app/agent.py", `from agents import Agent +from .prompts import SUPPORT_PROMPT +support = Agent(instructions=SUPPORT_PROMPT, model="gpt-5") +`) + writeTestFile(t, root, "app/prompts.py", `SUPPORT_PROMPT = "Original prompt"`+"\n") + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{ + File: "app/agent.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, id, reference) + prompt := "Updated imported prompt" + + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &prompt, + }, + }}) + + require.NoError(t, err) + assert.Equal(t, []string{"app/prompts.py"}, paths) + primary, err := os.ReadFile(filepath.Join(root, "app", "agent.py")) + require.NoError(t, err) + assert.Contains(t, string(primary), "instructions=SUPPORT_PROMPT") + dependency, err := os.ReadFile(filepath.Join(root, "app", "prompts.py")) + require.NoError(t, err) + assert.Contains(t, string(dependency), `"Updated imported prompt"`) + files, err := SourceFiles(root) + require.NoError(t, err) + assert.Contains(t, files, "app/prompts.py") +} + func TestWriteDescriptorCreatesUpdatesAndKeepsIdenticalContent(t *testing.T) { root := t.TempDir() - writeDescriptorTestFile(t, root, "first.md", "First.\n") - writeDescriptorTestFile(t, root, "second.md", "Second.\n") + writeTestFile(t, root, "first.md", "First.\n") + writeTestFile(t, root, "second.md", "Second.\n") id := syncdomain.VariationID("project", "support", "default") store := NewStore(root) @@ -35,9 +108,359 @@ func TestWriteDescriptorCreatesUpdatesAndKeepsIdenticalContent(t *testing.T) { assert.Contains(t, string(updated), "file: second.md") } -func writeDescriptorTestFile(t *testing.T, root, name, content string) { +func TestCompileSourceReportsMissingFile(t *testing.T) { + root := t.TempDir() + id := syncdomain.VariationID("project", "support", "default") + + _, err := CompileSource(root, id, Reference{File: "missing.md", Format: "markdown"}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorLinkedSourceMissing, problem.Code) + assert.Equal(t, "missing.md", problem.File) +} + +func TestCompileWorkspaceRejectsManagedAndDescriptorDefinitions(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "prompt.md", "Help.\n") + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.ref.yaml", `formatVersion: 1 +file: prompt.md +format: markdown +`) + writeTestFile(t, root, ".launchdarkly/project/configs/support/default.prompt.md", `--- +formatVersion: 1 +upsert: false +mode: agent +key: default +name: Default +--- + +Help. +`) + + _, err := CompileWorkspace(root) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorDefinitionConflict, problem.Code) +} + +func TestCompileAttachmentsReadsToolAndSkillDescriptors(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tools.ts", `import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +const server = new McpServer({ name: "orders", version: "1" }); +server.registerTool("lookup", { + description: "Look up an order.", + inputSchema: { type: "object" }, +}, handler); +`) + writeTestFile(t, root, ".agents/skills/refunds/SKILL.md", `--- +name: refunds +description: Explain refunds. +--- + +# Refunds + +Ask for the order ID. +`) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"} + skillID := syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"} + writeDescriptorForTest(t, root, toolID, Reference{ + File: "tools.ts", Format: "typescript", Select: `tools[id="lookup"]`, + Map: map[string]string{"description": "description", "schema": "inputSchema"}, + }) + writeDescriptorForTest(t, root, skillID, Reference{ + File: ".agents/skills/refunds/SKILL.md", Format: "agent-skill", + }) + + attachments, err := CompileAttachments(root, func(syncdomain.ResourceID) bool { return true }) + + require.NoError(t, err) + require.Len(t, attachments, 2) + byID := map[syncdomain.ResourceID]syncdomain.SyncedAttachment{} + for _, attachment := range attachments { + byID[attachment.ID()] = attachment + } + assert.Equal(t, "Look up an order.", *byID[toolID].Attachment.Tool.Description) + assert.Equal(t, "Explain refunds.", byID[skillID].Attachment.Skill.Description) +} + +func TestReplaceSourcesGroupsEditsAndPreservesOtherBytes(t *testing.T) { + root := t.TempDir() + original := `from agents import Agent + +support = Agent(instructions="Help with orders.") +refund = Agent(instructions='Explain refunds.') +` + writeTestFile(t, root, "agents.py", original) + supportID := syncdomain.VariationID("project", "support", "default") + refundID := syncdomain.VariationID("project", "refund", "default") + supportReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + refundReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="refund"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, supportID, supportReference) + writeDescriptorForTest(t, root, refundID, refundReference) + + supportPrompt := "Help with orders and shipping." + refundPrompt := "Explain refunds and escalation." + paths, err := NewStore(root).ReplaceSources([]SourceReplacement{ + { + ID: supportID, Mode: syncdomain.VariationModeAgent, Reference: supportReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &supportPrompt, + }, + }, + { + ID: refundID, Mode: syncdomain.VariationModeAgent, Reference: refundReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &refundPrompt, + }, + }, + }) + require.NoError(t, err) + assert.Equal(t, []string{"agents.py"}, paths) + content, err := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, err) + assert.Equal(t, `from agents import Agent + +support = Agent(instructions="Help with orders and shipping.") +refund = Agent(instructions='Explain refunds and escalation.') +`, string(content)) +} + +func TestReplaceSourcesGroupsStructuredFormatEdits(t *testing.T) { + tests := []struct { + name string + file string + format string + source string + firstID syncdomain.ResourceID + secondID syncdomain.ResourceID + firstSelect string + secondSelect string + fieldMap map[string]string + first syncdomain.Projection + second syncdomain.Projection + want []string + }{ + { + name: "JSON tools", file: "tools.json", format: "json", + source: `{"tools":[{"name":"lookup","description":"Old lookup"},{"name":"refund","description":"Old refund"}]}` + "\n", + firstID: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"}, + secondID: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "refund"}, + firstSelect: `tools[name="lookup"]`, secondSelect: `tools[name="refund"]`, + fieldMap: map[string]string{"description": "description"}, + first: syncdomain.ToolProjection{Description: stringPointer("New lookup")}, + second: syncdomain.ToolProjection{Description: stringPointer("New refund")}, + want: []string{`"description":"New lookup"`, `"description":"New refund"`}, + }, + { + name: "YAML agents", file: "agents.yaml", format: "yaml", + source: "agents:\n - name: support\n prompt: Old support\n - name: refunds\n prompt: Old refunds\n", + firstID: syncdomain.VariationID("project", "support", "default"), + secondID: syncdomain.VariationID("project", "refunds", "default"), + firstSelect: `agents[name="support"]`, secondSelect: `agents[name="refunds"]`, + fieldMap: map[string]string{"prompt": "prompt"}, + first: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New support")}, + second: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New refunds")}, + want: []string{"prompt: New support", "prompt: New refunds"}, + }, + { + name: "TOML agents", file: "agents.toml", format: "toml", + source: "[agents.support]\nprompt = \"Old support\"\n[agents.refunds]\nprompt = \"Old refunds\"\n", + firstID: syncdomain.VariationID("project", "support", "default"), + secondID: syncdomain.VariationID("project", "refunds", "default"), + firstSelect: "agents.support", secondSelect: "agents.refunds", + fieldMap: map[string]string{"prompt": "prompt"}, + first: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New support")}, + second: syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: stringPointer("New refunds")}, + want: []string{`prompt = "New support"`, `prompt = "New refunds"`}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, test.file, test.source) + firstReference := Reference{ + File: test.file, Format: test.format, Select: test.firstSelect, Map: test.fieldMap, + } + secondReference := Reference{ + File: test.file, Format: test.format, Select: test.secondSelect, Map: test.fieldMap, + } + writeDescriptorForTest(t, root, test.firstID, firstReference) + writeDescriptorForTest(t, root, test.secondID, secondReference) + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{ + {ID: test.firstID, Mode: syncdomain.VariationModeAgent, Reference: firstReference, Projection: test.first}, + {ID: test.secondID, Mode: syncdomain.VariationModeAgent, Reference: secondReference, Projection: test.second}, + }) + + require.NoError(t, err) + content, err := os.ReadFile(filepath.Join(root, test.file)) + require.NoError(t, err) + for _, expected := range test.want { + assert.Contains(t, string(content), expected) + } + }) + } +} + +func TestReplaceSourcesRejectsOverlappingEdits(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "tool.json", `{"description":"Old"}`+"\n") + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + reference := Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{"description": "description"}, + } + writeDescriptorForTest(t, root, id, reference) + description := "New" + replacement := SourceReplacement{ + ID: id, Reference: reference, Projection: syncdomain.ToolProjection{Description: &description}, + } + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{replacement, replacement}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorOverlappingEdits, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "tool.json")) + require.NoError(t, readErr) + assert.Equal(t, `{"description":"Old"}`+"\n", string(content)) +} + +func TestReplaceSourcesWritesNothingWhenOneFieldIsReadOnly(t *testing.T) { + root := t.TempDir() + original := `from agents import Agent +support = Agent(instructions="Help.") +dynamic = Agent(instructions=load_prompt()) +` + writeTestFile(t, root, "agents.py", original) + supportID := syncdomain.VariationID("project", "support", "default") + dynamicID := syncdomain.VariationID("project", "dynamic", "default") + supportReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + dynamicReference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="dynamic"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, supportID, supportReference) + writeDescriptorForTest(t, root, dynamicID, dynamicReference) + supportPrompt, dynamicPrompt := "Changed.", "Cannot write." + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{ + { + ID: supportID, Mode: syncdomain.VariationModeAgent, Reference: supportReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &supportPrompt, + }, + }, + { + ID: dynamicID, Mode: syncdomain.VariationModeAgent, Reference: dynamicReference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &dynamicPrompt, + }, + }, + }) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorFieldReadOnly, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, readErr) + assert.Equal(t, original, string(content)) +} + +func TestReplaceSourcesRejectsMappedChangeAfterReview(t *testing.T) { + root := t.TempDir() + writeTestFile(t, root, "agents.py", `from agents import Agent +support = Agent(instructions="Changed after review.") +`) + id := syncdomain.VariationID("project", "support", "default") + reference := Reference{ + File: "agents.py", Format: "python", Select: `agents[id="support"]`, + Map: map[string]string{"prompt": "instructions"}, + } + writeDescriptorForTest(t, root, id, reference) + desired := "Server value." + + _, err := NewStore(root).ReplaceSources([]SourceReplacement{{ + ID: id, Mode: syncdomain.VariationModeAgent, Reference: reference, + Projection: syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Instructions: &desired, + }, + ExpectedFingerprint: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }}) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorStateChanged, problem.Code) + content, readErr := os.ReadFile(filepath.Join(root, "agents.py")) + require.NoError(t, readErr) + assert.Contains(t, string(content), "Changed after review.") +} + +func TestCommitFilesRollsBackEarlierReplacement(t *testing.T) { + root := t.TempDir() + firstPath, secondPath := filepath.Join(root, "first.txt"), filepath.Join(root, "second.txt") + require.NoError(t, os.WriteFile(firstPath, []byte("first original"), 0o644)) + require.NoError(t, os.WriteFile(secondPath, []byte("second original"), 0o644)) + firstStaged, err := writeTempFile(firstPath, []byte("first updated"), 0o644) + require.NoError(t, err) + secondStaged, err := writeTempFile(secondPath, []byte("second updated"), 0o644) + require.NoError(t, err) + t.Cleanup(func() { + _ = os.Remove(firstStaged) + _ = os.Remove(secondStaged) + }) + files := []stagedFile{ + { + displayPath: "first.txt", path: firstPath, original: []byte("first original"), + originalExists: true, replacement: []byte("first updated"), mode: 0o644, stagedPath: firstStaged, + }, + { + displayPath: "second.txt", path: secondPath, original: []byte("second original"), + originalExists: true, replacement: []byte("second updated"), mode: 0o644, stagedPath: secondStaged, + }, + } + + err = commitFilesWithRename(files, func(source, destination string) error { + if source == secondStaged { + return errors.New("injected rename failure") + } + return os.Rename(source, destination) + }) + + require.ErrorContains(t, err, "injected rename failure") + first, readErr := os.ReadFile(firstPath) + require.NoError(t, readErr) + second, readErr := os.ReadFile(secondPath) + require.NoError(t, readErr) + assert.Equal(t, "first original", string(first)) + assert.Equal(t, "second original", string(second)) +} + +func writeDescriptorForTest(t *testing.T, root string, id syncdomain.ResourceID, reference Reference) { + t.Helper() + path, err := ReferencePath(id) + require.NoError(t, err) + content, err := RenderDescriptor(id, reference) + require.NoError(t, err) + writeTestFile(t, root, path, string(content)) +} + +func writeTestFile(t *testing.T, root, name, content string) { t.Helper() path := filepath.Join(root, filepath.FromSlash(name)) require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) } + +func stringPointer(value string) *string { + return &value +} diff --git a/internal/sync/local/sources.go b/internal/sync/local/sources.go index 4bf3716e..3ba8df50 100644 --- a/internal/sync/local/sources.go +++ b/internal/sync/local/sources.go @@ -8,12 +8,12 @@ import ( "slices" syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" ) // SourceFiles returns every file that can change the sync plan, relative to -// the repository root. The list has each file in a project directory and -// each managed file. A file that is not valid stays in the list, so that a -// fix to the file starts a new sync in watch mode. +// the repository root. The list has each managed or referenced file. A file +// that is not valid stays in the list, so a fix starts a new sync in watch mode. func SourceFiles(repositoryRoot string) ([]string, error) { managedRoot := filepath.Join(repositoryRoot, syncdomain.RootDir) var files []string @@ -35,6 +35,19 @@ func SourceFiles(repositoryRoot string) ([]string, error) { if filepath.Dir(filePath) != managedRoot { files = append(files, filepath.ToSlash(relative)) } + relative = filepath.ToSlash(relative) + if id, ok := ParseReferencePath(relative); ok { + if reference, err := ReadDescriptor(os.DirFS(repositoryRoot), relative, id); err == nil { + files = append(files, reference.File) + if content, err := ReadReference(repositoryRoot, reference); err == nil { + dependencies, _ := syncreference.DependencyPaths( + reference.Format, + syncreference.Document{Path: reference.File, Data: content}, + ) + files = append(files, dependencies...) + } + } + } return nil }) if errors.Is(err, os.ErrNotExist) { diff --git a/internal/sync/local/store.go b/internal/sync/local/store.go index 03a2c54c..f4fe51a9 100644 --- a/internal/sync/local/store.go +++ b/internal/sync/local/store.go @@ -75,21 +75,40 @@ func (store Store) Exists() (bool, error) { // VariationExists reports whether a variation file exists. func (store Store) VariationExists(projectKey, configKey, variationKey string) (bool, error) { - relativePath, err := variationPath(projectKey, configKey, variationKey) - if err != nil { - return false, err - } - return store.fileExists(relativePath, "variation "+variationKey) + return store.definitionExists( + syncdomain.VariationID(projectKey, configKey, variationKey), + "variation "+variationKey, + ) } // ResourceExists reports whether the file of a variation, a tool, or a skill // exists. func (store Store) ResourceExists(id syncdomain.ResourceID) (bool, error) { - relativePath, err := resourcePath(id) + return store.definitionExists(id, id.Label()) +} + +func (store Store) definitionExists(id syncdomain.ResourceID, label string) (bool, error) { + managedPath, err := resourcePath(id) + if err != nil { + return false, err + } + managed, err := store.fileExists(managedPath, label) + if err != nil { + return false, err + } + repositoryReferencePath, err := ReferencePath(id) if err != nil { return false, err } - return store.fileExists(relativePath, id.Label()) + referencePath := strings.TrimPrefix(repositoryReferencePath, syncdomain.RootDir+"/") + referenced, err := store.fileExists(referencePath, label+" descriptor") + if err != nil { + return false, err + } + if managed && referenced { + return false, definitionConflict(id, id.Label()+" has a managed file and a descriptor") + } + return managed || referenced, nil } // fileExists reports whether a managed file exists. label names the file in