From 98da134d1ac338dfd0f04939e0df4c53fa1f8fe3 Mon Sep 17 00:00:00 2001 From: Cliff Tawiah Date: Fri, 30 Oct 2026 18:45:08 -0400 Subject: [PATCH] feat(sync): add source descriptor storage --- internal/sync/local/create.go | 19 ++ internal/sync/local/layout.go | 57 +++++ internal/sync/local/reference.go | 246 +++++++++++++++++++ internal/sync/local/reference_test.go | 50 ++++ internal/sync/local/source_reference_test.go | 43 ++++ 5 files changed, 415 insertions(+) create mode 100644 internal/sync/local/reference.go create mode 100644 internal/sync/local/reference_test.go create mode 100644 internal/sync/local/source_reference_test.go diff --git a/internal/sync/local/create.go b/internal/sync/local/create.go index c6d78bfa2..02a2bace0 100644 --- a/internal/sync/local/create.go +++ b/internal/sync/local/create.go @@ -4,8 +4,11 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) // Creation records the result of a create. VariationPaths are the variation @@ -81,6 +84,22 @@ func (store Store) Add(files []VariationFile) (Creation, error) { if err != nil { return Creation{}, err } + for _, file := range rendered { + id, ok := ParseManagedPath(path.Join(syncdomain.RootDir, file.Path)) + if !ok { + continue + } + repositoryReferencePath, err := ReferencePath(id) + if err != nil { + return Creation{}, err + } + referencePath := strings.TrimPrefix(repositoryReferencePath, syncdomain.RootDir+"/") + if exists, err := store.fileExists(referencePath, id.Label()+" descriptor"); err != nil { + return Creation{}, err + } else if exists { + return Creation{}, definitionConflict(id, id.Label()+" already has a descriptor") + } + } var creation Creation for _, file := range rendered { diff --git a/internal/sync/local/layout.go b/internal/sync/local/layout.go index 8e86cb558..da6d46859 100644 --- a/internal/sync/local/layout.go +++ b/internal/sync/local/layout.go @@ -26,6 +26,7 @@ const ( variationFileSuffix = ".prompt.md" toolFileSuffix = ".json" skillFileSuffix = ".md" + referenceFileSuffix = ".ref.yaml" ) // ParseManagedPath returns the resource that a managed file stores. The path @@ -62,6 +63,37 @@ func ParseManagedPath(file string) (syncdomain.ResourceID, bool) { return id, true } +// ParseReferencePath returns the resource that a .ref.yaml descriptor links. +func ParseReferencePath(file string) (syncdomain.ResourceID, bool) { + parts := strings.Split(file, "/") + if len(parts) < 4 || parts[0] != syncdomain.RootDir { + return syncdomain.ResourceID{}, false + } + + var id syncdomain.ResourceID + var stem string + switch { + case len(parts) == 5 && parts[2] == configsDir && strings.HasSuffix(parts[4], referenceFileSuffix): + stem = strings.TrimSuffix(parts[4], referenceFileSuffix) + if syncdomain.ValidateKey(parts[3]) != nil { + return syncdomain.ResourceID{}, false + } + id = syncdomain.VariationID(parts[1], parts[3], stem) + case len(parts) == 4 && parts[2] == toolsDir && strings.HasSuffix(parts[3], referenceFileSuffix): + stem = strings.TrimSuffix(parts[3], referenceFileSuffix) + id = syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: parts[1], LookupKey: stem} + case len(parts) == 4 && parts[2] == skillsDir && strings.HasSuffix(parts[3], referenceFileSuffix): + stem = strings.TrimSuffix(parts[3], referenceFileSuffix) + id = syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: parts[1], LookupKey: stem} + default: + return syncdomain.ResourceID{}, false + } + if syncdomain.ValidateKey(parts[1]) != nil || syncdomain.ValidateKey(stem) != nil { + return syncdomain.ResourceID{}, false + } + return id, true +} + // ResourcePath returns the path of the file of a variation, a tool, or a // skill relative to the repository root, in the form that ParseManagedPath // reads. @@ -73,6 +105,31 @@ func ResourcePath(id syncdomain.ResourceID) (string, error) { return path.Join(syncdomain.RootDir, relativePath), nil } +// ReferencePath returns the repository-relative path of a resource descriptor. +func ReferencePath(id syncdomain.ResourceID) (string, error) { + if err := validateKey("project", id.ProjectKey); err != nil { + return "", err + } + if id.Kind == syncdomain.KindVariation { + configKey, variationKey, err := id.VariationKeys() + if err != nil { + return "", err + } + return path.Join(syncdomain.RootDir, id.ProjectKey, configsDir, configKey, variationKey+referenceFileSuffix), nil + } + if err := validateKey(string(id.Kind), id.LookupKey); err != nil { + return "", err + } + switch id.Kind { + case syncdomain.KindTool: + return path.Join(syncdomain.RootDir, id.ProjectKey, toolsDir, id.LookupKey+referenceFileSuffix), nil + case syncdomain.KindSkill: + return path.Join(syncdomain.RootDir, id.ProjectKey, skillsDir, id.LookupKey+referenceFileSuffix), nil + default: + return "", fmt.Errorf("unsupported resource kind %q", id.Kind) + } +} + // variationPath returns the path of a variation file relative to the managed // directory. It makes sure that each key is a safe path segment. func variationPath(projectKey, configKey, variationKey string) (string, error) { diff --git a/internal/sync/local/reference.go b/internal/sync/local/reference.go new file mode 100644 index 000000000..7d69d0652 --- /dev/null +++ b/internal/sync/local/reference.go @@ -0,0 +1,246 @@ +package local + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" + "gopkg.in/yaml.v3" +) + +// Reference identifies one repository source and its mapped fields. +type Reference = syncdomain.Reference + +func definitionConflict(id syncdomain.ResourceID, message string) error { + return syncdomain.NewProblem( + syncdomain.ErrorDefinitionConflict, + message, + "Keep one managed file or one .ref.yaml descriptor for this resource.", + ).WithResource(id) +} + +// NewSourceReference validates an external source and its mapping. +func NewSourceReference( + repositoryRoot, workingDirectory, file, format, selectPath string, + fieldMap map[string]string, +) (Reference, error) { + if file == "" { + return Reference{}, errors.New("linked file path is required") + } + if !filepath.IsAbs(file) { + file = filepath.Join(workingDirectory, file) + } + // Compare resolved paths. A path that looks like it is in the repository + // can go out of it through a symbolic link. + target, err := filepath.EvalSymlinks(file) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Reference{}, syncdomain.NewProblem( + syncdomain.ErrorLinkedSourceMissing, + fmt.Sprintf("source file %q does not exist", file), + "Restore the file or pass another --file.", + ).WithFile(file, 0, 0) + } + return Reference{}, fmt.Errorf("resolve linked file %q: %w", file, err) + } + root, err := filepath.EvalSymlinks(repositoryRoot) + if err != nil { + return Reference{}, fmt.Errorf("resolve repository root: %w", err) + } + if !isWithin(root, target) { + return Reference{}, errors.New("linked file must be inside the Git repository") + } + + relative, _ := filepath.Rel(root, target) + reference := Reference{ + File: filepath.ToSlash(relative), Format: format, Select: selectPath, Map: fieldMap, + } + if _, err := resolveReferencePath(root, reference); err != nil { + return Reference{}, err + } + return reference, nil +} + +const referenceFormatVersion = 1 + +type referenceDescriptor struct { + FormatVersion int `yaml:"formatVersion"` + Reference `yaml:",inline"` +} + +// ReadDescriptor reads and validates one .ref.yaml file. +func ReadDescriptor(fsys fs.FS, descriptorPath string, id syncdomain.ResourceID) (Reference, error) { + data, err := fs.ReadFile(fsys, descriptorPath) + if err != nil { + return Reference{}, err + } + var descriptor referenceDescriptor + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&descriptor); err != nil { + return Reference{}, fmt.Errorf("decode descriptor: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + err = errors.New("more than one YAML document") + } + return Reference{}, fmt.Errorf("decode descriptor: %w", err) + } + if descriptor.FormatVersion != referenceFormatVersion { + return Reference{}, fmt.Errorf("unsupported formatVersion %d", descriptor.FormatVersion) + } + if err := validateReference(descriptor.Reference); err != nil { + return Reference{}, err + } + return syncreference.NormalizeReference(id, descriptor.Reference) +} + +// RenderDescriptor encodes one editable .ref.yaml descriptor. +func RenderDescriptor(id syncdomain.ResourceID, reference Reference) ([]byte, error) { + reference, err := syncreference.NormalizeReference(id, reference) + if err != nil { + return nil, err + } + var content bytes.Buffer + encoder := yaml.NewEncoder(&content) + encoder.SetIndent(2) + if err := encoder.Encode(referenceDescriptor{ + FormatVersion: referenceFormatVersion, + Reference: reference, + }); err != nil { + return nil, fmt.Errorf("encode descriptor: %w", err) + } + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("encode descriptor: %w", err) + } + return content.Bytes(), nil +} + +// ValidateDescriptorTarget rejects a managed definition of the same resource. +func (store Store) ValidateDescriptorTarget(id syncdomain.ResourceID) error { + managed, err := resourcePath(id) + if err != nil { + return err + } + if exists, err := store.fileExists(managed, id.Label()); err != nil { + return err + } else if exists { + return definitionConflict(id, id.Label()+" already has a managed file") + } + return nil +} + +// WriteDescriptor creates or replaces one resource descriptor atomically. +func (store Store) WriteDescriptor(id syncdomain.ResourceID, reference Reference) (string, error) { + if err := store.ValidateDescriptorTarget(id); err != nil { + return "", err + } + + repositoryPath, err := ReferencePath(id) + if err != nil { + return "", err + } + relativePath := strings.TrimPrefix(repositoryPath, syncdomain.RootDir+"/") + target := store.absolute(relativePath) + content, err := RenderDescriptor(id, reference) + if err != nil { + return "", err + } + current, err := os.ReadFile(target) + switch { + case errors.Is(err, os.ErrNotExist): + if err := createFile(store.root, target, content); err != nil { + return "", err + } + case err != nil: + return "", fmt.Errorf("read descriptor %q: %w", repositoryPath, err) + case bytes.Equal(current, content): + return repositoryPath, nil + default: + info, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("inspect descriptor %q: %w", repositoryPath, err) + } + if err := ReplaceFileAtomically(target, repositoryPath, current, content, info.Mode().Perm()); err != nil { + return "", err + } + } + return repositoryPath, nil +} + +// validateReference makes sure that the file is a clean path in the +// repository and outside the managed directory. +func validateReference(reference Reference) error { + switch { + case reference.File == "": + return errors.New("ref.file is required") + case reference.Format == "": + return errors.New("ref.format is required") + case !fs.ValidPath(reference.File): + return fmt.Errorf("ref.file %q must be a repository-relative path", reference.File) + case reference.File == syncdomain.RootDir || strings.HasPrefix(reference.File, syncdomain.RootDir+"/"): + return fmt.Errorf("ref.file %q must be outside %s", reference.File, syncdomain.RootDir) + case path.Clean(reference.File) != reference.File: + return fmt.Errorf("ref.file %q must be a clean repository-relative path", reference.File) + default: + return nil + } +} + +// resolveReferencePath returns the absolute path of a linked file. It +// resolves symbolic links on each call, because a link can change after the +// variation was linked. +func resolveReferencePath(repositoryRoot string, reference Reference) (string, error) { + if err := validateReference(reference); err != nil { + return "", err + } + root, err := filepath.EvalSymlinks(repositoryRoot) + if err != nil { + return "", fmt.Errorf("resolve repository root: %w", err) + } + target, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(reference.File))) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", syncdomain.NewProblem( + syncdomain.ErrorLinkedSourceMissing, + fmt.Sprintf("source file %q does not exist", reference.File), + "Restore the file or run detach.", + ).WithFile(reference.File, 0, 0) + } + return "", fmt.Errorf("resolve referenced file %q: %w", reference.File, err) + } + if !isWithin(root, target) { + return "", fmt.Errorf("referenced file %q resolves outside the Git repository", reference.File) + } + info, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("inspect referenced file %q: %w", reference.File, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("referenced file %q is not a regular file", reference.File) + } + return target, nil +} + +// ReadReference reads a linked file. It resolves symbolic links and makes sure +// that the file is a regular file in the repository. +func ReadReference(repositoryRoot string, reference Reference) ([]byte, error) { + target, err := resolveReferencePath(repositoryRoot, reference) + if err != nil { + return nil, err + } + data, err := os.ReadFile(target) + if err != nil { + return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) + } + return data, nil +} diff --git a/internal/sync/local/reference_test.go b/internal/sync/local/reference_test.go new file mode 100644 index 000000000..f15948505 --- /dev/null +++ b/internal/sync/local/reference_test.go @@ -0,0 +1,50 @@ +package local + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestNewSourceReferenceStoresRepositoryRelativePath(t *testing.T) { + root := t.TempDir() + workingDirectory := filepath.Join(root, "app") + require.NoError(t, os.MkdirAll(filepath.Join(workingDirectory, "prompts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(workingDirectory, "prompts", "support.md"), []byte("Help"), 0o644)) + + reference, err := NewSourceReference(root, workingDirectory, "prompts/support.md", "markdown", "$", nil) + + require.NoError(t, err) + require.Equal(t, "app/prompts/support.md", reference.File) +} + +func TestNewSourceReferenceRejectsUnsafePaths(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "prompt.md") + require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) + + _, err := NewSourceReference(root, root, outside, "markdown", "$", nil) + require.ErrorContains(t, err, "inside the Git repository") + + managed := filepath.Join(root, syncdomain.RootDir, "prompt.md") + require.NoError(t, os.MkdirAll(filepath.Dir(managed), 0o755)) + require.NoError(t, os.WriteFile(managed, []byte("Help"), 0o644)) + _, err = NewSourceReference(root, root, managed, "markdown", "$", nil) + require.ErrorContains(t, err, "must be outside") +} + +func TestNewSourceReferenceRejectsSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "prompt.md") + require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) + link := filepath.Join(root, "prompt.md") + require.NoError(t, os.Symlink(outside, link)) + + _, err := NewSourceReference(root, root, link, "markdown", "$", nil) + + require.ErrorContains(t, err, "inside the Git repository") +} diff --git a/internal/sync/local/source_reference_test.go b/internal/sync/local/source_reference_test.go new file mode 100644 index 000000000..b111a9404 --- /dev/null +++ b/internal/sync/local/source_reference_test.go @@ -0,0 +1,43 @@ +package local + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestWriteDescriptorCreatesUpdatesAndKeepsIdenticalContent(t *testing.T) { + root := t.TempDir() + writeDescriptorTestFile(t, root, "first.md", "First.\n") + writeDescriptorTestFile(t, root, "second.md", "Second.\n") + id := syncdomain.VariationID("project", "support", "default") + store := NewStore(root) + + path, err := store.WriteDescriptor(id, Reference{File: "first.md", Format: "markdown"}) + require.NoError(t, err) + first, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + _, err = store.WriteDescriptor(id, Reference{File: "first.md", Format: "markdown"}) + require.NoError(t, err) + same, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Equal(t, first, same) + + _, err = store.WriteDescriptor(id, Reference{File: "second.md", Format: "markdown"}) + require.NoError(t, err) + updated, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + require.NoError(t, err) + assert.Contains(t, string(updated), "file: second.md") +} + +func writeDescriptorTestFile(t *testing.T, root, name, content string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(name)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +}