diff --git a/internal/sync/reference/adapter_conformance_test.go b/internal/sync/reference/adapter_conformance_test.go new file mode 100644 index 00000000..22246424 --- /dev/null +++ b/internal/sync/reference/adapter_conformance_test.go @@ -0,0 +1,135 @@ +package reference + +import ( + "bytes" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) { + tests := []struct { + name string + fixture string + realPath string + format string + id syncdomain.ResourceID + selectPath string + fieldMap map[string]string + want string + update string + }{ + { + name: "Markdown section", fixture: "markdown/agents-instructions.md.golden", format: "markdown", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: `sections[heading="Support agent"]`, fieldMap: map[string]string{"prompt": "body"}, + want: "Ask for an order ID before discussing an order.", update: "Ask for an order ID and region.", + }, + { + name: "Agent Skill", fixture: "agent-skill/refunds/skill-instructions.md.golden", + realPath: ".agents/skills/refunds/SKILL.md", format: "agent-skill", + id: syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"}, + want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := fixtureFile(t, test.fixture) + documentPath := test.fixture + if test.realPath != "" { + documentPath, source = materializeFixture(t, test.fixture, test.realPath) + } + reference := syncdomain.Reference{ + File: documentPath, Format: test.format, Select: test.selectPath, Map: test.fieldMap, + } + projection, err := ParseSource(sourceBundle(documentPath, source), test.id, syncdomain.VariationModeAgent, reference) + require.NoError(t, err) + assert.Equal(t, test.want, primaryText(t, projection)) + + desired := projectionWithText(projection, test.update) + result, err := UpdateSource(sourceBundle(documentPath, source), test.id, syncdomain.VariationModeAgent, reference, desired) + require.NoError(t, err) + edits := directEdits(t, result) + updated := applyConformanceEdits(t, source, edits) + reparsed, err := ParseSource(sourceBundle(documentPath, updated), test.id, syncdomain.VariationModeAgent, reference) + require.NoError(t, err) + assert.Equal(t, test.update, primaryText(t, reparsed)) + }) + } +} + +func sourceBundle(path string, data []byte) SourceBundle { + return SourceBundle{Primary: Document{Path: path, Data: data}} +} + +func primaryText(t *testing.T, projection syncdomain.Projection) string { + t.Helper() + switch value := projection.(type) { + case syncdomain.VariationProjection: + require.NotNil(t, value.Instructions) + return *value.Instructions + case syncdomain.ToolProjection: + require.NotNil(t, value.Description) + return *value.Description + case syncdomain.SkillProjection: + require.NotNil(t, value.Description) + return *value.Description + default: + t.Fatalf("unsupported projection %T", projection) + return "" + } +} + +func projectionWithText(projection syncdomain.Projection, text string) syncdomain.Projection { + switch value := projection.(type) { + case syncdomain.VariationProjection: + value.Instructions = &text + value.Messages = nil + return value + case syncdomain.ToolProjection: + value.Description = &text + return value + case syncdomain.SkillProjection: + value.Description = &text + return value + default: + return projection + } +} + +func directEdits(t *testing.T, result UpdateResult) []TextEdit { + t.Helper() + var edits []TextEdit + for field, update := range result.Fields { + require.Equal(t, CapabilityDirect, update.Capability, field) + edits = append(edits, update.Edits...) + } + require.NotEmpty(t, edits) + return edits +} + +func applyConformanceEdits(t *testing.T, source []byte, edits []TextEdit) []byte { + t.Helper() + slices.SortFunc(edits, func(left, right TextEdit) int { return right.Start - left.Start }) + updated := bytes.Clone(source) + for _, edit := range edits { + require.Equal(t, edit.Expected, source[edit.Start:edit.End]) + updated = append(append(append([]byte{}, updated[:edit.Start]...), edit.Replacement...), updated[edit.End:]...) + } + ascending := slices.Clone(edits) + slices.SortFunc(ascending, func(left, right TextEdit) int { return left.Start - right.Start }) + beforeOffset, afterOffset := 0, 0 + for _, edit := range ascending { + length := edit.Start - beforeOffset + require.Equal(t, source[beforeOffset:edit.Start], updated[afterOffset:afterOffset+length]) + beforeOffset = edit.End + afterOffset += length + len(edit.Replacement) + } + require.Equal(t, source[beforeOffset:], updated[afterOffset:]) + return updated +} diff --git a/internal/sync/reference/document.go b/internal/sync/reference/document.go new file mode 100644 index 00000000..3e888d85 --- /dev/null +++ b/internal/sync/reference/document.go @@ -0,0 +1,389 @@ +package reference + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type markdownAdapter struct{} + +func (markdownAdapter) Format() string { return "markdown" } +func (markdownAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseMarkdown(bundle.Primary) + return Inspection{root: root}, err +} +func (markdownAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindVariation { + return "$", map[string]string{string(syncdomain.FieldPrompt): "$"} + } + return "$", nil +} + +type agentSkillAdapter struct{} + +func (agentSkillAdapter) Format() string { return "agent-skill" } +func (agentSkillAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseAgentSkill(bundle.Primary) + return Inspection{root: root}, err +} +func (agentSkillAdapter) Defaults(kind syncdomain.Kind) (string, map[string]string) { + if kind == syncdomain.KindSkill { + return "$", map[string]string{ + string(syncdomain.FieldDescription): "frontmatter.description", + string(syncdomain.FieldMarkdown): "body", + } + } + return "$", nil +} + +func init() { + mustRegister(markdownAdapter{}) + mustRegister(agentSkillAdapter{}) +} + +func parseMarkdown(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + data := document.Data + root := markdownTextNode(data, 0, len(data)) + root.fields = map[string]*sourceNode{"body": root} + root.fields["sections"] = &sourceNode{items: markdownSections(data), collection: true} + return root, nil +} + +func markdownSections(data []byte) []*sourceNode { + type heading struct { + level int + text string + lineStart int + bodyStart int + } + var headings []heading + var fenceCharacter byte + fenceLength := 0 + for offset := 0; offset < len(data); { + end := bytes.IndexByte(data[offset:], '\n') + if end < 0 { + end = len(data) + } else { + end += offset + 1 + } + line := strings.TrimSuffix(strings.TrimSuffix(string(data[offset:end]), "\n"), "\r") + trimmed := strings.TrimLeft(line, " ") + indent := len(line) - len(trimmed) + if indent > 3 || strings.HasPrefix(trimmed, "\t") { + offset = end + continue + } + marker, markerLength := markdownFence(trimmed) + switch { + case fenceLength == 0 && markerLength >= 3: + fenceCharacter, fenceLength = marker, markerLength + case fenceLength != 0 && marker == fenceCharacter && markerLength >= fenceLength: + fenceCharacter, fenceLength = 0, 0 + case fenceLength == 0: + level := 0 + for level < len(trimmed) && trimmed[level] == '#' { + level++ + } + if level > 0 && level <= 6 && level < len(trimmed) && trimmed[level] == ' ' { + text := markdownHeadingText(trimmed[level:]) + headings = append(headings, heading{level: level, text: text, lineStart: offset, bodyStart: end}) + } + } + offset = end + } + + sections := make([]*sourceNode, 0, len(headings)) + for index, current := range headings { + bodyEnd := len(data) + for next := index + 1; next < len(headings); next++ { + if headings[next].level <= current.level { + bodyEnd = headings[next].lineStart + break + } + } + body := markdownTextNode(data[current.bodyStart:bodyEnd], current.bodyStart, bodyEnd) + sections = append(sections, &sourceNode{fields: map[string]*sourceNode{ + "heading": {value: current.text}, + "body": body, + }}) + } + return sections +} + +func markdownFence(line string) (byte, int) { + if line == "" || line[0] != '`' && line[0] != '~' { + return 0, 0 + } + length := 1 + for length < len(line) && line[length] == line[0] { + length++ + } + return line[0], length +} + +func markdownHeadingText(text string) string { + text = strings.TrimSpace(text) + lastSpace := strings.LastIndexByte(text, ' ') + if lastSpace < 0 || lastSpace == len(text)-1 { + return text + } + for _, character := range text[lastSpace+1:] { + if character != '#' { + return text + } + } + return strings.TrimSpace(text[:lastSpace]) +} + +func markdownTextNode(data []byte, start, end int) *sourceNode { + value := normalizeMarkdown(string(data)) + lineEnding := "\n" + if bytes.Contains(data, []byte("\r\n")) { + lineEnding = "\r\n" + } + contentStart, contentEnd := 0, len(data) + for contentStart < len(data) && isMarkdownSpace(data[contentStart]) { + contentStart++ + } + for contentEnd > contentStart && isMarkdownSpace(data[contentEnd-1]) { + contentEnd-- + } + prefix := bytes.Clone(data[:contentStart]) + suffix := bytes.Clone(data[contentEnd:]) + return stringNode(value, start, end, func(text string) ([]byte, error) { + text = normalizeMarkdown(text) + text = strings.ReplaceAll(text, "\n", lineEnding) + result := append(bytes.Clone(prefix), []byte(text)...) + return append(result, suffix...), nil + }) +} + +func isMarkdownSpace(value byte) bool { + return value == ' ' || value == '\t' || value == '\r' || value == '\n' +} + +func parseAgentSkill(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path) + } + frontMatter, bodyStart, err := splitFrontMatter(document.Data) + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + var yamlNode yaml.Node + decoder := yaml.NewDecoder(bytes.NewReader(frontMatter.data)) + if err := decoder.Decode(&yamlNode); err != nil { + return nil, fmt.Errorf("%s: invalid front matter: %w", document.Path, err) + } + description, err := yamlScalarNode(frontMatter, &yamlNode, "description") + if err != nil { + return nil, fmt.Errorf("%s: %w", document.Path, err) + } + body := document.Data[bodyStart:] + if len(body) == 0 || strings.TrimSpace(string(body)) == "" { + return nil, fmt.Errorf("%s: skill Markdown is required", document.Path) + } + bodyNode := exactMarkdownNode(body, bodyStart, len(document.Data)) + return &sourceNode{ + fields: map[string]*sourceNode{ + "frontmatter": {fields: map[string]*sourceNode{"description": description}}, + "body": bodyNode, + }, + }, nil +} + +type frontMatterRegion struct { + data []byte + startOffset int +} + +func splitFrontMatter(data []byte) (frontMatterRegion, int, error) { + offset := 0 + if bytes.HasPrefix(data, []byte("\ufeff")) { + offset = len([]byte("\ufeff")) + } + firstEnd := nextLineEnd(data, offset) + if strings.TrimSpace(string(data[offset:firstEnd])) != "---" { + return frontMatterRegion{}, 0, fmt.Errorf("missing YAML front matter") + } + contentStart := firstEnd + for lineStart := firstEnd; lineStart < len(data); { + lineEnd := nextLineEnd(data, lineStart) + if strings.TrimSpace(string(data[lineStart:lineEnd])) == "---" { + bodyStart := lineEnd + if bytes.HasPrefix(data[bodyStart:], []byte("\r\n")) { + bodyStart += 2 + } else if bytes.HasPrefix(data[bodyStart:], []byte("\n")) { + bodyStart++ + } + return frontMatterRegion{data: data[contentStart:lineStart], startOffset: contentStart}, bodyStart, nil + } + lineStart = lineEnd + } + return frontMatterRegion{}, 0, fmt.Errorf("unclosed YAML front matter") +} + +func nextLineEnd(data []byte, start int) int { + if index := bytes.IndexByte(data[start:], '\n'); index >= 0 { + return start + index + 1 + } + return len(data) +} + +func yamlScalarNode(region frontMatterRegion, document *yaml.Node, key string) (*sourceNode, error) { + if len(document.Content) == 0 || document.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf("front matter must be a mapping") + } + mapping := document.Content[0] + for index := 0; index+1 < len(mapping.Content); index += 2 { + name, value := mapping.Content[index], mapping.Content[index+1] + if name.Value != key { + continue + } + if value.Kind != yaml.ScalarNode || value.Tag != "!!str" { + return nil, fmt.Errorf("front matter %s must be a string", key) + } + start, end, raw, err := yamlScalarSpan(region, value) + if err != nil { + return nil, err + } + if value.Style == yaml.LiteralStyle || value.Style == yaml.FoldedStyle { + return &sourceNode{ + value: value.Value, start: start, end: end, capability: CapabilityReadOnly, + reason: "YAML block scalars are read-only", + }, nil + } + return stringNode(value.Value, start, end, func(text string) ([]byte, error) { + return encodeYAMLString(text, raw) + }), nil + } + return nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("front matter field %q was not found", key), + "Add the field or update the map path.", + ) +} + +func yamlScalarSpan(region frontMatterRegion, node *yaml.Node) (int, int, string, error) { + lines := bytes.SplitAfter(region.data, []byte("\n")) + if node.Line < 1 || node.Line > len(lines) || node.Column < 1 { + return 0, 0, "", fmt.Errorf("cannot locate YAML scalar") + } + start := region.startOffset + for index := 0; index < node.Line-1; index++ { + start += len(lines[index]) + } + line := lines[node.Line-1] + line = bytes.TrimRight(line, "\r\n") + localStart := byteOffsetForColumn(line, node.Column) + start += localStart + localEnd := yamlTokenEnd(line, localStart) + if localStart > localEnd { + return 0, 0, "", fmt.Errorf("cannot locate YAML scalar") + } + raw := string(line[localStart:localEnd]) + return start, start + len(raw), raw, nil +} + +func byteOffsetForColumn(line []byte, column int) int { + offset := 0 + for current := 1; current < column && offset < len(line); current++ { + _, width := utf8.DecodeRune(line[offset:]) + offset += width + } + return offset +} + +func yamlTokenEnd(line []byte, start int) int { + if start >= len(line) { + return start + } + quote := line[start] + if quote == '\'' || quote == '"' { + escaped := false + for index := start + 1; index < len(line); index++ { + if quote == '"' && line[index] == '\\' && !escaped { + escaped = true + continue + } + if line[index] == quote && !escaped { + if quote == '\'' && index+1 < len(line) && line[index+1] == '\'' { + index++ + continue + } + return index + 1 + } + escaped = false + } + return len(line) + } + end := len(line) + if comment := bytes.Index(line[start:], []byte(" #")); comment >= 0 { + end = start + comment + } + for _, delimiter := range []byte{',', ']', '}'} { + if index := bytes.IndexByte(line[start:end], delimiter); index >= 0 { + end = start + index + } + } + for end > start && (line[end-1] == ' ' || line[end-1] == '\t') { + end-- + } + return end +} + +func quoteYAML(value, previous string) string { + if strings.HasPrefix(previous, "'") { + if strings.ContainsAny(value, "\r\n") { + encoded, _ := json.Marshal(value) + return string(encoded) + } + return "'" + strings.ReplaceAll(value, "'", "''") + "'" + } + if strings.HasPrefix(previous, `"`) { + encoded, _ := json.Marshal(value) + return string(encoded) + } + if value != "" && !strings.ContainsAny(value, ":\n\r#{}[],&*!|>'\"%@`") && + value == strings.TrimSpace(value) { + var decoded any + if yaml.Unmarshal([]byte(value), &decoded) == nil { + if _, isString := decoded.(string); isString { + return value + } + } + } + encoded, _ := json.Marshal(value) + return string(encoded) +} + +func encodeYAMLString(value, previous string) ([]byte, error) { + for _, character := range value { + if unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported control character U+%04X", character) + } + } + return []byte(quoteYAML(value, previous)), nil +} + +func exactMarkdownNode(data []byte, start, end int) *sourceNode { + value := string(data) + finalNewline := bytes.HasSuffix(data, []byte("\n")) + return stringNode(value, start, end, func(text string) ([]byte, error) { + if finalNewline && text != "" && !strings.HasSuffix(text, "\n") { + text += "\n" + } + return []byte(text), nil + }) +} diff --git a/internal/sync/reference/fixture_test.go b/internal/sync/reference/fixture_test.go new file mode 100644 index 00000000..98d6c46e --- /dev/null +++ b/internal/sync/reference/fixture_test.go @@ -0,0 +1,25 @@ +package reference + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func fixtureFile(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", filepath.FromSlash(name))) + require.NoError(t, err) + return data +} + +func materializeFixture(t *testing.T, fixture, realPath string) (string, []byte) { + t.Helper() + data := fixtureFile(t, fixture) + path := filepath.Join(t.TempDir(), filepath.FromSlash(realPath)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, data, 0o644)) + return path, data +} diff --git a/internal/sync/reference/path.go b/internal/sync/reference/path.go new file mode 100644 index 00000000..5998cbc1 --- /dev/null +++ b/internal/sync/reference/path.go @@ -0,0 +1,270 @@ +package reference + +import ( + "encoding/json" + "fmt" + "strconv" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type sourcePath struct { + root bool + steps []pathStep +} + +type pathStep struct { + key string + filters []pathFilter + index *int +} + +type pathFilter struct { + key string + value string +} + +// ParsePath checks the shared select and map path grammar. +func ParsePath(text string) error { + _, err := parsePath(text) + return err +} + +func parsePath(text string) (sourcePath, error) { + if text == "$" { + return sourcePath{root: true}, nil + } + if text == "" { + return sourcePath{}, pathSyntaxError(text, 0, "path is empty") + } + + parser := pathParser{text: text} + key, err := parser.readKey() + if err != nil { + return sourcePath{}, err + } + result := sourcePath{steps: []pathStep{{key: key}}} + for parser.offset < len(text) { + switch text[parser.offset] { + case '.': + parser.offset++ + key, err = parser.readIdentifier() + if err != nil { + return sourcePath{}, err + } + result.steps = append(result.steps, pathStep{key: key}) + case '[': + step, kind, err := parser.readBracket() + if err != nil { + return sourcePath{}, err + } + if kind == bracketFilter { + last := len(result.steps) - 1 + if result.steps[last].index != nil { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a selector must follow a field") + } + if len(result.steps[last].filters) != 0 { + return sourcePath{}, pathSyntaxError(text, parser.offset, "a path step can have only one selector") + } + result.steps[last].filters = step.filters + } else { + result.steps = append(result.steps, step) + } + default: + return sourcePath{}, pathSyntaxError(text, parser.offset, "expected a dot or bracket") + } + } + return result, nil +} + +type pathParser struct { + text string + offset int +} + +type bracketKind int + +const ( + bracketField bracketKind = iota + bracketFilter + bracketIndex +) + +func (parser *pathParser) readKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '[' { + step, kind, err := parser.readBracket() + if err != nil { + return "", err + } + if kind != bracketField { + return "", pathSyntaxError(parser.text, parser.offset, "a path cannot start with a selector or index") + } + return step.key, nil + } + return parser.readIdentifier() +} + +func (parser *pathParser) readIdentifier() (string, error) { + start := parser.offset + if start >= len(parser.text) || !identifierStart(parser.text[start]) { + return "", pathSyntaxError(parser.text, start, "expected an identifier") + } + parser.offset++ + for parser.offset < len(parser.text) && identifierContinue(parser.text[parser.offset]) { + parser.offset++ + } + return parser.text[start:parser.offset], nil +} + +func (parser *pathParser) readBracket() (pathStep, bracketKind, error) { + start := parser.offset + parser.offset++ + if parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + return parser.readIndex(start) + } + key, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketField, err + } + if parser.offset >= len(parser.text) { + return pathStep{}, bracketField, pathSyntaxError(parser.text, start, "unclosed bracket") + } + if parser.text[parser.offset] == ']' { + parser.offset++ + return pathStep{key: key}, bracketField, nil + } + if parser.text[parser.offset] != '=' { + return pathStep{}, bracketField, pathSyntaxError(parser.text, parser.offset, "expected equals or closing bracket") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + value, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters := []pathFilter{{key: key, value: value}} + for parser.offset < len(parser.text) && parser.text[parser.offset] == ',' { + parser.offset++ + nextKey, err := parser.readBracketKey() + if err != nil { + return pathStep{}, bracketFilter, err + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '=' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected equals") + } + parser.offset++ + if parser.offset >= len(parser.text) || parser.text[parser.offset] != '"' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "selector value must be a JSON string") + } + nextValue, err := parser.readJSONString() + if err != nil { + return pathStep{}, bracketFilter, err + } + filters = append(filters, pathFilter{key: nextKey, value: nextValue}) + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketFilter, pathSyntaxError(parser.text, parser.offset, "expected a closing bracket") + } + parser.offset++ + return pathStep{filters: filters}, bracketFilter, nil +} + +func (parser *pathParser) readIndex(start int) (pathStep, bracketKind, error) { + numberStart := parser.offset + for parser.offset < len(parser.text) && parser.text[parser.offset] >= '0' && parser.text[parser.offset] <= '9' { + parser.offset++ + } + raw := parser.text[numberStart:parser.offset] + if len(raw) > 1 && raw[0] == '0' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "an index cannot have a leading zero") + } + if parser.offset >= len(parser.text) || parser.text[parser.offset] != ']' { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, parser.offset, "an index must be a decimal integer") + } + value, err := strconv.ParseUint(raw, 10, 31) + if err != nil { + return pathStep{}, bracketIndex, pathSyntaxError(parser.text, numberStart, "index exceeds 2147483647") + } + parser.offset++ + index := int(value) + return pathStep{index: &index}, bracketIndex, nil +} + +func (parser *pathParser) readBracketKey() (string, error) { + if parser.offset < len(parser.text) && parser.text[parser.offset] == '"' { + return parser.readJSONString() + } + return parser.readIdentifier() +} + +func (parser *pathParser) readJSONString() (string, error) { + start := parser.offset + parser.offset++ + for parser.offset < len(parser.text) { + switch parser.text[parser.offset] { + case '\\': + parser.offset += 2 + case '"': + parser.offset++ + quoted := parser.text[start:parser.offset] + if err := rejectUnpairedSurrogate(quoted); err != nil { + return "", pathSyntaxError(parser.text, start, err.Error()) + } + var value string + if err := json.Unmarshal([]byte(quoted), &value); err != nil { + return "", pathSyntaxError(parser.text, start, "invalid JSON string") + } + return value, nil + default: + parser.offset++ + } + } + return "", pathSyntaxError(parser.text, start, "unclosed JSON string") +} + +func rejectUnpairedSurrogate(quoted string) error { + for index := 0; index+5 < len(quoted); index++ { + if quoted[index:index+2] != `\u` { + continue + } + code, err := strconv.ParseUint(quoted[index+2:index+6], 16, 16) + if err != nil { + continue + } + switch { + case code >= 0xd800 && code <= 0xdbff: + if index+11 >= len(quoted) || quoted[index+6:index+8] != `\u` { + return fmt.Errorf("unpaired Unicode surrogate") + } + low, err := strconv.ParseUint(quoted[index+8:index+12], 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("unpaired Unicode surrogate") + } + index += 6 + case code >= 0xdc00 && code <= 0xdfff: + return fmt.Errorf("unpaired Unicode surrogate") + } + } + return nil +} + +func pathSyntaxError(path string, offset int, reason string) error { + if offset > len(path) { + offset = len(path) + } + return syncdomain.NewProblem( + syncdomain.ErrorInvalidPathSyntax, + fmt.Sprintf("invalid path at byte %d: %s", offset, reason), + "Use $, named fields, zero-based indexes, keyed selectors, or quoted literal keys.", + ) +} + +func identifierStart(value byte) bool { + return value == '_' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +} + +func identifierContinue(value byte) bool { + return identifierStart(value) || value >= '0' && value <= '9' +} diff --git a/internal/sync/reference/registry.go b/internal/sync/reference/registry.go new file mode 100644 index 00000000..8181b612 --- /dev/null +++ b/internal/sync/reference/registry.go @@ -0,0 +1,119 @@ +package reference + +import ( + "fmt" + "sync" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Adapter inspects one source format and supplies its conventional mapping. +// It receives bytes only and never reads files, runs code, or uses the network. +type Adapter interface { + Format() string + Inspect(SourceBundle) (Inspection, error) + Defaults(syncdomain.Kind) (selectPath string, fieldMap map[string]string) +} + +// DependencyReporter lists repository files that can contribute mapped values. +type DependencyReporter interface { + Dependencies(Document) ([]string, error) +} + +// Inspection is the logical document that an adapter exposes to select paths. +type Inspection struct { + root *sourceNode +} + +// Node is one logical value exposed by an adapter. +type Node struct { + Value any + Fields map[string]*Node + Items []*Node + Start int + End int + File string + Capability syncdomain.Capability + Reason string + Code syncdomain.ErrorCode + Encode func(any) ([]byte, error) +} + +// NewInspection creates an inspection from an adapter's logical root. +func NewInspection(root *Node) Inspection { + return Inspection{root: importNode(root)} +} + +func importNode(node *Node) *sourceNode { + if node == nil { + return nil + } + result := &sourceNode{ + value: node.Value, start: node.Start, end: node.End, file: node.File, + capability: node.Capability, reason: node.Reason, code: node.Code, encode: node.Encode, + } + if len(node.Fields) != 0 { + result.fields = make(map[string]*sourceNode, len(node.Fields)) + for key, child := range node.Fields { + result.fields[key] = importNode(child) + } + } + for _, child := range node.Items { + result.items = append(result.items, importNode(child)) + } + return result +} + +type adapterRegistry struct { + mu sync.RWMutex + adapters map[string]Adapter +} + +var formats = adapterRegistry{adapters: make(map[string]Adapter)} + +// Register adds one independent source format. +func Register(adapter Adapter) error { + if adapter == nil || adapter.Format() == "" { + return fmt.Errorf("adapter format is required") + } + formats.mu.Lock() + defer formats.mu.Unlock() + if _, exists := formats.adapters[adapter.Format()]; exists { + return fmt.Errorf("source format %q is already registered", adapter.Format()) + } + formats.adapters[adapter.Format()] = adapter + return nil +} + +func registeredAdapter(format string) (Adapter, error) { + formats.mu.RLock() + adapter := formats.adapters[format] + formats.mu.RUnlock() + if adapter == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorUnsupportedFormat, + fmt.Sprintf("source format %q is not supported", format), + "Use a registered format name.", + ) + } + return adapter, nil +} + +// DependencyPaths returns repository dependencies reported by one format. +func DependencyPaths(format string, primary Document) ([]string, error) { + adapter, err := registeredAdapter(format) + if err != nil { + return nil, err + } + reporter, ok := adapter.(DependencyReporter) + if !ok { + return nil, nil + } + return reporter.Dependencies(primary) +} + +func mustRegister(adapter Adapter) { + if err := Register(adapter); err != nil { + panic(err) + } +} diff --git a/internal/sync/reference/registry_external_test.go b/internal/sync/reference/registry_external_test.go new file mode 100644 index 00000000..72e37cc7 --- /dev/null +++ b/internal/sync/reference/registry_external_test.go @@ -0,0 +1,42 @@ +package reference_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +type externalFakeAdapter struct{} + +func (externalFakeAdapter) Format() string { return "external-fake" } +func (externalFakeAdapter) Inspect(bundle reference.SourceBundle) (reference.Inspection, error) { + document := bundle.Primary + return reference.NewInspection(&reference.Node{ + Value: string(document.Data), Start: 0, End: len(document.Data), + Capability: syncdomain.CapabilityDirect, + Encode: func(value any) ([]byte, error) { return []byte(value.(string)), nil }, + }), nil +} +func (externalFakeAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { + return "$", map[string]string{"prompt": "$"} +} + +func TestExternalAdapterNeedsOnlyRegistration(t *testing.T) { + require.NoError(t, reference.Register(externalFakeAdapter{})) + id := syncdomain.VariationID("project", "config", "default") + + projection, err := reference.ParseSource( + reference.SourceBundle{ + Primary: reference.Document{Path: "prompt.fake", Data: []byte("External adapter")}, + }, + id, + syncdomain.VariationModeAgent, + syncdomain.Reference{File: "prompt.fake", Format: "external-fake"}, + ) + + require.NoError(t, err) + require.Equal(t, "External adapter", *projection.(syncdomain.VariationProjection).Instructions) +} diff --git a/internal/sync/reference/repository_paths_test.go b/internal/sync/reference/repository_paths_test.go new file mode 100644 index 00000000..72265e8c --- /dev/null +++ b/internal/sync/reference/repository_paths_test.go @@ -0,0 +1,61 @@ +package reference_test + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAdapterFixturesHaveNoToolLoadedInstructionPaths(t *testing.T) { + root := filepath.Join(repositoryRoot(t), "internal", "sync") + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + require.NoError(t, err) + relative, err := filepath.Rel(root, path) + require.NoError(t, err) + relative = filepath.ToSlash(relative) + if !insideTestdata(relative) { + return nil + } + require.Falsef(t, forbiddenInstructionPath(relative), "forbidden adapter fixture path: %s", relative) + return nil + }) + require.NoError(t, err) +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + directory, err := os.Getwd() + require.NoError(t, err) + for { + if _, err := os.Stat(filepath.Join(directory, "go.mod")); err == nil { + return directory + } + parent := filepath.Dir(directory) + require.NotEqual(t, directory, parent, "repository root was not found") + directory = parent + } +} + +func forbiddenInstructionPath(path string) bool { + parts := strings.Split(path, "/") + for _, part := range parts { + switch part { + case ".cursor", ".claude", ".agents", ".cursorrules", "AGENTS.md", "CLAUDE.md", "SKILL.md": + return true + } + } + return path == ".github/copilot-instructions.md" || strings.HasPrefix(path, ".github/instructions/") +} + +func insideTestdata(path string) bool { + for _, part := range strings.Split(path, "/") { + if part == "testdata" { + return true + } + } + return false +} diff --git a/internal/sync/reference/source.go b/internal/sync/reference/source.go new file mode 100644 index 00000000..4c642280 --- /dev/null +++ b/internal/sync/reference/source.go @@ -0,0 +1,1006 @@ +package reference + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Capability is the shared mapped-field capability. +type Capability = syncdomain.Capability + +const ( + CapabilityDirect = syncdomain.CapabilityDirect + CapabilityPatch = syncdomain.CapabilityPatch + CapabilityReadOnly = syncdomain.CapabilityReadOnly + CapabilityReject = syncdomain.CapabilityReject +) + +// Document is one repository file supplied to an adapter. +type Document struct { + Path string + Data []byte +} + +// SourceBundle contains a primary source and optional repository dependencies. +type SourceBundle struct { + Primary Document + Dependencies map[string]Document +} + +// TextEdit replaces one exact byte range. +type TextEdit struct { + File string + Start int + End int + Expected []byte + Replacement []byte +} + +// FieldUpdate is the update capability and edits for one mapped field. +type FieldUpdate struct { + Capability Capability + Edits []TextEdit + Reason string +} + +// UpdateResult contains one result for each changed field. +type UpdateResult struct { + Fields map[syncdomain.Field]FieldUpdate +} + +// NormalizeReference validates a descriptor and fills conventional defaults. +func NormalizeReference(id syncdomain.ResourceID, reference syncdomain.Reference) (syncdomain.Reference, error) { + if reference.File == "" { + return syncdomain.Reference{}, invalidReference(id, "file is required", "Pass --file.") + } + adapter, err := registeredAdapter(reference.Format) + if err != nil { + return syncdomain.Reference{}, err + } + defaultSelect, defaultMap := adapter.Defaults(id.Kind) + if reference.Select == "" { + if defaultSelect == "" { + return syncdomain.Reference{}, invalidReference( + id, fmt.Sprintf("select is required for format %q", reference.Format), "Pass --select.", + ) + } + reference.Select = defaultSelect + } + if err := ParsePath(reference.Select); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("select %q: %w", reference.Select, err), id, reference.File, + ) + } + if len(reference.Map) == 0 { + if len(defaultMap) == 0 { + return syncdomain.Reference{}, invalidReference( + id, "map is required", "Pass at least one --map field=path value.", + ) + } + reference.Map = defaultMap + } + for target := range reference.Map { + if err := ParsePath(target); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map target %q: %w", target, err), id, reference.File, + ) + } + } + fields, err := syncdomain.ProjectionFields(reference) + if err != nil { + return syncdomain.Reference{}, invalidReference(id, err.Error(), "Use a field valid for this resource kind.") + } + wholeParameters := false + parameterKeys := map[string]syncdomain.Field{} + for _, field := range fields { + if !fieldAllowed(id.Kind, field) { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("field %q is not valid for %s", field, id.Kind), + "Use a mapped field supported by this resource kind.", + ) + } + if field == syncdomain.FieldModelParameters { + wholeParameters = true + } + if key, parameter := syncdomain.ModelParameterKey(field); parameter { + if key == "tools" { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map target %q duplicates the tools field", field), + "Map tools with the tools target.", + ) + } + if previous, duplicate := parameterKeys[key]; duplicate { + return syncdomain.Reference{}, invalidReference( + id, + fmt.Sprintf("map targets %q and %q select the same model parameter", previous, field), + "Map each model parameter key once.", + ) + } + parameterKeys[key] = field + } + if err := ParsePath(reference.Map[string(field)]); err != nil { + return syncdomain.Reference{}, annotateProblem( + fmt.Errorf("map field %q: %w", field, err), id, reference.File, + ) + } + } + if wholeParameters && len(parameterKeys) != 0 { + return syncdomain.Reference{}, invalidReference( + id, + "map target \"model.parameters\" overlaps an individual model parameter target", + "Map the full parameters object or individual parameter keys.", + ) + } + return reference, nil +} + +func invalidReference(id syncdomain.ResourceID, message, suggestion string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidReference, message, suggestion, + ).WithResource(id) +} + +// ParseSource reads one mapped resource from current source bytes. +func ParseSource( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, +) (syncdomain.Projection, error) { + projection, _, err := ParseSourceWithSelections(bundle, id, mode, reference) + return projection, err +} + +// ParseSourceWithSelections reads one mapped resource and its indexed selections. +func ParseSourceWithSelections( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, +) (syncdomain.Projection, []syncdomain.IndexedSelection, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return nil, nil, err + } + root, err := parseSourceBundle(bundle, reference.Format) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selected, selectedIndexes, err := resolveSourcePathTracked(root, reference.Select, "select") + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + selections, err := indexedSelections(selectedIndexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + fields := make([]string, 0, len(reference.Map)) + for field := range reference.Map { + fields = append(fields, field) + } + slices.Sort(fields) + for _, field := range fields { + _, indexes, err := resolveSourcePathTracked(selected, reference.Map[field], "map."+field) + if err != nil { + return nil, nil, annotateProblem(fmt.Errorf("map field %q: %w", field, err), id, bundle.Primary.Path) + } + mappedSelections, err := indexedSelections(indexes, selected) + if err != nil { + return nil, nil, annotateProblem(err, id, bundle.Primary.Path) + } + selections = append(selections, mappedSelections...) + } + slices.SortFunc(selections, func(left, right syncdomain.IndexedSelection) int { + return strings.Compare(left.Path, right.Path) + }) + projection, err := buildProjection(id.Kind, mode, selected, reference.Map) + return projection, selections, annotateProblem(err, id, bundle.Primary.Path) +} + +// UpdateSource plans exact edits for changed mapped fields. +func UpdateSource( + bundle SourceBundle, + id syncdomain.ResourceID, + mode syncdomain.VariationMode, + reference syncdomain.Reference, + desired syncdomain.Projection, +) (UpdateResult, error) { + reference, err := NormalizeReference(id, reference) + if err != nil { + return UpdateResult{}, err + } + if desired.Kind() != id.Kind { + return UpdateResult{}, fmt.Errorf("projection kind %q does not match %s", desired.Kind(), id.Label()) + } + root, err := parseSourceBundle(bundle, reference.Format) + if err != nil { + return UpdateResult{}, annotateProblem(err, id, bundle.Primary.Path) + } + selected, err := resolveSourcePath(root, reference.Select) + if err != nil { + return UpdateResult{}, annotateProblem(fmt.Errorf("select %q: %w", reference.Select, err), id, bundle.Primary.Path) + } + + result := UpdateResult{Fields: make(map[syncdomain.Field]FieldUpdate)} + for name, sourcePath := range reference.Map { + field := syncdomain.Field(name) + wanted, present, err := projectionField(desired, field) + if err != nil { + return UpdateResult{}, err + } + if !present { + continue + } + current, err := resolveSourcePath(selected, sourcePath) + if err != nil { + return UpdateResult{}, fmt.Errorf("%s: map field %q: %w", bundle.Primary.Path, field, err) + } + if field == syncdomain.FieldPrompt || field == syncdomain.FieldMessages { + if messages, ok := wanted.([]syncdomain.Message); ok { + if currentMessages, sourceIsMessages := current.value.([]syncdomain.Message); sourceIsMessages { + update := updateMessageContents(bundle, current, currentMessages, messages) + if len(update.Edits) != 0 || update.Capability != CapabilityDirect { + result.Fields[field] = update + } + continue + } + if _, sourceIsText := current.value.(string); sourceIsText { + if len(messages) != 1 || messages[0].Role != syncdomain.RoleSystem { + result.Fields[field] = FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "text sources support one system message", + } + continue + } + wanted = messages[0].Content + } + } + } + if semanticEqual(current.value, wanted) { + continue + } + update := FieldUpdate{Capability: current.capability, Reason: current.reason} + if current.capability == CapabilityDirect || current.capability == CapabilityPatch { + replacement, err := current.encode(wanted) + if err != nil { + update.Capability, update.Reason = CapabilityReadOnly, err.Error() + } else { + document, ok := bundleDocument(bundle, current.file) + if !ok { + update.Capability, update.Reason = CapabilityReadOnly, "source dependency is not available" + result.Fields[field] = update + continue + } + update.Edits = []TextEdit{{ + File: document.Path, + Start: current.start, + End: current.end, + Expected: bytes.Clone(document.Data[current.start:current.end]), + Replacement: replacement, + }} + } + } + result.Fields[field] = update + } + return result, nil +} + +func updateMessageContents( + bundle SourceBundle, + current *sourceNode, + have, want []syncdomain.Message, +) FieldUpdate { + if len(have) != len(want) { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "add or remove messages by hand", + } + } + var edits []TextEdit + for index := range have { + if have[index].Role != want[index].Role { + return FieldUpdate{ + Capability: CapabilityReadOnly, + Reason: "reorder messages or change roles by hand", + } + } + if have[index].Content == want[index].Content { + continue + } + if index >= len(current.items) || current.items[index].fields["content"] == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "message content is not a writable literal"} + } + content := current.items[index].fields["content"] + if content.capability != CapabilityDirect || content.encode == nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: content.reason} + } + replacement, err := content.encode(want[index].Content) + if err != nil { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: err.Error()} + } + document, ok := bundleDocument(bundle, content.file) + if !ok { + return FieldUpdate{Capability: CapabilityReadOnly, Reason: "source dependency is not available"} + } + edits = append(edits, TextEdit{ + File: document.Path, Start: content.start, End: content.end, + Expected: bytes.Clone(document.Data[content.start:content.end]), Replacement: replacement, + }) + } + return FieldUpdate{Capability: CapabilityDirect, Edits: edits} +} + +func bundleDocument(bundle SourceBundle, file string) (Document, bool) { + if file == "" || file == bundle.Primary.Path { + return bundle.Primary, true + } + document, ok := bundle.Dependencies[file] + return document, ok +} + +func annotateProblem(err error, id syncdomain.ResourceID, file string) error { + if err == nil { + return nil + } + if problem, ok := syncdomain.ProblemFrom(err); ok { + _ = problem.WithResource(id).WithFile(file, 0, 0) + } + return err +} + +type sourceNode struct { + value any + fields map[string]*sourceNode + items []*sourceNode + collection bool + start int + end int + file string + capability Capability + reason string + code syncdomain.ErrorCode + encode func(any) ([]byte, error) +} + +func resolveSourcePath(root *sourceNode, text string) (*sourceNode, error) { + node, _, err := resolveSourcePathTracked(root, text, "") + return node, err +} + +type indexedResolution struct { + path string + collectionLength int + item *sourceNode +} + +func resolveSourcePathTracked(root *sourceNode, text, scope string) (*sourceNode, []indexedResolution, error) { + path, err := parsePath(text) + if err != nil { + return nil, nil, err + } + if path.root { + return root, nil, nil + } + current := root + var indexes []indexedResolution + for _, step := range path.steps { + if step.index != nil { + if !sourceCollection(current) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("index %d needs a collection", *step.index), + "Select a collection before this index.", + ) + } + if *step.index >= len(current.items) { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorIndexOutOfRange, + fmt.Sprintf("index %d is out of range for collection of length %d", *step.index, len(current.items)), + "Use an index within the collection.", + ) + } + collectionLength := len(current.items) + current = current.items[*step.index] + indexes = append(indexes, indexedResolution{ + path: fmt.Sprintf("%s#%d", scope, len(indexes)), + collectionLength: collectionLength, + item: current, + }) + continue + } + if current.fields == nil { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathTypeMismatch, + fmt.Sprintf("%q does not contain fields", step.key), + "Select an object before this field.", + ) + } + next, ok := current.fields[step.key] + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("field %q was not found", step.key), + "Update the select or map path.", + ) + } + current = next + if len(step.filters) == 0 { + continue + } + var matches []*sourceNode + for _, item := range current.items { + matched := true + for _, filter := range step.filters { + identity, ok := item.fields[filter.key] + if !ok { + matched = false + break + } + text, ok := identity.value.(string) + if !ok { + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorFilterTypeMismatch, + fmt.Sprintf("selector field %q is not a string", filter.key), + "Select a direct string identity field.", + ) + } + if text != filter.value { + matched = false + break + } + } + if matched { + matches = append(matches, item) + } + } + selector := formatFilters(step.filters) + switch len(matches) { + case 0: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathNotFound, + fmt.Sprintf("selector %s matched no values", selector), + "Update the selector or restore the source value.", + ) + case 1: + current = matches[0] + default: + return nil, nil, syncdomain.NewProblem( + syncdomain.ErrorPathAmbiguous, + fmt.Sprintf("selector %s matched %d values", selector, len(matches)), + "Use a selector that matches one resource.", + ) + } + } + return current, indexes, nil +} + +func sourceCollection(node *sourceNode) bool { + if node == nil { + return false + } + if node.collection || node.items != nil { + return true + } + value := reflect.ValueOf(node.value) + return value.IsValid() && (value.Kind() == reflect.Array || value.Kind() == reflect.Slice) +} + +func indexedSelections(indexes []indexedResolution, selectedRoot *sourceNode) ([]syncdomain.IndexedSelection, error) { + selections := make([]syncdomain.IndexedSelection, 0, len(indexes)) + var ancestors []string + for _, index := range indexes { + identity := directSelectionIdentity(index.item) + if identity == nil { + // Without scalar identity, bind the nearest selected resource root and prior indexed ancestors. + identity = struct { + Root any `json:"root"` + Ancestors []string `json:"ancestors,omitempty"` + Item any `json:"item"` + }{ + Root: stableNodeValue(selectedRoot, map[*sourceNode]bool{}), + Ancestors: slices.Clone(ancestors), + Item: stableNodeValue(index.item, map[*sourceNode]bool{}), + } + } + fingerprint, err := selectionFingerprint(identity) + if err != nil { + return nil, err + } + selections = append(selections, syncdomain.IndexedSelection{ + Path: index.path, CollectionLength: index.collectionLength, Fingerprint: fingerprint, + }) + ancestors = append(ancestors, fingerprint) + } + return selections, nil +} + +func directSelectionIdentity(node *sourceNode) any { + if node == nil || node.fields == nil { + return nil + } + identity := map[string]any{} + for _, key := range []string{"id", "name", "role", "type"} { + field, ok := node.fields[key] + if !ok || !scalarValue(field.value) { + continue + } + identity[key] = field.value + } + if len(identity) == 0 { + return nil + } + return identity +} + +func scalarValue(value any) bool { + switch value.(type) { + case string, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return true + default: + return false + } +} + +func stableNodeValue(node *sourceNode, visited map[*sourceNode]bool) any { + if node == nil { + return nil + } + if node.value != nil { + return node.value + } + if visited[node] { + return "" + } + visited[node] = true + defer delete(visited, node) + if node.fields != nil { + value := make(map[string]any, len(node.fields)) + for key, field := range node.fields { + value[key] = stableNodeValue(field, visited) + } + return value + } + if sourceCollection(node) { + value := make([]any, len(node.items)) + for index, item := range node.items { + value[index] = stableNodeValue(item, visited) + } + return value + } + return nil +} + +func selectionFingerprint(value any) (string, error) { + canonical, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode indexed selection fingerprint: %w", err) + } + sum := sha256.Sum256(canonical) + return fmt.Sprintf("sha256:%x", sum), nil +} + +func formatFilters(filters []pathFilter) string { + parts := make([]string, len(filters)) + for index, filter := range filters { + parts[index] = fmt.Sprintf("%s=%q", filter.key, filter.value) + } + return strings.Join(parts, ",") +} + +func buildProjection( + kind syncdomain.Kind, + mode syncdomain.VariationMode, + selected *sourceNode, + fieldMap map[string]string, +) (syncdomain.Projection, error) { + switch kind { + case syncdomain.KindVariation: + projection := syncdomain.VariationProjection{ + Mode: mode, FieldStates: make(map[syncdomain.Field]syncdomain.FieldState), + } + if !projection.Mode.Valid() { + projection.Mode = syncdomain.VariationModeAgent + } + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + parameterKey, parameterField := syncdomain.ModelParameterKey(field) + if parameterField { + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: true, Reason: node.reason, Code: node.code, File: node.file, + } + value, ok := jsonValue(node.value) + if !ok { + return nil, mapTypeError(name, "a JSON-compatible value") + } + if projection.Parameters == nil { + parameters := map[string]any{} + projection.Parameters = ¶meters + } + (*projection.Parameters)[parameterKey] = value + continue + } + if field == syncdomain.FieldModelParameters && node.value == nil { + return nil, mapTypeError(name, "an object") + } + if node.value == nil { + continue + } + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + if field == syncdomain.FieldMessages || projection.Mode == syncdomain.VariationModeCompletion { + switch value := node.value.(type) { + case []syncdomain.Message: + for _, message := range value { + if !syncdomain.ValidMessageRole(message.Role) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("message role %q is not supported", message.Role), + "Use system, user, or assistant.", + ) + } + } + projection.Mode = syncdomain.VariationModeCompletion + projection.Messages = &value + case string: + messages := []syncdomain.Message{{Role: syncdomain.RoleSystem, Content: value}} + projection.Messages = &messages + default: + return nil, mapTypeError(name, "messages") + } + } else { + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Instructions = &value + } + case syncdomain.FieldName: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Name = &value + case syncdomain.FieldModel: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Model = &value + case syncdomain.FieldModelParameters: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + value = cloneObject(value) + delete(value, "tools") + projection.Parameters = &value + case syncdomain.FieldTools: + value, ok := stringList(node.value) + if !ok { + return nil, mapTypeError(name, "a string collection") + } + projection.Tools = &value + case syncdomain.FieldSkills: + value, ok := stringList(node.value) + if !ok { + return nil, mapTypeError(name, "a string collection") + } + projection.Skills = &value + } + } + _, promptMapped := projection.FieldStates[syncdomain.FieldPrompt] + _, messagesMapped := projection.FieldStates[syncdomain.FieldMessages] + if !promptMapped && !messagesMapped { + return nil, fmt.Errorf("map field %q is required for a variation", syncdomain.FieldPrompt) + } + return projection, nil + case syncdomain.KindTool: + projection := syncdomain.ToolProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + if node.value == nil { + continue + } + switch field { + case syncdomain.FieldDescription: + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + projection.Description = &value + case syncdomain.FieldSchema: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + projection.Schema = &value + case syncdomain.FieldCustomParameters: + value, ok := objectValue(node.value) + if !ok { + return nil, mapTypeError(name, "an object") + } + projection.CustomParameters = &value + case syncdomain.FieldTags: + value, ok := stringList(node.value) + if !ok { + return nil, mapTypeError(name, "a string collection") + } + projection.Tags = &value + } + } + return projection, nil + case syncdomain.KindSkill: + projection := syncdomain.SkillProjection{FieldStates: make(map[syncdomain.Field]syncdomain.FieldState)} + for name, path := range fieldMap { + node, err := mappedNode(selected, name, path) + if err != nil { + return nil, err + } + field := syncdomain.Field(name) + projection.FieldStates[field] = syncdomain.FieldState{ + Capability: node.capability, Known: node.value != nil, Reason: node.reason, Code: node.code, File: node.file, + } + if node.value == nil { + continue + } + value, ok := node.value.(string) + if !ok { + return nil, mapTypeError(name, "a string") + } + if field == syncdomain.FieldDescription { + projection.Description = &value + } else { + projection.Markdown = &value + } + } + return projection, nil + default: + return nil, fmt.Errorf("unsupported resource kind %q", kind) + } +} + +func mappedNode(root *sourceNode, name, path string) (*sourceNode, error) { + node, err := resolveSourcePath(root, path) + if err != nil { + return nil, fmt.Errorf("map field %q: %w", name, err) + } + return node, nil +} + +func projectionField(projection syncdomain.Projection, field syncdomain.Field) (any, bool, error) { + switch value := projection.(type) { + case syncdomain.VariationProjection: + switch field { + case syncdomain.FieldPrompt, syncdomain.FieldMessages: + if value.Instructions != nil { + return *value.Instructions, true, nil + } + if value.Messages != nil { + return *value.Messages, true, nil + } + case syncdomain.FieldName: + if value.Name != nil { + return *value.Name, true, nil + } + case syncdomain.FieldModel: + if value.Model != nil { + return *value.Model, true, nil + } + case syncdomain.FieldModelParameters: + if value.Parameters != nil { + return cloneObject(*value.Parameters), true, nil + } + case syncdomain.FieldTools: + if value.Tools != nil { + return *value.Tools, true, nil + } + case syncdomain.FieldSkills: + if value.Skills != nil { + return *value.Skills, true, nil + } + default: + if key, parameter := syncdomain.ModelParameterKey(field); parameter && value.Parameters != nil { + item, present := (*value.Parameters)[key] + return item, present, nil + } + } + case syncdomain.ToolProjection: + switch field { + case syncdomain.FieldDescription: + if value.Description != nil { + return *value.Description, true, nil + } + case syncdomain.FieldSchema: + if value.Schema != nil { + return *value.Schema, true, nil + } + case syncdomain.FieldCustomParameters: + if value.CustomParameters != nil { + return *value.CustomParameters, true, nil + } + case syncdomain.FieldTags: + if value.Tags != nil { + return *value.Tags, true, nil + } + } + case syncdomain.SkillProjection: + if field == syncdomain.FieldDescription && value.Description != nil { + return *value.Description, true, nil + } + if field == syncdomain.FieldMarkdown && value.Markdown != nil { + return *value.Markdown, true, nil + } + default: + return nil, false, fmt.Errorf("unsupported projection %T", projection) + } + return nil, false, nil +} + +func parseSourceBundle(bundle SourceBundle, format string) (*sourceNode, error) { + adapter, err := registeredAdapter(format) + if err != nil { + return nil, err + } + inspection, err := adapter.Inspect(bundle) + if err != nil { + if _, ok := syncdomain.ProblemFrom(err); !ok { + err = syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + err.Error(), + "Fix the source syntax or use a supported static form.", + ).WithFile(bundle.Primary.Path, 0, 0), err) + } + } + markSourceFile(inspection.root, bundle.Primary.Path) + return inspection.root, err +} + +func markSourceFile(node *sourceNode, file string) { + markSourceFileOnce(node, file, map[*sourceNode]struct{}{}) +} + +func markSourceFileOnce(node *sourceNode, file string, visited map[*sourceNode]struct{}) { + if node == nil { + return + } + if _, seen := visited[node]; seen { + return + } + visited[node] = struct{}{} + if node.file == "" { + node.file = file + } + for _, child := range node.fields { + markSourceFileOnce(child, file, visited) + } + for _, child := range node.items { + markSourceFileOnce(child, file, visited) + } +} + +func fieldAllowed(kind syncdomain.Kind, field syncdomain.Field) bool { + switch kind { + case syncdomain.KindVariation: + if _, parameter := syncdomain.ModelParameterKey(field); parameter { + return true + } + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldPrompt, syncdomain.FieldMessages, syncdomain.FieldName, syncdomain.FieldModel, + syncdomain.FieldModelParameters, syncdomain.FieldTools, syncdomain.FieldSkills, + }, field) + case syncdomain.KindTool: + return slices.Contains([]syncdomain.Field{ + syncdomain.FieldDescription, syncdomain.FieldSchema, + syncdomain.FieldCustomParameters, syncdomain.FieldTags, + }, field) + case syncdomain.KindSkill: + return field == syncdomain.FieldDescription || field == syncdomain.FieldMarkdown + default: + return false + } +} + +func semanticEqual(left, right any) bool { + leftJSON, leftErr := json.Marshal(left) + rightJSON, rightErr := json.Marshal(right) + return leftErr == nil && rightErr == nil && bytes.Equal(leftJSON, rightJSON) +} + +func mapTypeError(field, expected string) error { + return syncdomain.NewProblem( + syncdomain.ErrorMapTypeMismatch, + fmt.Sprintf("mapped field %q needs %s", field, expected), + "Map the field to a compatible static value.", + ) +} + +func stringList(value any) ([]string, bool) { + switch values := value.(type) { + case []string: + return slices.Clone(values), true + case []any: + result := make([]string, len(values)) + for index, value := range values { + text, ok := value.(string) + if !ok { + return nil, false + } + result[index] = text + } + return result, true + default: + return nil, false + } +} + +func objectValue(value any) (map[string]any, bool) { + object, ok := value.(map[string]any) + if !ok { + return nil, false + } + return object, true +} + +func cloneObject(value map[string]any) map[string]any { + cloned, ok := jsonValue(value) + if !ok { + return map[string]any{} + } + object, ok := cloned.(map[string]any) + if !ok { + return map[string]any{} + } + return object +} + +func jsonValue(value any) (any, bool) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, false + } + var decoded any + if err := json.Unmarshal(encoded, &decoded); err != nil { + return nil, false + } + return decoded, true +} + +func stringNode(value string, start, end int, encode func(string) ([]byte, error)) *sourceNode { + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(value any) ([]byte, error) { + text, ok := value.(string) + if !ok { + return nil, mapTypeError("source", "a string") + } + return encode(text) + }, + } +} + +func normalizeMarkdown(text string) string { + return strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")) +} diff --git a/internal/sync/reference/source_test.go b/internal/sync/reference/source_test.go new file mode 100644 index 00000000..73dbc2ad --- /dev/null +++ b/internal/sync/reference/source_test.go @@ -0,0 +1,120 @@ +package reference + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestPathParserReadsTwoFieldFilter(t *testing.T) { + path, err := parsePath(`items[id="one",role="system"]`) + require.NoError(t, err) + require.Len(t, path.steps, 1) + assert.Equal(t, "items", path.steps[0].key) + assert.Equal(t, []pathFilter{{key: "id", value: "one"}, {key: "role", value: "system"}}, path.steps[0].filters) +} + +func TestResolvePathSupportsChainedAndMixedIndexes(t *testing.T) { + root := &sourceNode{fields: map[string]*sourceNode{ + "agents": {items: []*sourceNode{ + {fields: map[string]*sourceNode{ + "id": {value: "support"}, + "messages": {items: []*sourceNode{ + {fields: map[string]*sourceNode{"content": {value: "First"}}}, + {fields: map[string]*sourceNode{"content": {value: "Second"}}}, + }, collection: true}, + }}, + }, collection: true}, + }} + + value, err := resolveSourcePath(root, `agents[id="support"].messages[1].content`) + + require.NoError(t, err) + assert.Equal(t, "Second", value.value) +} + +func TestResolvePathIndexErrorsUseStableCodes(t *testing.T) { + root := &sourceNode{fields: map[string]*sourceNode{ + "items": {items: []*sourceNode{{value: "only"}}, collection: true}, + "value": {value: "text"}, + }} + tests := []struct { + path string + code syncdomain.ErrorCode + }{ + {path: "items[1]", code: syncdomain.ErrorIndexOutOfRange}, + {path: "value[0]", code: syncdomain.ErrorPathTypeMismatch}, + } + for _, test := range tests { + _, err := resolveSourcePath(root, test.path) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + } +} + +func TestIndexedSelectionIdentityUsesStableFieldsAndFallback(t *testing.T) { + identified := &sourceNode{fields: map[string]*sourceNode{ + "id": {value: "support"}, + "n": {value: 1}, + }} + assert.Equal(t, map[string]any{"id": "support"}, directSelectionIdentity(identified)) + assert.Nil(t, directSelectionIdentity(nil)) + assert.True(t, scalarValue(uint64(2))) + assert.False(t, scalarValue(nil)) + assert.False(t, sourceCollection(nil)) + assert.True(t, sourceCollection(&sourceNode{value: []string{"one"}})) + + cycle := &sourceNode{fields: map[string]*sourceNode{}} + cycle.fields["self"] = cycle + fallback := &sourceNode{fields: map[string]*sourceNode{"prompt": {value: "Help"}}} + selections, err := indexedSelections( + []indexedResolution{ + {path: "items[0]", collectionLength: 1, item: identified}, + {path: "items[0].children[0]", collectionLength: 1, item: fallback}, + {path: "items[0].cycle[0]", collectionLength: 1, item: cycle}, + }, + fallback, + ) + require.NoError(t, err) + require.Len(t, selections, 3) + for _, selection := range selections { + assert.NotEmpty(t, selection.Fingerprint) + } +} + +func TestMarkdownSectionUpdatePreservesOtherSections(t *testing.T) { + source := []byte("# Support\nOld prompt\n\n# Other\nKeep this.\n") + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "AGENTS.md", Format: "markdown", + Select: `sections[heading="Support"]`, Map: map[string]string{"prompt": "body"}, + } + prompt := "New prompt" + result, err := UpdateSource( + sourceBundle("AGENTS.md", source), + id, + syncdomain.VariationModeAgent, + reference, + syncdomain.VariationProjection{Instructions: &prompt}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Contains(t, string(updated), "# Other\nKeep this.") +} + +func FuzzPathParserNeverPanics(f *testing.F) { + for _, seed := range []string{ + "$", "field.child", `items[id="one"]`, `object["literal.key"]`, + "items[0]", "items[-1]", "items[1.5]", "items[]", "items[2147483648]", + "items[0][1]", `items[id="support"].messages[0]`, + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, path string) { + _ = ParsePath(path) + }) +} diff --git a/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md b/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md new file mode 100644 index 00000000..52b96251 --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/references/policy.md @@ -0,0 +1,3 @@ +# Refund policy + +Escalate refunds above $500. diff --git a/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py b/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py new file mode 100644 index 00000000..e28fe6cd --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/scripts/check_refund.py @@ -0,0 +1,2 @@ +def eligible(total: float) -> bool: + return total <= 500 diff --git a/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden b/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden new file mode 100644 index 00000000..95237b04 --- /dev/null +++ b/internal/sync/reference/testdata/agent-skill/refunds/skill-instructions.md.golden @@ -0,0 +1,12 @@ +--- +name: refunds +description: Explain the refund policy and escalation rules. +license: Apache-2.0 +metadata: + owner: support-platform +--- + +# Refund handling + +Confirm the order ID first. +Escalate refunds above $500. diff --git a/internal/sync/reference/testdata/markdown/agents-instructions.md.golden b/internal/sync/reference/testdata/markdown/agents-instructions.md.golden new file mode 100644 index 00000000..16f2d0c4 --- /dev/null +++ b/internal/sync/reference/testdata/markdown/agents-instructions.md.golden @@ -0,0 +1,9 @@ +# Repository guide + +## Support agent + +Ask for an order ID before discussing an order. + +## Build + +Run `make test`.