diff --git a/internal/sync/reference/adapter_conformance_test.go b/internal/sync/reference/adapter_conformance_test.go index 22246424..0eee2569 100644 --- a/internal/sync/reference/adapter_conformance_test.go +++ b/internal/sync/reference/adapter_conformance_test.go @@ -35,6 +35,24 @@ func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) { id: syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"}, want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.", }, + { + name: "JSON tool", fixture: "json/mcp.json", format: "json", + id: syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup_order"}, + selectPath: `tools[name="lookup_order"]`, fieldMap: map[string]string{"description": "description"}, + want: "Look up one order.", update: "Look up one order and status.", + }, + { + name: "YAML agent", fixture: "yaml/agents.yaml", format: "yaml", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: `agents[name="support"]`, fieldMap: map[string]string{"prompt": "prompt"}, + want: "Ask for an order ID.", update: "Ask for an order ID and region.", + }, + { + name: "TOML agent", fixture: "toml/pyproject.toml", format: "toml", + id: syncdomain.VariationID("project", "support", "default"), + selectPath: "tool.agent.support", fieldMap: map[string]string{"prompt": "prompt"}, + want: "Ask for an order ID.\nKeep the answer short.", update: "Ask for an order ID and region.", + }, } for _, test := range tests { diff --git a/internal/sync/reference/defensive_test.go b/internal/sync/reference/defensive_test.go new file mode 100644 index 00000000..064511af --- /dev/null +++ b/internal/sync/reference/defensive_test.go @@ -0,0 +1,35 @@ +package reference + +import ( + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestStructuredAdaptersRejectMalformedSources(t *testing.T) { + tests := []struct { + name, format, source string + }{ + {name: "JSON duplicate", format: "json", source: `{"key":1,"key":2}`}, + {name: "JSON trailing", format: "json", source: `{"key":1} trailing`}, + {name: "YAML duplicate", format: "yaml", source: "key: one\nkey: two\n"}, + {name: "TOML duplicate", format: "toml", source: "key = 1\nkey = 2\n"}, + } + id := syncdomain.VariationID("project", "config", "default") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := ParseSource( + sourceBundle("source", []byte(test.source)), + id, + syncdomain.VariationModeAgent, + syncdomain.Reference{ + File: "source", Format: test.format, Select: "$", + Map: map[string]string{"prompt": "key"}, + }, + ) + require.Error(t, err) + }) + } +} diff --git a/internal/sync/reference/document.go b/internal/sync/reference/document.go index 3e888d85..fc3d55b2 100644 --- a/internal/sync/reference/document.go +++ b/internal/sync/reference/document.go @@ -370,8 +370,12 @@ func quoteYAML(value, previous string) string { func encodeYAMLString(value, previous string) ([]byte, error) { for _, character := range value { - if unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { - return nil, fmt.Errorf("YAML string contains unsupported control character U+%04X", character) + valid := character == '\n' || character == '\r' || character == '\t' || + character >= 0x20 && character <= 0xd7ff || + character >= 0xe000 && character <= 0xfffd || + character >= 0x10000 && character <= utf8.MaxRune + if !valid || unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' { + return nil, fmt.Errorf("YAML string contains unsupported character U+%04X", character) } } return []byte(quoteYAML(value, previous)), nil diff --git a/internal/sync/reference/json.go b/internal/sync/reference/json.go new file mode 100644 index 00000000..98a1b1b3 --- /dev/null +++ b/internal/sync/reference/json.go @@ -0,0 +1,238 @@ +package reference + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "unicode/utf8" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type jsonAdapter struct{} + +func (jsonAdapter) Format() string { return "json" } +func (jsonAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseJSONDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (jsonAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(jsonAdapter{}) } + +type jsonParser struct { + data []byte + offset int +} + +func parseJSONDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "JSON source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + parser := jsonParser{data: document.Data} + if bytes.HasPrefix(parser.data, []byte("\ufeff")) { + parser.offset = len([]byte("\ufeff")) + } + parser.skipSpace() + root, err := parser.value() + if err != nil { + return nil, err + } + parser.skipSpace() + if parser.offset != len(parser.data) { + return nil, parser.problem("JSON source has trailing input") + } + return root, nil +} + +func (parser *jsonParser) value() (*sourceNode, error) { + parser.skipSpace() + if parser.offset >= len(parser.data) { + return nil, parser.problem("JSON value is missing") + } + switch parser.data[parser.offset] { + case '{': + return parser.object() + case '[': + return parser.array() + case '"': + return parser.string() + case 't': + return parser.keyword("true", true) + case 'f': + return parser.keyword("false", false) + case 'n': + return parser.keyword("null", nil) + default: + return parser.number() + } +} + +func (parser *jsonParser) object() (*sourceNode, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + fields := map[string]*sourceNode{} + value := map[string]any{} + if parser.take('}') { + return jsonObjectNode(value, fields, start, parser.offset), nil + } + for { + keyNode, err := parser.string() + if err != nil { + return nil, err + } + key := keyNode.value.(string) + if _, duplicate := fields[key]; duplicate { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("JSON object contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + parser.skipSpace() + if !parser.take(':') { + return nil, parser.problem("JSON object key needs a colon") + } + item, err := parser.value() + if err != nil { + return nil, err + } + fields[key], value[key] = item, item.value + parser.skipSpace() + if parser.take('}') { + return jsonObjectNode(value, fields, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON object needs a comma or closing brace") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) array() (*sourceNode, error) { + start := parser.offset + parser.offset++ + parser.skipSpace() + var items []*sourceNode + var values []any + if parser.take(']') { + return jsonArrayNode(values, items, start, parser.offset), nil + } + for { + item, err := parser.value() + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.value) + parser.skipSpace() + if parser.take(']') { + return jsonArrayNode(values, items, start, parser.offset), nil + } + if !parser.take(',') { + return nil, parser.problem("JSON array needs a comma or closing bracket") + } + parser.skipSpace() + } +} + +func (parser *jsonParser) string() (*sourceNode, error) { + start := parser.offset + if !parser.take('"') { + return nil, parser.problem("JSON string must start with a quote") + } + escaped := false + for parser.offset < len(parser.data) { + next := parser.data[parser.offset] + parser.offset++ + switch { + case escaped: + escaped = false + case next == '\\': + escaped = true + case next == '"': + raw := parser.data[start:parser.offset] + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON string is invalid") + } + return stringNode(value, start, parser.offset, func(text string) ([]byte, error) { + return json.Marshal(text) + }), nil + case next < 0x20: + return nil, parser.problem("JSON string contains a control character") + } + } + return nil, parser.problem("JSON string is not closed") +} + +func (parser *jsonParser) number() (*sourceNode, error) { + start := parser.offset + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte("-+0123456789.eE"), rune(parser.data[parser.offset])) { + parser.offset++ + } + if start == parser.offset { + return nil, parser.problem("JSON value is invalid") + } + raw := parser.data[start:parser.offset] + var value float64 + if err := json.Unmarshal(raw, &value); err != nil { + return nil, parser.problem("JSON number is invalid") + } + return numberNode(value, start, parser.offset, func(number float64) ([]byte, error) { + return []byte(strconv.FormatFloat(number, 'g', -1, 64)), nil + }), nil +} + +func (parser *jsonParser) keyword(keyword string, value any) (*sourceNode, error) { + start := parser.offset + if !bytes.HasPrefix(parser.data[start:], []byte(keyword)) { + return nil, parser.problem("JSON value is invalid") + } + parser.offset += len(keyword) + return &sourceNode{ + value: value, start: start, end: parser.offset, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + }, nil +} + +func (parser *jsonParser) skipSpace() { + for parser.offset < len(parser.data) && bytes.ContainsRune([]byte(" \t\r\n"), rune(parser.data[parser.offset])) { + parser.offset++ + } +} + +func (parser *jsonParser) take(expected byte) bool { + if parser.offset >= len(parser.data) || parser.data[parser.offset] != expected { + return false + } + parser.offset++ + return true +} + +func (parser *jsonParser) problem(message string) error { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + fmt.Sprintf("%s at byte %d", message, parser.offset), + "Fix the JSON syntax.", + ) +} + +func jsonObjectNode(value map[string]any, fields map[string]*sourceNode, start, end int) *sourceNode { + return &sourceNode{ + value: value, fields: fields, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + } +} + +func jsonArrayNode(value []any, items []*sourceNode, start, end int) *sourceNode { + return &sourceNode{ + value: value, items: items, collection: true, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { return json.Marshal(next) }, + } +} diff --git a/internal/sync/reference/parameters_test.go b/internal/sync/reference/parameters_test.go new file mode 100644 index 00000000..83ab0b84 --- /dev/null +++ b/internal/sync/reference/parameters_test.go @@ -0,0 +1,203 @@ +package reference + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestJSONIndividualModelParameterMappingsRoundTrip(t *testing.T) { + tests := []struct { + name string + key string + source any + replacement any + }{ + {name: "temperature", key: "temperature", source: 0.2, replacement: 0.7}, + {name: "top k", key: "top_k", source: 20, replacement: 40}, + {name: "top p", key: "top_p", source: 0.9, replacement: 0.8}, + {name: "max tokens", key: "max_tokens", source: 256, replacement: 512}, + {name: "future provider key", key: "Provider.Future-Key", source: "v1", replacement: "v2"}, + {name: "string", key: "response_style", source: "brief", replacement: "detailed"}, + {name: "boolean", key: "reasoning", source: true, replacement: false}, + {name: "null", key: "seed", source: nil, replacement: true}, + { + name: "nested object", key: "provider", + source: map[string]any{"mode": "fast"}, + replacement: map[string]any{"mode": "careful", "retry": true}, + }, + { + name: "array", key: "stop", + source: []any{"END"}, + replacement: []any{"DONE", map[string]any{"token": 2}}, + }, + } + id := syncdomain.VariationID("project", "config", "default") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source, err := json.Marshal(map[string]any{"prompt": "Help", "value": test.source}) + require.NoError(t, err) + target := modelParameterTarget(test.key) + reference := syncdomain.Reference{ + File: "agent.json", Format: "json", + Map: map[string]string{"prompt": "prompt", target: "value"}, + } + + projection, err := ParseSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + value, present := syncdomain.ProjectionFieldValue(projection, syncdomain.Field(target)) + assert.True(t, present) + assertJSONValueEqual(t, test.source, value) + + parameters := map[string]any{test.key: test.replacement} + update, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Parameters: ¶meters, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.Field(target): {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + ) + require.NoError(t, err) + require.Len(t, update.Fields[syncdomain.Field(target)].Edits, 1) + updated := applyParameterEdits(source, update.Fields[syncdomain.Field(target)].Edits...) + reparsed, err := ParseSource( + sourceBundle(reference.File, updated), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + value, present = syncdomain.ProjectionFieldValue(reparsed, syncdomain.Field(target)) + assert.True(t, present) + assertJSONValueEqual(t, test.replacement, value) + }) + } +} + +func TestJSONWholeModelParametersOwnsCanonicalMap(t *testing.T) { + source := []byte(`{"prompt":"Help","parameters":{"temperature":0.2,"tools":[{"name":"search"}]}}`) + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "agent.json", Format: "json", + Map: map[string]string{"prompt": "prompt", "model.parameters": "parameters"}, + } + projection, err := ParseSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + ) + require.NoError(t, err) + parameters := projection.(syncdomain.VariationProjection).Parameters + require.NotNil(t, parameters) + assert.Equal(t, map[string]any{"temperature": 0.2}, *parameters) + + replacement := map[string]any{ + "top_k": 30, + "nested": map[string]any{"enabled": true}, + "stop": []any{"END"}, + "seed": nil, + } + update, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{ + Mode: syncdomain.VariationModeAgent, Parameters: &replacement, + FieldStates: map[syncdomain.Field]syncdomain.FieldState{ + syncdomain.FieldModelParameters: {Capability: syncdomain.CapabilityDirect, Known: true}, + }, + }, + ) + require.NoError(t, err) + require.Len(t, update.Fields[syncdomain.FieldModelParameters].Edits, 1) + updated := applyParameterEdits(source, update.Fields[syncdomain.FieldModelParameters].Edits...) + assert.JSONEq(t, `{"prompt":"Help","parameters":{"top_k":30,"nested":{"enabled":true},"stop":["END"],"seed":null}}`, string(updated)) +} + +func TestNormalizeReferenceRejectsOverlappingOrInvalidParameterTargets(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + tests := []struct { + name string + targets map[string]string + code syncdomain.ErrorCode + message string + }{ + { + name: "whole and child overlap", + targets: map[string]string{ + "prompt": "prompt", "model.parameters": "parameters", + `model.parameters["temperature"]`: "temperature", + }, + code: syncdomain.ErrorInvalidReference, message: "overlaps", + }, + { + name: "same exact key through JSON escape", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["top_k"]`: "top_k", + `model.parameters["top\u005fk"]`: "other", + }, + code: syncdomain.ErrorInvalidReference, message: "same model parameter", + }, + { + name: "expanded tools child", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["tools"]`: "tools", + }, + code: syncdomain.ErrorInvalidReference, message: "duplicates the tools field", + }, + { + name: "child needs exact bracket key", + targets: map[string]string{ + "prompt": "prompt", "model.parameters.temperature": "temperature", + }, + code: syncdomain.ErrorInvalidReference, message: "unsupported mapped field", + }, + { + name: "target uses shared path syntax", + targets: map[string]string{ + "prompt": "prompt", `model.parameters["temperature"`: "temperature", + }, + code: syncdomain.ErrorInvalidPathSyntax, message: "invalid path", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NormalizeReference(id, syncdomain.Reference{ + File: "agent.json", Format: "json", Map: test.targets, + }) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, test.code, problem.Code) + assert.ErrorContains(t, err, test.message) + }) + } +} + +func modelParameterTarget(key string) string { + encoded, _ := json.Marshal(key) + return fmt.Sprintf("model.parameters[%s]", encoded) +} + +func assertJSONValueEqual(t *testing.T, expected, actual any) { + t.Helper() + expectedJSON, err := json.Marshal(expected) + require.NoError(t, err) + actualJSON, err := json.Marshal(actual) + require.NoError(t, err) + assert.JSONEq(t, string(expectedJSON), string(actualJSON)) +} + +func applyParameterEdits(source []byte, edits ...TextEdit) []byte { + updated := append([]byte(nil), source...) + for _, edit := range edits { + next := make([]byte, 0, len(updated)-edit.End+edit.Start+len(edit.Replacement)) + next = append(next, updated[:edit.Start]...) + next = append(next, edit.Replacement...) + next = append(next, updated[edit.End:]...) + updated = next + } + return updated +} diff --git a/internal/sync/reference/source.go b/internal/sync/reference/source.go index 4c642280..74a1a495 100644 --- a/internal/sync/reference/source.go +++ b/internal/sync/reference/source.go @@ -859,6 +859,10 @@ func projectionField(projection syncdomain.Projection, field syncdomain.Field) ( return nil, false, nil } +func parseSourceDocument(document Document, format string) (*sourceNode, error) { + return parseSourceBundle(SourceBundle{Primary: document}, format) +} + func parseSourceBundle(bundle SourceBundle, format string) (*sourceNode, error) { adapter, err := registeredAdapter(format) if err != nil { @@ -937,6 +941,19 @@ func mapTypeError(field, expected string) error { ) } +func sourceNumber(value any) (float64, bool) { + switch value := value.(type) { + case float64: + return value, true + case int: + return float64(value), true + case int64: + return float64(value), true + default: + return 0, false + } +} + func stringList(value any) ([]string, bool) { switch values := value.(type) { case []string: @@ -988,6 +1005,10 @@ func jsonValue(value any) (any, bool) { return decoded, true } +func readOnlyNode(start, end int, reason string) *sourceNode { + return &sourceNode{start: start, end: end, capability: CapabilityReadOnly, reason: reason} +} + func stringNode(value string, start, end int, encode func(string) ([]byte, error)) *sourceNode { return &sourceNode{ value: value, start: start, end: end, capability: CapabilityDirect, @@ -1001,6 +1022,19 @@ func stringNode(value string, start, end int, encode func(string) ([]byte, error } } +func numberNode(value float64, start, end int, encode func(float64) ([]byte, error)) *sourceNode { + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(value any) ([]byte, error) { + number, ok := sourceNumber(value) + if !ok { + return nil, mapTypeError("source", "a number") + } + return encode(number) + }, + } +} + func normalizeMarkdown(text string) string { return strings.TrimSpace(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")) } diff --git a/internal/sync/reference/structured_test.go b/internal/sync/reference/structured_test.go new file mode 100644 index 00000000..b79a1afc --- /dev/null +++ b/internal/sync/reference/structured_test.go @@ -0,0 +1,437 @@ +package reference + +import ( + "encoding/json" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestStructuredPathsReadNestedListsAndLiteralKeys(t *testing.T) { + tests := []struct { + name string + format string + source string + path string + want any + }{ + { + name: "JSON keyed list", format: "json", + source: `{"items":[{"name":"one","enabled":false},{"name":"two","enabled":true}]}`, + path: `items[name="two"].enabled`, want: true, + }, + { + name: "YAML nested field", format: "yaml", + source: "agent:\n model:\n temperature: 0.3\n", + path: "agent.model.temperature", want: 0.3, + }, + { + name: "TOML literal key", format: "toml", + source: "[metadata]\n\"display.name\" = \"Support\"\n", + path: `metadata["display.name"]`, want: "Support", + }, + { + name: "TOML inline table list", format: "toml", + source: "items = [{ name = \"one\", prompt = \"First\" }, { name = \"two\", prompt = \"Second\" }]\n", + path: `items[name="two"].prompt`, want: "Second", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root, err := parseSourceDocument(Document{Path: "source", Data: []byte(test.source)}, test.format) + require.NoError(t, err) + value, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.want, value.value) + }) + } +} + +func FuzzJSONStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quotes \" and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + quoted, err := json.Marshal(value) + require.NoError(t, err) + source := append([]byte(`{"value":`), quoted...) + source = append(source, '}') + root, err := parseSourceDocument(Document{Path: "value.json", Data: source}, "json") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + encoded, err := node.encode(node.value) + require.NoError(t, err) + var decoded string + require.NoError(t, json.Unmarshal(encoded, &decoded)) + require.Equal(t, value, decoded) + }) +} + +func FuzzYAMLStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quote ' and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + for _, previous := range []string{"plain", "'plain'", `"plain"`} { + encoded, err := encodeYAMLString(value, previous) + if err != nil { + t.Skip() + } + source := append([]byte("value: "), encoded...) + source = append(source, '\n') + root, err := parseSourceDocument(Document{Path: "value.yaml", Data: source}, "yaml") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + require.Equal(t, value, node.value) + } + }) +} + +func TestYAMLStringCodecRejectsUnsupportedCodePoint(t *testing.T) { + _, err := encodeYAMLString("\uffff", "plain") + require.ErrorContains(t, err, "unsupported character U+FFFF") +} + +func FuzzTOMLStringCodec(f *testing.F) { + for _, value := range []string{"plain", "quotes \" and slash \\", "line one\nline two", "Unicode 🌍"} { + f.Add(value) + } + f.Fuzz(func(t *testing.T, value string) { + if !utf8.ValidString(value) { + t.Skip() + } + encoded, err := encodeTOMLString(value, `"plain"`) + require.NoError(t, err) + source := append([]byte("value = "), encoded...) + source = append(source, '\n') + root, err := parseSourceDocument(Document{Path: "value.toml", Data: source}, "toml") + require.NoError(t, err) + node, err := resolveSourcePath(root, "value") + require.NoError(t, err) + require.Equal(t, value, node.value) + }) +} + +func TestStructuredAdaptersRejectDuplicateKeys(t *testing.T) { + tests := []struct { + name string + format string + source string + }{ + {name: "JSON", format: "json", source: `{"name":"first","name":"second"}`}, + {name: "YAML", format: "yaml", source: "name: first\nname: second\n"}, + {name: "TOML", format: "toml", source: "name = \"first\"\nname = \"second\"\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := parseSourceDocument(Document{Path: "source", Data: []byte(test.source)}, test.format) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorDuplicateKey, problem.Code) + }) + } +} + +func TestStructuredAdaptersRejectInvalidSyntax(t *testing.T) { + tests := []struct { + name, format, source string + }{ + {name: "empty JSON", format: "json", source: ""}, + {name: "JSON trailing input", format: "json", source: `{"a":1} true`}, + {name: "JSON missing colon", format: "json", source: `{"a" 1}`}, + {name: "JSON missing comma", format: "json", source: `[1 2]`}, + {name: "JSON unclosed string", format: "json", source: `{"a":"value}`}, + {name: "invalid YAML", format: "yaml", source: "value: [\n"}, + {name: "empty YAML", format: "yaml", source: ""}, + {name: "invalid TOML", format: "toml", source: "value = [\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := parseSourceDocument( + Document{Path: "source", Data: []byte(test.source)}, test.format, + ) + require.Error(t, err) + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorInvalidSource, problem.Code) + }) + } +} + +func TestYAMLUnsafeFormsAreReadOnly(t *testing.T) { + tests := []struct { + name string + source string + path string + }{ + {name: "block scalar", source: "prompt: |\n Help the user.\n", path: "prompt"}, + {name: "anchor", source: "prompt: &shared \"Help\"\n", path: "prompt"}, + {name: "alias", source: "base: &shared \"Help\"\nprompt: *shared\n", path: "prompt"}, + {name: "merge", source: "base: &base\n prompt: Help\nagent:\n <<: *base\n", path: "agent.prompt"}, + {name: "tag", source: "prompt: !custom Help\n", path: "prompt"}, + {name: "multiple documents", source: "prompt: First\n---\nprompt: Second\n", path: "prompt"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root, err := parseSourceDocument(Document{Path: "agents.yaml", Data: []byte(test.source)}, "yaml") + require.NoError(t, err) + value, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, CapabilityReadOnly, value.capability) + assert.NotEmpty(t, value.reason) + }) + } +} + +func TestTOMLArrayTableFieldsAreReadOnly(t *testing.T) { + source := `[[tool.plugins]] +name = "audit" +enabled = true +` + root, err := parseSourceDocument(Document{Path: "pyproject.toml", Data: []byte(source)}, "toml") + require.NoError(t, err) + + value, err := resolveSourcePath(root, `tool.plugins[name="audit"].enabled`) + + require.NoError(t, err) + assert.Equal(t, CapabilityReadOnly, value.capability) +} + +func TestStructuredEditsPreserveBOMAndCRLFOutsideSpan(t *testing.T) { + source := []byte("\ufeff{\r\n \"description\": \"Old\",\r\n \"enabled\": true\r\n}\r\n") + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "tool"} + reference := syncdomain.Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{"description": "description"}, + } + description := "New 🌍" + + result, err := UpdateSource( + sourceBundle(reference.File, source), id, "", reference, + syncdomain.ToolProjection{Description: &description}, + ) + + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Contains(t, string(updated), "\ufeff{\r\n") + assert.Contains(t, string(updated), "\"enabled\": true\r\n") + assert.Contains(t, string(updated), `"description": "New 🌍"`) +} + +func TestStructuredIndexWriteRoundTripsPreserveOtherItems(t *testing.T) { + tests := []struct { + name, format, source, unchanged string + }{ + { + name: "JSON", format: "json", + source: "{\n \"tools\": [{\"name\":\"first\",\"description\":\"Keep\"}, {\"name\":\"second\",\"description\":\"Old\"}]\n}\n", + unchanged: `{"name":"first","description":"Keep"}`, + }, + { + name: "YAML", format: "yaml", + source: "tools:\n - {name: first, description: Keep}\n - {name: second, description: Old}\n", + unchanged: "{name: first, description: Keep}", + }, + { + name: "TOML", format: "toml", + source: "tools = [{ name = \"first\", description = \"Keep\" }, { name = \"second\", description = \"Old\" }]\n", + unchanged: `{ name = "first", description = "Keep" }`, + }, + } + id := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "second"} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := []byte(test.source) + reference := syncdomain.Reference{ + File: "tools." + test.format, Format: test.format, Select: "tools[1]", + Map: map[string]string{"description": "description"}, + } + projection, selections, err := ParseSourceWithSelections(sourceBundle(reference.File, source), id, "", reference) + require.NoError(t, err) + assert.Equal(t, "Old", *projection.(syncdomain.ToolProjection).Description) + require.Len(t, selections, 1) + assert.Equal(t, 2, selections[0].CollectionLength) + + description := "Updated" + update, err := UpdateSource( + sourceBundle(reference.File, source), id, "", reference, + syncdomain.ToolProjection{Description: &description}, + ) + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, update)) + assert.Contains(t, string(updated), test.unchanged) + assert.Contains(t, string(updated), "Updated") + + roundTrip, _, err := ParseSourceWithSelections( + sourceBundle(reference.File, updated), id, "", reference, + ) + require.NoError(t, err) + assert.Equal(t, description, *roundTrip.(syncdomain.ToolProjection).Description) + }) + } +} + +func TestIndexedSelectionsDetectNestedReorders(t *testing.T) { + id := syncdomain.VariationID("project", "completion", "default") + reference := syncdomain.Reference{ + File: "agents.json", Format: "json", Select: "agents[0]", + Map: map[string]string{"prompt": "messages[0].content"}, + } + before := []byte(`{"agents":[ + {"id":"support","messages":[{"role":"system","content":"Help"},{"role":"user","content":"Example"}]}, + {"id":"other","messages":[{"role":"system","content":"Other"}]} + ]}`) + after := []byte(`{"agents":[ + {"id":"other","messages":[{"role":"system","content":"Other"}]}, + {"id":"support","messages":[{"role":"user","content":"Example"},{"role":"system","content":"Help"}]} + ]}`) + + _, baseline, err := ParseSourceWithSelections( + sourceBundle(reference.File, before), id, syncdomain.VariationModeCompletion, reference, + ) + require.NoError(t, err) + _, reordered, err := ParseSourceWithSelections( + sourceBundle(reference.File, after), id, syncdomain.VariationModeCompletion, reference, + ) + require.NoError(t, err) + + require.Len(t, baseline, 2) + require.Len(t, reordered, 2) + assert.NotEqual(t, baseline, reordered) +} + +func TestYAMLUnicodeColumnsAndFlowScalarsUseExactSpans(t *testing.T) { + source := []byte("étiquette: {prompt: ancien, enabled: true}\n") + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "agents.yaml", Format: "yaml", Select: `["étiquette"]`, + Map: map[string]string{"prompt": "prompt"}, + } + prompt := "nouveau" + + result, err := UpdateSource( + sourceBundle(reference.File, source), id, syncdomain.VariationModeAgent, reference, + syncdomain.VariationProjection{Mode: syncdomain.VariationModeAgent, Instructions: &prompt}, + ) + + require.NoError(t, err) + updated := applyConformanceEdits(t, source, directEdits(t, result)) + assert.Equal(t, "étiquette: {prompt: nouveau, enabled: true}\n", string(updated)) +} + +func TestStructuredValueFormsRoundTrip(t *testing.T) { + tests := []struct { + name string + format string + source string + path string + want any + next any + }{ + {name: "JSON escaped Unicode string", format: "json", source: `{"value":"Hello \u4e16\u754c"}`, path: "value", want: "Hello 世界", next: "Hello 🌍"}, + {name: "JSON number", format: "json", source: `{"value":1.25}`, path: "value", want: 1.25, next: 2.5}, + {name: "JSON boolean", format: "json", source: `{"value":true}`, path: "value", want: true, next: false}, + {name: "JSON object", format: "json", source: `{"value":{"enabled":true}}`, path: "value", want: map[string]any{"enabled": true}, next: map[string]any{"enabled": false}}, + {name: "YAML plain string", format: "yaml", source: "value: hello\n", path: "value", want: "hello", next: "updated"}, + {name: "YAML single quoted string", format: "yaml", source: "value: 'hello'\n", path: "value", want: "hello", next: "updated"}, + {name: "YAML double quoted string", format: "yaml", source: "value: \"hello\\nworld\"\n", path: "value", want: "hello\nworld", next: "updated\nworld"}, + {name: "YAML number", format: "yaml", source: "value: 1.25\n", path: "value", want: 1.25, next: 2.5}, + {name: "YAML boolean", format: "yaml", source: "value: true\n", path: "value", want: true, next: false}, + {name: "TOML basic string", format: "toml", source: "value = \"hello\"\n", path: "value", want: "hello", next: "updated"}, + {name: "TOML literal string", format: "toml", source: "value = 'hello'\n", path: "value", want: "hello", next: "updated"}, + {name: "TOML multiline string", format: "toml", source: "value = \"\"\"hello\nworld\"\"\"\n", path: "value", want: "hello\nworld", next: "updated\nworld"}, + {name: "TOML number", format: "toml", source: "value = 1.25\n", path: "value", want: 1.25, next: 2.5}, + {name: "TOML boolean", format: "toml", source: "value = true\n", path: "value", want: true, next: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + source := []byte(test.source) + root, err := parseSourceDocument(Document{Path: "source", Data: source}, test.format) + require.NoError(t, err) + node, err := resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.want, node.value) + require.Equal(t, CapabilityDirect, node.capability) + replacement, err := node.encode(test.next) + require.NoError(t, err) + updated := append(append(append([]byte{}, source[:node.start]...), replacement...), source[node.end:]...) + root, err = parseSourceDocument(Document{Path: "source", Data: updated}, test.format) + require.NoError(t, err) + node, err = resolveSourcePath(root, test.path) + require.NoError(t, err) + assert.Equal(t, test.next, node.value) + }) + } +} + +func TestStructuredProjectionCollectionsAndObjects(t *testing.T) { + variationSource := []byte(`{ + "prompt": "Help", + "tools": ["lookup"], + "skills": ["refunds"] +}`) + variationID := syncdomain.VariationID("project", "config", "default") + variationRef := syncdomain.Reference{ + File: "variation.json", Format: "json", Select: "$", + Map: map[string]string{"prompt": "prompt", "tools": "tools", "skills": "skills"}, + } + projection, err := ParseSource( + sourceBundle(variationRef.File, variationSource), variationID, + syncdomain.VariationModeAgent, variationRef, + ) + require.NoError(t, err) + variation := projection.(syncdomain.VariationProjection) + assert.Equal(t, []string{"lookup"}, *variation.Tools) + assert.Equal(t, []string{"refunds"}, *variation.Skills) + + toolSource := []byte(`{ + "description": "Lookup", + "schema": {"type": "object"}, + "customParameters": {"timeout": 5}, + "tags": ["orders", "support"] +}`) + toolID := syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup"} + toolRef := syncdomain.Reference{ + File: "tool.json", Format: "json", Select: "$", + Map: map[string]string{ + "description": "description", "schema": "schema", + "customParameters": "customParameters", "tags": "tags", + }, + } + projection, err = ParseSource(sourceBundle(toolRef.File, toolSource), toolID, "", toolRef) + require.NoError(t, err) + tool := projection.(syncdomain.ToolProjection) + assert.Equal(t, "object", (*tool.Schema)["type"]) + assert.Equal(t, float64(5), (*tool.CustomParameters)["timeout"]) + assert.Equal(t, []string{"orders", "support"}, *tool.Tags) +} + +func TestMapTypeMismatchUsesStableCode(t *testing.T) { + id := syncdomain.VariationID("project", "config", "default") + reference := syncdomain.Reference{ + File: "value.json", Format: "json", Select: "$", + Map: map[string]string{"prompt": "prompt"}, + } + + _, err := ParseSource( + sourceBundle(reference.File, []byte(`{"prompt":42}`)), + id, syncdomain.VariationModeAgent, reference, + ) + + problem, ok := syncdomain.ProblemFrom(err) + require.True(t, ok) + assert.Equal(t, syncdomain.ErrorMapTypeMismatch, problem.Code) +} diff --git a/internal/sync/reference/testdata/json/mcp.json b/internal/sync/reference/testdata/json/mcp.json new file mode 100644 index 00000000..34146950 --- /dev/null +++ b/internal/sync/reference/testdata/json/mcp.json @@ -0,0 +1,32 @@ +{ + "server": "orders", + "tools": [ + { + "name": "lookup_order", + "description": "Look up one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" } + }, + "required": ["orderId"] + } + }, + { + "name": "refund_order", + "description": "Refund one order.", + "inputSchema": { + "type": "object", + "properties": { + "orderId": { "type": "string" }, + "reason": { "type": "string" } + }, + "required": ["orderId", "reason"] + } + } + ], + "metadata": { + "display.name": "Order tools", + "enabled": true + } +} diff --git a/internal/sync/reference/testdata/toml/pyproject.toml b/internal/sync/reference/testdata/toml/pyproject.toml new file mode 100644 index 00000000..689ed0c2 --- /dev/null +++ b/internal/sync/reference/testdata/toml/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "support-agents" +version = "1.0.0" + +[tool.agent.support] +prompt = """Ask for an order ID. +Keep the answer short.""" +model = "gpt-5" +temperature = 0.2 +enabled = true + +[tool.agent.refunds] +prompt = 'Explain the refund policy.' +model = 'gpt-5' +temperature = 0.1 +enabled = false + +[[tool.agent.plugins]] +name = "audit" +enabled = true diff --git a/internal/sync/reference/testdata/yaml/agents.yaml b/internal/sync/reference/testdata/yaml/agents.yaml new file mode 100644 index 00000000..2bb224b4 --- /dev/null +++ b/internal/sync/reference/testdata/yaml/agents.yaml @@ -0,0 +1,16 @@ +service: support +agents: + - name: support + prompt: "Ask for an order ID." + model: + name: gpt-5 + temperature: 0.2 + enabled: true + - name: refunds + prompt: 'Explain the refund policy.' + model: + name: gpt-5 + temperature: 0.1 + enabled: false +metadata: + "display.name": Support agents diff --git a/internal/sync/reference/toml.go b/internal/sync/reference/toml.go new file mode 100644 index 00000000..6bb34248 --- /dev/null +++ b/internal/sync/reference/toml.go @@ -0,0 +1,307 @@ +package reference + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + toml "github.com/pelletier/go-toml/v2" + "github.com/pelletier/go-toml/v2/unstable" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type tomlAdapter struct{} + +func (tomlAdapter) Format() string { return "toml" } +func (tomlAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseTOMLDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (tomlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(tomlAdapter{}) } + +func parseTOMLDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "TOML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := document.Data + offset := 0 + if bytes.HasPrefix(data, []byte("\ufeff")) { + offset = len([]byte("\ufeff")) + data = data[offset:] + } + var parser unstable.Parser + parser.Reset(data) + root := &sourceNode{value: map[string]any{}, fields: map[string]*sourceNode{}} + table := root + for parser.NextExpression() { + expression := parser.Expression() + switch expression.Kind { + case unstable.KeyValue: + keys := tomlKeys(expression) + if err := addTOMLValue(data, offset, table, keys, expression.Value()); err != nil { + return nil, err + } + case unstable.Table: + var err error + table, err = tomlTable(root, tomlKeys(expression), false) + if err != nil { + return nil, err + } + case unstable.ArrayTable: + var err error + table, err = tomlTable(root, tomlKeys(expression), true) + if err != nil { + return nil, err + } + } + } + if err := parser.Error(); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML source is invalid", "Fix the TOML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + return root, nil +} + +func tomlKeys(node *unstable.Node) []string { + var keys []string + iterator := node.Key() + for iterator.Next() { + keys = append(keys, string(iterator.Node().Data)) + } + return keys +} + +func addTOMLValue(data []byte, offset int, table *sourceNode, keys []string, valueNode *unstable.Node) error { + if len(keys) == 0 { + return syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML key is missing", "Add a key to the value.", + ) + } + parent, err := tomlObjectPath(table, keys[:len(keys)-1]) + if err != nil { + return err + } + key := keys[len(keys)-1] + if _, duplicate := parent.fields[key]; duplicate { + return syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML table contains duplicate key %q", strings.Join(keys, ".")), + "Remove the duplicate key.", + ) + } + value, err := tomlValueNode(data, offset, valueNode) + if err != nil { + return err + } + if table.capability == CapabilityReadOnly { + markReadOnly(value, table.reason) + } + parent.fields[key] = value + parent.value.(map[string]any)[key] = value.value + return nil +} + +func tomlTable(root *sourceNode, keys []string, array bool) (*sourceNode, error) { + if !array { + return tomlObjectPath(root, keys) + } + if len(keys) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "TOML array table key is missing", "Add a table key.", + ) + } + parent, err := tomlObjectPath(root, keys[:len(keys)-1]) + if err != nil { + return nil, err + } + key := keys[len(keys)-1] + collection := parent.fields[key] + if collection == nil { + collection = &sourceNode{ + value: []any{}, capability: CapabilityReadOnly, + reason: "TOML arrays of tables are read-only", + } + parent.fields[key] = collection + parent.value.(map[string]any)[key] = collection.value + } + if collection.items == nil && len(collection.value.([]any)) != 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q has incompatible definitions", strings.Join(keys, ".")), + "Keep one table definition.", + ) + } + item := &sourceNode{ + value: map[string]any{}, fields: map[string]*sourceNode{}, + capability: CapabilityReadOnly, reason: "TOML arrays of tables are read-only", + } + collection.items = append(collection.items, item) + collection.value = append(collection.value.([]any), item.value) + parent.value.(map[string]any)[key] = collection.value + return item, nil +} + +func tomlObjectPath(root *sourceNode, keys []string) (*sourceNode, error) { + current := root + for _, key := range keys { + child := current.fields[key] + if child == nil { + child = &sourceNode{value: map[string]any{}, fields: map[string]*sourceNode{}} + current.fields[key] = child + current.value.(map[string]any)[key] = child.value + } + if child.fields == nil { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("TOML key %q is already a value", key), + "Keep one compatible key definition.", + ) + } + current = child + } + return current, nil +} + +func tomlValueNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + start := offset + int(node.Raw.Offset) + end := start + int(node.Raw.Length) + raw := data[node.Raw.Offset : node.Raw.Offset+node.Raw.Length] + switch node.Kind { + case unstable.String: + return stringNode(string(node.Data), start, end, func(value string) ([]byte, error) { + return encodeTOMLString(value, string(raw)) + }), nil + case unstable.Bool: + value := string(node.Data) == "true" + return &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { + value, ok := next.(bool) + if !ok { + return nil, mapTypeError("source", "a boolean") + } + return []byte(strconv.FormatBool(value)), nil + }, + }, nil + case unstable.Integer, unstable.Float: + var decoded map[string]any + if err := toml.Unmarshal(append([]byte("value = "), raw...), &decoded); err != nil { + return nil, err + } + number, ok := sourceNumber(decoded["value"]) + if !ok { + return readOnlyNode(start, end, "TOML number is not representable"), nil + } + return numberNode(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }), nil + case unstable.Array: + return tomlArrayNode(data, offset, node) + case unstable.InlineTable: + return tomlInlineTableNode(data, offset, node) + default: + result := readOnlyNode(start, end, "TOML date and time values are read-only") + result.value = string(node.Data) + return result, nil + } +} + +func tomlArrayNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + var items []*sourceNode + var values []any + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind == unstable.Comment { + continue + } + item, err := tomlValueNode(data, offset, child) + if err != nil { + return nil, err + } + items, values = append(items, item), append(values, item.value) + } + return &sourceNode{ + value: values, items: items, collection: true, capability: CapabilityReadOnly, + reason: "TOML structural changes are read-only", + }, nil +} + +func tomlInlineTableNode(data []byte, offset int, node *unstable.Node) (*sourceNode, error) { + result := &sourceNode{ + value: map[string]any{}, fields: map[string]*sourceNode{}, + capability: CapabilityDirect, + } + iterator := node.Children() + for iterator.Next() { + child := iterator.Node() + if child.Kind != unstable.KeyValue { + continue + } + if err := addTOMLValue(data, offset, result, tomlKeys(child), child.Value()); err != nil { + return nil, err + } + } + result.capability = CapabilityReadOnly + result.reason = "TOML structural changes are read-only" + return result, nil +} + +func encodeTOMLString(value, previous string) ([]byte, error) { + switch { + case strings.HasPrefix(previous, `"""`): + return []byte(`"""` + strings.ReplaceAll(value, `"""`, `\"""`) + `"""`), nil + case strings.HasPrefix(previous, `'''`): + if strings.Contains(value, `'''`) { + return nil, fmt.Errorf("multiline literal string cannot contain three apostrophes") + } + return []byte(`'''` + value + `'''`), nil + case strings.HasPrefix(previous, `'`): + if strings.ContainsAny(value, "'\r\n") { + return nil, fmt.Errorf("literal string cannot represent the new value") + } + return []byte("'" + value + "'"), nil + default: + return []byte(encodeTOMLBasicString(value)), nil + } +} + +func encodeTOMLBasicString(value string) string { + var encoded strings.Builder + encoded.WriteByte('"') + for _, character := range value { + switch character { + case '"', '\\': + encoded.WriteByte('\\') + encoded.WriteRune(character) + case '\b': + encoded.WriteString(`\b`) + case '\t': + encoded.WriteString(`\t`) + case '\n': + encoded.WriteString(`\n`) + case '\f': + encoded.WriteString(`\f`) + case '\r': + encoded.WriteString(`\r`) + default: + if character < 0x20 || character >= 0x7f && character <= 0x9f { + fmt.Fprintf(&encoded, `\u%04X`, character) + } else { + encoded.WriteRune(character) + } + } + } + encoded.WriteByte('"') + return encoded.String() +} diff --git a/internal/sync/reference/yaml.go b/internal/sync/reference/yaml.go new file mode 100644 index 00000000..72253f57 --- /dev/null +++ b/internal/sync/reference/yaml.go @@ -0,0 +1,218 @@ +package reference + +import ( + "bytes" + "fmt" + "io" + "strconv" + "strings" + "unicode/utf8" + + "gopkg.in/yaml.v3" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +type yamlAdapter struct{} + +func (yamlAdapter) Format() string { return "yaml" } +func (yamlAdapter) Inspect(bundle SourceBundle) (Inspection, error) { + root, err := parseYAMLDocument(bundle.Primary) + return Inspection{root: root}, err +} +func (yamlAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil } + +func init() { mustRegister(yamlAdapter{}) } + +func parseYAMLDocument(document Document) (*sourceNode, error) { + if !utf8.Valid(document.Data) { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML source is not valid UTF-8", + "Save the file as UTF-8.", + ).WithFile(document.Path, 0, 0) + } + data := bytes.TrimPrefix(document.Data, []byte("\ufeff")) + offset := len(document.Data) - len(data) + decoder := yaml.NewDecoder(bytes.NewReader(data)) + var first yaml.Node + if err := decoder.Decode(&first); err != nil { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), err) + } + var extra yaml.Node + extraErr := decoder.Decode(&extra) + multiple := extraErr == nil + if extraErr != nil && extraErr != io.EOF { + return nil, syncdomain.WrapProblem(syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML source is invalid", "Fix the YAML syntax.", + ).WithFile(document.Path, 0, 0), extraErr) + } + if len(first.Content) == 0 { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML document is empty", "Add one YAML value.", + ) + } + root, err := yamlSourceNode(data, first.Content[0], offset) + if err != nil { + return nil, err + } + if multiple { + markReadOnly(root, "multi-document YAML is read-only") + } + return root, nil +} + +func yamlSourceNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + switch node.Kind { + case yaml.MappingNode: + return yamlMappingNode(data, node, offset) + case yaml.SequenceNode: + return yamlSequenceNode(data, node, offset) + case yaml.ScalarNode: + return yamlValueNode(data, node, offset) + case yaml.AliasNode: + value, err := yamlSourceNode(data, node.Alias, offset) + if value != nil { + markReadOnly(value, "YAML aliases are read-only") + } + return value, err + default: + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, "YAML node is not supported", "Use mappings, lists, or scalar values.", + ) + } +} + +func yamlMappingNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + fields := make(map[string]*sourceNode, len(node.Content)/2) + value := make(map[string]any, len(node.Content)/2) + unsafe := node.Anchor != "" || customYAMLTag(node.Tag) + reason := "YAML anchors and tags are read-only" + for index := 0; index+1 < len(node.Content); index += 2 { + keyNode, valueNode := node.Content[index], node.Content[index+1] + if keyNode.Kind != yaml.ScalarNode || keyNode.Tag != "!!str" && keyNode.Tag != "!!merge" { + return nil, syncdomain.NewProblem( + syncdomain.ErrorInvalidSource, + "YAML mapping key is not a string", + "Use direct string keys.", + ) + } + key := keyNode.Value + if _, duplicate := fields[key]; duplicate { + return nil, syncdomain.NewProblem( + syncdomain.ErrorDuplicateKey, + fmt.Sprintf("YAML mapping contains duplicate key %q", key), + "Remove the duplicate key.", + ) + } + child, err := yamlSourceNode(data, valueNode, offset) + if err != nil { + return nil, err + } + fields[key], value[key] = child, child.value + if key == "<<" { + unsafe, reason = true, "YAML merge keys are read-only" + for mergedKey, mergedValue := range child.fields { + if _, exists := fields[mergedKey]; !exists { + fields[mergedKey] = mergedValue + value[mergedKey] = mergedValue.value + } + } + } + } + result := &sourceNode{ + value: value, fields: fields, capability: CapabilityReadOnly, + reason: "YAML structural changes are read-only", + } + if unsafe { + markReadOnly(result, reason) + } + return result, nil +} + +func yamlSequenceNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + items := make([]*sourceNode, 0, len(node.Content)) + values := make([]any, 0, len(node.Content)) + for _, childNode := range node.Content { + child, err := yamlSourceNode(data, childNode, offset) + if err != nil { + return nil, err + } + items, values = append(items, child), append(values, child.value) + } + result := &sourceNode{ + value: values, items: items, collection: true, capability: CapabilityReadOnly, + reason: "YAML structural changes are read-only", + } + if node.Anchor != "" || customYAMLTag(node.Tag) { + markReadOnly(result, "YAML anchors and tags are read-only") + } + return result, nil +} + +func yamlValueNode(data []byte, node *yaml.Node, offset int) (*sourceNode, error) { + region := frontMatterRegion{data: data, startOffset: offset} + start, end, raw, err := yamlScalarSpan(region, node) + if err != nil { + return nil, err + } + readOnly := node.Anchor != "" || customYAMLTag(node.Tag) || + node.Style == yaml.LiteralStyle || node.Style == yaml.FoldedStyle + reason := "YAML anchors, tags, and block scalars are read-only" + var result *sourceNode + switch node.Tag { + case "!!str": + result = stringNode(node.Value, start, end, func(text string) ([]byte, error) { + return encodeYAMLString(text, raw) + }) + case "!!bool": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return nil, err + } + result = &sourceNode{ + value: value, start: start, end: end, capability: CapabilityDirect, + encode: func(next any) ([]byte, error) { + value, ok := next.(bool) + if !ok { + return nil, mapTypeError("source", "a boolean") + } + return []byte(strconv.FormatBool(value)), nil + }, + } + case "!!int", "!!float": + number, err := strconv.ParseFloat(strings.ReplaceAll(node.Value, "_", ""), 64) + if err != nil { + return nil, err + } + result = numberNode(number, start, end, func(next float64) ([]byte, error) { + return []byte(strconv.FormatFloat(next, 'g', -1, 64)), nil + }) + default: + result = readOnlyNode(start, end, "YAML scalar type is read-only") + result.value = node.Value + } + if readOnly { + result.capability, result.reason, result.encode = CapabilityReadOnly, reason, nil + } + return result, nil +} + +func customYAMLTag(tag string) bool { + return tag != "" && !strings.HasPrefix(tag, "!!") +} + +func markReadOnly(node *sourceNode, reason string) { + if node == nil { + return + } + node.capability, node.reason, node.encode = CapabilityReadOnly, reason, nil + for _, child := range node.fields { + markReadOnly(child, reason) + } + for _, child := range node.items { + markReadOnly(child, reason) + } +}