feat(sync): add structured document adapters

internal/sync/reference/adapter_conformance_test.go

@@ -35,6 +35,24 @@ func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) {
 			id:   syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"},
 			want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.",
 		},
+		{
+			name: "JSON tool", fixture: "json/mcp.json", format: "json",
+			id:         syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup_order"},
+			selectPath: `tools[name="lookup_order"]`, fieldMap: map[string]string{"description": "description"},
+			want: "Look up one order.", update: "Look up one order and status.",
+		},
+		{
+			name: "YAML agent", fixture: "yaml/agents.yaml", format: "yaml",
+			id:         syncdomain.VariationID("project", "support", "default"),
+			selectPath: `agents[name="support"]`, fieldMap: map[string]string{"prompt": "prompt"},
+			want: "Ask for an order ID.", update: "Ask for an order ID and region.",
+		},
+		{
+			name: "TOML agent", fixture: "toml/pyproject.toml", format: "toml",
+			id:         syncdomain.VariationID("project", "support", "default"),
+			selectPath: "tool.agent.support", fieldMap: map[string]string{"prompt": "prompt"},
+			want: "Ask for an order ID.\nKeep the answer short.", update: "Ask for an order ID and region.",
+		},
 	}
 
 	for _, test := range tests {

internal/sync/reference/defensive_test.go

@@ -0,0 +1,35 @@
+package reference
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
+)
+
+func TestStructuredAdaptersRejectMalformedSources(t *testing.T) {
+	tests := []struct {
+		name, format, source string
+	}{
+		{name: "JSON duplicate", format: "json", source: `{"key":1,"key":2}`},
+		{name: "JSON trailing", format: "json", source: `{"key":1} trailing`},
+		{name: "YAML duplicate", format: "yaml", source: "key: one\nkey: two\n"},
+		{name: "TOML duplicate", format: "toml", source: "key = 1\nkey = 2\n"},
+	}
+	id := syncdomain.VariationID("project", "config", "default")
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			_, err := ParseSource(
+				sourceBundle("source", []byte(test.source)),
+				id,
+				syncdomain.VariationModeAgent,
+				syncdomain.Reference{
+					File: "source", Format: test.format, Select: "$",
+					Map: map[string]string{"prompt": "key"},
+				},
+			)
+			require.Error(t, err)
+		})
+	}
+}

internal/sync/reference/document.go

@@ -370,8 +370,12 @@ func quoteYAML(value, previous string) string {
 
 func encodeYAMLString(value, previous string) ([]byte, error) {
 	for _, character := range value {
-		if unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' {
-			return nil, fmt.Errorf("YAML string contains unsupported control character U+%04X", character)
+		valid := character == '\n' || character == '\r' || character == '\t' ||
+			character >= 0x20 && character <= 0xd7ff ||
+			character >= 0xe000 && character <= 0xfffd ||
+			character >= 0x10000 && character <= utf8.MaxRune
+		if !valid || unicode.IsControl(character) && character != '\n' && character != '\r' && character != '\t' {
+			return nil, fmt.Errorf("YAML string contains unsupported character U+%04X", character)
 		}
 	}
 	return []byte(quoteYAML(value, previous)), nil

internal/sync/reference/json.go

@@ -0,0 +1,238 @@
+package reference
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"unicode/utf8"
+
+	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
+)
+
+type jsonAdapter struct{}
+
+func (jsonAdapter) Format() string { return "json" }
+func (jsonAdapter) Inspect(bundle SourceBundle) (Inspection, error) {
+	root, err := parseJSONDocument(bundle.Primary)
+	return Inspection{root: root}, err
+}
+func (jsonAdapter) Defaults(syncdomain.Kind) (string, map[string]string) { return "$", nil }
+
+func init() { mustRegister(jsonAdapter{}) }
+
+type jsonParser struct {
+	data   []byte
+	offset int
+}
+
+func parseJSONDocument(document Document) (*sourceNode, error) {
+	if !utf8.Valid(document.Data) {
+		return nil, syncdomain.NewProblem(
+			syncdomain.ErrorInvalidSource,
+			"JSON source is not valid UTF-8",
+			"Save the file as UTF-8.",
+		).WithFile(document.Path, 0, 0)
+	}
+	parser := jsonParser{data: document.Data}
+	if bytes.HasPrefix(parser.data, []byte("\ufeff")) {
+		parser.offset = len([]byte("\ufeff"))
+	}
+	parser.skipSpace()
+	root, err := parser.value()
+	if err != nil {
+		return nil, err
+	}
+	parser.skipSpace()
+	if parser.offset != len(parser.data) {
+		return nil, parser.problem("JSON source has trailing input")
+	}
+	return root, nil
+}
+
+func (parser *jsonParser) value() (*sourceNode, error) {
+	parser.skipSpace()
+	if parser.offset >= len(parser.data) {
+		return nil, parser.problem("JSON value is missing")
+	}
+	switch parser.data[parser.offset] {
+	case '{':
+		return parser.object()
+	case '[':
+		return parser.array()
+	case '"':
+		return parser.string()
+	case 't':
+		return parser.keyword("true", true)
+	case 'f':
+		return parser.keyword("false", false)
+	case 'n':
+		return parser.keyword("null", nil)
+	default:
+		return parser.number()
+	}
+}
+
+func (parser *jsonParser) object() (*sourceNode, error) {
+	start := parser.offset
+	parser.offset++
+	parser.skipSpace()
+	fields := map[string]*sourceNode{}
+	value := map[string]any{}
+	if parser.take('}') {
+		return jsonObjectNode(value, fields, start, parser.offset), nil
+	}
+	for {
+		keyNode, err := parser.string()
+		if err != nil {
+			return nil, err
+		}
+		key := keyNode.value.(string)
+		if _, duplicate := fields[key]; duplicate {
+			return nil, syncdomain.NewProblem(
+				syncdomain.ErrorDuplicateKey,
+				fmt.Sprintf("JSON object contains duplicate key %q", key),
+				"Remove the duplicate key.",
+			)
+		}
+		parser.skipSpace()
+		if !parser.take(':') {
+			return nil, parser.problem("JSON object key needs a colon")
+		}
+		item, err := parser.value()
+		if err != nil {
+			return nil, err
+		}
+		fields[key], value[key] = item, item.value
+		parser.skipSpace()
+		if parser.take('}') {
+			return jsonObjectNode(value, fields, start, parser.offset), nil
+		}
+		if !parser.take(',') {
+			return nil, parser.problem("JSON object needs a comma or closing brace")
+		}
+		parser.skipSpace()
+	}
+}
+
+func (parser *jsonParser) array() (*sourceNode, error) {
+	start := parser.offset
+	parser.offset++
+	parser.skipSpace()
+	var items []*sourceNode
+	var values []any
+	if parser.take(']') {
+		return jsonArrayNode(values, items, start, parser.offset), nil
+	}
+	for {
+		item, err := parser.value()
+		if err != nil {
+			return nil, err
+		}
+		items, values = append(items, item), append(values, item.value)
+		parser.skipSpace()
+		if parser.take(']') {
+			return jsonArrayNode(values, items, start, parser.offset), nil
+		}
+		if !parser.take(',') {
+			return nil, parser.problem("JSON array needs a comma or closing bracket")
+		}
+		parser.skipSpace()
+	}
+}
+
+func (parser *jsonParser) string() (*sourceNode, error) {
+	start := parser.offset
+	if !parser.take('"') {
+		return nil, parser.problem("JSON string must start with a quote")
+	}
+	escaped := false
+	for parser.offset < len(parser.data) {
+		next := parser.data[parser.offset]
+		parser.offset++
+		switch {
+		case escaped:
+			escaped = false
+		case next == '\\':
+			escaped = true
+		case next == '"':
+			raw := parser.data[start:parser.offset]
+			var value string
+			if err := json.Unmarshal(raw, &value); err != nil {
+				return nil, parser.problem("JSON string is invalid")
+			}
+			return stringNode(value, start, parser.offset, func(text string) ([]byte, error) {
+				return json.Marshal(text)
+			}), nil
+		case next < 0x20:
+			return nil, parser.problem("JSON string contains a control character")
+		}
+	}
+	return nil, parser.problem("JSON string is not closed")
+}
+
+func (parser *jsonParser) number() (*sourceNode, error) {
+	start := parser.offset
+	for parser.offset < len(parser.data) && bytes.ContainsRune([]byte("-+0123456789.eE"), rune(parser.data[parser.offset])) {
+		parser.offset++
+	}
+	if start == parser.offset {
+		return nil, parser.problem("JSON value is invalid")
+	}
+	raw := parser.data[start:parser.offset]
+	var value float64
+	if err := json.Unmarshal(raw, &value); err != nil {
+		return nil, parser.problem("JSON number is invalid")
+	}
+	return numberNode(value, start, parser.offset, func(number float64) ([]byte, error) {
+		return []byte(strconv.FormatFloat(number, 'g', -1, 64)), nil
+	}), nil
+}
+
+func (parser *jsonParser) keyword(keyword string, value any) (*sourceNode, error) {
+	start := parser.offset
+	if !bytes.HasPrefix(parser.data[start:], []byte(keyword)) {
+		return nil, parser.problem("JSON value is invalid")
+	}
+	parser.offset += len(keyword)
+	return &sourceNode{
+		value: value, start: start, end: parser.offset, capability: CapabilityDirect,
+		encode: func(next any) ([]byte, error) { return json.Marshal(next) },
+	}, nil
+}
+
+func (parser *jsonParser) skipSpace() {
+	for parser.offset < len(parser.data) && bytes.ContainsRune([]byte(" \t\r\n"), rune(parser.data[parser.offset])) {
+		parser.offset++
+	}
+}
+
+func (parser *jsonParser) take(expected byte) bool {
+	if parser.offset >= len(parser.data) || parser.data[parser.offset] != expected {
+		return false
+	}
+	parser.offset++
+	return true
+}
+
+func (parser *jsonParser) problem(message string) error {
+	return syncdomain.NewProblem(
+		syncdomain.ErrorInvalidSource,
+		fmt.Sprintf("%s at byte %d", message, parser.offset),
+		"Fix the JSON syntax.",
+	)
+}
+
+func jsonObjectNode(value map[string]any, fields map[string]*sourceNode, start, end int) *sourceNode {
+	return &sourceNode{
+		value: value, fields: fields, start: start, end: end, capability: CapabilityDirect,
+		encode: func(next any) ([]byte, error) { return json.Marshal(next) },
+	}
+}
+
+func jsonArrayNode(value []any, items []*sourceNode, start, end int) *sourceNode {
+	return &sourceNode{
+		value: value, items: items, collection: true, start: start, end: end, capability: CapabilityDirect,
+		encode: func(next any) ([]byte, error) { return json.Marshal(next) },
+	}
+}