feat(sync): add the Python source adapter

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/atotto/clipboard v0.1.4
 	github.com/aws/aws-sdk-go-v2 v1.47.3
 	github.com/aws/aws-sdk-go-v2/config v1.33.5
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.5
 	github.com/aws/aws-sdk-go-v2/service/devopsagent v1.17.0
 	github.com/aws/aws-sdk-go-v2/service/iam v1.64.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4
@@ -46,6 +47,10 @@ require (
 	github.com/spf13/pflag v1.0.10
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
+	github.com/tree-sitter/go-tree-sitter v0.25.0
+	github.com/tree-sitter/tree-sitter-javascript v0.23.1
+	github.com/tree-sitter/tree-sitter-python v0.25.0
+	github.com/tree-sitter/tree-sitter-typescript v0.23.2
 	go.uber.org/mock v0.6.0
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa
 	golang.org/x/term v0.40.0
@@ -57,7 +62,6 @@ require (
 	github.com/alecthomas/chroma/v2 v2.20.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.5 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
@@ -99,6 +103,7 @@ require (
 	github.com/mailru/easyjson v0.9.1 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-pointer v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.17 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect

go.sum

@@ -215,6 +215,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
 github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=
+github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.17 h1:78v8ZlW0bP43XfmAfPsdXcoNCelfMHsDmd/pkENfrjQ=
 github.com/mattn/go-runewidth v0.0.17/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -317,6 +319,34 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tree-sitter/go-tree-sitter v0.25.0 h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU=
+github.com/tree-sitter/go-tree-sitter v0.25.0/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU=
+github.com/tree-sitter/tree-sitter-c v0.23.4 h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c=
+github.com/tree-sitter/tree-sitter-c v0.23.4/go.mod h1:MkI5dOiIpeN94LNjeCp8ljXN/953JCwAby4bClMr6bw=
+github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w=
+github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8=
+github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg=
+github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ=
+github.com/tree-sitter/tree-sitter-go v0.23.4 h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA=
+github.com/tree-sitter/tree-sitter-go v0.23.4/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040=
+github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc=
+github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og=
+github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig=
+github.com/tree-sitter/tree-sitter-java v0.23.5/go.mod h1:NRKlI8+EznxA7t1Yt3xtraPk1Wzqh3GAIC46wxvc320=
+github.com/tree-sitter/tree-sitter-javascript v0.23.1 h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo=
+github.com/tree-sitter/tree-sitter-javascript v0.23.1/go.mod h1:lmGD1EJdCA+v0S1u2fFgepMg/opzSg/4pgFym2FPGAs=
+github.com/tree-sitter/tree-sitter-json v0.24.8 h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ=
+github.com/tree-sitter/tree-sitter-json v0.24.8/go.mod h1:F351KK0KGvCaYbZ5zxwx/gWWvZhIDl0eMtn+1r+gQbo=
+github.com/tree-sitter/tree-sitter-php v0.23.11 h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8=
+github.com/tree-sitter/tree-sitter-php v0.23.11/go.mod h1:T/kbfi+UcCywQfUNAJnGTN/fMSUjnwPXA8k4yoIks74=
+github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0=
+github.com/tree-sitter/tree-sitter-python v0.25.0/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM=
+github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo=
+github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA=
+github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90=
+github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI=
+github.com/tree-sitter/tree-sitter-typescript v0.23.2 h1:/Odvphn18PniVixb9e97X0DbNVsU6Qocv9mfkyzdXwU=
+github.com/tree-sitter/tree-sitter-typescript v0.23.2/go.mod h1:zjzMXT/Ulffel2xfOcAkQQkiAkmgnbtPGlFQw/5X4xA=
 github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk=
 github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=

internal/sync/reference/adapter_conformance_test.go

@@ -35,6 +35,12 @@ func TestRegisteredFormatsReadAndWriteRealisticSources(t *testing.T) {
 			id:   syncdomain.ResourceID{Kind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "refunds"},
 			want: "Explain the refund policy and escalation rules.", update: "Explain refunds and escalation rules.",
 		},
+		{
+			name: "Python agent", fixture: "python/agents.py", format: "python",
+			id:         syncdomain.VariationID("project", "support", "default"),
+			selectPath: `agents[id="support"]`, fieldMap: map[string]string{"prompt": "instructions"},
+			want: "Ask for an order ID before discussing an order.", update: "Ask for an order ID and region.",
+		},
 		{
 			name: "JSON tool", fixture: "json/mcp.json", format: "json",
 			id:         syncdomain.ResourceID{Kind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "lookup_order"},

internal/sync/reference/code.go

@@ -0,0 +1,239 @@
+package reference
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	sitter "github.com/tree-sitter/go-tree-sitter"
+
+	syncdomain "github.com/launchdarkly/ldcli/internal/sync"
+)
+
+const maxSourceBytes = 4 << 20
+
+func parseTree(document Document, language *sitter.Language) (*sitter.Tree, error) {
+	if !utf8.Valid(document.Data) {
+		return nil, fmt.Errorf("%s: source is not valid UTF-8", document.Path)
+	}
+	if len(document.Data) > maxSourceBytes {
+		return nil, fmt.Errorf("%s: source exceeds %d bytes", document.Path, maxSourceBytes)
+	}
+	parser := sitter.NewParser()
+	defer parser.Close()
+	if err := parser.SetLanguage(language); err != nil {
+		return nil, fmt.Errorf("%s: set parser language: %w", document.Path, err)
+	}
+	deadline := time.Now().Add(2 * time.Second)
+	tree := parser.ParseWithOptions(func(offset int, _ sitter.Point) []byte {
+		if offset >= len(document.Data) {
+			return nil
+		}
+		return document.Data[offset:]
+	}, nil, &sitter.ParseOptions{
+		ProgressCallback: func(sitter.ParseState) bool {
+			return time.Now().After(deadline)
+		},
+	})
+	if tree == nil {
+		return nil, fmt.Errorf("%s: parser timed out", document.Path)
+	}
+	if tree.RootNode().HasError() {
+		tree.Close()
+		return nil, fmt.Errorf("%s: source contains syntax errors", document.Path)
+	}
+	return tree, nil
+}
+
+func logicalRoot(collections map[string][]*sourceNode) *sourceNode {
+	fields := make(map[string]*sourceNode, len(collections))
+	for name, items := range collections {
+		fields[name] = &sourceNode{items: items, collection: true}
+	}
+	return &sourceNode{fields: fields}
+}
+
+func logicalResource(id string, fields map[string]*sourceNode) *sourceNode {
+	fields["id"] = &sourceNode{value: id}
+	return &sourceNode{fields: fields}
+}
+
+func walkTree(root *sitter.Node, visit func(*sitter.Node)) {
+	stack := []*sitter.Node{root}
+	for len(stack) != 0 {
+		node := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		visit(node)
+		for index := node.NamedChildCount(); index > 0; index-- {
+			stack = append(stack, node.NamedChild(index-1))
+		}
+	}
+}
+
+func namedChildren(node *sitter.Node) []*sitter.Node {
+	if node == nil {
+		return nil
+	}
+	children := make([]*sitter.Node, 0, node.NamedChildCount())
+	for index := uint(0); index < node.NamedChildCount(); index++ {
+		child := node.NamedChild(index)
+		if child.Kind() != "comment" {
+			children = append(children, child)
+		}
+	}
+	return children
+}
+
+func nodeText(node *sitter.Node, source []byte) string {
+	if node == nil {
+		return ""
+	}
+	return string(source[node.StartByte():node.EndByte()])
+}
+
+func messageListFromCollection(collection *sourceNode) *sourceNode {
+	if collection == nil || collection.items == nil {
+		return nil
+	}
+	result := &sourceNode{
+		items: collection.items, collection: true, start: collection.start, end: collection.end,
+		capability: CapabilityDirect, reason: "message structure is read-only",
+	}
+	var messages []syncdomain.Message
+	for _, item := range collection.items {
+		fields := item.fields
+		if fields == nil && len(item.items) == 2 {
+			fields = map[string]*sourceNode{"role": item.items[0], "content": item.items[1]}
+			item.fields = fields
+		}
+		role, roleOK := fields["role"].value.(string)
+		content, contentOK := fields["content"].value.(string)
+		if !roleOK || !contentOK {
+			result.capability = CapabilityReadOnly
+			result.reason = "dynamic message content is read-only"
+			return result
+		}
+		messages = append(messages, syncdomain.Message{
+			Role: normalizeMessageRole(role), Content: content,
+		})
+	}
+	result.value = messages
+	return result
+}
+
+func decodePythonString(raw []byte) (string, string, error) {
+	text := string(raw)
+	quoteAt := strings.IndexAny(text, `"'`)
+	if quoteAt < 0 {
+		return "", "", fmt.Errorf("unsupported Python string")
+	}
+	prefix := strings.ToLower(text[:quoteAt])
+	if strings.ContainsAny(prefix, "fbr") || strings.Trim(prefix, "u") != "" {
+		return "", "", fmt.Errorf("Python string prefix %q is read-only", prefix)
+	}
+	quote := text[quoteAt : quoteAt+1]
+	if strings.HasPrefix(text[quoteAt:], quote+quote+quote) {
+		quote = quote + quote + quote
+	}
+	if len(text) < quoteAt+2*len(quote) || !strings.HasSuffix(text, quote) {
+		return "", "", fmt.Errorf("invalid Python string")
+	}
+	value, err := decodeEscapes(text[quoteAt+len(quote):len(text)-len(quote)], true)
+	return value, quote, err
+}
+
+func encodePythonString(value, quote string) []byte {
+	body := encodeEscapes(value, quote[0], len(quote) == 3)
+	if len(quote) == 3 {
+		body = strings.ReplaceAll(body, quote, `\`+quote)
+	}
+	return []byte(quote + body + quote)
+}
+
+func decodeEscapes(text string, python bool) (string, error) {
+	var result strings.Builder
+	for offset := 0; offset < len(text); {
+		if text[offset] != '\\' {
+			r, width := utf8.DecodeRuneInString(text[offset:])
+			result.WriteRune(r)
+			offset += width
+			continue
+		}
+		offset++
+		if offset == len(text) {
+			return "", fmt.Errorf("string ends with a backslash")
+		}
+		escape := text[offset]
+		offset++
+		switch escape {
+		case '\\', '\'', '"', '`':
+			result.WriteByte(escape)
+		case 'n':
+			result.WriteByte('\n')
+		case 'r':
+			result.WriteByte('\r')
+		case 't':
+			result.WriteByte('\t')
+		case 'b':
+			result.WriteByte('\b')
+		case 'f':
+			result.WriteByte('\f')
+		case 'v':
+			result.WriteByte('\v')
+		case '\n':
+		case 'x', 'u', 'U':
+			digits := map[byte]int{'x': 2, 'u': 4, 'U': 8}[escape]
+			if !python && escape == 'U' {
+				return "", fmt.Errorf("TypeScript does not support \\U escapes")
+			}
+			if offset+digits > len(text) {
+				return "", fmt.Errorf("short hexadecimal escape")
+			}
+			value, err := strconv.ParseUint(text[offset:offset+digits], 16, 32)
+			if err != nil || value > utf8.MaxRune || value >= 0xd800 && value <= 0xdfff {
+				return "", fmt.Errorf("invalid Unicode escape")
+			}
+			result.WriteRune(rune(value))
+			offset += digits
+		default:
+			if !python {
+				return "", fmt.Errorf("unsupported TypeScript escape \\%c", escape)
+			}
+			result.WriteByte('\\')
+			result.WriteByte(escape)
+		}
+	}
+	return result.String(), nil
+}
+
+func encodeEscapes(value string, quote byte, multiline bool) string {
+	var result strings.Builder
+	for _, character := range value {
+		switch character {
+		case '\\':
+			result.WriteString(`\\`)
+		case '\n':
+			if multiline {
+				result.WriteByte('\n')
+			} else {
+				result.WriteString(`\n`)
+			}
+		case '\r':
+			result.WriteString(`\r`)
+		case '\t':
+			result.WriteString(`\t`)
+		case rune(quote):
+			result.WriteByte('\\')
+			result.WriteRune(character)
+		default:
+			if character < 0x20 {
+				fmt.Fprintf(&result, `\u%04x`, character)
+			} else {
+				result.WriteRune(character)
+			}
+		}
+	}
+	return result.String()
+}