bundle: delete libs/dyn; configuration is typed structs with structvar refs and locations

.agents/rules/style-guide-go.md

@@ -113,7 +113,7 @@ return fieldPaths
 
 ### Encoding
 
-**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys. `libs/dyn/jsonloader` preserves key order but also lacks `UseNumber`, so it shares the int64 hazard.
+**RULE: When mutating an API response by round-tripping JSON, decode with `json.Decoder` + `UseNumber()`.** The naive `json.Marshal` → `map[string]any` → `Marshal` path corrupts any int64 larger than 2^53 (it degrades to a float64 mantissa — e.g. a real `spark_context_id`) and alphabetizes object keys.
 
 **RULE: Be careful with `encoding/csv` `Writer.UseCRLF = true`.** It rewrites both record terminators AND embedded newlines inside quoted fields to `\r\n`, so tests for quoted multiline fields must expect `\r\n`, not just the line endings between rows.
 
@@ -191,7 +191,7 @@ func NewService(deps ServiceDeps) *Service { ... }
 
 ### Configuration patterns
 
-- Bundle config uses `dyn.Value` for dynamic typing
+- Bundle config is typed structs; `libs/structs/structvar` carries `${...}` references and source locations
 - Config loading supports includes, variable interpolation, and target overrides
 - Schema generation is automated from Go struct tags
 

.github/workflows/python_push.yml

@@ -39,7 +39,7 @@ jobs:
             acceptance/bundle/python/ \
             go.mod \
             go.sum \
-            libs/dyn/dynvar/testdata/reference_vectors.json \
+            libs/structs/structvar/testdata/reference_vectors.json \
             python/ \
             tools/task/ \
           )

.nextchanges/bundles/6954.md

@@ -0,0 +1 @@
+* Speed up bundle commands on large bundles: configuration is kept as typed structs instead of being converted to and from a dynamic tree (`bundle validate` and `bundle plan` with 1000 jobs are about 10x faster and use about 3.5x less memory). ([#6954](https://github.com/databricks/cli/pull/6954))

AGENTS.md

@@ -14,7 +14,7 @@ This is the Databricks CLI, a command-line interface for interacting with Databr
 
 **RULE: Keep each PR focused on one change.** If you notice an unrelated cleanup, bug fix, or refactor while making your primary change, leave it alone or put it in a separate PR. Reviewers consistently ask to split mixed PRs, especially when a dependency bump or schema diff rides along with a feature change.
 
-**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structpath/` and `libs/dyn/` (path / dynamic values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name.
+**RULE: Before adding a new helper, search the codebase for an existing one.** Common homes: `libs/` (shared utilities), `libs/databrickscfg/` (config), `libs/git/`, `libs/filer/`, `libs/cmdio/` (CLI I/O, spinners, prompts), `libs/env/` (env vars), `libs/testserver/`, `libs/structs/structpath/` and `libs/structs/structvar/` (paths / typed config values), `acceptance/bin/` (acceptance test helpers), `internal/mocks/` (generated mocks). A function that duplicates an existing name and signature in the same package is a compile error waiting to happen; grep before you name.
 
 # Development Commands
 
@@ -73,7 +73,7 @@ GIT_EDITOR=true GIT_SEQUENCE_EDITOR=true VISUAL=true GIT_PAGER=cat git rebase or
 - `bundle/phases/` - High-level deployment phases
 
 **libs/** - Shared libraries and utilities
-- `libs/dyn/` - Dynamic configuration value manipulation
+- `libs/structs/structvar/` - Typed configuration values with references and locations
 - `libs/filer/` - File system abstraction (local, DBFS, workspace)
 - `libs/auth/` - Databricks authentication handling
 - `libs/sync/` - File synchronization between local and remote

acceptance/bundle/run/job-ref-job-id/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    bar:
+      name: job bar
+    foo:
+      name: job foo
+      tasks:
+        - task_key: job_task
+          run_job_task:
+            job_id: ${resources.jobs.bar.id}

acceptance/bundle/run/job-ref-job-id/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/run/job-ref-job-id/output.txt

@@ -0,0 +1,14 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created jobs.bar
+Created jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== run resolves the referenced job id into the int job_id field
+>>> [CLI] bundle run foo
+Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID]
+
+[TIMESTAMP] "job foo" RUNNING
+[TIMESTAMP] "job foo" TERMINATED SUCCESS

acceptance/bundle/run/job-ref-job-id/script

@@ -0,0 +1,5 @@
+trace $CLI bundle deploy
+replace_ids.py
+
+title "run resolves the referenced job id into the int job_id field"
+trace $CLI bundle run foo

acceptance/bundle/run/job-ref-job-id/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = '/runs/\d+'
+New = '/runs/[RUN_ID]'

acceptance/invariant_test.go

@@ -8,10 +8,9 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/cli/libs/dyn/yamlloader"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.yaml.in/yaml/v3"
 )
 
 const invariantConfigsDir = "bundle/invariant/configs"
@@ -88,31 +87,22 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma
 		contents, err := os.ReadFile(path)
 		require.NoError(t, err)
 
-		v, err := yamlloader.LoadYAML(path, strings.NewReader(string(contents)))
-		require.NoError(t, err, "failed to parse %s", path)
-
-		resources := v.Get("resources")
-		if resources.Kind() != dyn.KindMap {
-			// Some configs (e.g. PyDABs) declare resources outside of YAML.
-			continue
+		var doc struct {
+			Resources map[string]any `yaml:"resources"`
 		}
+		require.NoError(t, yaml.Unmarshal(contents, &doc), "failed to parse %s", path)
 
-		for _, group := range resources.MustMap().Pairs() {
-			groupName := group.Key.MustString()
+		// Some configs (e.g. PyDABs) declare resources outside of YAML.
+		for groupName, group := range doc.Resources {
 			present[groupName] = true
 
-			if group.Value.Kind() != dyn.KindMap {
-				continue
-			}
-			for _, resource := range group.Value.MustMap().Pairs() {
-				cfg := resource.Value
-				if cfg.Kind() != dyn.KindMap {
-					continue
-				}
-				if cfg.Get("permissions").Kind() != dyn.KindInvalid {
+			resources, _ := group.(map[string]any)
+			for _, resource := range resources {
+				cfg, _ := resource.(map[string]any)
+				if _, ok := cfg["permissions"]; ok {
 					withPermissions[groupName] = true
 				}
-				if cfg.Get("grants").Kind() != dyn.KindInvalid {
+				if _, ok := cfg["grants"]; ok {
 					withGrants[groupName] = true
 				}
 			}

bundle/apps/validate.go

@@ -8,8 +8,8 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/structs/structpath"
+	"github.com/databricks/cli/libs/structs/structvar"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 )
 
@@ -28,7 +28,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Missing app source code path or git source",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field", key),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
+				Locations: b.Config.GetLocations("resources.apps." + key),
 			})
 			continue
 		}
@@ -38,7 +38,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Both source_code_path and git_source fields are set",
 				Detail:    fmt.Sprintf("app resource '%s' should have either source_code_path or git_source field, not both", key),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key)),
+				Locations: b.Config.GetLocations("resources.apps." + key),
 			})
 			continue
 		}
@@ -48,7 +48,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 				Severity:  diag.Error,
 				Summary:   "Duplicate app source code path",
 				Detail:    fmt.Sprintf("app resource '%s' has the same source code path as app resource '%s', this will lead to the app configuration being overridden by each other", key, usedSourceCodePaths[app.SourceCodePath]),
-				Locations: b.Config.GetLocationsOf(structpath.NewPath(nil, "resources", "apps", key, "source_code_path")),
+				Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
 			})
 		}
 		usedSourceCodePaths[app.SourceCodePath] = key
@@ -94,32 +94,26 @@ type appResourceReference struct {
 
 // hasPermissions checks if a bundle resource at the given dyn path has a non-empty permissions list.
 func hasPermissions(b *bundle.Bundle, resourcePath string) bool {
-	pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions")
+	pv, err := valueAt(b, resourcePath+".permissions")
 	if err != nil {
 		return false
 	}
-	s, ok := pv.AsSequence()
-	return ok && len(s) > 0
+	for range pv.Sequence() {
+		return true
+	}
+	return false
 }
 
 // hasAppSPInPermissions checks if any permission entry for the given resource
 // references the app's service principal via variable interpolation.
 func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool {
 	appSPRef := fmt.Sprintf("${resources.apps.%s.service_principal_client_id}", appKey)
-	pv, err := dyn.Get(b.Config.Value(), resourcePath+".permissions")
+	pv, err := valueAt(b, resourcePath+".permissions")
 	if err != nil {
 		return false
 	}
-	s, ok := pv.AsSequence()
-	if !ok {
-		return false
-	}
-	for _, entry := range s {
-		spn, err := dyn.Get(entry, "service_principal_name")
-		if err != nil {
-			continue
-		}
-		if str, ok := spn.AsString(); ok && str == appSPRef {
+	for _, entry := range pv.Sequence() {
+		if str, ok := entry.Get("service_principal_name").AsString(); ok && str == appSPRef {
 			return true
 		}
 	}
@@ -155,7 +149,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 			continue
 		}
 
-		appPath := structpath.NewPath(nil, "resources", "apps", appKey)
+		appPath := "resources.apps." + appKey
 		diags = append(diags, diag.Diagnostic{
 			Severity: diag.Warning,
 			Summary:  fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType),
@@ -173,8 +167,8 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc
 				ref.permission,
 				appKey,
 			),
-			Paths:     []*structpath.PathNode{appPath},
-			Locations: b.Config.GetLocationsOf(appPath),
+			Paths:     structpath.NewPathSlice("resources", "apps", appKey),
+			Locations: b.Config.GetLocations(appPath),
 		})
 	}
 
@@ -188,3 +182,12 @@ func (v *validate) Name() string {
 func Validate() bundle.Mutator {
 	return &validate{}
 }
+
+// valueAt returns the view of the configuration value at path.
+func valueAt(b *bundle.Bundle, path string) (structvar.View, error) {
+	p, err := structpath.ParsePath(path)
+	if err != nil {
+		return structvar.View{}, err
+	}
+	return b.Config.View().Lookup(p), nil
+}