Simulate app service principal and job permissions in test server (#4644)

## Summary

- Make the test server assign a service principal to apps and grant
permissions on referenced resources, mimicking real platform behavior
- Add an acceptance test that exercises the permission lifecycle across
multiple deploys
- Refactor `SetPermissions` dedup logic into a shared `upsertACL` helper

The acceptance test is restricted to the terraform engine. The direct
engine fails during planning:

```
cannot set (*dresources.PermissionsState).[0].service_principal_name
to string: failed to navigate to parent [0]: cannot index struct
```

## Test plan

- [x] All acceptance tests pass
- [x] Verified direct engine failure mode separately

Relates to #4309

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: pietern

acceptance/bundle/apps/job_permissions/app/app.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/apps/job_permissions/databricks.yml

@@ -0,0 +1,30 @@
+bundle:
+  name: test-bundle
+
+permissions:
+  - level: CAN_MANAGE
+    user_name: ${workspace.current_user.userName}
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: hello
+          environment_key: default
+          spark_python_task:
+            python_file: ./hello.py
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+
+  apps:
+    my_app:
+      name: my-app
+      source_code_path: ./app
+      resources:
+        - name: my-job
+          job:
+            id: ${resources.jobs.my_job.id}
+            permission: CAN_MANAGE_RUN

acceptance/bundle/apps/job_permissions/fix.patch

@@ -0,0 +1,11 @@
+--- a/databricks.yml
++++ b/databricks.yml
+@@ -10,6 +10,9 @@
+     my_job:
+       name: my-job
++      permissions:
++        - level: CAN_MANAGE_RUN
++          service_principal_name: ${resources.apps.my_app.service_principal_client_id}
+       tasks:
+         - task_key: hello
+           environment_key: default

acceptance/bundle/apps/job_permissions/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/apps/job_permissions/out.test.toml

@@ -0,0 +1,40 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== After first deploy
+>>> has_manage_run
+true
+
+=== After second deploy
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> has_manage_run
+false
+
+=== Apply fix and redeploy
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> has_manage_run
+true
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.apps.my_app
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/apps/job_permissions/output.txt

@@ -0,0 +1,26 @@
+
+# Test that app-granted permissions on a job survive a second deploy.
+# Issue: https://github.com/databricks/cli/issues/4309
+
+trace $CLI bundle deploy
+
+job_id=$(read_id.py my_job)
+
+has_manage_run() {
+  MSYS_NO_PATHCONV=1 $CLI api get /api/2.0/permissions/jobs/$job_id \
+    | jq 'any(.access_control_list[].all_permissions[]; .permission_level == "CAN_MANAGE_RUN")'
+}
+
+title "After first deploy"
+trace has_manage_run
+
+title "After second deploy"
+trace $CLI bundle deploy
+trace has_manage_run
+
+title "Apply fix and redeploy"
+patch -s --no-backup-if-mismatch databricks.yml < fix.patch
+trace $CLI bundle deploy
+trace has_manage_run
+
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/apps/job_permissions/script

@@ -0,0 +1,11 @@
+# Direct engine error: cannot plan resources.jobs.my_job.permissions: cannot update
+# [0].service_principal_name: failed to navigate to parent [0]: [0]: cannot index struct.
+# This is a bug in structaccess.Set() where it fails to index into a struct when
+# setting permissions with service_principal_name.
+# See https://github.com/databricks/cli/pull/4644
+Badness = "Direct engine fails to plan permissions with service_principal_name on jobs"
+Cloud = true
+RecordRequests = false
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/apps/job_permissions/test.toml

@@ -11,6 +11,9 @@
   },
   "id":"1000",
   "name":"my-app",
+  "service_principal_client_id":"[UUID]",
+  "service_principal_id":[NUMID],
+  "service_principal_name":"app-my-app",
   "url":"my-app-123.cloud.databricksapps.com"
 }
 

acceptance/bundle/generate/app_not_yet_deployed/output.txt

@@ -11,6 +11,9 @@
   },
   "id":"1000",
   "name":"test-app-already-exists",
+  "service_principal_client_id":"[UUID]",
+  "service_principal_id":[NUMID],
+  "service_principal_name":"app-test-app-already-exists",
   "url":"test-app-already-exists-123.cloud.databricksapps.com"
 }