From cd158f0e2170f9daef205cf5c2efbcda8c6d294a Mon Sep 17 00:00:00 2001 From: SpectralDragon Date: Sat, 7 Nov 2026 22:17:28 +0300 Subject: [PATCH] Add YAML interaction flows for Ada Studio --- .../Designs/studio-interaction-flows.md | 70 +++++ .../Plans/studio-interaction-flows.md | 10 + Editor/Documentation/AgentCatalog.md | 4 + .../Documentation/StudioInteractionFlows.md | 37 +++ Editor/Package.swift | 31 +- Editor/README.md | 4 + Editor/Sources/AdaEditor/AdaEditorApp.swift | 2 +- .../Assets/Documentation/catalog.json | 92 ++++++ .../Tooling/EditorXCTestReport.swift | 4 +- .../UI/Editor/EditorSceneViewportView.swift | 4 + .../UI/Editor/EditorWorkbenchInspection.swift | 81 +++++ .../StudioInteractionHost/Fixtures/Main.ascn | 24 ++ .../Fixtures/workbench.yaml | 30 ++ .../StudioInteractionFlow.swift | 135 ++++++++ .../StudioInteractionHost.swift | 151 +++++++++ .../StudioInteractionRunner.swift | 292 ++++++++++++++++++ .../Tests/StudioInteractionHost/check_flow.py | 104 +++++++ .../scripts/update-offline-documentation.py | 2 +- README.md | 2 + .../Skeletal/SkeletalAnimationPlayer.swift | 1 + .../Assets/Shaders/flat3d.glsl | 2 +- .../Shaders/screen_space_reflection.glsl | 2 +- .../RenderNodes3D/Main3DRenderNode.swift | 4 +- .../ScreenSpaceReflectionRenderNode.swift | 2 +- .../Shaders/Public/DistanceField3D.glsl | 2 +- Sources/AdaUI/CoreUI/UIInspection.swift | 14 + .../Render3DDistanceFieldTests.swift | 4 +- 27 files changed, 1097 insertions(+), 13 deletions(-) create mode 100644 Documentation/Designs/studio-interaction-flows.md create mode 100644 Documentation/Plans/studio-interaction-flows.md create mode 100644 Editor/Documentation/StudioInteractionFlows.md create mode 100644 Editor/Sources/AdaEditor/UI/Editor/EditorWorkbenchInspection.swift create mode 100644 Editor/Tests/StudioInteractionHost/Fixtures/Main.ascn create mode 100644 Editor/Tests/StudioInteractionHost/Fixtures/workbench.yaml create mode 100644 Editor/Tests/StudioInteractionHost/StudioInteractionFlow.swift create mode 100644 Editor/Tests/StudioInteractionHost/StudioInteractionHost.swift create mode 100644 Editor/Tests/StudioInteractionHost/StudioInteractionRunner.swift create mode 100644 Editor/Tests/StudioInteractionHost/check_flow.py diff --git a/Documentation/Designs/studio-interaction-flows.md b/Documentation/Designs/studio-interaction-flows.md new file mode 100644 index 000000000..bcb6d5bf1 --- /dev/null +++ b/Documentation/Designs/studio-interaction-flows.md @@ -0,0 +1,70 @@ +# Ada Studio interaction flows + +Status **Proposed** · 2026-11-07 + +An agent can inspect a running Studio workbench, write a short YAML flow, replay it without desktop control, and get a pass or a reproducible failure. The first flow verifies that a user can move a scene entity with a viewport gizmo and undo the resulting document edit. A second flow exercises hierarchy reparenting with the same drag command. + +## Existing building blocks + +- The adjacent AdaMCP checkout provides `ui.get_tree`, `ui.hit_test`, and YAML/JSON execution through `automation.run` and `adamcp.run_file`. Its low-level input commands queue events. A successful queue operation gives the caller no evidence that Studio processed the input or that a gesture succeeded. Its selector-based tap dispatches directly to the resolved AdaUI container. The exact capabilities of the staged desktop dependency need verification. +- Studio already exposes AdaUI identifiers for hierarchy rows and viewport panels. Gizmo handles are drawn and hit-tested using viewport geometry. They need an inspection adapter that can describe an actionable point within the current viewport. +- `editor.play.run_scenario` targets the game running in Play or an offscreen game runtime. Studio interaction flows target the editor window and assert against the editor document. Both can use the existing conventions for bounded execution, captures, and structured diagnostics. [Game scenario runner](../../Editor/Sources/AdaEditor/Agent/EditorPlayScenarioRunner.swift) +- UI Designer tests already drive press, move, release, and cancel through the AdaUI container and verify document changes and undo. They are useful focused coverage for the behavior beneath a full workbench flow. [UI Designer drag tests](../../Editor/Tests/AdaEditorTests/EditorUIDesignerDragTests.swift) + +## Execution model + +Use AdaMCP as the entry point. Extend its existing flow format with a versioned, allowlisted Studio test profile, and expose the same runner to a local CLI for CI. Keep existing `automation.run` arrays compatible. Advertise the test profile and its command schemas only when the connected host supports them. Agents inspect before writing a flow, run inline YAML while debugging, and replay the saved file through the same runner. + +Run flows in a separate Studio test executable composed from the real workbench, editor models, and rendering/input plugins. Give each run a temporary copy of a fixture project, isolated settings and caches, and a session-scoped local connection. Keep fixtures and harness lifecycle in the test executable. Start with a rendered macOS workbench on a host that can create a window and GPU surface. A developer can run it alongside their normal workspace. Windows is the next validation target. Linux, mobile, browser Studio, OS dialogs, and dragging files from Finder or Explorer require separate coverage. + +AdaMCP owns parsing, sequencing, and reporting. Studio supplies workbench target discovery, dispatch acknowledgement, and read-only document probes. AdaUI owns normal hit-testing, routing, focus, and gesture capture. The profile takes an explicit session and window binding before the first step. It validates the complete flow before acting, serializes runs per session, bounds time and output, and accepts a small set of input, assertion, and capture commands. Project setup happens at launch. Assertions cannot invoke scene edits, ECS mutations, arbitrary scripts, or shell commands. + +## Target resolution and input + +- Resolve UI targets using stable accessibility IDs scoped to the bound window and, when necessary, the workbench pane. A missing, ambiguous, clipped, or covered target produces a diagnostic. Resolve canvas points relative to a named surface. Use logical window coordinates internally, applying each container's transform and the display scale only at the appropriate boundary. +- Expose named anchors such as `translate.x` through a read-only viewport inspection adapter. Wait for the presented geometry matching the current selection, camera, tool, and viewport. Choose a point in the handle's active region, convert it from rendered geometry into window coordinates, and include the geometry generation in the result. Re-resolve if layout changes before the gesture starts. Validate that the point reaches the viewport and that the production picker identifies the intended handle. An obstructed or unavailable handle fails with the reason and geometry. The pointer-down still runs normal picking. [Presented gizmo geometry](../../Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportGizmos.swift) +- Submit synthetic events to the same window-scoped input queue the platform uses. This preserves UI interception, hit-testing, pointer capture, and forwarding into the hosted scene viewport. A drag is a single flow step that sends down, ordered moves with the pressed button and correct deltas, and up. Preserve the path through the gesture threshold and onto the destination. Bare hover events can terminate existing mouse capture. Use monotonic timing and release or cancel any held input on error, cancellation, or timeout. [Window input routing](../../Sources/AdaUI/CoreUI/UIWindow+InputEvents.swift) · [Viewport drag handling](../../Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportModel.swift) +- Advance after the target window has dispatched the step's input and the relevant workbench or hosted viewport update has observed it. Captures await a rendered frame newer than the triggering action. Use deadline-bound predicate waits for loading and assertions. Continue driving the real frame loop during the wait. Report timeouts with the last observed frame and state. + +## Example + +The names and fields below illustrate the proposed Studio profile. The fixture opens a scene with a cube at the origin, an unrotated parent, and a camera facing the scene. It has stable entity IDs. + +```yaml +version: 1 +profile: studio-ui +steps: + - tapOn: { id: AdaEditor.SceneHierarchy.Select.fixture-cube } + - tapOn: { id: AdaEditor.SceneViewport.Control.Move } + - drag: + from: + id: AdaEditor.SceneEditor.ViewportPanel + anchor: translate.x + by: { x: 100, y: 0 } + durationMs: 350 + - assertScene: + entity: fixture-cube + localPosition: + x: { greaterThan: 0.1 } + y: { near: 0, tolerance: 0.001 } + z: { near: 0, tolerance: 0.001 } + - pressKey: { key: z, modifiers: [main] } + - assertScene: + entity: fixture-cube + localPosition: { near: [0, 0, 0], tolerance: 0.001 } +``` + +The hierarchy flow uses the same `drag` step with `from.id` and `to.id` set to two row identifiers. It checks the resulting parent ID and undo. The source and destination describe input geometry. The existing hierarchy gesture determines whether the reparent is valid. + +## Results and acceptance + +- Return a structured outcome and a CLI exit status. On failure, include the step index, resolved targets, coordinates, last observed frame, expected and actual state, and a bounded event trace. Attach the test window's capture and a relevant UI subtree when available. Identify editor-window captures explicitly because the generic capture tool can prefer the running game's camera. +- Prove that the same saved flow passes through the MCP and CLI entry points on the rendered workbench. Verify that an intermediate move updates the preview while the document revision stays stable, release moves the cube only along X in the document, and one undo restores it. Use a fixture with prior history to distinguish one undo entry from several. Repeat at two window sizes and across English and Russian to exercise layout and selector stability. +- Test cancellation after an intermediate move, a covered start point, and a missing or ambiguous selector. Cancellation restores the initial document and leaves the next flow able to click and drag. Obstruction and bad selectors fail with distinct results. A deliberate wrong transform expectation produces a failing exit status and actionable evidence. +- Drag one hierarchy row onto another and verify its document parent and undo. Verify an invalid drop leaves the document intact. Use real input, a real fixture, and a real workbench for these checks. + +## Alternatives and delivery + +Direct authoring commands remain useful for arranging fixtures. They cannot validate hit-testing or gesture ownership. Raw screen coordinates work for fixed diagnostic replays and make durable tests dependent on window layout and scaling. An external Maestro driver would also need access to custom-rendered gizmo geometry. Reuse Maestro's inspect, replay, and assert workflow through Studio's own surfaces. + +Ship a user guide in `Editor/Documentation/StudioInteractionFlows.md` and link it from the Studio README and agent catalog. Include a verified fixture flow, instructions for MCP and CLI, selector discovery, supported targets, and sample failure output. Add the guide to the Studio offline documentation and verify its navigation, the offline snapshot, and replay of the published example as part of implementation. diff --git a/Documentation/Plans/studio-interaction-flows.md b/Documentation/Plans/studio-interaction-flows.md new file mode 100644 index 000000000..ad35bb073 --- /dev/null +++ b/Documentation/Plans/studio-interaction-flows.md @@ -0,0 +1,10 @@ +# Studio interaction flow MVP + +Approved design [Studio interaction flows](../Designs/studio-interaction-flows.md). The contributor approved the design on 2026-11-07 and prioritized the X gizmo, undo, cancellation, then hierarchy drag. + +1. Create a separate macOS test executable using the production workbench and the existing AdaMCP plugin. Expose the versioned YAML runner through an additional MCP tool and a CLI entry to the same runner. Validate the entire flow before injecting input and keep the legacy AdaMCP automation schema compatible. +2. Exercise a 2D X-axis move against a disposable scene. Resolve the handle against presented geometry, drive the window input path and acknowledge viewport processing. Verify preview versus committed state, one undo and mid-gesture cancellation. +3. Use the same drag command with row IDs and verify hierarchy reparent, undo and invalid drop through the real document. +4. Publish the fixture and user guide, update navigation and offline documentation, and exercise the same saved fixture through both entry points. Record negative assertion, missing target and teardown evidence. Use the shared validation runner and independent review. + +The harness is macOS-first. Windows and other platforms require their own runtime verification. Report the exact build, renderer, window sizes, languages and entry points exercised. diff --git a/Editor/Documentation/AgentCatalog.md b/Editor/Documentation/AgentCatalog.md index a83113eb9..f9d3012f9 100644 --- a/Editor/Documentation/AgentCatalog.md +++ b/Editor/Documentation/AgentCatalog.md @@ -88,3 +88,7 @@ remaining checks. It does not require a new planning ceremony for each feature. The public [make-game workflow](https://github.com/PlayableIntelligence/game-creator/tree/main/skills/make-game) was reviewed as a reference for game-development phases and iteration; the bundled instructions are written for AdaEngine's own authoring and runtime paths. + +## Studio interaction flows + +Use a separate macOS workbench session to replay [Studio interaction flows](StudioInteractionFlows.md) from YAML. The same file can run through the CLI or the host's AdaMCP tool. diff --git a/Editor/Documentation/StudioInteractionFlows.md b/Editor/Documentation/StudioInteractionFlows.md new file mode 100644 index 000000000..6cefb015d --- /dev/null +++ b/Editor/Documentation/StudioInteractionFlows.md @@ -0,0 +1,37 @@ +# Studio interaction flows + +Run repeatable workbench gestures from YAML on macOS with a graphical session and a supported GPU. The separate interaction host opens Studio's workbench on a disposable scene project. It uses the window input queue, including the same hit testing and pointer capture as a desktop gesture. + +## Run the example + +From the repository's `Editor` directory, build and run the interaction host: + +```sh +ADAEDITOR_INTERACTION_HOST=1 swift build --product adastudio-flow +ADAEDITOR_INTERACTION_HOST=1 swift run adastudio-flow Tests/StudioInteractionHost/Fixtures/workbench.yaml +``` + +The example selects a cube, moves it along the X gizmo, checks the committed position and undo depth, then undoes it. It cancels another move after sending intermediate motion and verifies that the preview and document are restored. It then drags the cube onto a group in Hierarchy and verifies reparenting, rejects a cycle and undoes the valid drop. + +## Run through AdaMCP + +Start a fresh `adastudio-flow --stdio` process with an AdaMCP-compatible Content-Length framed client. For example, configure the AdaMCP router with `--stdio-command "/absolute/path/to/adastudio-flow --stdio"`. Call `editor.ui.run_flow` with `source` set to the contents of the same YAML file. Use `ui.find_nodes` to look up identifiers or `ui.get_tree` to inspect the full AdaUI hierarchy. The tool and CLI execute the same runner. Use a fresh host for a repeatable initial scene. + +The repository contains a check that runs the example through both interfaces: + +```sh +python3 Tests/StudioInteractionHost/check_flow.py .build/debug/adastudio-flow +``` + +## Flow format + +Set `version: 1` and `profile: studio-ui`. A flow contains up to 64 steps and 64 KiB of YAML. Each step has one command. + +- `tapOn` takes an accessibility `id` in the workbench window. +- `drag` takes `from`, either `to` or `by`, and `durationMs`. Targets use an `id`. For the X gizmo use the viewport surface ID with `anchor: translate.x` on `from`. Offsets use logical points. Add `end: cancel` to cancel after intermediate motion. +- `pressKey` supports `key: z` with `modifiers: [main]` for undo or `[main, shift]` for redo. +- `assertScene` takes an entity ID and checks its local `position`, `parent` or `undoCount`. Position uses an optional absolute `tolerance`. + +A flow returns a JSON report. A failure includes the step number, command, diagnostic, recent input trace and, when available, the workbench screenshot path. The CLI returns a failing exit status. For an unavailable gizmo, check that the fixture scene is active, the cube is selected and the Move tool has drawn its handles. + +The initial profile covers the macOS workbench, one fixture project and the X translation handle. Play game scenarios, OS dialogs, and file drags use separate verification paths. diff --git a/Editor/Package.swift b/Editor/Package.swift index bd418027c..16e0ad5bf 100644 --- a/Editor/Package.swift +++ b/Editor/Package.swift @@ -9,6 +9,11 @@ let adaMCPURL = ProcessInfo.processInfo.environment["ADA_MCP_LOCAL_PATH"].map { URL(fileURLWithPath: $0, relativeTo: editorPackageURL).standardizedFileURL } ?? (FileManager.default.fileExists(atPath: adjacentAdaMCP.appendingPathComponent("Package.swift").path) ? adjacentAdaMCP : nil) let isAndroidStudio = ProcessInfo.processInfo.environment["ADAEDITOR_ANDROID"] == "1" +#if os(macOS) +let isStudioInteractionHost = !isAndroidStudio && ProcessInfo.processInfo.environment["ADAEDITOR_INTERACTION_HOST"] == "1" +#else +let isStudioInteractionHost = false +#endif #if (os(macOS) && arch(arm64)) || (os(Windows) && arch(x86_64)) || (os(Linux) && (arch(x86_64) || arch(arm64))) #if os(Windows) || os(Linux) // Prepared by the host's ensure_*_lore.py --package helper. The shared native @@ -198,6 +203,27 @@ let package = Package( ] ) +#if os(macOS) +if isStudioInteractionHost { + package.products.removeAll { $0.name == "AdaEditor" } + package.products.append(.executable(name: "adastudio-flow", targets: ["StudioInteractionHost"])) + package.targets.append(.executableTarget( + name: "StudioInteractionHost", + dependencies: [ + "AdaEditor", + "Yams", + .product(name: "AdaEngine", package: "AdaEngine"), + .product(name: "AdaMCPPlugin", package: "AdaMCP"), + .product(name: "AdaMCPCore", package: "AdaMCP"), + .product(name: "MCP", package: "AdaMCP"), + ], + path: "Tests/StudioInteractionHost", + exclude: ["check_flow.py"], + resources: [.copy("Fixtures")] + )) +} +#endif + let editorSwiftSettings: [SwiftSetting] = [ .define("MACOS", .when(platforms: [.macOS])), .define("WINDOWS", .when(platforms: [.windows])), @@ -229,8 +255,9 @@ private func makeEditorTarget( swiftSettings: [SwiftSetting], linkerSettings: [LinkerSetting] ) -> Target { - if isAndroidStudio { - return .target(name: name, dependencies: dependencies, exclude: exclude, resources: resources, swiftSettings: swiftSettings, linkerSettings: linkerSettings) + if isAndroidStudio || isStudioInteractionHost { + let settings: [SwiftSetting] = isStudioInteractionHost ? [.define("EDITOR_INTERACTION_HOST")] : swiftSettings + return .target(name: name, dependencies: dependencies, exclude: exclude, resources: resources, swiftSettings: settings, linkerSettings: linkerSettings) } return .executableTarget(name: name, dependencies: dependencies, exclude: exclude, resources: resources, swiftSettings: swiftSettings, linkerSettings: linkerSettings) } diff --git a/Editor/README.md b/Editor/README.md index 8e0fb1111..6b122e2f5 100644 --- a/Editor/README.md +++ b/Editor/README.md @@ -204,3 +204,7 @@ See [Examples.md](Documentation/Examples.md) for downloading editable projects a Desktop projects can use **Git** or **Lore**, selected during project creation and in the Source Control panel. See [Source control](Documentation/SourceControl.md) for supported platforms, repository behavior and Windows build preparation. + +## Studio interaction flows + +Run workbench gestures and assertions from one YAML file through [the Studio flow CLI or AdaMCP](Documentation/StudioInteractionFlows.md). diff --git a/Editor/Sources/AdaEditor/AdaEditorApp.swift b/Editor/Sources/AdaEditor/AdaEditorApp.swift index 2a7a33b21..9f7853127 100644 --- a/Editor/Sources/AdaEditor/AdaEditorApp.swift +++ b/Editor/Sources/AdaEditor/AdaEditorApp.swift @@ -17,7 +17,7 @@ import Logging import AdaMCPPlugin #endif -#if !os(visionOS) && !os(Android) +#if !os(visionOS) && !os(Android) && !EDITOR_INTERACTION_HOST @main #endif enum AdaApplicationEntry { diff --git a/Editor/Sources/AdaEditor/Assets/Documentation/catalog.json b/Editor/Sources/AdaEditor/Assets/Documentation/catalog.json index b0fef0dab..635aeaa6a 100644 --- a/Editor/Sources/AdaEditor/Assets/Documentation/catalog.json +++ b/Editor/Sources/AdaEditor/Assets/Documentation/catalog.json @@ -2934,6 +2934,98 @@ } ] }, + { + "id": "StudioInteractionFlows", + "title": "Studio interaction flows", + "section": "Editor", + "source": "Editor/Documentation/StudioInteractionFlows.md", + "blocks": [ + { + "kind": "text", + "text": "Run repeatable workbench gestures from YAML on macOS with a graphical session and a supported GPU. The separate interaction host opens Studio's workbench on a disposable scene project. It uses the window input queue, including the same hit testing and pointer capture as a desktop gesture.", + "level": 0, + "links": [] + }, + { + "kind": "heading", + "text": "Run the example", + "level": 2, + "links": [] + }, + { + "kind": "text", + "text": "From the repository's `Editor` directory, build and run the interaction host:", + "level": 0, + "links": [] + }, + { + "kind": "code", + "text": "ADAEDITOR_INTERACTION_HOST=1 swift build --product adastudio-flow\nADAEDITOR_INTERACTION_HOST=1 swift run adastudio-flow Tests/StudioInteractionHost/Fixtures/workbench.yaml", + "language": "sh", + "links": [] + }, + { + "kind": "text", + "text": "The example selects a cube, moves it along the X gizmo, checks the committed position and undo depth, then undoes it. It cancels another move after sending intermediate motion and verifies that the preview and document are restored. It then drags the cube onto a group in Hierarchy and verifies reparenting, rejects a cycle and undoes the valid drop.", + "level": 0, + "links": [] + }, + { + "kind": "heading", + "text": "Run through AdaMCP", + "level": 2, + "links": [] + }, + { + "kind": "text", + "text": "Start a fresh `adastudio-flow --stdio` process with an AdaMCP-compatible Content-Length framed client. For example, configure the AdaMCP router with `--stdio-command \"/absolute/path/to/adastudio-flow --stdio\"`. Call `editor.ui.run_flow` with `source` set to the contents of the same YAML file. Use `ui.find_nodes` to look up identifiers or `ui.get_tree` to inspect the full AdaUI hierarchy. The tool and CLI execute the same runner. Use a fresh host for a repeatable initial scene.", + "level": 0, + "links": [] + }, + { + "kind": "text", + "text": "The repository contains a check that runs the example through both interfaces:", + "level": 0, + "links": [] + }, + { + "kind": "code", + "text": "python3 Tests/StudioInteractionHost/check_flow.py .build/debug/adastudio-flow", + "language": "sh", + "links": [] + }, + { + "kind": "heading", + "text": "Flow format", + "level": 2, + "links": [] + }, + { + "kind": "text", + "text": "Set `version: 1` and `profile: studio-ui`. A flow contains up to 64 steps and 64 KiB of YAML. Each step has one command.", + "level": 0, + "links": [] + }, + { + "kind": "text", + "text": "• `tapOn` takes an accessibility `id` in the workbench window.\n• `drag` takes `from`, either `to` or `by`, and `durationMs`. Targets use an `id`. For the X gizmo use the viewport surface ID with `anchor: translate.x` on `from`. Offsets use logical points. Add `end: cancel` to cancel after intermediate motion.\n• `pressKey` supports `key: z` with `modifiers: [main]` for undo or `[main, shift]` for redo.\n• `assertScene` takes an entity ID and checks its local `position`, `parent` or `undoCount`. Position uses an optional absolute `tolerance`.", + "level": 0, + "links": [] + }, + { + "kind": "text", + "text": "A flow returns a JSON report. A failure includes the step number, command, diagnostic, recent input trace and, when available, the workbench screenshot path. The CLI returns a failing exit status. For an unavailable gizmo, check that the fixture scene is active, the cube is selected and the Move tool has drawn its handles.", + "level": 0, + "links": [] + }, + { + "kind": "text", + "text": "The initial profile covers the macOS workbench, one fixture project and the X translation handle. Play game scenarios, OS dialogs, and file drags use separate verification paths.", + "level": 0, + "links": [] + } + ] + }, { "id": "AnimationAuthoring", "title": "Authoring skeletal animation", diff --git a/Editor/Sources/AdaEditor/Tooling/EditorXCTestReport.swift b/Editor/Sources/AdaEditor/Tooling/EditorXCTestReport.swift index 766a8a378..0607340c7 100644 --- a/Editor/Sources/AdaEditor/Tooling/EditorXCTestReport.swift +++ b/Editor/Sources/AdaEditor/Tooling/EditorXCTestReport.swift @@ -1,5 +1,7 @@ import Foundation -import FoundationXML +#if canImport(FoundationXML) + import FoundationXML +#endif /// SwiftPM's parallel XCTest runner records per-test results in its xUnit report. final class EditorXCTestReport: NSObject, XMLParserDelegate { diff --git a/Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportView.swift b/Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportView.swift index 9a3795da5..5406abf84 100644 --- a/Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportView.swift +++ b/Editor/Sources/AdaEditor/UI/Editor/EditorSceneViewportView.swift @@ -33,6 +33,7 @@ struct EditorSceneViewportView: View { @State var viewportModel = EditorSceneViewportModel() @Environment(\.theme) var theme @Environment(\.viewProxy) var viewProxy + @Environment(\.workbenchInspection) private var workbenchInspection var body: some View { // A declaration is required inside AdaUI's result builder. @@ -79,6 +80,7 @@ struct EditorSceneViewportView: View { viewportModel.setPresentationActive(isDocumentActive) guard isDocumentActive else { return } guard !document.isLoading, document.errorMessage == nil else { return } + workbenchInspection?.viewport = viewportModel if isPlayingThisDocument { preparePlayModeViewport() } else { configureViewportModel() } } @@ -118,6 +120,7 @@ struct EditorSceneViewportView: View { if let input = world.getResource(Input.self) { for event in input.getInputEvents() { let handled = viewportModel.handleInput(event) + if let mouse = event as? MouseEvent { workbenchInspection?.observedInput(mouse) } didChangeViewport = didChangeViewport || handled } } @@ -133,6 +136,7 @@ struct EditorSceneViewportView: View { viewportPresentation.prepareFrame(viewportModel: viewportModel, theme: viewportPresentation.annotationTheme) } .frame(width: geometry.size.width, height: geometry.size.height) + .accessibilityIdentifier("AdaEditor.SceneViewport.Surface") // Retained scenes stay prepared behind asset tabs. Their custom drawing // layers must be removed while inactive; inherited opacity does not hide them. diff --git a/Editor/Sources/AdaEditor/UI/Editor/EditorWorkbenchInspection.swift b/Editor/Sources/AdaEditor/UI/Editor/EditorWorkbenchInspection.swift new file mode 100644 index 000000000..9b2834a5c --- /dev/null +++ b/Editor/Sources/AdaEditor/UI/Editor/EditorWorkbenchInspection.swift @@ -0,0 +1,81 @@ +@_spi(AdaEngine) import AdaEngine +import AdaInput +import Foundation +import Math + +/// Read-only access to the rendered scene surface and open document of a workbench. +@MainActor +package final class EditorWorkbenchInspection { + package struct SceneState: Codable, Equatable { + package let position: [Float]? + package let parent: String? + package let undoCount: Int + } + + private let project: EditorProjectReference + private let model: EditorViewModel + weak var viewport: EditorSceneViewportModel? + package private(set) var viewportInputCount = 0 + package private(set) var lastViewportInput: MouseEvent? + + package init(project: EditorProjectReference) { + EditorComponentRegistry.registerBuiltIns() + self.project = project + model = EditorViewModel(project: project) + } + + package var content: some View { + EditorView(project: project, viewModel: model) + .environment(\.workbenchInspection, self) + } + + package func sceneState(entity id: String, preview: Bool = false) -> SceneState? { + guard let document = model.workbench.activeSceneDocument, + let scene = preview ? viewport?.sceneModel : document.sceneModel, + let entity = scene.entities.first(where: { $0.id == id }) else { return nil } + let transform = entity.components[EditorBuiltInComponentType.transform].flatMap { + try? EditorComponentPayloadDecoder.decode(Transform.self, payload: $0) as? Transform + } + return SceneState( + position: transform.map { [$0.position.x, $0.position.y, $0.position.z] }, + parent: entity.parent, + undoCount: model.workbench.sceneUndoHistory[document.id]?.count ?? 0 + ) + } + + package var hasLoadedScene: Bool { + guard let scene = model.workbench.activeSceneDocument else { + return false + } + return !scene.isLoading && scene.sceneModel != nil + } + + package var selectedEntity: String? { viewport?.selectedEditorID } + + package func translateXAnchor() -> Point? { + guard let viewport, let id = viewport.selectedEditorID, + let committed = sceneState(entity: id)?.position, + sceneState(entity: id, preview: true)?.position == committed, + viewport.presentedGizmo != nil, let gizmo = viewport.interactionGizmo(), + let current = viewport.transformGizmo(size: gizmo.size), current.viewProjection == gizmo.viewProjection, + (current.origin - gizmo.origin).squaredLength < 0.000001, + gizmo.tool == .translate, let shape = gizmo.shapes.first(where: { $0.handle == .x }), + let start = shape.points.first, let end = shape.points.last else { return nil } + for fraction: Float in [0.5, 0.75, 0.25, 1] { + let point = start + (end - start) * fraction + if gizmo.hitTest(point) == .x { + return Point(point.x / gizmo.size.width, point.y / gizmo.size.height) + } + } + return nil + } + + func observedInput(_ event: MouseEvent) { + lastViewportInput = event + viewportInputCount += 1 + } +} + +extension EnvironmentValues { + @Entry var workbenchInspection: EditorWorkbenchInspection? +} diff --git a/Editor/Tests/StudioInteractionHost/Fixtures/Main.ascn b/Editor/Tests/StudioInteractionHost/Fixtures/Main.ascn new file mode 100644 index 000000000..8de13a02d --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/Fixtures/Main.ascn @@ -0,0 +1,24 @@ +format: ada.scene +schemaVersion: 1 +scene: { id: fixture-scene, name: Flow Fixture } +entities: + - id: fixture-root + name: Root + enabled: true + components: {} + - id: fixture-cube + name: Cube + enabled: true + parent: fixture-root + components: + AdaTransform.Transform: + position: [0, 0, 0] + rotation: [0, 0, 0, 1] + scale: [1, 1, 1] + - id: fixture-group + name: Group + enabled: true + parent: fixture-root + components: {} +editor: + expandedEntities: [fixture-root] diff --git a/Editor/Tests/StudioInteractionHost/Fixtures/workbench.yaml b/Editor/Tests/StudioInteractionHost/Fixtures/workbench.yaml new file mode 100644 index 000000000..d4fd3275f --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/Fixtures/workbench.yaml @@ -0,0 +1,30 @@ +version: 1 +profile: studio-ui +steps: + - tapOn: { id: AdaEditor.SceneHierarchy.Select.fixture-cube } + - tapOn: { id: AdaEditor.SceneViewport.Control.Move } + - drag: + from: { id: AdaEditor.SceneViewport.Surface, anchor: translate.x } + by: { x: 60, y: 0 } + durationMs: 250 + - assertScene: { entity: fixture-cube, position: [0.4, 0, 0], tolerance: 0.02, undoCount: 1 } + - pressKey: { key: z, modifiers: [main] } + - assertScene: { entity: fixture-cube, position: [0, 0, 0], undoCount: 0 } + - drag: + from: { id: AdaEditor.SceneViewport.Surface, anchor: translate.x } + by: { x: 60, y: 0 } + durationMs: 250 + end: cancel + - assertScene: { entity: fixture-cube, position: [0, 0, 0], undoCount: 0 } + - drag: + from: { id: AdaEditor.SceneHierarchy.Row.fixture-cube } + to: { id: AdaEditor.SceneHierarchy.Row.fixture-group } + durationMs: 250 + - assertScene: { entity: fixture-cube, parent: fixture-group, undoCount: 1 } + - drag: + from: { id: AdaEditor.SceneHierarchy.Row.fixture-group } + to: { id: AdaEditor.SceneHierarchy.Row.fixture-cube } + durationMs: 250 + - assertScene: { entity: fixture-group, parent: fixture-root, undoCount: 1 } + - pressKey: { key: z, modifiers: [main] } + - assertScene: { entity: fixture-cube, parent: fixture-root, undoCount: 0 } diff --git a/Editor/Tests/StudioInteractionHost/StudioInteractionFlow.swift b/Editor/Tests/StudioInteractionHost/StudioInteractionFlow.swift new file mode 100644 index 000000000..544b3e04f --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/StudioInteractionFlow.swift @@ -0,0 +1,135 @@ +import Foundation +import MCP +import Yams + +struct FlowError: LocalizedError { + let errorDescription: String? + + init(_ message: String) { errorDescription = message } +} + +struct StudioInteractionFlow { + struct Target: Decodable { + let id: String + let anchor: String? + + var isValid: Bool { + !id.isEmpty && (anchor == nil || (anchor == "translate.x" && id == "AdaEditor.SceneViewport.Surface")) + } + } + + struct Offset: Decodable { + let x: Float + let y: Float + } + + struct Drag: Decodable { + enum End: String, Decodable { case release, cancel } + + let from: Target + let to: Target? + let by: Offset? + let durationMs: Int + let end: End? + } + + struct Key: Decodable { + let key: String + let modifiers: [String]? + } + + struct Assertion: Decodable { + let entity: String + let position: [Float]? + let tolerance: Float? + let parent: String? + let undoCount: Int? + } + + enum Step { + case tapOn(Target) + case drag(Drag) + case pressKey(Key) + case assertScene(Assertion) + + var command: String { + switch self { + case .tapOn: "tapOn" + case .drag: "drag" + case .pressKey: "pressKey" + case .assertScene: "assertScene" + } + } + } + + let steps: [Step] + + init(source: String) throws { + guard source.utf8.count <= 65_536 else { throw FlowError("Flow exceeds 64 KiB.") } + let envelope = try YAMLDecoder().decode(Value.self, from: source) + guard let root = envelope.objectValue, Set(root.keys) == ["version", "profile", "steps"], + root["version"]?.intValue == 1, root["profile"]?.stringValue == "studio-ui", + let steps = root["steps"]?.arrayValue, (1...64).contains(steps.count) else { + throw FlowError("Expected version 1, profile studio-ui and 1–64 steps.") + } + self.steps = try steps.enumerated().map { index, value in + do { + return try Self.parseStep(value) + } catch { + throw FlowError("Step \(index + 1): \(error.localizedDescription)") + } + } + } + + private static func parseStep(_ value: Value) throws -> Step { + guard let object = value.objectValue, object.count == 1, let (command, args) = object.first else { + throw FlowError("Expected one command per step.") + } + switch command { + case "tapOn": + let target = try decode(Target.self, args, keys: ["id"]) + guard target.isValid else { throw FlowError("tapOn needs a non-empty id.") } + return .tapOn(target) + case "drag": + let drag = try decode(Drag.self, args, keys: ["from", "to", "by", "durationMs", "end"]) + _ = try decode(Target.self, args.objectValue?["from"], keys: ["id", "anchor"]) + if let to = args.objectValue?["to"] { + _ = try decode(Target.self, to, keys: ["id"]) + } + if let by = args.objectValue?["by"] { + _ = try decode(Offset.self, by, keys: ["x", "y"]) + } + guard drag.from.isValid, drag.to?.isValid != false, (drag.to == nil) != (drag.by == nil), + (80...2000).contains(drag.durationMs), + [drag.by?.x, drag.by?.y].compactMap({ $0 }).allSatisfy({ $0.isFinite && abs($0) <= 4000 }) else { + throw FlowError("drag needs a valid from, one destination and durationMs from 80 to 2000.") + } + return .drag(drag) + case "pressKey": + let key = try decode(Key.self, args, keys: ["key", "modifiers"]) + guard key.key == "z", key.modifiers == ["main"] || key.modifiers == ["main", "shift"] else { + throw FlowError("This profile supports main+z and main+shift+z.") + } + return .pressKey(key) + case "assertScene": + let assertion = try decode(Assertion.self, args, keys: ["entity", "position", "tolerance", "parent", "undoCount"]) + guard !assertion.entity.isEmpty, + assertion.position != nil || assertion.parent != nil || assertion.undoCount != nil, + assertion.position.map({ $0.count == 3 && $0.allSatisfy(\.isFinite) }) != false, + assertion.undoCount.map({ $0 >= 0 }) != false, + assertion.tolerance.map({ $0.isFinite && $0 >= 0 }) != false else { + throw FlowError("assertScene needs an entity and a valid position, parent or undoCount expectation.") + } + return .assertScene(assertion) + default: + throw FlowError("Unknown command '\(command)'.") + } + } + + private static func decode(_ type: T.Type, _ value: Value?, keys: Set) throws -> T { + guard let value, let object = value.objectValue, Set(object.keys).isSubset(of: keys) else { + throw FlowError("Unexpected fields or missing command arguments.") + } + return try JSONDecoder().decode(type, from: JSONEncoder().encode(value)) + } +} diff --git a/Editor/Tests/StudioInteractionHost/StudioInteractionHost.swift b/Editor/Tests/StudioInteractionHost/StudioInteractionHost.swift new file mode 100644 index 000000000..292cb8c31 --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/StudioInteractionHost.swift @@ -0,0 +1,151 @@ +import AdaEditor +import AdaEngine +import AdaMCPCore +import AdaMCPPlugin +import Darwin +import Dispatch +import Foundation +import MCP + +@main +enum StudioInteractionHost { + @MainActor + static func main() async throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + if arguments == ["--help"] { + print("adastudio-flow | --stdio") + return + } + guard arguments.count == 1, let argument = arguments.first else { + throw FlowError("Usage: adastudio-flow | --stdio") + } + let source: String? + if argument == "--stdio" { + source = nil + } else { + let flowURL = URL(fileURLWithPath: argument) + guard let size = try flowURL.resourceValues(forKeys: [.fileSizeKey]).fileSize, size <= 65_536 else { + throw FlowError("Flow exceeds 64 KiB.") + } + source = try String(contentsOf: flowURL, encoding: .utf8) + } + + let temporary = FileManager.default.temporaryDirectory.appendingPathComponent("ada-studio-flow-\(UUID())", isDirectory: true) + defer { try? FileManager.default.removeItem(at: temporary) } + try FileManager.default.createDirectory(at: temporary, withIntermediateDirectories: true) + guard setenv("CFFIXED_USER_HOME", temporary.path, 1) == 0, + FileManager.default.homeDirectoryForCurrentUser.standardizedFileURL.path == temporary.standardizedFileURL.path else { + throw FlowError("Could not isolate the host's settings and caches.") + } + let shutdown = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) + signal(SIGTERM, SIG_IGN) + shutdown.setEventHandler { + try? FileManager.default.removeItem(at: temporary) + Foundation.exit(source == nil ? 0 : 130) + } + shutdown.resume() + defer { shutdown.cancel() } + let inputMonitor = source == nil ? Task { await Self.exitWhenMCPInputCloses(project: temporary) } : nil + defer { inputMonitor?.cancel() } + let projectDirectory = temporary.appendingPathComponent("Project", isDirectory: true) + let assets = projectDirectory.appendingPathComponent("Assets/Scenes", isDirectory: true) + try FileManager.default.createDirectory(at: assets, withIntermediateDirectories: true) + guard let fixture = Bundle.module.url(forResource: "Main", withExtension: "ascn", subdirectory: "Fixtures") else { + throw FlowError("Missing scene fixture.") + } + try FileManager.default.copyItem(at: fixture, to: assets.appendingPathComponent("Main.ascn")) + try ProjectSystem.createDefaultProject(at: projectDirectory, buildSystem: .adaScript) + let project = EditorProjectReference(name: "Studio Flow Fixture", path: projectDirectory.path) + let runner = StudioInteractionRunner(workbench: EditorWorkbenchInspection(project: project)) + try await AppRuntime.run(StudioInteractionApp(runner: runner, source: source, project: temporary)) + } + + private static func exitWhenMCPInputCloses(project: URL) async { + // Observe pipe closure without consuming the bytes owned by AdaMCP's transport. + var descriptor = pollfd(fd: STDIN_FILENO, events: Int16(POLLHUP), revents: 0) + let closed = Int16(POLLHUP | POLLERR | POLLNVAL) + while !Task.isCancelled { + if poll(&descriptor, 1, 0) > 0, descriptor.revents & closed != 0 { + try? FileManager.default.removeItem(at: project) + Foundation.exit(0) + } + do { + try await Task.sleep(for: .milliseconds(200)) + } catch { + return + } + } + } +} + +private struct StudioInteractionApp: App { + let runner: StudioInteractionRunner + let source: String? + let project: URL + + var body: some AppScene { + WindowGroup { + runner.workbench.content + .onAppear { runner.workbenchMounted = true } + .task { + guard let source else { + return + } + let report = await runner.run(source: source) + if let output = try? encode(report) { print(output) } + try? FileManager.default.removeItem(at: project) + Foundation.exit(report.status == "passed" ? 0 : 1) + } + } + .windowMode(.windowed) + .singleWindow(true) + .windowTitle("Ada Studio Interaction Host") + .minimumSize(width: 1160, height: 760) + .transformAppWorlds { runner.worlds = $0 } + .addPlugins(MCPPlugin(configuration: .init( + enableHTTP: false, + enableStdio: source == nil, + serverName: "Ada Studio Interaction Host", + instructions: "Run a version 1 studio-ui YAML flow with editor.ui.run_flow. This host uses a disposable fixture scene.", + additionalTools: { + [ + Tool( + name: "editor.ui.run_flow", + description: "Run version 1 studio-ui YAML using tapOn, drag, pressKey and assertScene.", + inputSchema: [ + "type": "object", + "required": ["source"], + "properties": ["source": ["type": "string", "description": "Contents of the same YAML accepted by adastudio-flow."]] + ] + ) + ] + }, + additionalToolHandler: { name, arguments in + guard name == "editor.ui.run_flow" else { + return nil + } + guard let source = arguments["source"]?.stringValue else { + return result("A source string is required.", failed: true) + } + let report = await runner.run(source: source) + let payload = try? encode(report) + return result(payload ?? "Could not encode the flow report.", failed: report.status != "passed" || payload == nil) + }, + traceRecorder: nil, + enableDiscovery: false + ))) + } +} + +private func encode(_ report: StudioInteractionRunner.Report) throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + guard let text = String(bytes: try encoder.encode(report), encoding: .utf8) else { + throw FlowError("Could not encode the flow report as UTF-8.") + } + return text +} + +private func result(_ text: String, failed: Bool) -> CallTool.Result { + CallTool.Result(content: [.text(text: text, annotations: nil, _meta: nil)], isError: failed) +} diff --git a/Editor/Tests/StudioInteractionHost/StudioInteractionRunner.swift b/Editor/Tests/StudioInteractionHost/StudioInteractionRunner.swift new file mode 100644 index 000000000..69205ff74 --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/StudioInteractionRunner.swift @@ -0,0 +1,292 @@ +import AdaEditor +@_spi(Internal) import AdaEngine +import AdaMCPCore +import Foundation + +@MainActor +final class StudioInteractionRunner { + struct Report: Encodable { + var status = "passed" + var completedSteps = 0 + var failedStep: Int? + var command: String? + var message: String? + var screenshot: String? + var trace: [String] = [] + } + + let workbench: EditorWorkbenchInspection + weak var worlds: AppWorlds? + var workbenchMounted = false + private var running = false + private var usedFixture = false + private var trace: [String] = [] + private let clock = ContinuousClock() + private var runDeadline: ContinuousClock.Instant? + + init(workbench: EditorWorkbenchInspection) { + self.workbench = workbench + } + + func run(source: String) async -> Report { + guard !running else { + return Report(status: "failed", message: "A flow is already running in this host.") + } + guard !usedFixture else { + return Report(status: "failed", message: "Start a fresh host to run a new flow against the original fixture.") + } + running = true + trace = [] + runDeadline = clock.now.advanced(by: .seconds(35)) + defer { + runDeadline = nil + running = false + } + var report = Report() + do { + let flow = try StudioInteractionFlow(source: source) + usedFixture = true + let window = try await wait("loaded Studio workbench", timeout: .seconds(22)) { + guard self.workbenchMounted, self.workbench.hasLoadedScene else { + return nil as UIWindow? + } + return self.worlds?.getResource(PrimaryWindow.self)?.window + } + for step in flow.steps { + report.command = step.command + try await perform(step, window: window) + report.completedSteps += 1 + } + report.command = nil + } catch { + report.status = "failed" + report.failedStep = report.command == nil ? nil : report.completedSteps + 1 + report.message = error.localizedDescription + if let worlds, report.command != nil, !Task.isCancelled { + let capture = try? await RenderCaptureService(appWorlds: worlds) + .capture(pauseBeforeCapture: false, refreshFrame: false) + report.screenshot = capture?.path + } + } + report.trace = trace + return report + } + + private func perform(_ step: StudioInteractionFlow.Step, window: UIWindow) async throws { + switch step { + case let .tapOn(target): + let point = try await resolve(target, window: window) + do { + try await mouse(.began, at: point, window: window) + try await mouse(.ended, at: point, window: window) + } catch { + await cancelMouse(at: point, window: window) + throw error + } + case let .drag(drag): + try await performDrag(drag, window: window) + case let .pressKey(key): + let modifiers: KeyModifier = key.modifiers?.contains("shift") == true ? [.main, .shift] : .main + try await pressUndoKey(modifiers: modifiers, window: window) + case let .assertScene(expectation): + try await assertScene(expectation) + } + } + + private func performDrag(_ drag: StudioInteractionFlow.Drag, window: UIWindow) async throws { + let viewport = drag.from.anchor != nil + let endTarget: Point? = if let destination = drag.to { + try await resolve(destination, window: window) + } else { + nil + } + let start = try await resolve(drag.from, window: window) + let end = endTarget ?? Point(start.x + (drag.by?.x ?? 0), start.y + (drag.by?.y ?? 0)) + guard window.bounds.contains(point: end) else { throw FlowError("Drag destination \(end) is outside the window.") } + let entity = viewport ? workbench.selectedEntity : nil + let original = entity.flatMap { workbench.sceneState(entity: $0) } + guard !viewport || original?.position != nil else { throw FlowError("Selected gizmo entity has no document transform.") } + var last = start + do { + try await mouse(.began, at: start, window: window, awaitViewport: viewport) + let samples = max(2, drag.durationMs / 40) + let moves = drag.end == .cancel ? max(1, samples / 2) : samples + let started = clock.now + for index in 1...moves { + try await clock.sleep(until: started.advanced(by: .milliseconds(drag.durationMs * index / samples))) + let point = start + (end - start) * (Float(index) / Float(samples)) + let delta = point - last + last = point + try await mouse(.changed, at: point, delta: delta, window: window, awaitViewport: viewport) + } + if let entity, let original { + guard let previewPosition = workbench.sceneState(entity: entity, preview: true)?.position, + workbench.sceneState(entity: entity) == original, previewPosition != original.position else { + throw FlowError("Gizmo drag must change preview position before changing the document.") + } + } + try await mouse(drag.end == .cancel ? .cancelled : .ended, at: last, window: window, awaitViewport: viewport) + if drag.end == .cancel, let entity, let original { + guard workbench.sceneState(entity: entity) == original, + workbench.sceneState(entity: entity, preview: true)?.position == original.position else { + throw FlowError("Cancelled gizmo drag did not restore the preview and document.") + } + } + } catch { + await cancelMouse(at: last, window: window) + throw error + } + } + + private func resolve(_ target: StudioInteractionFlow.Target, window: UIWindow) async throws -> Point { + var observation = "not mounted" + do { + return try await wait("target \(target.id) \(target.anchor ?? "")") { + let matches = window.uiInspectableContainers().flatMap { container in + container.uiFindNodes(matching: .accessibilityIdentifier(target.id)).map { (container, $0) } + } + guard matches.count < 2 else { throw FlowError("Ambiguous target '\(target.id)' (\(matches.count) matches).") } + guard let (container, node) = matches.first, let view = container as? UIView else { + observation = "not mounted" + return nil + } + var fraction = Point(0.5, 0.5) + if target.anchor != nil { + guard let anchor = self.workbench.translateXAnchor() else { + observation = "X gizmo has no actionable point in the presented frame" + return nil + } + fraction = anchor + } + let frame = node.absoluteFrame + let local = Point(frame.minX + frame.width * fraction.x, frame.minY + frame.height * fraction.y) + let point = view.convert(local, to: window) + guard window.bounds.contains(point: point) else { + observation = "clipped at \(point)" + return nil + } + let hit = window.uiInspectionHitTest(at: point) + guard hit?.path.contains(where: { $0.runtimeId == node.runtimeId }) == true else { + observation = "blocked at \(point). Hit \(hit?.node.accessibilityIdentifier ?? hit?.node.viewType ?? "nothing")" + return nil + } + self.record("target \(target.id) \(target.anchor ?? "") at \(point)") + return point + } + } catch { + throw FlowError("\(error.localizedDescription) Last target observation: \(observation).") + } + } + + private func assertScene(_ expectation: StudioInteractionFlow.Assertion) async throws { + var actual: EditorWorkbenchInspection.SceneState? + do { + let _: Bool = try await wait("scene assertion for \(expectation.entity)", timeout: .seconds(2)) { + actual = self.workbench.sceneState(entity: expectation.entity) + guard let actual, + expectation.parent.map({ actual.parent == $0 }) != false, + expectation.undoCount.map({ actual.undoCount == $0 }) != false else { + return nil + } + if let expected = expectation.position { + guard let position = actual.position, + zip(position, expected).allSatisfy({ abs($0 - $1) <= (expectation.tolerance ?? 0.001) }) else { return nil } + } + return true + } + } catch { + throw FlowError("Assertion for \(expectation.entity): expected \(expectation), actual \(String(describing: actual)). \(error.localizedDescription)") + } + } + + private func mouse( + _ phase: MouseEvent.Phase, at point: Point, delta: Vector2? = nil, window: UIWindow, awaitViewport: Bool = false + ) async throws { + let observed = workbench.viewportInputCount + record("\(phase) \(point)") + let event = MouseEvent( + window: window.id, + button: .left, + motionDelta: delta, + mousePosition: point, + phase: phase, + modifierKeys: [], + time: timestamp() + ) + try await send(event) + if awaitViewport { + let _: Bool = try await wait("viewport acknowledgement of \(phase)", timeout: .seconds(2)) { + guard self.workbench.viewportInputCount > observed, + let received = self.workbench.lastViewportInput, + received.phase == phase, received.time == event.time, received.button == .left else { return nil } + return true + } + } + } + + private func cancelMouse(at point: Point, window: UIWindow) async { + await Task { try? await self.mouse(.cancelled, at: point, window: window) }.value + } + + private func send(_ event: T) async throws { + guard let worlds else { throw FlowError("Studio runtime detached.") } + if let runDeadline, clock.now >= runDeadline, !isRelease(event) { + throw FlowError("Studio flow exceeded 35 seconds.") + } + try await worlds.withWorldAccess { + guard worlds.main.getResource(Input.self) != nil, + worlds.getResource(PrimaryWindow.self)?.window.id == event.window else { + throw FlowError("The bound Studio window is no longer available.") + } + worlds.main.getRefResource(Input.self).wrappedValue.receiveEvent(event) + } + let _: Bool = try await wait("window input dispatch", timeout: .seconds(2)) { + try await worlds.withWorldAccess { + let input = worlds.main.getResource(Input.self) + return input?.pendingEventsPool.contains(where: { $0.id == event.id }) == false ? true : nil + } + } + } + + private func wait( + _ description: String, timeout: Duration = .seconds(8), until predicate: @MainActor () async throws -> T? + ) async throws -> T { + let localDeadline = clock.now.advanced(by: timeout) + let deadline = min(localDeadline, runDeadline ?? localDeadline) + repeat { + try Task.checkCancellation() + if let result = try await predicate() { + return result + } + try await Task.sleep(for: .milliseconds(16)) + } while clock.now < deadline + throw FlowError("Timed out waiting for \(description).") + } + + private func record(_ entry: String) { + if trace.count == 128 { trace.removeFirst() } + trace.append(entry) + } + + private func pressUndoKey(modifiers: KeyModifier, window: UIWindow) async throws { + let down = KeyEvent(window: window.id, keyCode: .z, modifiers: modifiers, status: .down, time: timestamp(), isRepeated: false) + do { + try await send(down) + try await send(KeyEvent(window: window.id, keyCode: .z, modifiers: modifiers, status: .up, time: timestamp(), isRepeated: false)) + } catch { + let release = KeyEvent(window: window.id, keyCode: .z, modifiers: modifiers, status: .up, time: timestamp(), isRepeated: false) + await Task { try? await self.send(release) }.value + throw error + } + } + + private func isRelease(_ event: any InputEvent) -> Bool { + switch event { + case let mouse as MouseEvent: mouse.phase == .cancelled || mouse.phase == .ended + case let key as KeyEvent: key.status == .up + default: false + } + } + + private func timestamp() -> Float { Float(Time.absolute) } +} diff --git a/Editor/Tests/StudioInteractionHost/check_flow.py b/Editor/Tests/StudioInteractionHost/check_flow.py new file mode 100644 index 000000000..a486bd415 --- /dev/null +++ b/Editor/Tests/StudioInteractionHost/check_flow.py @@ -0,0 +1,104 @@ +"""Run the workbench flow through both CLI and AdaMCP stdio.""" +import json +import select +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def send(process, message): + data = json.dumps({"jsonrpc": "2.0", **message}).encode() + process.stdin.write(f"Content-Length: {len(data)}\r\n\r\n".encode() + data) + process.stdin.flush() + + +def receive(process, request_id, timeout=45): + deadline = time.monotonic() + timeout + pending = bytearray() + while time.monotonic() < deadline: + ready, _, _ = select.select([process.stdout], [], [], deadline - time.monotonic()) + if not ready: + break + chunk = process.stdout.read(8192) + if not chunk: + raise RuntimeError("AdaMCP closed its output before returning a response.") + pending.extend(chunk) + while b"\r\n\r\n" in pending: + header, body = pending.split(b"\r\n\r\n", 1) + length = int(header.removeprefix(b"Content-Length: ")) + if len(body) < length: + break + response = json.loads(body[:length]) + pending = body[length:] + if response.get("id") == request_id: + if "error" in response: + raise RuntimeError(response["error"]) + return response["result"] + raise TimeoutError(f"Missing AdaMCP response for request {request_id}.") + + +def run_mcp(executable, source): + with subprocess.Popen( + [executable, "--stdio"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0 + ) as process: + try: + send(process, { + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "studio-flow-check", "version": "1"}, + }, + }) + receive(process, 1) + send(process, {"method": "notifications/initialized"}) + send(process, { + "id": 2, + "method": "tools/call", + "params": {"name": "editor.ui.run_flow", "arguments": {"source": source}}, + }) + result = receive(process, 2) + process.stdin.close() + if process.wait(timeout=5) != 0: + raise RuntimeError("AdaMCP host failed while shutting down after input closed.") + return json.loads(result["content"][0]["text"]) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + + +def main(): + executable = sys.argv[1] + flow = Path(__file__).parent / "Fixtures" / "workbench.yaml" + cli = subprocess.run([executable, str(flow)], capture_output=True, text=True, timeout=60, check=False) + if cli.returncode: + raise RuntimeError(f"CLI failed ({cli.returncode}):\n{cli.stdout}\n{cli.stderr}") + cli_report = json.loads(cli.stdout) + mcp_report = run_mcp(executable, flow.read_text(encoding="utf-8")) + for report in (cli_report, mcp_report): + if report["status"] != "passed" or report["completedSteps"] != 14: + raise AssertionError(f"Workbench flow failed: {report}") + with tempfile.TemporaryDirectory(prefix="ada-flow-check-") as directory: + negative = Path(directory) / "wrong-transform.yaml" + negative.write_text( + flow.read_text(encoding="utf-8").replace("[0.4, 0, 0]", "[100, 0, 0]"), + encoding="utf-8", + ) + failed = subprocess.run( + [executable, str(negative)], capture_output=True, text=True, timeout=60, check=False + ) + report = json.loads(failed.stdout) + if failed.returncode != 1 or report["failedStep"] != 4 or report["command"] != "assertScene": + raise AssertionError(f"Wrong transform did not fail step four: {report}") + print("Passed the same 14-step flow through CLI and AdaMCP. Wrong transform failed step four.") + + +if __name__ == "__main__": + main() diff --git a/Editor/scripts/update-offline-documentation.py b/Editor/scripts/update-offline-documentation.py index c09cb424c..930d204ba 100644 --- a/Editor/scripts/update-offline-documentation.py +++ b/Editor/scripts/update-offline-documentation.py @@ -12,7 +12,7 @@ def articles(): sources = [(EDITOR / f'Documentation/{name}.md', 'Editor') - for name in ['EditorGuide', 'Agent3DWorkflow', 'A2UI', 'AdaScriptGameplayRecipes', 'InspectorGroups', 'BuildExport']] + for name in ['EditorGuide', 'Agent3DWorkflow', 'A2UI', 'AdaScriptGameplayRecipes', 'InspectorGroups', 'BuildExport', 'StudioInteractionFlows']] for module, section in [('AdaEngine', 'AdaEngine'), ('AdaScripting', 'AdaScript')]: catalog = ROOT / f'Sources/{module}/{module}.docc' sources += [(path, section) for path in sorted(catalog.rglob('*.md')) diff --git a/README.md b/README.md index a52cb2c59..dc7e6c655 100644 --- a/README.md +++ b/README.md @@ -160,3 +160,5 @@ python ada_tool.py install studio --debug # Debug alongside Release Requires Python 3.9+ (`python3` on macOS/Linux if needed). See the [CLI guide](Documentation/Tooling.md) for build, WASM, playground and preview commands. + +For editor interaction testing, see [Studio interaction flows](Editor/Documentation/StudioInteractionFlows.md). diff --git a/Sources/AdaAnimation/Skeletal/SkeletalAnimationPlayer.swift b/Sources/AdaAnimation/Skeletal/SkeletalAnimationPlayer.swift index d1df46b27..df12af5a3 100644 --- a/Sources/AdaAnimation/Skeletal/SkeletalAnimationPlayer.swift +++ b/Sources/AdaAnimation/Skeletal/SkeletalAnimationPlayer.swift @@ -174,6 +174,7 @@ public struct SkeletalAnimationPlayer: Sendable { public mutating func finalizePoseModifiers(inputs: [String: AnimationPoseModifierInput] = [:]) { for index in poses.indices { poses[index] = basePoses[index] } rebuildTransforms() + let rig = self.rig poseModifierState?.apply(rig: rig, inputs: inputs, poses: &poses, locals: &localTransforms, globals: &globalTransforms, diagnostics: &modifierDiagnostics) } diff --git a/Sources/AdaCorePipelines/Assets/Shaders/flat3d.glsl b/Sources/AdaCorePipelines/Assets/Shaders/flat3d.glsl index 40ef8e646..efb64b640 100644 --- a/Sources/AdaCorePipelines/Assets/Shaders/flat3d.glsl +++ b/Sources/AdaCorePipelines/Assets/Shaders/flat3d.glsl @@ -282,7 +282,7 @@ layout (binding = 29) uniform LightTileMasks3DUniform { uvec4 u_TileMasks[1024]; }; layout (binding = 41) uniform texture2D u_ContactSceneDepth; -layout (binding = 42) uniform SurfaceLighting3DUniform { +layout (binding = 23) uniform SurfaceLighting3DUniform { mat4 u_ContactProjection; mat4 u_ContactInverseProjection; vec4 u_Contact; // enabled, ray length, thickness, bias (view-space meters) diff --git a/Sources/AdaCorePipelines/Assets/Shaders/screen_space_reflection.glsl b/Sources/AdaCorePipelines/Assets/Shaders/screen_space_reflection.glsl index 2380e1a74..abca96d91 100644 --- a/Sources/AdaCorePipelines/Assets/Shaders/screen_space_reflection.glsl +++ b/Sources/AdaCorePipelines/Assets/Shaders/screen_space_reflection.glsl @@ -34,7 +34,7 @@ layout (binding = 15) uniform texture2D u_ExposureState; layout (binding = 16) uniform texture2D u_Volumetric; layout (binding = 17) uniform texture2D u_ReflectionResponse; layout (binding = 18) uniform texture2D u_PrefilteredEnvironment; -layout (binding = 19) uniform sampler u_IBLSampler; +layout (binding = 10) uniform sampler u_IBLSampler; layout (binding = 4) uniform Environment3DUniform { mat4 u_Projection; diff --git a/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/Main3DRenderNode.swift b/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/Main3DRenderNode.swift index 40289a65b..01d80a618 100644 --- a/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/Main3DRenderNode.swift +++ b/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/Main3DRenderNode.swift @@ -353,8 +353,8 @@ public struct Main3DRenderNode: RenderNode { renderPass.setFragmentBuffer(localMatrices, offset: 0, slot: 28) renderPass.setFragmentBuffer(tileConstants, offset: 0, slot: 27) renderPass.setFragmentBuffer(tileMaskBuffer, offset: 0, slot: 29) - renderPass.setFragmentBuffer(surfaceConstants, offset: 0, slot: 42) - renderPass.setFragmentBuffer(distanceFieldConstants, offset: 0, slot: 50) + renderPass.setFragmentBuffer(surfaceConstants, offset: 0, slot: 23) + renderPass.setFragmentBuffer(distanceFieldConstants, offset: 0, slot: 30) renderPass.setResourceSet(.init(bindings: [ .init(binding: 41, shaderStages: .fragment, resource: .texture(contactTexture)), .init(binding: 49, shaderStages: .fragment, diff --git a/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/ScreenSpaceReflectionRenderNode.swift b/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/ScreenSpaceReflectionRenderNode.swift index e1f3693f6..3bf9b8284 100644 --- a/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/ScreenSpaceReflectionRenderNode.swift +++ b/Sources/AdaCorePipelines/CorePipelines/RenderNodes3D/ScreenSpaceReflectionRenderNode.swift @@ -210,7 +210,7 @@ public struct ScreenSpaceReflectionRenderNode: RenderNode { .init(binding: 20, shaderStages: .fragment, resource: .texture(giTexture ?? Texture2D.whiteTexture)), .init(binding: 18, shaderStages: .fragment, resource: .texture(environment.imageBasedLighting?.asset.asset?.specular ?? Texture2D.whiteTexture)), // Match the PBR pass's atlas filtering exactly, including programmatically supplied maps. - .init(binding: 19, shaderStages: .fragment, resource: .sampler((environment.imageBasedLighting?.asset.asset?.irradiance ?? Texture2D.whiteTexture).sampler)), + .init(binding: 10, shaderStages: .fragment, resource: .sampler((environment.imageBasedLighting?.asset.asset?.irradiance ?? Texture2D.whiteTexture).sampler)), .init(binding: 13, shaderStages: .fragment, resource: .texture(gradingLUT?.texture ?? Texture2D.whiteTexture)), .init(binding: 14, shaderStages: .fragment, resource: .sampler(pipeline.sampler)), .init(binding: 15, shaderStages: .fragment, resource: .texture(autoExposureTexture ?? Texture2D.whiteTexture)), diff --git a/Sources/AdaRender/Assets/Shaders/Public/DistanceField3D.glsl b/Sources/AdaRender/Assets/Shaders/Public/DistanceField3D.glsl index fbf291b39..6a5bfdc15 100644 --- a/Sources/AdaRender/Assets/Shaders/Public/DistanceField3D.glsl +++ b/Sources/AdaRender/Assets/Shaders/Public/DistanceField3D.glsl @@ -1,7 +1,7 @@ // World-space SDF shadow queries. R32F node samples, tiled Z slices, no filtering across slices. // The caller supplies its existing nearest sampler as u_DirectionalShadowSampler. layout(binding=49) uniform texture2D u_DistanceFieldTexture; -layout(binding=50) uniform DistanceField3DUniform { +layout(binding=30) uniform DistanceField3DUniform { vec4 u_SDFMinimumVoxel; // world minimum.xyz, cubic voxel size vec4 u_SDFCountsDistance; // sample counts.xyz, maximum shadow ray distance vec4 u_SDFTrace; // maximum steps, normal bias, cone tangent, 0 off / 1 shadows / 2 debug diff --git a/Sources/AdaUI/CoreUI/UIInspection.swift b/Sources/AdaUI/CoreUI/UIInspection.swift index f40cd8172..c4861c8eb 100644 --- a/Sources/AdaUI/CoreUI/UIInspection.swift +++ b/Sources/AdaUI/CoreUI/UIInspection.swift @@ -262,6 +262,20 @@ public protocol UIInspectableViewContainer: AnyObject { } extension UIWindow { + /// Inspects the same container that receives a pointer press in this window. + @_spi(Internal) + public func uiInspectionHitTest(at point: Point) -> UIHitTestResult? { + let event = MouseEvent(window: id, button: .left, mousePosition: point, phase: .began, modifierKeys: [], time: 0) + var responder = findFirstResponder(for: event) + while let view = responder { + if let container = view as? any UIInspectableViewContainer { + return container.uiHitTest(at: point) + } + responder = view.parentView + } + return nil + } + public func uiInspectableContainers() -> [any UIInspectableViewContainer] { var result: [any UIInspectableViewContainer] = [] diff --git a/Tests/AdaEngineTests/Render3DDistanceFieldTests.swift b/Tests/AdaEngineTests/Render3DDistanceFieldTests.swift index e0e26c582..e961365ce 100644 --- a/Tests/AdaEngineTests/Render3DDistanceFieldTests.swift +++ b/Tests/AdaEngineTests/Render3DDistanceFieldTests.swift @@ -294,7 +294,7 @@ extension Render3DGPUTests { let pipeline = device.createRenderPipeline(from: .init(vertex: try #require(module.getShader(for: .vertex)), fragment: module.getShader(for: .fragment), debugName: "Production SDF query regression", backfaceCulling: false, depthPixelFormat: .none, colorAttachments: [.init(format: .bgra8)])) - let uniform = device.createUniformBuffer(Vector4.self, count: 3, binding: 50) + let uniform = device.createUniformBuffer(Vector4.self, count: 3, binding: 30) var constants: [Vector4] = [Vector4(minimum, voxelSize), [Float(counts[0]), Float(counts[1]), Float(counts[2]), 6], [steps, bias, 0, 1]] uniform.setElements(&constants) @@ -306,7 +306,7 @@ extension Render3DGPUTests { ])) pass.setViewport(.init(x: 0, y: 0, width: Float(width), height: Float(height))) pass.setRenderPipelineState(pipeline) - pass.setFragmentBuffer(uniform, offset: 0, slot: 50) + pass.setFragmentBuffer(uniform, offset: 0, slot: 30) pass.setFragmentTexture(texture, slot: 49) pass.setFragmentSamplerState(sampler, slot: 11) pass.draw(type: .triangle, vertexStart: 0, vertexCount: 3, instanceCount: 1)