Use the cloned window's layer stack space for raw coordinates

prabirmsp · prabirmsp · commit 72c6990eb8e8 · 2025-01-06T20:06:35.000Z
Use the cloned window's layer stack (screen) space as the
raw coordinate space for input going to clones. This ensures the raw
coordinates are consistent between the clone and cloned windows while
mirroring.

Flag: com.android.input.flags.use_cloned_screen_coordinates_as_raw
Test: Presubmit
Bug: 377846505
Change-Id: I7370ac64585315d9c754eaa1eb0077c8555fc6a9
diff --git a/libs/gui/WindowInfo.cpp b/libs/gui/WindowInfo.cpp
@@ -59,6 +59,32 @@ std::ostream& operator<<(std::ostream& out, const Region& region) {
     return out;
 }
 
+status_t writeTransform(android::Parcel* parcel, const ui::Transform& transform) {
+    return parcel->writeFloat(transform.dsdx()) ?:
+            parcel->writeFloat(transform.dtdx()) ?:
+            parcel->writeFloat(transform.tx()) ?:
+            parcel->writeFloat(transform.dtdy()) ?:
+            parcel->writeFloat(transform.dsdy()) ?:
+            parcel->writeFloat(transform.ty());
+}
+
+status_t readTransform(const android::Parcel* parcel, ui::Transform& transform) {
+    float dsdx, dtdx, tx, dtdy, dsdy, ty;
+
+    const status_t status = parcel->readFloat(&dsdx) ?:
+            parcel->readFloat(&dtdx) ?:
+            parcel->readFloat(&tx) ?:
+            parcel->readFloat(&dtdy) ?:
+            parcel->readFloat(&dsdy) ?:
+            parcel->readFloat(&ty);
+    if (status != OK) {
+        return status;
+    }
+
+    transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
+    return OK;
+}
+
 } // namespace
 
 void WindowInfo::setInputConfig(ftl::Flags<InputConfig> config, bool value) {
@@ -135,12 +161,7 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const {
         parcel->writeInt32(surfaceInset) ?:
         parcel->writeFloat(globalScaleFactor) ?:
         parcel->writeFloat(alpha) ?:
-        parcel->writeFloat(transform.dsdx()) ?:
-        parcel->writeFloat(transform.dtdx()) ?:
-        parcel->writeFloat(transform.tx()) ?:
-        parcel->writeFloat(transform.dtdy()) ?:
-        parcel->writeFloat(transform.dsdy()) ?:
-        parcel->writeFloat(transform.ty()) ?:
+        writeTransform(parcel, transform) ?:
         parcel->writeInt32(static_cast<int32_t>(touchOcclusionMode)) ?:
         parcel->writeInt32(ownerPid.val()) ?:
         parcel->writeInt32(ownerUid.val()) ?:
@@ -153,8 +174,12 @@ status_t WindowInfo::writeToParcel(android::Parcel* parcel) const {
         parcel->writeStrongBinder(touchableRegionCropHandle.promote()) ?:
         parcel->writeStrongBinder(windowToken) ?:
         parcel->writeStrongBinder(focusTransferTarget) ?:
-        parcel->writeBool(canOccludePresentation);
+        parcel->writeBool(canOccludePresentation) ?:
+        parcel->writeBool(cloneLayerStackTransform.has_value());
     // clang-format on
+    if (cloneLayerStackTransform) {
+        status = status ?: writeTransform(parcel, *cloneLayerStackTransform);
+    }
     return status;
 }
 
@@ -174,10 +199,10 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         return status;
     }
 
-    float dsdx, dtdx, tx, dtdy, dsdy, ty;
     int32_t lpFlags, lpType, touchOcclusionModeInt, inputConfigInt, ownerPidInt, ownerUidInt,
             displayIdInt;
     sp<IBinder> touchableRegionCropHandleSp;
+    bool hasCloneLayerStackTransform = false;
 
     // clang-format off
     status = parcel->readInt32(&lpFlags) ?:
@@ -188,12 +213,7 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         parcel->readInt32(&surfaceInset) ?:
         parcel->readFloat(&globalScaleFactor) ?:
         parcel->readFloat(&alpha) ?:
-        parcel->readFloat(&dsdx) ?:
-        parcel->readFloat(&dtdx) ?:
-        parcel->readFloat(&tx) ?:
-        parcel->readFloat(&dtdy) ?:
-        parcel->readFloat(&dsdy) ?:
-        parcel->readFloat(&ty) ?:
+        readTransform(parcel, /*byRef*/ transform) ?:
         parcel->readInt32(&touchOcclusionModeInt) ?:
         parcel->readInt32(&ownerPidInt) ?:
         parcel->readInt32(&ownerUidInt) ?:
@@ -206,8 +226,8 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
         parcel->readNullableStrongBinder(&touchableRegionCropHandleSp) ?:
         parcel->readNullableStrongBinder(&windowToken) ?:
         parcel->readNullableStrongBinder(&focusTransferTarget) ?:
-        parcel->readBool(&canOccludePresentation);
-
+        parcel->readBool(&canOccludePresentation)?:
+        parcel->readBool(&hasCloneLayerStackTransform);
     // clang-format on
 
     if (status != OK) {
@@ -216,14 +236,22 @@ status_t WindowInfo::readFromParcel(const android::Parcel* parcel) {
 
     layoutParamsFlags = ftl::Flags<Flag>(lpFlags);
     layoutParamsType = static_cast<Type>(lpType);
-    transform.set({dsdx, dtdx, tx, dtdy, dsdy, ty, 0, 0, 1});
     touchOcclusionMode = static_cast<TouchOcclusionMode>(touchOcclusionModeInt);
     inputConfig = ftl::Flags<InputConfig>(inputConfigInt);
     ownerPid = Pid{ownerPidInt};
     ownerUid = Uid{static_cast<uid_t>(ownerUidInt)};
     touchableRegionCropHandle = touchableRegionCropHandleSp;
     displayId = ui::LogicalDisplayId{displayIdInt};
 
+    cloneLayerStackTransform =
+            hasCloneLayerStackTransform ? std::make_optional<ui::Transform>() : std::nullopt;
+    if (cloneLayerStackTransform) {
+        status = readTransform(parcel, /*byRef*/ *cloneLayerStackTransform);
+        if (status != OK) {
+            return status;
+        }
+    }
+
     return OK;
 }
 
diff --git a/libs/gui/include/gui/WindowInfo.h b/libs/gui/include/gui/WindowInfo.h
@@ -220,9 +220,14 @@ struct WindowInfo : public Parcelable {
     // An alpha of 1.0 means fully opaque and 0.0 means fully transparent.
     float alpha;
 
-    // Transform applied to individual windows.
+    // Transform applied to individual windows for input.
+    // Maps display coordinates to the window's input coordinate space.
     ui::Transform transform;
 
+    // Transform applied to get to the layer stack space of the cloned window for input.
+    // Maps display coordinates of the clone window to the layer stack space of the cloned window.
+    std::optional<ui::Transform> cloneLayerStackTransform;
+
     /*
      * This is filled in by the WM relative to the frame and then translated
      * to absolute coordinates by SurfaceFlinger once the frame is computed.
diff --git a/libs/gui/tests/WindowInfo_test.cpp b/libs/gui/tests/WindowInfo_test.cpp
@@ -40,7 +40,18 @@ TEST(WindowInfo, ParcellingWithoutToken) {
     ASSERT_EQ(OK, i.writeToParcel(&p));
     p.setDataPosition(0);
     i2.readFromParcel(&p);
-    ASSERT_TRUE(i2.token == nullptr);
+    ASSERT_EQ(i2.token, nullptr);
+}
+
+TEST(WindowInfo, ParcellingWithoutCloneTransform) {
+    WindowInfo i, i2;
+    i.cloneLayerStackTransform.reset();
+
+    Parcel p;
+    ASSERT_EQ(OK, i.writeToParcel(&p));
+    p.setDataPosition(0);
+    i2.readFromParcel(&p);
+    ASSERT_EQ(i2.cloneLayerStackTransform, std::nullopt);
 }
 
 TEST(WindowInfo, Parcelling) {
@@ -71,6 +82,8 @@ TEST(WindowInfo, Parcelling) {
     i.applicationInfo.token = new BBinder();
     i.applicationInfo.dispatchingTimeoutMillis = 0x12345678ABCD;
     i.focusTransferTarget = new BBinder();
+    i.cloneLayerStackTransform = ui::Transform();
+    i.cloneLayerStackTransform->set({5, -1, 100, 4, 0, 40, 0, 0, 1});
 
     Parcel p;
     i.writeToParcel(&p);
@@ -100,6 +113,7 @@ TEST(WindowInfo, Parcelling) {
     ASSERT_EQ(i.touchableRegionCropHandle, i2.touchableRegionCropHandle);
     ASSERT_EQ(i.applicationInfo, i2.applicationInfo);
     ASSERT_EQ(i.focusTransferTarget, i2.focusTransferTarget);
+    ASSERT_EQ(i.cloneLayerStackTransform, i2.cloneLayerStackTransform);
 }
 
 TEST(InputApplicationInfo, Parcelling) {
diff --git a/libs/input/input_flags.aconfig b/libs/input/input_flags.aconfig
@@ -224,3 +224,13 @@ flag {
   description: "Allow cursor to transition across multiple connected displays"
   bug: "362719483"
 }
+
+flag {
+  name: "use_cloned_screen_coordinates_as_raw"
+  namespace: "input"
+  description: "Use the cloned window's layer stack (screen) space as the raw coordinate space for input going to clones"
+  bug: "377846505"
+  metadata {
+    purpose: PURPOSE_BUGFIX
+  }
+}
diff --git a/services/inputflinger/dispatcher/InputDispatcher.cpp b/services/inputflinger/dispatcher/InputDispatcher.cpp
@@ -3013,11 +3013,10 @@ void InputDispatcher::addWindowTargetLocked(const sp<WindowInfoHandle>& windowHa
                   windowHandle->getName().c_str());
             return;
         }
-        inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
-                                                 targetFlags,
-                                                 mWindowInfos.getDisplayTransform(
-                                                         windowHandle->getInfo()->displayId),
-                                                 firstDownTimeInTarget));
+        inputTargets.push_back(
+                createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+                                  mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+                                  firstDownTimeInTarget));
         it = inputTargets.end() - 1;
     }
 
@@ -3068,11 +3067,10 @@ void InputDispatcher::addPointerWindowTargetLocked(
                   windowHandle->getName().c_str());
             return;
         }
-        inputTargets.push_back(createInputTarget(connection, windowHandle, dispatchMode,
-                                                 targetFlags,
-                                                 mWindowInfos.getDisplayTransform(
-                                                         windowHandle->getInfo()->displayId),
-                                                 firstDownTimeInTarget));
+        inputTargets.push_back(
+                createInputTarget(connection, windowHandle, dispatchMode, targetFlags,
+                                  mWindowInfos.getRawTransform(*windowHandle->getInfo()),
+                                  firstDownTimeInTarget));
         it = inputTargets.end() - 1;
     }
 
@@ -3104,7 +3102,8 @@ void InputDispatcher::addGlobalMonitoringTargetsLocked(std::vector<InputTarget>&
     for (const Monitor& monitor : selectResponsiveMonitorsLocked(monitorsIt->second)) {
         InputTarget target{monitor.connection};
         // target.firstDownTimeInTarget is not set for global monitors. It is only required in split
-        // touch and global monitoring works as intended even without setting firstDownTimeInTarget
+        // touch and global monitoring works as intended even without setting firstDownTimeInTarget.
+        // Since global monitors don't have windows, use the display transform as the raw transform.
         target.rawTransform = mWindowInfos.getDisplayTransform(displayId);
         target.setDefaultPointerTransform(target.rawTransform);
         inputTargets.push_back(target);
@@ -4291,6 +4290,7 @@ void InputDispatcher::synthesizeCancelationEventsForConnectionLocked(
                                                  motionEntry.downTime, targets);
                 } else {
                     targets.emplace_back(fallbackTarget);
+                    // Since we don't have a window, use the display transform as the raw transform.
                     const ui::Transform displayTransform =
                             mWindowInfos.getDisplayTransform(motionEntry.displayId);
                     targets.back().rawTransform = displayTransform;
@@ -4376,6 +4376,7 @@ void InputDispatcher::synthesizePointerDownEventsForConnectionLocked(
                                                  targets);
                 } else {
                     targets.emplace_back(connection, targetFlags);
+                    // Since we don't have a window, use the display transform as the raw transform.
                     const ui::Transform displayTransform =
                             mWindowInfos.getDisplayTransform(motionEntry.displayId);
                     targets.back().rawTransform = displayTransform;
@@ -5289,6 +5290,16 @@ ui::Transform InputDispatcher::DispatcherWindowInfo::getDisplayTransform(
                                                 : kIdentityTransform;
 }
 
+ui::Transform InputDispatcher::DispatcherWindowInfo::getRawTransform(
+        const android::gui::WindowInfo& windowInfo) const {
+    // If the window has a cloneLayerStackTransform, always use it as the transform for the "getRaw"
+    // APIs. If not, fall back to using the DisplayInfo transform of the window's display.
+    return (input_flags::use_cloned_screen_coordinates_as_raw() &&
+            windowInfo.cloneLayerStackTransform)
+            ? *windowInfo.cloneLayerStackTransform
+            : getDisplayTransform(windowInfo.displayId);
+}
+
 std::string InputDispatcher::DispatcherWindowInfo::dumpDisplayAndWindowInfo() const {
     std::string dump;
     if (!mWindowHandlesByDisplay.empty()) {
@@ -6567,9 +6578,8 @@ void InputDispatcher::doDispatchCycleFinishedCommand(nsecs_t finishTime,
                                            createInputTarget(connection, windowHandle,
                                                              InputTarget::DispatchMode::AS_IS,
                                                              dispatchEntry->targetFlags,
-                                                             mWindowInfos.getDisplayTransform(
-                                                                     windowHandle->getInfo()
-                                                                             ->displayId),
+                                                             mWindowInfos.getRawTransform(
+                                                                     *windowHandle->getInfo()),
                                                              downTime));
             }
         }
diff --git a/services/inputflinger/dispatcher/InputDispatcher.h b/services/inputflinger/dispatcher/InputDispatcher.h
@@ -384,6 +384,9 @@ class InputDispatcher : public android::InputDispatcherInterface {
         // Get the transform for display, returns Identity-transform if display is missing.
         ui::Transform getDisplayTransform(ui::LogicalDisplayId displayId) const;
 
+        // Get the raw transform to use for motion events going to the given window.
+        ui::Transform getRawTransform(const android::gui::WindowInfo&) const;
+
         // Lookup for WindowInfoHandle from token and optionally a display-id. In cases where
         // display-id is not provided lookup is done for all displays.
         sp<android::gui::WindowInfoHandle> findWindowHandle(
diff --git a/services/inputflinger/tests/InputDispatcher_test.cpp b/services/inputflinger/tests/InputDispatcher_test.cpp
@@ -6501,19 +6501,47 @@ TEST_F(InputDispatcherDisplayProjectionTest, WindowGetsEventsInCorrectCoordinate
                                ui::LogicalDisplayId::DEFAULT, {PointF{150, 220}}));
 
     firstWindow->assertNoEvents();
-    std::unique_ptr<MotionEvent> event = secondWindow->consumeMotionEvent();
-    ASSERT_NE(nullptr, event);
-    EXPECT_EQ(AMOTION_EVENT_ACTION_DOWN, event->getAction());
+    secondWindow->consumeMotionEvent(
+            AllOf(WithMotionAction(ACTION_DOWN),
+                  // Ensure that the events from the "getRaw" API are in logical display
+                  // coordinates, which has an x-scale of 2 and y-scale of 4.
+                  WithRawCoords(300, 880),
+                  // Ensure that the x and y values are in the window's coordinate space.
+                  // The left-top of the second window is at (100, 200) in display space, which is
+                  // (200, 800) in the logical display space. This will be the origin of the window
+                  // space.
+                  WithCoords(100, 80)));
+}
 
-    // Ensure that the events from the "getRaw" API are in logical display coordinates.
-    EXPECT_EQ(300, event->getRawX(0));
-    EXPECT_EQ(880, event->getRawY(0));
+TEST_F(InputDispatcherDisplayProjectionTest, UseCloneLayerStackTransformForRawCoordinates) {
+    SCOPED_FLAG_OVERRIDE(use_cloned_screen_coordinates_as_raw, true);
 
-    // Ensure that the x and y values are in the window's coordinate space.
-    // The left-top of the second window is at (100, 200) in display space, which is (200, 800) in
-    // the logical display space. This will be the origin of the window space.
-    EXPECT_EQ(100, event->getX(0));
-    EXPECT_EQ(80, event->getY(0));
+    auto [firstWindow, secondWindow] = setupScaledDisplayScenario();
+
+    const std::array<float, 9> matrix = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 0.0, 0.0, 1.0};
+    ui::Transform secondDisplayTransform;
+    secondDisplayTransform.set(matrix);
+    addDisplayInfo(SECOND_DISPLAY_ID, secondDisplayTransform);
+
+    // When a clone layer stack transform is provided for a window, we should use that as the
+    // "display transform" for input going to that window.
+    sp<FakeWindowHandle> secondWindowClone = secondWindow->clone(SECOND_DISPLAY_ID);
+    secondWindowClone->editInfo()->cloneLayerStackTransform = ui::Transform();
+    secondWindowClone->editInfo()->cloneLayerStackTransform->set(0.5, 0, 0, 0.25);
+    addWindow(secondWindowClone);
+
+    // Touch down on the clone window, and ensure its raw coordinates use
+    // the clone layer stack transform.
+    mDispatcher->notifyMotion(generateMotionArgs(ACTION_DOWN, AINPUT_SOURCE_TOUCHSCREEN,
+                                                 SECOND_DISPLAY_ID, {PointF{150, 220}}));
+    secondWindowClone->consumeMotionEvent(
+            AllOf(WithMotionAction(ACTION_DOWN),
+                  // Ensure the x and y coordinates are in the window's coordinate space.
+                  // See previous test case for calculation.
+                  WithCoords(100, 80),
+                  // Ensure the "getRaw" API uses the clone layer stack transform when it is
+                  // provided for the window. It has an x-scale of 0.5 and y-scale of 0.25.
+                  WithRawCoords(75, 55)));
 }
 
 TEST_F(InputDispatcherDisplayProjectionTest, CancelMotionWithCorrectCoordinates) {
diff --git a/services/surfaceflinger/FrontEnd/LayerHierarchy.h b/services/surfaceflinger/FrontEnd/LayerHierarchy.h
@@ -102,6 +102,8 @@ class LayerHierarchy {
         // Returns true if the node is a clone.
         bool isClone() const { return !mirrorRootIds.empty(); }
 
+        TraversalPath getClonedFrom() const { return {.id = id, .variant = variant}; }
+
         bool operator==(const TraversalPath& other) const {
             return id == other.id && mirrorRootIds == other.mirrorRootIds;
         }
diff --git a/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp b/services/surfaceflinger/FrontEnd/LayerSnapshotBuilder.cpp
@@ -16,6 +16,8 @@
 
 // #define LOG_NDEBUG 0
 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
+#include "FrontEnd/LayerSnapshot.h"
+#include "ui/Transform.h"
 #undef LOG_TAG
 #define LOG_TAG "SurfaceFlinger"
 
@@ -1205,13 +1207,27 @@ void LayerSnapshotBuilder::updateInput(LayerSnapshot& snapshot,
     snapshot.inputInfo.contentSize = {snapshot.croppedBufferSize.getHeight(),
                                       snapshot.croppedBufferSize.getWidth()};
 
-    // If the layer is a clone, we need to crop the input region to cloned root to prevent
-    // touches from going outside the cloned area.
+    snapshot.inputInfo.cloneLayerStackTransform.reset();
+
     if (path.isClone()) {
         snapshot.inputInfo.inputConfig |= InputConfig::CLONE;
         // Cloned layers shouldn't handle watch outside since their z order is not determined by
         // WM or the client.
         snapshot.inputInfo.inputConfig.clear(InputConfig::WATCH_OUTSIDE_TOUCH);
+
+        // Compute the transform that maps the clone's display to the layer stack space of the
+        // cloned window.
+        const LayerSnapshot* clonedSnapshot = getSnapshot(path.getClonedFrom());
+        if (clonedSnapshot != nullptr) {
+            const auto& [clonedInputBounds, s] =
+                    getInputBounds(*clonedSnapshot, /*fillParentBounds=*/false);
+            ui::Transform inputToLayer;
+            inputToLayer.set(clonedInputBounds.left, clonedInputBounds.top);
+            const ui::Transform& layerToLayerStack = getInputTransform(*clonedSnapshot);
+            const auto& displayToInput = snapshot.inputInfo.transform;
+            snapshot.inputInfo.cloneLayerStackTransform =
+                    layerToLayerStack * inputToLayer * displayToInput;
+        }
     }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -102,6 +102,8 @@ class LayerHierarchy {`
`102`	`102`	`// Returns true if the node is a clone.`
`103`	`103`	`bool isClone() const { return !mirrorRootIds.empty(); }`
`104`	`104`
	`105`	`+ TraversalPath getClonedFrom() const { return {.id = id, .variant = variant}; }`
	`106`	`+`
`105`	`107`	`bool operator==(const TraversalPath& other) const {`
`106`	`108`	`return id == other.id && mirrorRootIds == other.mirrorRootIds;`
`107`	`109`	`}`