Merge "Enable cursor to transition across multiple displays" into main

Author: Arpit Singh

services/inputflinger/PointerChoreographer.cpp

@@ -103,6 +103,9 @@ std::unordered_set<ui::LogicalDisplayId> getPrivacySensitiveDisplaysFromWindowIn
 
 // --- PointerChoreographer ---
 
+const bool PointerChoreographer::IS_TOPOLOGY_AWARE =
+        com::android::input::flags::connected_displays_cursor();
+
 PointerChoreographer::PointerChoreographer(InputListenerInterface& inputListener,
                                            PointerChoreographerPolicyInterface& policy)
       : PointerChoreographer(
@@ -204,20 +207,30 @@ void PointerChoreographer::fadeMouseCursorOnKeyPress(const android::NotifyKeyArg
 }
 
 NotifyMotionArgs PointerChoreographer::processMotion(const NotifyMotionArgs& args) {
-    std::scoped_lock _l(mLock);
+    NotifyMotionArgs newArgs(args);
+    PointerDisplayChange pointerDisplayChange;
+    { // acquire lock
+        std::scoped_lock _l(mLock);
+        if (isFromMouse(args)) {
+            newArgs = processMouseEventLocked(args);
+            pointerDisplayChange = calculatePointerDisplayChangeToNotify();
+        } else if (isFromTouchpad(args)) {
+            newArgs = processTouchpadEventLocked(args);
+            pointerDisplayChange = calculatePointerDisplayChangeToNotify();
+        } else if (isFromDrawingTablet(args)) {
+            processDrawingTabletEventLocked(args);
+        } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) {
+            processStylusHoverEventLocked(args);
+        } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) {
+            processTouchscreenAndStylusEventLocked(args);
+        }
+    } // release lock
 
-    if (isFromMouse(args)) {
-        return processMouseEventLocked(args);
-    } else if (isFromTouchpad(args)) {
-        return processTouchpadEventLocked(args);
-    } else if (isFromDrawingTablet(args)) {
-        processDrawingTabletEventLocked(args);
-    } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) {
-        processStylusHoverEventLocked(args);
-    } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) {
-        processTouchscreenAndStylusEventLocked(args);
+    if (pointerDisplayChange) {
+        // pointer display may have changed if mouse crossed display boundary
+        notifyPointerDisplayChange(pointerDisplayChange, mPolicy);
     }
-    return args;
+    return newArgs;
 }
 
 NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotionArgs& args) {
@@ -242,16 +255,10 @@ NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotio
         pc.setPosition(args.xCursorPosition, args.yCursorPosition);
     } else {
         // This is a relative mouse, so move the cursor by the specified amount.
-        const float deltaX = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);
-        const float deltaY = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);
-        pc.move(deltaX, deltaY);
-        const auto [x, y] = pc.getPosition();
-        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);
-        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);
-        newArgs.xCursorPosition = x;
-        newArgs.yCursorPosition = y;
+        processPointerDeviceMotionEventLocked(/*byref*/ newArgs, /*byref*/ pc);
     }
-    if (canUnfadeOnDisplay(displayId)) {
+    // Note displayId may have changed if the cursor moved to a different display
+    if (canUnfadeOnDisplay(newArgs.displayId)) {
         pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);
     }
     return newArgs;
@@ -265,24 +272,9 @@ NotifyMotionArgs PointerChoreographer::processTouchpadEventLocked(const NotifyMo
     newArgs.displayId = displayId;
     if (args.getPointerCount() == 1 && args.classification == MotionClassification::NONE) {
         // This is a movement of the mouse pointer.
-        const float deltaX = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);
-        const float deltaY = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);
-        pc.move(deltaX, deltaY);
-        if (canUnfadeOnDisplay(displayId)) {
-            pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);
-        }
-
-        const auto [x, y] = pc.getPosition();
-        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);
-        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);
-        newArgs.xCursorPosition = x;
-        newArgs.yCursorPosition = y;
+        processPointerDeviceMotionEventLocked(/*byref*/ newArgs, /*byref*/ pc);
     } else {
         // This is a trackpad gesture with fake finger(s) that should not move the mouse pointer.
-        if (canUnfadeOnDisplay(displayId)) {
-            pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);
-        }
-
         const auto [x, y] = pc.getPosition();
         for (uint32_t i = 0; i < newArgs.getPointerCount(); i++) {
             newArgs.pointerCoords[i].setAxisValue(AMOTION_EVENT_AXIS_X,
@@ -293,9 +285,61 @@ NotifyMotionArgs PointerChoreographer::processTouchpadEventLocked(const NotifyMo
         newArgs.xCursorPosition = x;
         newArgs.yCursorPosition = y;
     }
+
+    // Note displayId may have changed if the cursor moved to a different display
+    if (canUnfadeOnDisplay(newArgs.displayId)) {
+        pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);
+    }
     return newArgs;
 }
 
+void PointerChoreographer::processPointerDeviceMotionEventLocked(NotifyMotionArgs& newArgs,
+                                                                 PointerControllerInterface& pc) {
+    const float deltaX = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);
+    const float deltaY = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);
+    vec2 unconsumedDelta = pc.move(deltaX, deltaY);
+    if (IS_TOPOLOGY_AWARE && (std::abs(unconsumedDelta.x) > 0 || std::abs(unconsumedDelta.y) > 0)) {
+        handleUnconsumedDeltaLocked(pc, unconsumedDelta);
+        // pointer may have moved to a different viewport
+        newArgs.displayId = pc.getDisplayId();
+    }
+    const auto [x, y] = pc.getPosition();
+    newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);
+    newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);
+    newArgs.xCursorPosition = x;
+    newArgs.yCursorPosition = y;
+}
+
+void PointerChoreographer::handleUnconsumedDeltaLocked(PointerControllerInterface& pc,
+                                                       const vec2& unconsumedDelta) {
+    const ui::LogicalDisplayId sourceDisplayId = pc.getDisplayId();
+    const auto& sourceViewport = *findViewportByIdLocked(sourceDisplayId);
+    std::optional<AdjacentDisplay> destinationDisplay =
+            findDestinationDisplayLocked(sourceViewport, unconsumedDelta);
+    if (!destinationDisplay) {
+        // no adjacent display
+        return;
+    }
+
+    const DisplayViewport* destinationViewport =
+            findViewportByIdLocked(destinationDisplay->displayId);
+    if (destinationViewport == nullptr) {
+        // Topology is likely out of sync with viewport info, wait for them to be updated
+        LOG(WARNING) << "Cannot find viewport for adjacent display "
+                     << destinationDisplay->displayId << "of source display " << sourceDisplayId;
+        return;
+    }
+
+    mDefaultMouseDisplayId = destinationDisplay->displayId;
+    auto pcNode = mMousePointersByDisplay.extract(sourceDisplayId);
+    pcNode.key() = destinationDisplay->displayId;
+    mMousePointersByDisplay.insert(std::move(pcNode));
+
+    // This will place cursor at the center of the target display for now
+    // TODO (b/367660694) place the cursor at the appropriate position in destination display
+    pc.setDisplayViewport(*destinationViewport);
+}
+
 void PointerChoreographer::processDrawingTabletEventLocked(const android::NotifyMotionArgs& args) {
     if (args.displayId == ui::LogicalDisplayId::INVALID) {
         return;
@@ -441,7 +485,8 @@ void PointerChoreographer::processDeviceReset(const NotifyDeviceResetArgs& args)
 }
 
 void PointerChoreographer::onControllerAddedOrRemovedLocked() {
-    if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows()) {
+    if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows() &&
+        !IS_TOPOLOGY_AWARE) {
         return;
     }
     bool requireListener = !mTouchPointersByDevice.empty() || !mMousePointersByDisplay.empty() ||
@@ -674,6 +719,10 @@ PointerChoreographer::calculatePointerDisplayChangeToNotify() {
 }
 
 void PointerChoreographer::setDefaultMouseDisplayId(ui::LogicalDisplayId displayId) {
+    if (IS_TOPOLOGY_AWARE) {
+        // default display will be set based on the topology
+        return;
+    }
     PointerDisplayChange pointerDisplayChange;
 
     { // acquire lock
@@ -887,6 +936,7 @@ void PointerChoreographer::PointerChoreographerDisplayInfoListener::onWindowInfo
         mPrivacySensitiveDisplays = std::move(newPrivacySensitiveDisplays);
         mPointerChoreographer->onPrivacySensitiveDisplaysChanged(mPrivacySensitiveDisplays);
     }
+    mPointerChoreographer->populateFakeDisplayTopology(windowInfosUpdate.displayInfos);
 }
 
 void PointerChoreographer::PointerChoreographerDisplayInfoListener::setInitialDisplayInfos(
@@ -907,4 +957,93 @@ void PointerChoreographer::PointerChoreographerDisplayInfoListener::
     mPointerChoreographer = nullptr;
 }
 
+void PointerChoreographer::populateFakeDisplayTopology(
+        const std::vector<gui::DisplayInfo>& displayInfos) {
+    if (!IS_TOPOLOGY_AWARE) {
+        return;
+    }
+    std::scoped_lock _lock(mLock);
+
+    if (displayInfos.size() == mTopology.size()) {
+        bool displaysChanged = false;
+        for (const auto& displayInfo : displayInfos) {
+            if (mTopology.find(displayInfo.displayId) == mTopology.end()) {
+                displaysChanged = true;
+                break;
+            }
+        }
+
+        if (!displaysChanged) {
+            return;
+        }
+    }
+
+    // create a fake topology assuming following order
+    // default-display (top-edge) -> next-display (right-edge) -> next-display (right-edge) ...
+    // ┌─────────┬─────────┐
+    // │ next    │ next 2  │ ...
+    // ├─────────┼─────────┘
+    // │ default │
+    // └─────────┘
+    mTopology.clear();
+
+    // treat default display as base, in real topology it should be the primary-display
+    ui::LogicalDisplayId previousDisplay = ui::LogicalDisplayId::DEFAULT;
+    for (const auto& displayInfo : displayInfos) {
+        if (displayInfo.displayId == ui::LogicalDisplayId::DEFAULT) {
+            continue;
+        }
+        if (previousDisplay == ui::LogicalDisplayId::DEFAULT) {
+            mTopology[previousDisplay].push_back({displayInfo.displayId, DisplayPosition::TOP, 0});
+            mTopology[displayInfo.displayId].push_back(
+                    {previousDisplay, DisplayPosition::BOTTOM, 0});
+        } else {
+            mTopology[previousDisplay].push_back(
+                    {displayInfo.displayId, DisplayPosition::RIGHT, 0});
+            mTopology[displayInfo.displayId].push_back({previousDisplay, DisplayPosition::LEFT, 0});
+        }
+        previousDisplay = displayInfo.displayId;
+    }
+
+    // update default pointer display. In real topology it should be the primary-display
+    if (mTopology.find(mDefaultMouseDisplayId) == mTopology.end()) {
+        mDefaultMouseDisplayId = ui::LogicalDisplayId::DEFAULT;
+    }
+}
+
+std::optional<PointerChoreographer::AdjacentDisplay>
+PointerChoreographer::findDestinationDisplayLocked(const DisplayViewport& sourceViewport,
+                                                   const vec2& unconsumedDelta) const {
+    DisplayPosition sourceBoundary;
+    if (unconsumedDelta.x > 0) {
+        sourceBoundary = DisplayPosition::RIGHT;
+    } else if (unconsumedDelta.x < 0) {
+        sourceBoundary = DisplayPosition::LEFT;
+    } else if (unconsumedDelta.y > 0) {
+        sourceBoundary = DisplayPosition::BOTTOM;
+    } else {
+        sourceBoundary = DisplayPosition::TOP;
+    }
+
+    // Choreographer works in un-rotate coordinate space so we need to rotate boundary by viewport
+    // orientation to find the rotated boundary
+    constexpr int MOD = ftl::to_underlying(ui::Rotation::ftl_last) + 1;
+    sourceBoundary = static_cast<DisplayPosition>(
+            (ftl::to_underlying(sourceBoundary) + ftl::to_underlying(sourceViewport.orientation)) %
+            MOD);
+
+    if (mTopology.find(sourceViewport.displayId) == mTopology.end()) {
+        // Topology is likely out of sync with viewport info, wait for them to be updated
+        LOG(WARNING) << "Source display missing from topology " << sourceViewport.displayId;
+        return std::nullopt;
+    }
+
+    for (const auto& adjacentDisplay : mTopology.at(sourceViewport.displayId)) {
+        if (adjacentDisplay.position == sourceBoundary) {
+            return adjacentDisplay;
+        }
+    }
+    return std::nullopt;
+}
+
 } // namespace android

services/inputflinger/PointerChoreographer.h

@@ -137,13 +137,41 @@ class PointerChoreographer : public PointerChoreographerInterface {
     void processTouchscreenAndStylusEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);
     void processStylusHoverEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);
     void processDeviceReset(const NotifyDeviceResetArgs& args);
+    void processPointerDeviceMotionEventLocked(NotifyMotionArgs& newArgs,
+                                               PointerControllerInterface& pc) REQUIRES(mLock);
     void onControllerAddedOrRemovedLocked() REQUIRES(mLock);
     void onPrivacySensitiveDisplaysChangedLocked(
             const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays)
             REQUIRES(mLock);
     void onPrivacySensitiveDisplaysChanged(
             const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays);
 
+    void handleUnconsumedDeltaLocked(PointerControllerInterface& pc, const vec2& unconsumedDelta)
+            REQUIRES(mLock);
+
+    // TODO(b/362719483) remove these when real topology is available
+    enum class DisplayPosition : int32_t {
+        RIGHT = 0,
+        TOP = 1,
+        LEFT = 2,
+        BOTTOM = 3,
+        ftl_last = BOTTOM,
+    };
+
+    struct AdjacentDisplay {
+        ui::LogicalDisplayId displayId;
+        DisplayPosition position;
+        float offsetPx;
+    };
+    void populateFakeDisplayTopology(const std::vector<gui::DisplayInfo>& displayInfos);
+
+    std::optional<AdjacentDisplay> findDestinationDisplayLocked(
+            const DisplayViewport& sourceViewport, const vec2& unconsumedDelta) const
+            REQUIRES(mLock);
+
+    std::unordered_map<ui::LogicalDisplayId, std::vector<AdjacentDisplay>> mTopology
+            GUARDED_BY(mLock);
+
     /* This listener keeps tracks of visible privacy sensitive displays and updates the
      * choreographer if there are any changes.
      *
@@ -211,6 +239,7 @@ class PointerChoreographer : public PointerChoreographerInterface {
                                   const WindowListenerUnregisterConsumer& unregisterListener);
 
 private:
+    const static bool IS_TOPOLOGY_AWARE;
     const WindowListenerRegisterConsumer mRegisterListener;
     const WindowListenerUnregisterConsumer mUnregisterListener;
 };

services/inputflinger/include/PointerControllerInterface.h

@@ -72,8 +72,12 @@ class PointerControllerInterface {
     /* Dumps the state of the pointer controller. */
     virtual std::string dump() = 0;
 
-    /* Move the pointer. */
-    virtual void move(float deltaX, float deltaY) = 0;
+    /* Move the pointer and return unconsumed delta if the pointer has crossed the current
+     * viewport bounds .
+     *
+     * Return value may be used to move pointer to corresponding adjacent display, if it exists in
+     * the display-topology */
+    [[nodiscard]] virtual vec2 move(float deltaX, float deltaY) = 0;
 
     /* Sets the absolute location of the pointer. */
     virtual void setPosition(float x, float y) = 0;

services/inputflinger/tests/FakePointerController.cpp

@@ -148,15 +148,17 @@ bool FakePointerController::isPointerShown() {
     return mIsPointerShown;
 }
 
-void FakePointerController::move(float deltaX, float deltaY) {
-    if (!mEnabled) return;
+vec2 FakePointerController::move(float deltaX, float deltaY) {
+    if (!mEnabled) return {};
 
     mX += deltaX;
     if (mX < mMinX) mX = mMinX;
     if (mX > mMaxX) mX = mMaxX;
     mY += deltaY;
     if (mY < mMinY) mY = mMinY;
     if (mY > mMaxY) mY = mMaxY;
+
+    return {};
 }
 
 void FakePointerController::fade(Transition) {

services/inputflinger/tests/FakePointerController.h

@@ -65,7 +65,7 @@ class FakePointerController : public PointerControllerInterface {
 
 private:
     std::string dump() override { return ""; }
-    void move(float deltaX, float deltaY) override;
+    vec2 move(float deltaX, float deltaY) override;
     void unfade(Transition) override;
     void setPresentation(Presentation) override {}
     void setSpots(const PointerCoords*, const uint32_t*, BitSet32 spotIdBits,