Merge changes I41cb79ea,I12eb8eae into main

* changes:
  Add logic to overwrite pointer coordinates if event time is too old
  Add logic to overwrite pointer coordinates in motion event

Author: Treehugger Robot

include/input/Resampler.h

@@ -99,6 +99,17 @@ class LegacyResampler final : public Resampler {
      */
     RingBuffer<Sample> mLatestSamples{/*capacity=*/2};
 
+    /**
+     * Latest sample in mLatestSamples after resampling motion event. Used to compare if a pointer
+     * does not move between samples.
+     */
+    std::optional<Sample> mLastRealSample;
+
+    /**
+     * Latest prediction. Used to overwrite motion event samples if a set of conditions is met.
+     */
+    std::optional<Sample> mPreviousPrediction;
+
     /**
      * Adds up to mLatestSamples.capacity() of motionEvent's latest samples to mLatestSamples. If
      * motionEvent has fewer samples than mLatestSamples.capacity(), then the available samples are
@@ -144,6 +155,23 @@ class LegacyResampler final : public Resampler {
      */
     std::optional<Sample> attemptExtrapolation(std::chrono::nanoseconds resampleTime) const;
 
+    /**
+     * Iterates through motion event samples, and calls overwriteStillPointers on each sample.
+     */
+    void overwriteMotionEventSamples(MotionEvent& motionEvent) const;
+
+    /**
+     * Overwrites with resampled data the pointer coordinates that did not move between motion event
+     * samples, that is, both x and y values are identical to mLastRealSample.
+     */
+    void overwriteStillPointers(MotionEvent& motionEvent, size_t sampleIndex) const;
+
+    /**
+     * Overwrites the pointer coordinates of a sample with event time older than
+     * that of mPreviousPrediction.
+     */
+    void overwriteOldPointers(MotionEvent& motionEvent, size_t sampleIndex) const;
+
     inline static void addSampleToMotionEvent(const Sample& sample, MotionEvent& motionEvent);
 };
 } // namespace android

libs/input/Resampler.cpp

@@ -18,6 +18,7 @@
 
 #include <algorithm>
 #include <chrono>
+#include <ostream>
 
 #include <android-base/logging.h>
 #include <android-base/properties.h>
@@ -26,10 +27,7 @@
 #include <input/Resampler.h>
 #include <utils/Timers.h>
 
-using std::chrono::nanoseconds;
-
 namespace android {
-
 namespace {
 
 const bool IS_DEBUGGABLE_BUILD =
@@ -49,6 +47,8 @@ bool debugResampling() {
     return __android_log_is_loggable(ANDROID_LOG_DEBUG, LOG_TAG "Resampling", ANDROID_LOG_INFO);
 }
 
+using std::chrono::nanoseconds;
+
 constexpr std::chrono::milliseconds RESAMPLE_LATENCY{5};
 
 constexpr std::chrono::milliseconds RESAMPLE_MIN_DELTA{2};
@@ -75,6 +75,31 @@ PointerCoords calculateResampledCoords(const PointerCoords& a, const PointerCoor
     resampledCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, lerp(a.getY(), b.getY(), alpha));
     return resampledCoords;
 }
+
+bool equalXY(const PointerCoords& a, const PointerCoords& b) {
+    return (a.getX() == b.getX()) && (a.getY() == b.getY());
+}
+
+void setMotionEventPointerCoords(MotionEvent& motionEvent, size_t sampleIndex, size_t pointerIndex,
+                                 const PointerCoords& pointerCoords) {
+    // Ideally, we should not cast away const. In this particular case, it's safe to cast away const
+    // and dereference getHistoricalRawPointerCoords returned pointer because motionEvent is a
+    // nonconst reference to a MotionEvent object, so mutating the object should not be undefined
+    // behavior; moreover, the invoked method guarantees to return a valid pointer. Otherwise, it
+    // fatally logs. Alternatively, we could've created a new MotionEvent from scratch, but this
+    // approach is simpler and more efficient.
+    PointerCoords& motionEventCoords = const_cast<PointerCoords&>(
+            *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex)));
+    motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_X, pointerCoords.getX());
+    motionEventCoords.setAxisValue(AMOTION_EVENT_AXIS_Y, pointerCoords.getY());
+    motionEventCoords.isResampled = pointerCoords.isResampled;
+}
+
+std::ostream& operator<<(std::ostream& os, const PointerCoords& pointerCoords) {
+    os << "(" << pointerCoords.getX() << ", " << pointerCoords.getY() << ")";
+    return os;
+}
+
 } // namespace
 
 void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {
@@ -85,12 +110,9 @@ void LegacyResampler::updateLatestSamples(const MotionEvent& motionEvent) {
         std::vector<Pointer> pointers;
         const size_t numPointers = motionEvent.getPointerCount();
         for (size_t pointerIndex = 0; pointerIndex < numPointers; ++pointerIndex) {
-            // getSamplePointerCoords is the vector representation of a getHistorySize by
-            // getPointerCount matrix.
-            const PointerCoords& pointerCoords =
-                    motionEvent.getSamplePointerCoords()[sampleIndex * numPointers + pointerIndex];
-            pointers.push_back(
-                    Pointer{*motionEvent.getPointerProperties(pointerIndex), pointerCoords});
+            pointers.push_back(Pointer{*(motionEvent.getPointerProperties(pointerIndex)),
+                                       *(motionEvent.getHistoricalRawPointerCoords(pointerIndex,
+                                                                                   sampleIndex))});
         }
         mLatestSamples.pushBack(
                 Sample{nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)}, pointers});
@@ -245,6 +267,47 @@ nanoseconds LegacyResampler::getResampleLatency() const {
     return RESAMPLE_LATENCY;
 }
 
+void LegacyResampler::overwriteMotionEventSamples(MotionEvent& motionEvent) const {
+    const size_t numSamples = motionEvent.getHistorySize() + 1;
+    for (size_t sampleIndex = 0; sampleIndex < numSamples; ++sampleIndex) {
+        overwriteStillPointers(motionEvent, sampleIndex);
+        overwriteOldPointers(motionEvent, sampleIndex);
+    }
+}
+
+void LegacyResampler::overwriteStillPointers(MotionEvent& motionEvent, size_t sampleIndex) const {
+    for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount(); ++pointerIndex) {
+        const PointerCoords& pointerCoords =
+                *(motionEvent.getHistoricalRawPointerCoords(pointerIndex, sampleIndex));
+        if (equalXY(mLastRealSample->pointers[pointerIndex].coords, pointerCoords)) {
+            LOG_IF(INFO, debugResampling())
+                    << "Pointer ID: " << motionEvent.getPointerId(pointerIndex)
+                    << " did not move. Overwriting its coordinates from " << pointerCoords << " to "
+                    << mLastRealSample->pointers[pointerIndex].coords;
+            setMotionEventPointerCoords(motionEvent, sampleIndex, pointerIndex,
+                                        mPreviousPrediction->pointers[pointerIndex].coords);
+        }
+    }
+}
+
+void LegacyResampler::overwriteOldPointers(MotionEvent& motionEvent, size_t sampleIndex) const {
+    if (!mPreviousPrediction.has_value()) {
+        return;
+    }
+    if (nanoseconds{motionEvent.getHistoricalEventTime(sampleIndex)} <
+        mPreviousPrediction->eventTime) {
+        LOG_IF(INFO, debugResampling())
+                << "Motion event sample older than predicted sample. Overwriting event time from "
+                << motionEvent.getHistoricalEventTime(sampleIndex) << "ns to "
+                << mPreviousPrediction->eventTime.count() << "ns.";
+        for (size_t pointerIndex = 0; pointerIndex < motionEvent.getPointerCount();
+             ++pointerIndex) {
+            setMotionEventPointerCoords(motionEvent, sampleIndex, pointerIndex,
+                                        mPreviousPrediction->pointers[pointerIndex].coords);
+        }
+    }
+}
+
 void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& motionEvent,
                                           const InputMessage* futureSample) {
     const nanoseconds resampleTime = frameTime - RESAMPLE_LATENCY;
@@ -261,6 +324,16 @@ void LegacyResampler::resampleMotionEvent(nanoseconds frameTime, MotionEvent& mo
             : (attemptExtrapolation(resampleTime));
     if (sample.has_value()) {
         addSampleToMotionEvent(*sample, motionEvent);
+        if (mPreviousPrediction.has_value()) {
+            overwriteMotionEventSamples(motionEvent);
+        }
+        // mPreviousPrediction is only updated whenever extrapolation occurs because extrapolation
+        // is about predicting upcoming scenarios.
+        if (futureSample == nullptr) {
+            mPreviousPrediction = sample;
+        }
     }
+    mLastRealSample = *(mLatestSamples.end() - 1);
 }
+
 } // namespace android

libs/input/tests/InputConsumerResampling_test.cpp

@@ -197,8 +197,6 @@ TEST_F(InputConsumerResamplingTest, EventIsResampled) {
     mClientTestChannel->enqueueMessage(nextPointerMessage(
             {0ms, {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}}, AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent({InputEventEntry{0ms,
                                                {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}},
@@ -238,8 +236,6 @@ TEST_F(InputConsumerResamplingTest, EventIsResampledWithDifferentId) {
     mClientTestChannel->enqueueMessage(nextPointerMessage(
             {0ms, {Pointer{.id = 1, .x = 10.0f, .y = 20.0f}}, AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent({InputEventEntry{0ms,
                                                {Pointer{.id = 1, .x = 10.0f, .y = 20.0f}},
@@ -280,8 +276,6 @@ TEST_F(InputConsumerResamplingTest, StylusEventIsResampled) {
              {Pointer{.id = 0, .x = 10.0f, .y = 20.0f, .toolType = ToolType::STYLUS}},
              AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent({InputEventEntry{0ms,
                                                {Pointer{.id = 0,
@@ -338,8 +332,6 @@ TEST_F(InputConsumerResamplingTest, MouseEventIsResampled) {
              {Pointer{.id = 0, .x = 10.0f, .y = 20.0f, .toolType = ToolType::MOUSE}},
              AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent({InputEventEntry{0ms,
                                                {Pointer{.id = 0,
@@ -396,8 +388,6 @@ TEST_F(InputConsumerResamplingTest, PalmEventIsNotResampled) {
              {Pointer{.id = 0, .x = 10.0f, .y = 20.0f, .toolType = ToolType::PALM}},
              AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent(
             {InputEventEntry{0ms,
@@ -438,8 +428,6 @@ TEST_F(InputConsumerResamplingTest, SampleTimeEqualsEventTime) {
     mClientTestChannel->enqueueMessage(nextPointerMessage(
             {0ms, {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}}, AMOTION_EVENT_ACTION_DOWN}));
 
-    mClientTestChannel->assertNoSentMessages();
-
     invokeLooperCallback();
     assertReceivedMotionEvent({InputEventEntry{0ms,
                                                {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}},
@@ -468,4 +456,114 @@ TEST_F(InputConsumerResamplingTest, SampleTimeEqualsEventTime) {
     mClientTestChannel->assertFinishMessage(/*seq=*/3, /*handled=*/true);
 }
 
+/**
+ * Once we send a resampled value to the app, we should continue to send the last predicted value if
+ * a pointer does not move. Only real values are used to determine if a pointer does not move.
+ */
+TEST_F(InputConsumerResamplingTest, ResampledValueIsUsedForIdenticalCoordinates) {
+    // Send the initial ACTION_DOWN separately, so that the first consumed event will only return an
+    // InputEvent with a single action.
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {0ms, {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}}, AMOTION_EVENT_ACTION_DOWN}));
+
+    invokeLooperCallback();
+    assertReceivedMotionEvent({InputEventEntry{0ms,
+                                               {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}},
+                                               AMOTION_EVENT_ACTION_DOWN}});
+
+    // Two ACTION_MOVE events 10 ms apart that move in X direction and stay still in Y
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {10ms, {Pointer{.id = 0, .x = 20.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {20ms, {Pointer{.id = 0, .x = 30.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+
+    invokeLooperCallback();
+    mConsumer->consumeBatchedInputEvents(nanoseconds{35ms}.count());
+    assertReceivedMotionEvent(
+            {InputEventEntry{10ms,
+                             {Pointer{.id = 0, .x = 20.0f, .y = 30.0f}},
+                             AMOTION_EVENT_ACTION_MOVE},
+             InputEventEntry{20ms,
+                             {Pointer{.id = 0, .x = 30.0f, .y = 30.0f}},
+                             AMOTION_EVENT_ACTION_MOVE},
+             InputEventEntry{25ms,
+                             {Pointer{.id = 0, .x = 35.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}});
+
+    // Coordinate value 30 has been resampled to 35. When a new event comes in with value 30 again,
+    // the system should still report 35.
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {40ms, {Pointer{.id = 0, .x = 30.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+
+    invokeLooperCallback();
+    mConsumer->consumeBatchedInputEvents(nanoseconds{45ms + 5ms /*RESAMPLE_LATENCY*/}.count());
+    assertReceivedMotionEvent(
+            {InputEventEntry{40ms,
+                             {Pointer{.id = 0, .x = 35.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}, // original event, rewritten
+             InputEventEntry{45ms,
+                             {Pointer{.id = 0, .x = 35.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}}); // resampled event, rewritten
+
+    mClientTestChannel->assertFinishMessage(/*seq=*/1, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/2, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/3, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/4, /*handled=*/true);
+}
+
+TEST_F(InputConsumerResamplingTest, OldEventReceivedAfterResampleOccurs) {
+    // Send the initial ACTION_DOWN separately, so that the first consumed event will only return an
+    // InputEvent with a single action.
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {0ms, {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}}, AMOTION_EVENT_ACTION_DOWN}));
+
+    invokeLooperCallback();
+    assertReceivedMotionEvent({InputEventEntry{0ms,
+                                               {Pointer{.id = 0, .x = 10.0f, .y = 20.0f}},
+                                               AMOTION_EVENT_ACTION_DOWN}});
+
+    // Two ACTION_MOVE events 10 ms apart that move in X direction and stay still in Y
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {10ms, {Pointer{.id = 0, .x = 20.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {20ms, {Pointer{.id = 0, .x = 30.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+
+    invokeLooperCallback();
+    mConsumer->consumeBatchedInputEvents(nanoseconds{35ms}.count());
+    assertReceivedMotionEvent(
+            {InputEventEntry{10ms,
+                             {Pointer{.id = 0, .x = 20.0f, .y = 30.0f}},
+                             AMOTION_EVENT_ACTION_MOVE},
+             InputEventEntry{20ms,
+                             {Pointer{.id = 0, .x = 30.0f, .y = 30.0f}},
+                             AMOTION_EVENT_ACTION_MOVE},
+             InputEventEntry{25ms,
+                             {Pointer{.id = 0, .x = 35.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}});
+
+    // Above, the resampled event is at 25ms rather than at 30 ms = 35ms - RESAMPLE_LATENCY
+    // because we are further bound by how far we can extrapolate by the "last time delta".
+    // That's 50% of (20 ms - 10ms) => 5ms. So we can't predict more than 5 ms into the future
+    // from the event at 20ms, which is why the resampled event is at t = 25 ms.
+
+    // We resampled the event to 25 ms. Now, an older 'real' event comes in.
+    mClientTestChannel->enqueueMessage(nextPointerMessage(
+            {24ms, {Pointer{.id = 0, .x = 40.0f, .y = 30.0f}}, AMOTION_EVENT_ACTION_MOVE}));
+
+    invokeLooperCallback();
+    mConsumer->consumeBatchedInputEvents(nanoseconds{50ms}.count());
+    assertReceivedMotionEvent(
+            {InputEventEntry{24ms,
+                             {Pointer{.id = 0, .x = 35.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}, // original event, rewritten
+             InputEventEntry{26ms,
+                             {Pointer{.id = 0, .x = 45.0f, .y = 30.0f, .isResampled = true}},
+                             AMOTION_EVENT_ACTION_MOVE}}); // resampled event, rewritten
+
+    mClientTestChannel->assertFinishMessage(/*seq=*/1, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/2, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/3, /*handled=*/true);
+    mClientTestChannel->assertFinishMessage(/*seq=*/4, /*handled=*/true);
+}
+
 } // namespace android