Resample scroll event in compositor thread event queue

This CL adds scroll predictor in compositor thread under flag
kResamplingScrollEvents.

design doc:
https://docs.google.com/document/d/1g6eGYdLZKznF3sCiktc5BbAPeiudWk8xTzx3qdrxowI/edit#

Bug: 836352
Change-Id: Iecfad7ad75dd4b871426e00a2f2d542996544b07
Reviewed-on: https://chromium-review.googlesource.com/919662
Commit-Queue: Ella Ge <eirage@chromium.org>
Reviewed-by: Sahel Sharifymoghaddam <sahel@chromium.org>
Reviewed-by: Dave Tapuska <dtapuska@chromium.org>
Reviewed-by: Navid Zolghadr <nzolghadr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#568003}

ui/events/BUILD.gn

@@ -447,6 +447,7 @@ if (!is_ios) {
       "blink/input_handler_proxy_unittest.cc",
       "blink/input_scroll_elasticity_controller_unittest.cc",
       "blink/prediction/least_squares_predictor_unittest.cc",
+      "blink/scroll_predictor_unittest.cc",
       "blink/web_input_event_traits_unittest.cc",
       "blink/web_input_event_unittest.cc",
       "cocoa/events_mac_unittest.mm",

ui/events/blink/BUILD.gn

@@ -29,6 +29,8 @@ jumbo_source_set("blink") {
     "prediction/input_predictor.h",
     "prediction/least_squares_predictor.cc",
     "prediction/least_squares_predictor.h",
+    "scroll_predictor.cc",
+    "scroll_predictor.h",
     "synchronous_input_handler_proxy.h",
     "web_input_event.cc",
     "web_input_event.h",

ui/events/blink/blink_features.cc

@@ -11,6 +11,9 @@ namespace features {
 const base::Feature kVsyncAlignedInputEvents{"VsyncAlignedInput",
                                              base::FEATURE_ENABLED_BY_DEFAULT};
 
+const base::Feature kResamplingScrollEvents{"ResamplingScrollEvents",
+                                            base::FEATURE_DISABLED_BY_DEFAULT};
+
 const base::Feature kSendMouseLeaveEvents{"SendMouseLeaveEvents",
                                           base::FEATURE_ENABLED_BY_DEFAULT};
 }

ui/events/blink/blink_features.h

@@ -11,6 +11,9 @@ namespace features {
 
 extern const base::Feature kVsyncAlignedInputEvents;
 
+// Enables resampling GestureScroll events on compositor thread.
+extern const base::Feature kResamplingScrollEvents;
+
 // This feature allows native ET_MOUSE_EXIT events to be passed
 // through to blink as mouse leave events. Traditionally these events were
 // converted to mouse move events due to a number of inconsistencies on

ui/events/blink/event_with_callback.h

@@ -49,6 +49,7 @@ class EventWithCallback {
                     std::unique_ptr<DidOverscrollParams>);
 
   const blink::WebInputEvent& event() const { return *event_; }
+  blink::WebInputEvent* event_pointer() { return event_.get(); }
   const LatencyInfo latency_info() const { return latency_; }
   base::TimeTicks creation_timestamp() const { return creation_timestamp_; }
   base::TimeTicks last_coalesced_timestamp() const {

ui/events/blink/input_handler_proxy.cc

@@ -28,6 +28,7 @@
 #include "ui/events/blink/fling_booster.h"
 #include "ui/events/blink/input_handler_proxy_client.h"
 #include "ui/events/blink/input_scroll_elasticity_controller.h"
+#include "ui/events/blink/scroll_predictor.h"
 #include "ui/events/blink/web_input_event_traits.h"
 #include "ui/gfx/geometry/point_conversions.h"
 #include "ui/latency/latency_info.h"
@@ -216,6 +217,10 @@ InputHandlerProxy::InputHandlerProxy(
       base::FeatureList::IsEnabled(features::kVsyncAlignedInputEvents)
           ? std::make_unique<CompositorThreadEventQueue>()
           : nullptr;
+  scroll_predictor_ =
+      base::FeatureList::IsEnabled(features::kResamplingScrollEvents)
+          ? std::make_unique<ScrollPredictor>()
+          : nullptr;
 }
 
 InputHandlerProxy::~InputHandlerProxy() {}
@@ -303,6 +308,10 @@ void InputHandlerProxy::DispatchSingleInputEvent(
     const base::TimeTicks now) {
   if (compositor_event_queue_ &&
       IsGestureScrollOrFlingOrPinch(event_with_callback->event().GetType())) {
+    if (scroll_predictor_)
+      scroll_predictor_->HandleEvent(event_with_callback->original_events(),
+                                     now, event_with_callback->event_pointer());
+
     // Report the coalesced count only for continuous events to avoid the noise
     // from non-continuous events.
     if (IsContinuousGestureEvent(event_with_callback->event().GetType())) {

ui/events/blink/input_handler_proxy.h

@@ -42,6 +42,7 @@ class EventWithCallback;
 class FlingBooster;
 class InputHandlerProxyClient;
 class InputScrollElasticityController;
+class ScrollPredictor;
 class SynchronousInputHandler;
 class SynchronousInputHandlerProxy;
 struct DidOverscrollParams;
@@ -277,6 +278,8 @@ class InputHandlerProxy : public cc::InputHandlerClient,
 
   std::unique_ptr<cc::SnapFlingController> snap_fling_controller_;
 
+  std::unique_ptr<ScrollPredictor> scroll_predictor_;
+
   DISALLOW_COPY_AND_ASSIGN(InputHandlerProxy);
 };
 

ui/events/blink/input_handler_proxy_unittest.cc

@@ -32,6 +32,7 @@
 #include "ui/events/blink/event_with_callback.h"
 #include "ui/events/blink/input_handler_proxy.h"
 #include "ui/events/blink/input_handler_proxy_client.h"
+#include "ui/events/blink/scroll_predictor.h"
 #include "ui/events/blink/web_input_event_traits.h"
 #include "ui/gfx/geometry/scroll_offset.h"
 #include "ui/gfx/geometry/size_f.h"
@@ -515,6 +516,8 @@ class InputHandlerProxyEventQueueTest : public testing::TestWithParam<bool> {
     if (input_handler_proxy_->compositor_event_queue_)
       input_handler_proxy_->compositor_event_queue_ =
           std::make_unique<CompositorThreadEventQueue>();
+    input_handler_proxy_->scroll_predictor_ =
+        std::make_unique<ScrollPredictor>();
   }
 
   void HandleGestureEvent(WebInputEvent::Type type,
@@ -558,6 +561,13 @@ class InputHandlerProxyEventQueueTest : public testing::TestWithParam<bool> {
     input_handler_proxy_->SetTickClockForTesting(tick_clock);
   }
 
+  bool GestureScrollEventPredictionAvailable(
+      ui::InputPredictor::InputData* result) {
+    return input_handler_proxy_->scroll_predictor_->predictor_
+        ->GeneratePrediction(WebInputEvent::GetStaticTimeStampForTests(),
+                             result);
+  }
+
  protected:
   base::test::ScopedFeatureList feature_list_;
   testing::StrictMock<MockInputHandler> mock_input_handler_;
@@ -2143,6 +2153,43 @@ TEST_P(InputHandlerProxyEventQueueTest, CoalescedEventSwitchToMainThread) {
   testing::Mock::VerifyAndClearExpectations(&mock_input_handler_);
 }
 
+TEST_P(InputHandlerProxyEventQueueTest, ScrollPredictorTest) {
+  cc::InputHandlerScrollResult scroll_result_did_scroll_;
+  scroll_result_did_scroll_.did_scroll = true;
+  EXPECT_CALL(mock_input_handler_, ScrollBegin(testing::_, testing::_))
+      .WillOnce(testing::Return(kImplThreadScrollState));
+  EXPECT_CALL(mock_input_handler_, SetNeedsAnimateInput()).Times(1);
+  EXPECT_CALL(
+      mock_input_handler_,
+      ScrollBy(testing::Property(&cc::ScrollState::delta_y, testing::Gt(0))))
+      .WillOnce(testing::Return(scroll_result_did_scroll_));
+
+  // No prediction when start with a GSB
+  ui::InputPredictor::InputData result;
+  HandleGestureEvent(WebInputEvent::kGestureScrollBegin);
+  input_handler_proxy_->DeliverInputForBeginFrame();
+  EXPECT_FALSE(GestureScrollEventPredictionAvailable(&result));
+
+  // Test predictor returns last GSU delta.
+  HandleGestureEvent(WebInputEvent::kGestureScrollUpdate, -20);
+  HandleGestureEvent(WebInputEvent::kGestureScrollUpdate, -15);
+  input_handler_proxy_->DeliverInputForBeginFrame();
+  EXPECT_TRUE(GestureScrollEventPredictionAvailable(&result));
+  EXPECT_EQ(-35, result.pos_y);
+
+  testing::Mock::VerifyAndClearExpectations(&mock_input_handler_);
+
+  // Predictor has been reset after a new GSB.
+  EXPECT_CALL(mock_input_handler_, SetNeedsAnimateInput()).Times(1);
+  EXPECT_CALL(mock_input_handler_, ScrollBegin(testing::_, testing::_))
+      .WillOnce(testing::Return(kImplThreadScrollState));
+  HandleGestureEvent(WebInputEvent::kGestureScrollBegin);
+  input_handler_proxy_->DeliverInputForBeginFrame();
+  EXPECT_FALSE(GestureScrollEventPredictionAvailable(&result));
+
+  testing::Mock::VerifyAndClearExpectations(&mock_input_handler_);
+}
+
 INSTANTIATE_TEST_CASE_P(AnimateInput,
                         InputHandlerProxyTest,
                         testing::ValuesIn(test_types));

ui/events/blink/scroll_predictor.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ui/events/blink/scroll_predictor.h"
+
+#include "base/metrics/field_trial.h"
+#include "base/metrics/field_trial_params.h"
+#include "ui/events/blink/prediction/empty_predictor.h"
+#include "ui/events/blink/prediction/least_squares_predictor.h"
+
+using blink::WebInputEvent;
+using blink::WebGestureEvent;
+
+namespace ui {
+
+namespace {
+
+constexpr char kPredictor[] = "predictor";
+constexpr char kScrollPredictorTypeLsq[] = "lsq";
+
+}  // namespace
+
+ScrollPredictor::ScrollPredictor() {
+  std::string predictor_type_ = GetFieldTrialParamValueByFeature(
+      features::kResamplingScrollEvents, kPredictor);
+  if (predictor_type_ == kScrollPredictorTypeLsq)
+    predictor_ = std::make_unique<LeastSquaresPredictor>();
+  else
+    predictor_ = std::make_unique<EmptyPredictor>();
+}
+
+ScrollPredictor::~ScrollPredictor() = default;
+
+void ScrollPredictor::HandleEvent(
+    const EventWithCallback::OriginalEventList& original_events,
+    base::TimeTicks frame_time,
+    WebInputEvent* event) {
+  if (event->GetType() == WebInputEvent::kGestureScrollBegin) {
+    predictor_->Reset();
+    current_accumulated_delta_ = gfx::PointF();
+    last_accumulated_delta_ = gfx::PointF();
+  } else if (event->GetType() == WebInputEvent::kGestureScrollUpdate) {
+    // TODO(eirage): When scroll events are coalesced with pinch, we can have
+    // empty original event list. In that case, we can't use the original events
+    // to update the prediction. We don't want to use the aggregated event to
+    // update because of the event time stamp, so skip the prediction for now.
+    if (original_events.empty())
+      return;
+
+    for (auto& coalesced_event : original_events)
+      UpdatePrediction(coalesced_event.event_);
+    ResampleEvent(frame_time, event);
+  }
+}
+
+void ScrollPredictor::UpdatePrediction(const WebScopedInputEvent& event) {
+  DCHECK(event->GetType() == WebInputEvent::kGestureScrollUpdate);
+  const WebGestureEvent& gesture_event =
+      static_cast<const WebGestureEvent&>(*event);
+  current_accumulated_delta_.Offset(gesture_event.data.scroll_update.delta_x,
+                                    gesture_event.data.scroll_update.delta_y);
+  InputPredictor::InputData data = {current_accumulated_delta_.x(),
+                                    current_accumulated_delta_.y(),
+                                    gesture_event.TimeStamp()};
+  predictor_->Update(data);
+}
+
+void ScrollPredictor::ResampleEvent(base::TimeTicks time_stamp,
+                                    WebInputEvent* event) {
+  DCHECK(event->GetType() == WebInputEvent::kGestureScrollUpdate);
+  WebGestureEvent* gesture_event = static_cast<WebGestureEvent*>(event);
+
+  InputPredictor::InputData result;
+  if (predictor_->HasPrediction() &&
+      predictor_->GeneratePrediction(time_stamp, &result)) {
+    gfx::PointF predicted_accumulated_delta_ =
+        gfx::PointF(result.pos_x, result.pos_y);
+    gesture_event->data.scroll_update.delta_x =
+        predicted_accumulated_delta_.x() - last_accumulated_delta_.x();
+    gesture_event->data.scroll_update.delta_y =
+        predicted_accumulated_delta_.y() - last_accumulated_delta_.y();
+    gesture_event->SetTimeStamp(time_stamp);
+    last_accumulated_delta_ = predicted_accumulated_delta_;
+  } else {
+    last_accumulated_delta_.Offset(gesture_event->data.scroll_update.delta_x,
+                                   gesture_event->data.scroll_update.delta_y);
+  }
+}
+
+}  // namespace ui

ui/events/blink/scroll_predictor.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef UI_EVENTS_BLINK_SCROLL_PREDICTOR_H_
+#define UI_EVENTS_BLINK_SCROLL_PREDICTOR_H_
+
+#include <vector>
+
+#include "ui/events/base_event_utils.h"
+#include "ui/events/blink/event_with_callback.h"
+#include "ui/events/blink/prediction/input_predictor.h"
+
+namespace ui {
+
+namespace test {
+class ScrollPredictorTest;
+}
+
+// This class handles resampling GestureScrollUpdate events on InputHandlerProxy
+// at |BeginFrame| signal, before events been dispatched. The predictor use
+// original events to update the prediction and align the aggregated event
+// timestamp and delta_x/y to the VSync time.
+class ScrollPredictor {
+ public:
+  ScrollPredictor();
+  ~ScrollPredictor();
+
+  // Resampling gesture scroll events. Each prediction starts with a GSB.
+  // On each GSU, updates the prediction with events in original events list,
+  // and apply the prediction to the GSU event.
+  void HandleEvent(const EventWithCallback::OriginalEventList& original_events,
+                   base::TimeTicks frame_time,
+                   blink::WebInputEvent* event);
+
+ private:
+  friend class test::InputHandlerProxyEventQueueTest;
+  friend class test::ScrollPredictorTest;
+
+  // Update the prediction with GestureScrollUpdate deltaX and deltaY
+  void UpdatePrediction(const WebScopedInputEvent& event);
+
+  // Apply resampled deltaX/deltaY to gesture events
+  void ResampleEvent(base::TimeTicks frame_time, blink::WebInputEvent* event);
+
+  std::unique_ptr<InputPredictor> predictor_;
+
+  // Total scroll delta, used for prediction. Reset when GestureScrollBegin
+  gfx::PointF current_accumulated_delta_;
+  // Accumulated delta from last vsync, use to calculate delta_x and delta_y for
+  // the aggregated event.
+  gfx::PointF last_accumulated_delta_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScrollPredictor);
+};
+
+}  // namespace ui
+
+#endif  // UI_EVENTS_BLINK_SCROLL_PREDICTOR_H_