Filter noise from the time interval between the events in the Kalman predictor.

Time intervals are inconsistent due to the event's timestamp being determined by the
OS once its free to process such event. This causes artifacts in the prediction,
specifically overshoot. In order to mitigate this, time intervals are fed into a
Kalman filter instead of being used directly.

Change-Id: Iec1050804939a56dea8b8593287351371a26ef81
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1653650Reviewed-by: Ella Ge <eirage@chromium.org>
Reviewed-by: Navid Zolghadr <nzolghadr@chromium.org>
Reviewed-by: Avi Drissman <avi@chromium.org>
Commit-Queue: Luis Sanchez Padilla <lusanpad@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#671775}

chrome/browser/about_flags.cc

@@ -968,11 +968,17 @@ const FeatureEntry::FeatureParam kResamplingInputEventsLSQEnabled[] = {
 const FeatureEntry::FeatureParam kResamplingInputEventsKalmanEnabled[] = {
     {"predictor", "kalman"}};
 
+const FeatureEntry::FeatureParam
+    kResamplingInputEventsKalmanTimeFilteredEnabled[] = {
+        {"predictor", "kalman_time_filtered"}};
+
 const FeatureEntry::FeatureVariation kResamplingInputEventsFeatureVariations[] =
     {{"lsq", kResamplingInputEventsLSQEnabled,
       base::size(kResamplingInputEventsLSQEnabled), nullptr},
      {"kalman", kResamplingInputEventsKalmanEnabled,
-      base::size(kResamplingInputEventsKalmanEnabled), nullptr}};
+      base::size(kResamplingInputEventsKalmanEnabled), nullptr},
+     {"kalman time filtered", kResamplingInputEventsKalmanTimeFilteredEnabled,
+      base::size(kResamplingInputEventsKalmanTimeFilteredEnabled), nullptr}};
 
 #if defined(OS_ANDROID)
 const FeatureEntry::FeatureParam kBottomOfflineIndicatorEnabled[] = {

content/renderer/input/input_event_prediction.cc

@@ -26,6 +26,8 @@ namespace {
 constexpr char kPredictor[] = "predictor";
 constexpr char kInputEventPredictorTypeLsq[] = "lsq";
 constexpr char kInputEventPredictorTypeEmpty[] = "empty";
+constexpr char kInputEventPredictorTypeKalmanTimeFiltered[] =
+    "kalman_time_filtered";
 
 constexpr uint32_t kPredictEventCount = 3;
 constexpr base::TimeDelta kPredictionInterval =
@@ -54,6 +56,8 @@ void InputEventPrediction::SetUpPredictorType() {
     selected_predictor_type_ = PredictorType::kLsq;
   else if (predictor_type == kInputEventPredictorTypeEmpty)
     selected_predictor_type_ = PredictorType::kEmpty;
+  else if (predictor_type == kInputEventPredictorTypeKalmanTimeFiltered)
+    selected_predictor_type_ = PredictorType::kKalmanTimeFiltered;
   else
     selected_predictor_type_ = PredictorType::kKalman;
 
@@ -107,7 +111,11 @@ std::unique_ptr<ui::InputPredictor> InputEventPrediction::CreatePredictor()
     case PredictorType::kLsq:
       return std::make_unique<ui::LeastSquaresPredictor>();
     case PredictorType::kKalman:
-      return std::make_unique<ui::KalmanPredictor>();
+      return std::make_unique<ui::KalmanPredictor>(
+          false /* enable_time_filtering */);
+    case PredictorType::kKalmanTimeFiltered:
+      return std::make_unique<ui::KalmanPredictor>(
+          true /* enable_time_filtering */);
   }
 }
 

content/renderer/input/input_event_prediction.h

@@ -44,7 +44,7 @@ class CONTENT_EXPORT InputEventPrediction {
   FRIEND_TEST_ALL_PREFIXES(InputEventPredictionTest, PredictorType);
   FRIEND_TEST_ALL_PREFIXES(InputEventPredictionTest, ResamplingDisabled);
 
-  enum class PredictorType { kEmpty, kLsq, kKalman };
+  enum class PredictorType { kEmpty, kLsq, kKalman, kKalmanTimeFiltered };
 
   // Initialize selected_predictor_type_ from field trial parameters of
   // kResamplingInputEvent flag if resampling is enable. Otherwise set it

ui/events/blink/prediction/kalman_predictor.cc

@@ -18,7 +18,8 @@ namespace ui {
 constexpr base::TimeDelta InputPredictor::kMaxTimeDelta;
 constexpr base::TimeDelta InputPredictor::kMaxResampleTime;
 
-KalmanPredictor::KalmanPredictor() = default;
+KalmanPredictor::KalmanPredictor(const bool enable_time_filtering)
+    : enable_time_filtering_(enable_time_filtering) {}
 
 KalmanPredictor::~KalmanPredictor() = default;
 
@@ -30,6 +31,7 @@ void KalmanPredictor::Reset() {
   x_predictor_.Reset();
   y_predictor_.Reset();
   last_point_.time_stamp = base::TimeTicks();
+  time_filter_.Reset();
 }
 
 void KalmanPredictor::Update(const InputData& cur_input) {
@@ -39,9 +41,12 @@ void KalmanPredictor::Update(const InputData& cur_input) {
     dt = cur_input.time_stamp - last_point_.time_stamp;
     if (dt > kMaxTimeDelta)
       Reset();
+    else if (enable_time_filtering_)
+      time_filter_.Update(dt.InMillisecondsF(), 0);
   }
 
-  double dt_ms = dt.InMillisecondsF();
+  double dt_ms = enable_time_filtering_ ? time_filter_.GetPosition()
+                                        : dt.InMillisecondsF();
   last_point_ = cur_input;
   x_predictor_.Update(cur_input.pos.x(), dt_ms);
   y_predictor_.Update(cur_input.pos.y(), dt_ms);

ui/events/blink/prediction/kalman_predictor.h

@@ -19,7 +19,7 @@ namespace ui {
 // be used to predict one dimension (x, y).
 class KalmanPredictor : public InputPredictor {
  public:
-  explicit KalmanPredictor();
+  explicit KalmanPredictor(bool enable_time_filtering);
   ~KalmanPredictor() override;
 
   const char* GetName() const override;
@@ -45,10 +45,14 @@ class KalmanPredictor : public InputPredictor {
   gfx::Vector2dF PredictVelocity() const;
   gfx::Vector2dF PredictAcceleration() const;
 
-  // Prdictor for each axis.
+  // Predictor for each axis.
   KalmanFilter x_predictor_;
   KalmanFilter y_predictor_;
 
+  // Filter to smooth time intervals.
+  KalmanFilter time_filter_;
+  bool enable_time_filtering_;
+
   // The last input point.
   InputData last_point_;
 

ui/events/blink/prediction/kalman_predictor_unittest.cc

@@ -44,7 +44,8 @@ class KalmanPredictorTest : public InputPredictorTest {
   explicit KalmanPredictorTest() {}
 
   void SetUp() override {
-    predictor_ = std::make_unique<ui::KalmanPredictor>();
+    predictor_ = std::make_unique<ui::KalmanPredictor>(
+        false /* enable_time_filtering */);
   }
 
   DISALLOW_COPY_AND_ASSIGN(KalmanPredictorTest);
@@ -89,6 +90,16 @@ TEST_F(KalmanPredictorTest, PredictConstantValue) {
   ValidatePredictor(x, y, t);
 }
 
+// Tests the kalman predictor constant position with time filtering.
+TEST_F(KalmanPredictorTest, PredictConstantValueTimeFiltered) {
+  predictor_ =
+      std::make_unique<ui::KalmanPredictor>(true /* enable_time_filtering */);
+  std::vector<double> x = {50, 50, 50, 50, 50, 50};
+  std::vector<double> y = {-50, -50, -50, -50, -50, -50};
+  std::vector<double> t = {8, 16, 24, 32, 40, 48};
+  ValidatePredictor(x, y, t);
+}
+
 // Tests the kalman predictor predict constant velocity.
 TEST_F(KalmanPredictorTest, PredictLinearValue) {
   std::vector<double> x = {0, 8, 16, 20, 23, 27, 31, 38, 48, 60};
@@ -97,6 +108,37 @@ TEST_F(KalmanPredictorTest, PredictLinearValue) {
   ValidatePredictor(x, y, t);
 }
 
+// Tests the time filtering as the kalman predictor predicts constant velocity.
+// Position is changing on a fixed rate assuming a fixed hardware resample
+// frequency. Due to timestamp noise generated by the OS event delivery,
+// timestamps are inconsistent.
+TEST_F(KalmanPredictorTest, PredictLinearValueTimeFiltered) {
+  std::unique_ptr<ui::KalmanPredictor> filtered_predictor =
+      std::make_unique<ui::KalmanPredictor>(true /* enable_time_filtering */);
+  std::vector<double> x = {0, 8, 16, 24, 32, 40, 48, 60};
+  std::vector<double> y = {30, 38, 46, 54, 62, 70, 78, 90};
+  std::vector<double> t = {0, 8, 16, 23, 33, 39, 49, 60};
+  for (size_t i = 0; i < t.size(); i++) {
+    if (filtered_predictor->HasPrediction() && predictor_->HasPrediction()) {
+      ui::InputPredictor::InputData filtered_result;
+      ui::InputPredictor::InputData unfiltered_result;
+      EXPECT_TRUE(filtered_predictor->GeneratePrediction(
+          FromMilliseconds(t[i]), false /* is_resampling */, &filtered_result));
+      EXPECT_TRUE(predictor_->GeneratePrediction(FromMilliseconds(t[i]),
+                                                 false /* is_resampling */,
+                                                 &unfiltered_result));
+      EXPECT_LT(std::abs(filtered_result.pos.x() - x[i]),
+                std::abs(unfiltered_result.pos.x() - x[i]));
+      EXPECT_LT(std::abs(filtered_result.pos.y() - y[i]),
+                std::abs(unfiltered_result.pos.y() - y[i]));
+    }
+    InputPredictor::InputData data = {gfx::PointF(x[i], y[i]),
+                                      FromMilliseconds(t[i])};
+    filtered_predictor->Update(data);
+    predictor_->Update(data);
+  }
+}
+
 // Tests the kalman predictor predict constant acceleration.
 TEST_F(KalmanPredictorTest, PredictQuadraticValue) {
   std::vector<double> x = {0, 2, 8, 18, 32, 50, 72, 98};
@@ -105,5 +147,15 @@ TEST_F(KalmanPredictorTest, PredictQuadraticValue) {
   ValidatePredictor(x, y, t);
 }
 
+// Tests the kalman predictor predict constant acceleration with time filtering.
+TEST_F(KalmanPredictorTest, PredictQuadraticValueTimeFiltered) {
+  predictor_ =
+      std::make_unique<ui::KalmanPredictor>(true /* enable_time_filtering */);
+  std::vector<double> x = {0, 2, 8, 18, 32, 50, 72, 98};
+  std::vector<double> y = {10, 11, 14, 19, 26, 35, 46, 59};
+  std::vector<double> t = {8, 16, 24, 32, 40, 48, 56, 64};
+  ValidatePredictor(x, y, t);
+}
+
 }  // namespace test
 }  // namespace ui

ui/events/blink/scroll_predictor.cc

@@ -22,6 +22,8 @@ namespace {
 constexpr char kPredictor[] = "predictor";
 constexpr char kScrollPredictorTypeLsq[] = "lsq";
 constexpr char kScrollPredictorTypeKalman[] = "kalman";
+constexpr char kScrollPredictorTypeKalmanTimeFiltered[] =
+    "kalman_time_filtered";
 
 }  // namespace
 
@@ -39,7 +41,11 @@ ScrollPredictor::ScrollPredictor(bool enable_resampling)
   if (predictor_type == kScrollPredictorTypeLsq)
     predictor_ = std::make_unique<LeastSquaresPredictor>();
   else if (predictor_type == kScrollPredictorTypeKalman)
-    predictor_ = std::make_unique<KalmanPredictor>();
+    predictor_ =
+        std::make_unique<KalmanPredictor>(false /* enable_time_filtering */);
+  else if (predictor_type == kScrollPredictorTypeKalmanTimeFiltered)
+    predictor_ =
+        std::make_unique<KalmanPredictor>(true /* enable_time_filtering */);
   else
     predictor_ = std::make_unique<EmptyPredictor>();
 }