Commit a7f6d9d5 authored by Ella Ge's avatar Ella Ge Committed by Commit Bot

Add PredictedEvents to WebCoalescedInputEvent

This CL adds a vector predicted_events_ to WebCoalescedInputEvent, to
support pointer event  getPredictedEvent API.
This CL also changes the InputEventPrediction class used for resampling
to also generate 3 predicted event even when resampling is disabled. The
3 predicted event will be generated by Kalman filter predictor for now,
but in the following CL (add accuracy metrics), we change it to be controll
by another flag, so we can gather accuracy data for different prediction
algorithms on finch without turning on resampling.


Intent to implement:
https://groups.google.com/a/chromium.org/forum/#!topic/blink-dev/CgFyxYikn6A

Bug: 885299
Change-Id: I89b50ceea2847ea501b04ceb0da58f03741c379d
Reviewed-on: https://chromium-review.googlesource.com/c/1232884Reviewed-by: default avatarDave Tapuska <dtapuska@chromium.org>
Reviewed-by: default avatarRick Byers <rbyers@chromium.org>
Reviewed-by: default avatarNavid Zolghadr <nzolghadr@chromium.org>
Commit-Queue: Ella Ge <eirage@chromium.org>
Cr-Commit-Position: refs/heads/master@{#600178}
parent 71db5985
......@@ -26,28 +26,43 @@ constexpr char kPredictor[] = "predictor";
constexpr char kInputEventPredictorTypeLsq[] = "lsq";
constexpr char kInputEventPredictorTypeKalman[] = "kalman";
constexpr uint32_t kPredictEventCount = 3;
constexpr base::TimeDelta kPredictionInterval =
base::TimeDelta::FromMilliseconds(8);
} // namespace
InputEventPrediction::InputEventPrediction() {
std::string predictor_type = GetFieldTrialParamValueByFeature(
features::kResamplingInputEvents, kPredictor);
if (predictor_type == kInputEventPredictorTypeLsq)
selected_predictor_type_ = PredictorType::kLsq;
else if (predictor_type == kInputEventPredictorTypeKalman)
InputEventPrediction::InputEventPrediction(bool enable_resampling)
: enable_resampling_(enable_resampling) {
SetUpPredictorType();
}
InputEventPrediction::~InputEventPrediction() {}
void InputEventPrediction::SetUpPredictorType() {
// Resampling predictor type is set from field trial parameters.
// When resampling is disable, use kalman filter predictor for
// creating predicted events.
if (enable_resampling_) {
std::string predictor_type = GetFieldTrialParamValueByFeature(
features::kResamplingInputEvents, kPredictor);
if (predictor_type == kInputEventPredictorTypeLsq)
selected_predictor_type_ = PredictorType::kLsq;
else if (predictor_type == kInputEventPredictorTypeKalman)
selected_predictor_type_ = PredictorType::kKalman;
else
selected_predictor_type_ = PredictorType::kEmpty;
} else {
selected_predictor_type_ = PredictorType::kKalman;
else
selected_predictor_type_ = PredictorType::kEmpty;
}
mouse_predictor_ = CreatePredictor();
}
InputEventPrediction::~InputEventPrediction() {}
void InputEventPrediction::HandleEvents(
const blink::WebCoalescedInputEvent& coalesced_event,
base::TimeTicks frame_time,
blink::WebInputEvent* event) {
switch (event->GetType()) {
blink::WebCoalescedInputEvent& coalesced_event,
base::TimeTicks frame_time) {
switch (coalesced_event.Event().GetType()) {
case WebInputEvent::kMouseMove:
case WebInputEvent::kTouchMove:
case WebInputEvent::kPointerMove: {
......@@ -55,7 +70,20 @@ void InputEventPrediction::HandleEvents(
for (size_t i = 0; i < coalesced_size; i++)
UpdatePrediction(coalesced_event.CoalescedEvent(i));
ApplyResampling(frame_time, event);
if (enable_resampling_)
ApplyResampling(frame_time, coalesced_event.EventPointer());
base::TimeTicks predict_time =
enable_resampling_
? coalesced_event.EventPointer()->TimeStamp() +
kPredictionInterval
: std::max(frame_time,
coalesced_event.EventPointer()->TimeStamp());
for (uint32_t i = 0; i < kPredictEventCount; i++) {
if (!AddPredictedEvent(predict_time, coalesced_event))
break;
predict_time += kPredictionInterval;
}
break;
}
case WebInputEvent::kTouchScrollStarted:
......@@ -63,7 +91,7 @@ void InputEventPrediction::HandleEvents(
pointer_id_predictor_map_.clear();
break;
default:
ResetPredictor(*event);
ResetPredictor(coalesced_event.Event());
}
}
......@@ -104,14 +132,14 @@ void InputEventPrediction::ApplyResampling(base::TimeTicks frame_time,
if (event->GetType() == WebInputEvent::kTouchMove) {
WebTouchEvent* touch_event = static_cast<WebTouchEvent*>(event);
for (unsigned i = 0; i < touch_event->touches_length; ++i) {
if (ResampleSinglePointer(frame_time, &touch_event->touches[i]))
if (GetPointerPrediction(frame_time, &touch_event->touches[i]))
event->SetTimeStamp(frame_time);
}
} else if (event->GetType() == WebInputEvent::kMouseMove) {
if (ResampleSinglePointer(frame_time, static_cast<WebMouseEvent*>(event)))
if (GetPointerPrediction(frame_time, static_cast<WebMouseEvent*>(event)))
event->SetTimeStamp(frame_time);
} else if (event->GetType() == WebInputEvent::kPointerMove) {
if (ResampleSinglePointer(frame_time, static_cast<WebPointerEvent*>(event)))
if (GetPointerPrediction(frame_time, static_cast<WebPointerEvent*>(event)))
event->SetTimeStamp(frame_time);
}
}
......@@ -132,6 +160,35 @@ void InputEventPrediction::ResetPredictor(const WebInputEvent& event) {
}
}
bool InputEventPrediction::AddPredictedEvent(
base::TimeTicks predict_time,
blink::WebCoalescedInputEvent& coalesced_event) {
ui::WebScopedInputEvent predicted_event =
ui::WebInputEventTraits::Clone(coalesced_event.Event());
bool success = false;
if (predicted_event->GetType() == WebInputEvent::kTouchMove) {
WebTouchEvent& touch_event = static_cast<WebTouchEvent&>(*predicted_event);
success = true;
for (unsigned i = 0; i < touch_event.touches_length; ++i) {
if (!GetPointerPrediction(predict_time, &touch_event.touches[i]))
success = false;
}
} else if (predicted_event->GetType() == WebInputEvent::kMouseMove) {
if (GetPointerPrediction(predict_time,
&static_cast<WebMouseEvent&>(*predicted_event)))
success = true;
} else if (predicted_event->GetType() == WebInputEvent::kPointerMove) {
if (GetPointerPrediction(predict_time,
&static_cast<WebPointerEvent&>(*predicted_event)))
success = true;
}
if (success) {
predicted_event->SetTimeStamp(predict_time);
coalesced_event.AddPredictedEvent(*predicted_event);
}
return success;
}
void InputEventPrediction::UpdateSinglePointer(
const WebPointerProperties& event,
base::TimeTicks event_time) {
......@@ -152,12 +209,12 @@ void InputEventPrediction::UpdateSinglePointer(
}
}
bool InputEventPrediction::ResampleSinglePointer(base::TimeTicks frame_time,
WebPointerProperties* event) {
bool InputEventPrediction::GetPointerPrediction(base::TimeTicks predict_time,
WebPointerProperties* event) {
ui::InputPredictor::InputData predict_result;
if (event->pointer_type == WebPointerProperties::PointerType::kMouse) {
if (mouse_predictor_->HasPrediction() &&
mouse_predictor_->GeneratePrediction(frame_time, &predict_result)) {
mouse_predictor_->GeneratePrediction(predict_time, &predict_result)) {
event->SetPositionInWidget(predict_result.pos);
return true;
}
......@@ -168,7 +225,7 @@ bool InputEventPrediction::ResampleSinglePointer(base::TimeTicks frame_time,
auto predictor = pointer_id_predictor_map_.find(event->id);
if (predictor != pointer_id_predictor_map_.end() &&
predictor->second->HasPrediction() &&
predictor->second->GeneratePrediction(frame_time, &predict_result)) {
predictor->second->GeneratePrediction(predict_time, &predict_result)) {
event->SetPositionInWidget(predict_result.pos);
return true;
}
......
......@@ -21,22 +21,35 @@ namespace content {
// This class stores prediction of all active pointers.
class CONTENT_EXPORT InputEventPrediction {
public:
InputEventPrediction();
// enable_resampling is true when kResamplingInputEvents is enabled.
explicit InputEventPrediction(bool enable_resampling);
~InputEventPrediction();
void HandleEvents(const blink::WebCoalescedInputEvent& coalesced_event,
base::TimeTicks frame_time,
blink::WebInputEvent* event);
// Handle Resampling/Prediction of WebInputEvents. This function is mainly
// doing three things:
// 1. Maintain/Updates predictor using current CoalescedEvents vector.
// 2. When enable_resampling is true, change coalesced_event->EventPointer()'s
// coordinates to the position at frame time.
// 3. Generates 3 predicted events when prediction is available, add the
// PredictedEvent to coalesced_event.
void HandleEvents(blink::WebCoalescedInputEvent& coalesced_event,
base::TimeTicks frame_time);
// Initialize predictor for different pointer.
std::unique_ptr<ui::InputPredictor> CreatePredictor() const;
private:
friend class InputEventPredictionTest;
FRIEND_TEST_ALL_PREFIXES(PredictedEventTest, ResamplingDisabled);
FRIEND_TEST_ALL_PREFIXES(InputEventPredictionTest, PredictorType);
enum class PredictorType { kEmpty, kLsq, kKalman };
// The following three function is for handling multiple TouchPoints in a
// Set predictor type from field parameters of kResamplingInputEvent flag if
// it's enable. Otherwise use Kalman filter predictor.
void SetUpPredictorType();
// The following functions are for handling multiple TouchPoints in a
// WebTouchEvent. They should be more neat when WebTouchEvent is elimated.
// Cast events from WebInputEvent to WebPointerProperties. Call
// UpdateSinglePointer for each pointer.
......@@ -44,17 +57,22 @@ class CONTENT_EXPORT InputEventPrediction {
// Cast events from WebInputEvent to WebPointerProperties. Call
// ResamplingSinglePointer for each poitner.
void ApplyResampling(base::TimeTicks frame_time, WebInputEvent* event);
// Cast events from WebInputEvent to WebPointerProperties. Call
// ResetSinglePredictor for each pointer.
// Reset predictor for each pointer in WebInputEvent by ResetSinglePredictor.
void ResetPredictor(const WebInputEvent& event);
// Add predicted event to WebCoalescedInputEvent if prediction is available.
bool AddPredictedEvent(base::TimeTicks predict_time,
blink::WebCoalescedInputEvent& coalesced_event);
// Get single predictor based on event id and type, and update the predictor
// with new events coords.
void UpdateSinglePointer(const WebPointerProperties& event,
base::TimeTicks time);
// Get single predictor based on event id and type, apply resampling event
// coordinates.
bool ResampleSinglePointer(base::TimeTicks time, WebPointerProperties* event);
// Get prediction result of a single predictor based on the predict_time,
// and apply predicted result to the event. Return false if no prediction
// available.
bool GetPointerPrediction(base::TimeTicks predict_time,
WebPointerProperties* event);
// Get single predictor based on event id and type. For mouse, reset the
// predictor, for other pointer type, remove it from mapping.
void ResetSinglePredictor(const WebPointerProperties& event);
......@@ -67,6 +85,8 @@ class CONTENT_EXPORT InputEventPrediction {
// predictor.
PredictorType selected_predictor_type_;
bool enable_resampling_ = false;
DISALLOW_COPY_AND_ASSIGN(InputEventPrediction);
};
......
......@@ -4,7 +4,12 @@
#include "content/renderer/input/input_event_prediction.h"
#include "base/metrics/field_trial_param_associator.h"
#include "base/metrics/field_trial_params.h"
#include "base/test/scoped_feature_list.h"
#include "content/common/input/synthetic_web_input_event_builders.h"
#include "content/public/common/content_features.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "ui/events/base_event_utils.h"
#include "ui/events/blink/prediction/empty_predictor.h"
......@@ -14,12 +19,15 @@ using blink::WebMouseEvent;
using blink::WebPointerProperties;
using blink::WebTouchEvent;
namespace {} // namespace
namespace content {
class InputEventPredictionTest : public testing::Test {
public:
InputEventPredictionTest() {
event_predictor_ = std::make_unique<InputEventPrediction>();
event_predictor_ =
std::make_unique<InputEventPrediction>(true /* enable_resampling */);
}
int GetPredictorMapSize() const {
......@@ -49,16 +57,66 @@ class InputEventPredictionTest : public testing::Test {
void HandleEvents(const WebInputEvent& event) {
blink::WebCoalescedInputEvent coalesced_event(event);
event_predictor_->HandleEvents(coalesced_event,
WebInputEvent::GetStaticTimeStampForTests(),
coalesced_event.EventPointer());
WebInputEvent::GetStaticTimeStampForTests());
}
void ConfigureFieldTrial(const std::string& predictor_type) {
const std::string kTrialName = "TestTrial";
const std::string kGroupName = "TestGroup";
field_trial_list_.reset();
field_trial_list_.reset(new base::FieldTrialList(nullptr));
scoped_refptr<base::FieldTrial> trial =
base::FieldTrialList::CreateFieldTrial(kTrialName, kGroupName);
base::FieldTrialParamAssociator::GetInstance()->ClearAllParamsForTesting();
std::map<std::string, std::string> params;
params["predictor"] = predictor_type;
base::FieldTrialParamAssociator::GetInstance()->AssociateFieldTrialParams(
kTrialName, kGroupName, params);
std::unique_ptr<base::FeatureList> feature_list(new base::FeatureList);
feature_list->RegisterFieldTrialOverride(
features::kResamplingInputEvents.name,
base::FeatureList::OVERRIDE_ENABLE_FEATURE, trial.get());
base::FeatureList::ClearInstanceForTesting();
scoped_feature_list_.InitWithFeatureList(std::move(feature_list));
EXPECT_EQ(params["predictor"],
GetFieldTrialParamValueByFeature(features::kResamplingInputEvents,
"predictor"));
}
protected:
std::unique_ptr<InputEventPrediction> event_predictor_;
base::test::ScopedFeatureList scoped_feature_list_;
std::unique_ptr<base::FieldTrialList> field_trial_list_;
DISALLOW_COPY_AND_ASSIGN(InputEventPredictionTest);
};
TEST_F(InputEventPredictionTest, PredictorType) {
EXPECT_TRUE(event_predictor_->enable_resampling_);
EXPECT_EQ(event_predictor_->selected_predictor_type_,
InputEventPrediction::PredictorType::kEmpty);
ConfigureFieldTrial("empty");
event_predictor_->SetUpPredictorType();
EXPECT_EQ(event_predictor_->selected_predictor_type_,
InputEventPrediction::PredictorType::kEmpty);
ConfigureFieldTrial("kalman");
event_predictor_->SetUpPredictorType();
EXPECT_EQ(event_predictor_->selected_predictor_type_,
InputEventPrediction::PredictorType::kKalman);
ConfigureFieldTrial("lsq");
event_predictor_->SetUpPredictorType();
EXPECT_EQ(event_predictor_->selected_predictor_type_,
InputEventPrediction::PredictorType::kLsq);
}
TEST_F(InputEventPredictionTest, MouseEvent) {
WebMouseEvent mouse_move = SyntheticWebMouseEventBuilder::Build(
WebInputEvent::kMouseMove, 10, 10, 0);
......@@ -210,4 +268,46 @@ TEST_F(InputEventPredictionTest, TouchScrollStartedRemoveAllTouchPoints) {
EXPECT_EQ(GetPredictorMapSize(), 0);
}
class PredictedEventTest : public InputEventPredictionTest {
public:
PredictedEventTest() {
event_predictor_ =
std::make_unique<InputEventPrediction>(false /* enable_resampling */);
}
};
TEST_F(PredictedEventTest, ResamplingDisabled) {
// When resampling is disable, use kalman filter predictor to generate
// predicted event.
EXPECT_FALSE(event_predictor_->enable_resampling_);
EXPECT_EQ(event_predictor_->selected_predictor_type_,
InputEventPrediction::PredictorType::kKalman);
// Send 3 mouse move to get kalman predictor ready.
WebMouseEvent mouse_move = SyntheticWebMouseEventBuilder::Build(
WebInputEvent::kMouseMove, 10, 10, 0);
HandleEvents(mouse_move);
mouse_move =
SyntheticWebMouseEventBuilder::Build(WebInputEvent::kMouseMove, 11, 9, 0);
HandleEvents(mouse_move);
mouse_move =
SyntheticWebMouseEventBuilder::Build(WebInputEvent::kMouseMove, 12, 8, 0);
HandleEvents(mouse_move);
// The 4th move event should generate predicted events.
mouse_move =
SyntheticWebMouseEventBuilder::Build(WebInputEvent::kMouseMove, 13, 7, 0);
blink::WebCoalescedInputEvent coalesced_event(mouse_move);
event_predictor_->HandleEvents(coalesced_event, ui::EventTimeForNow());
EXPECT_GT(coalesced_event.PredictedEventSize(), 0u);
// Verify when resampling event is disabled, event coordinate doesn't change.
const WebMouseEvent& event =
static_cast<const blink::WebMouseEvent&>(coalesced_event.Event());
EXPECT_EQ(event.PositionInWidget().x, 13);
EXPECT_EQ(event.PositionInWidget().y, 7);
}
} // namespace content
......@@ -236,10 +236,8 @@ MainThreadEventQueue::MainThreadEventQueue(
use_raf_fallback_timer_(true) {
raf_fallback_timer_.SetTaskRunner(main_task_runner);
event_predictor_ =
base::FeatureList::IsEnabled(features::kResamplingInputEvents)
? std::make_unique<InputEventPrediction>()
: nullptr;
event_predictor_ = std::make_unique<InputEventPrediction>(
base::FeatureList::IsEnabled(features::kResamplingInputEvents));
}
MainThreadEventQueue::~MainThreadEventQueue() {}
......@@ -586,8 +584,7 @@ void MainThreadEventQueue::HandleEventResampling(
base::TimeTicks frame_time) {
if (item->IsWebInputEvent() && allow_raf_aligned_input_ && event_predictor_) {
QueuedWebInputEvent* event = static_cast<QueuedWebInputEvent*>(item.get());
event_predictor_->HandleEvents(event->coalesced_event(), frame_time,
&event->event());
event_predictor_->HandleEvents(event->coalesced_event(), frame_time);
}
}
......
......@@ -53,4 +53,9 @@ ScopedWebInputEventWithLatencyInfo::coalesced_event() const {
return *event_;
}
blink::WebCoalescedInputEvent&
ScopedWebInputEventWithLatencyInfo::coalesced_event() {
return *event_;
}
} // namespace content
......@@ -31,6 +31,7 @@ class ScopedWebInputEventWithLatencyInfo {
const blink::WebInputEvent& event() const;
const blink::WebCoalescedInputEvent& coalesced_event() const;
blink::WebInputEvent& event();
blink::WebCoalescedInputEvent& coalesced_event();
const ui::LatencyInfo latencyInfo() const { return latency_; }
void CoalesceWith(const ScopedWebInputEventWithLatencyInfo& other);
......
......@@ -33,6 +33,11 @@ class BLINK_PLATFORM_EXPORT WebCoalescedInputEvent {
const WebInputEvent& CoalescedEvent(size_t index) const;
std::vector<const WebInputEvent*> GetCoalescedEventsPointers() const;
void AddPredictedEvent(const blink::WebInputEvent&);
size_t PredictedEventSize() const;
const WebInputEvent& PredictedEvent(size_t index) const;
std::vector<const WebInputEvent*> GetPredictedEventsPointers() const;
private:
// TODO(hans): Remove this once clang-cl knows to not inline dtors that
// call operator(), https://crbug.com/691714
......@@ -47,6 +52,7 @@ class BLINK_PLATFORM_EXPORT WebCoalescedInputEvent {
WebScopedInputEvent event_;
std::vector<WebScopedInputEvent> coalesced_events_;
std::vector<WebScopedInputEvent> predicted_events_;
};
using WebScopedCoalescedInputEvent = std::unique_ptr<WebCoalescedInputEvent>;
......
......@@ -82,6 +82,28 @@ WebCoalescedInputEvent::GetCoalescedEventsPointers() const {
return events;
}
void WebCoalescedInputEvent::AddPredictedEvent(
const blink::WebInputEvent& event) {
predicted_events_.push_back(MakeWebScopedInputEvent(event));
}
size_t WebCoalescedInputEvent::PredictedEventSize() const {
return predicted_events_.size();
}
const WebInputEvent& WebCoalescedInputEvent::PredictedEvent(
size_t index) const {
return *predicted_events_[index].get();
}
std::vector<const WebInputEvent*>
WebCoalescedInputEvent::GetPredictedEventsPointers() const {
std::vector<const WebInputEvent*> events;
for (const auto& event : predicted_events_)
events.push_back(event.get());
return events;
}
WebCoalescedInputEvent::WebCoalescedInputEvent(const WebInputEvent& event) {
event_ = MakeWebScopedInputEvent(event);
coalesced_events_.push_back(MakeWebScopedInputEvent(event));
......
......@@ -47,15 +47,15 @@ bool KalmanPredictor::HasPrediction() const {
return x_predictor_.Stable() && y_predictor_.Stable();
}
bool KalmanPredictor::GeneratePrediction(base::TimeTicks frame_time,
bool KalmanPredictor::GeneratePrediction(base::TimeTicks predict_time,
InputData* result) const {
std::vector<InputData> pred_points;
base::TimeDelta dt = frame_time - last_point_.time_stamp;
base::TimeDelta dt = predict_time - last_point_.time_stamp;
// Kalman filter is not very good when predicting backwards. Besides,
// predicting backwards means increasing latency. Thus disable prediction when
// dt < 0.
if (!HasPrediction() || dt < base::TimeDelta::Min() || dt > kMaxResampleTime)
if (!HasPrediction() || dt < base::TimeDelta() || dt > kMaxResampleTime)
return false;
gfx::Vector2dF position(last_point_.pos.x(), last_point_.pos.y());
......
......@@ -33,7 +33,7 @@ class KalmanPredictor : public InputPredictor {
// Generate the prediction based on stored points and given time_stamp.
// Return false if no prediction available.
bool GeneratePrediction(base::TimeTicks frame_time,
bool GeneratePrediction(base::TimeTicks predict_time,
InputData* result) const override;
private:
......
......@@ -75,14 +75,14 @@ gfx::Matrix3F LeastSquaresPredictor::GetXMatrix() const {
return x;
}
bool LeastSquaresPredictor::GeneratePrediction(base::TimeTicks frame_time,
bool LeastSquaresPredictor::GeneratePrediction(base::TimeTicks predict_time,
InputData* result) const {
if (!HasPrediction() || frame_time - time_.back() > kMaxResampleTime)
if (!HasPrediction() || predict_time - time_.back() > kMaxResampleTime)
return false;
gfx::Matrix3F time_matrix = GetXMatrix();
double dt = (frame_time - time_[0]).InMillisecondsF();
double dt = (predict_time - time_[0]).InMillisecondsF();
if (dt > 0) {
gfx::Vector3dF b1, b2;
if (SolveLeastSquares(time_matrix, x_queue_, b1) &&
......
......@@ -33,7 +33,7 @@ class LeastSquaresPredictor : public InputPredictor {
// Generate the prediction based on stored points and given time_stamp.
// Return false if no prediction available.
bool GeneratePrediction(base::TimeTicks frame_time,
bool GeneratePrediction(base::TimeTicks predict_time,
InputData* result) const override;
private:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment