Add input coordinate information to InputLatency

The coordinates information will be used in trace-viewer to display input
coordinates within the chrome compositor view.

Original patch by Vicentiu Ciorbaru <cvicentiu@chromium.org>

Review URL: https://codereview.chromium.org/569653002

Cr-Commit-Position: refs/heads/master@{#295993}

cc/trees/layer_tree_impl.cc

@@ -862,6 +862,11 @@ void LayerTreeImpl::AsValueInto(base::debug::TracedValue* state) const {
     TracedValue::AppendIDRef(*it, state);
   }
   state->EndArray();
+
+  state->BeginArray("swap_promise_trace_ids");
+  for (size_t i = 0; i < swap_promise_list_.size(); i++)
+    state->AppendDouble(swap_promise_list_[i]->TraceId());
+  state->EndArray();
 }
 
 void LayerTreeImpl::SetRootLayerScrollOffsetDelegate(

content/browser/renderer_host/render_widget_host_impl.cc

@@ -857,9 +857,11 @@ void RenderWidgetHostImpl::ForwardMouseEventWithLatencyInfo(
       const ui::LatencyInfo& ui_latency) {
   TRACE_EVENT2("input", "RenderWidgetHostImpl::ForwardMouseEvent",
                "x", mouse_event.x, "y", mouse_event.y);
+  ui::LatencyInfo::InputCoordinate logical_coordinate(mouse_event.x,
+                                                      mouse_event.y);
 
-  ui::LatencyInfo latency_info =
-      CreateRWHLatencyInfoIfNotExist(&ui_latency, mouse_event.type);
+  ui::LatencyInfo latency_info = CreateRWHLatencyInfoIfNotExist(
+      &ui_latency, mouse_event.type, &logical_coordinate, 1);
 
   for (size_t i = 0; i < mouse_event_callbacks_.size(); ++i) {
     if (mouse_event_callbacks_[i].Run(mouse_event))
@@ -889,8 +891,11 @@ void RenderWidgetHostImpl::ForwardWheelEventWithLatencyInfo(
       const ui::LatencyInfo& ui_latency) {
   TRACE_EVENT0("input", "RenderWidgetHostImpl::ForwardWheelEvent");
 
-  ui::LatencyInfo latency_info =
-      CreateRWHLatencyInfoIfNotExist(&ui_latency, wheel_event.type);
+  ui::LatencyInfo::InputCoordinate logical_coordinate(wheel_event.x,
+                                                      wheel_event.y);
+
+  ui::LatencyInfo latency_info = CreateRWHLatencyInfoIfNotExist(
+      &ui_latency, wheel_event.type, &logical_coordinate, 1);
 
   if (IgnoreInputEvents())
     return;
@@ -918,8 +923,11 @@ void RenderWidgetHostImpl::ForwardGestureEventWithLatencyInfo(
   if (delegate_->PreHandleGestureEvent(gesture_event))
     return;
 
-  ui::LatencyInfo latency_info =
-      CreateRWHLatencyInfoIfNotExist(&ui_latency, gesture_event.type);
+  ui::LatencyInfo::InputCoordinate logical_coordinate(gesture_event.x,
+                                                      gesture_event.y);
+
+  ui::LatencyInfo latency_info = CreateRWHLatencyInfoIfNotExist(
+      &ui_latency, gesture_event.type, &logical_coordinate, 1);
 
   if (gesture_event.type == blink::WebInputEvent::GestureScrollUpdate) {
     latency_info.AddLatencyNumber(
@@ -950,8 +958,19 @@ void RenderWidgetHostImpl::ForwardGestureEventWithLatencyInfo(
 void RenderWidgetHostImpl::ForwardEmulatedTouchEvent(
       const blink::WebTouchEvent& touch_event) {
   TRACE_EVENT0("input", "RenderWidgetHostImpl::ForwardEmulatedTouchEvent");
-  ui::LatencyInfo latency_info =
-      CreateRWHLatencyInfoIfNotExist(NULL, touch_event.type);
+
+  ui::LatencyInfo::InputCoordinate
+      logical_coordinates[ui::LatencyInfo::kMaxInputCoordinates];
+  size_t logical_coordinates_size =
+      std::min(arraysize(logical_coordinates),
+               static_cast<size_t>(touch_event.touchesLength));
+  for (size_t i = 0; i < logical_coordinates_size; i++) {
+    logical_coordinates[i] = ui::LatencyInfo::InputCoordinate(
+        touch_event.touches[i].position.x, touch_event.touches[i].position.y);
+  }
+
+  ui::LatencyInfo latency_info = CreateRWHLatencyInfoIfNotExist(
+      NULL, touch_event.type, logical_coordinates, logical_coordinates_size);
   TouchEventWithLatencyInfo touch_with_latency(touch_event, latency_info);
   input_router_->SendTouchEvent(touch_with_latency);
 }
@@ -964,8 +983,21 @@ void RenderWidgetHostImpl::ForwardTouchEventWithLatencyInfo(
   // Always forward TouchEvents for touch stream consistency. They will be
   // ignored if appropriate in FilterInputEvent().
 
+  ui::LatencyInfo::InputCoordinate
+      logical_coordinates[ui::LatencyInfo::kMaxInputCoordinates];
+  size_t logical_coordinates_size =
+      std::min(arraysize(logical_coordinates),
+               static_cast<size_t>(touch_event.touchesLength));
+  for (size_t i = 0; i < logical_coordinates_size; i++) {
+    logical_coordinates[i] = ui::LatencyInfo::InputCoordinate(
+        touch_event.touches[i].position.x, touch_event.touches[i].position.y);
+  }
+
   ui::LatencyInfo latency_info =
-      CreateRWHLatencyInfoIfNotExist(&ui_latency, touch_event.type);
+      CreateRWHLatencyInfoIfNotExist(&ui_latency,
+                                     touch_event.type,
+                                     logical_coordinates,
+                                     logical_coordinates_size);
   TouchEventWithLatencyInfo touch_with_latency(touch_event, latency_info);
 
   if (touch_emulator_ &&
@@ -1046,7 +1078,7 @@ void RenderWidgetHostImpl::ForwardKeyboardEvent(
 
   input_router_->SendKeyboardEvent(
       key_event,
-      CreateRWHLatencyInfoIfNotExist(NULL, key_event.type),
+      CreateRWHLatencyInfoIfNotExist(NULL, key_event.type, NULL, 0),
       is_shortcut);
 }
 
@@ -1089,7 +1121,10 @@ void RenderWidgetHostImpl::DisableResizeAckCheckForTesting() {
 }
 
 ui::LatencyInfo RenderWidgetHostImpl::CreateRWHLatencyInfoIfNotExist(
-    const ui::LatencyInfo* original, WebInputEvent::Type type) {
+    const ui::LatencyInfo* original,
+    WebInputEvent::Type type,
+    const ui::LatencyInfo::InputCoordinate* logical_coordinates,
+    size_t logical_coordinates_size) {
   ui::LatencyInfo info;
   if (original)
     info = *original;
@@ -1102,7 +1137,21 @@ ui::LatencyInfo RenderWidgetHostImpl::CreateRWHLatencyInfoIfNotExist(
                           GetLatencyComponentId(),
                           ++last_input_number_);
     info.TraceEventType(WebInputEventTraits::GetName(type));
+
+    // Convert logical coordinates to physical coordinates, based on the
+    // device scale factor.
+    float device_scale_factor =
+        screen_info_ ? screen_info_->deviceScaleFactor : 1;
+    DCHECK(logical_coordinates_size <= ui::LatencyInfo::kMaxInputCoordinates);
+    info.input_coordinates_size = logical_coordinates_size;
+    for (size_t i = 0; i < info.input_coordinates_size; i++) {
+      info.input_coordinates[i].x =
+          logical_coordinates[i].x * device_scale_factor;
+      info.input_coordinates[i].y =
+          logical_coordinates[i].y * device_scale_factor;
+    }
   }
+
   return info;
 }
 

content/browser/renderer_host/render_widget_host_impl.h

@@ -494,7 +494,10 @@ class CONTENT_EXPORT RenderWidgetHostImpl
   // component if it is not already in |original|. And if |original| is
   // not NULL, it is also merged into the resulting LatencyInfo.
   ui::LatencyInfo CreateRWHLatencyInfoIfNotExist(
-      const ui::LatencyInfo* original, blink::WebInputEvent::Type type);
+      const ui::LatencyInfo* original,
+      blink::WebInputEvent::Type type,
+      const ui::LatencyInfo::InputCoordinate* logical_coordinates,
+      size_t logical_coordinates_size);
 
   // Called when we receive a notification indicating that the renderer
   // process has gone. This will reset our state so that our state will be

content/browser/renderer_host/render_widget_host_unittest.cc

@@ -1334,21 +1334,24 @@ TEST_F(RenderWidgetHostTest, InputRouterReceivesHasTouchEventHandlers) {
   EXPECT_TRUE(host_->mock_input_router()->message_received_);
 }
 
-
-void CheckLatencyInfoComponentInMessage(RenderWidgetHostProcess* process,
-                                        int64 component_id,
-                                        WebInputEvent::Type input_type) {
+ui::LatencyInfo GetLatencyInfoFromInputEvent(RenderWidgetHostProcess* process) {
   const IPC::Message* message = process->sink().GetUniqueMessageMatching(
       InputMsg_HandleInputEvent::ID);
-  ASSERT_TRUE(message);
+  EXPECT_TRUE(message);
   InputMsg_HandleInputEvent::Param params;
   EXPECT_TRUE(InputMsg_HandleInputEvent::Read(message, &params));
-  ui::LatencyInfo latency_info = params.b;
+  process->sink().ClearMessages();
+  return params.b;
+}
+
+void CheckLatencyInfoComponentInMessage(RenderWidgetHostProcess* process,
+                                        int64 component_id,
+                                        WebInputEvent::Type input_type) {
+  ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process);
   EXPECT_TRUE(latency_info.FindLatency(
       ui::INPUT_EVENT_LATENCY_BEGIN_RWH_COMPONENT,
       component_id,
       NULL));
-  process->sink().ClearMessages();
 }
 
 // Tests that after input event passes through RWHI through ForwardXXXEvent()
@@ -1407,6 +1410,79 @@ TEST_F(RenderWidgetHostTest, InputEventRWHLatencyComponent) {
   SendInputEventACK(WebInputEvent::TouchStart, INPUT_EVENT_ACK_STATE_CONSUMED);
 }
 
+// Tests that after input event passes through RWHI through
+// ForwardXXXEventWithLatencyInfo(), input event coordinates will be present in
+// the latency info.
+TEST_F(RenderWidgetHostTest, InputEventRWHLatencyInfoCoordinates) {
+  host_->OnMessageReceived(ViewHostMsg_HasTouchEventHandlers(0, true));
+  process_->sink().ClearMessages();
+
+  {
+    WebMouseWheelEvent event =
+        SyntheticWebMouseWheelEventBuilder::Build(-5, 0, 0, true);
+    event.x = 100;
+    event.y = 200;
+    host_->ForwardWheelEvent(event);
+    ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process_);
+    EXPECT_EQ(1u, latency_info.input_coordinates_size);
+    EXPECT_EQ(100, latency_info.input_coordinates[0].x);
+    EXPECT_EQ(200, latency_info.input_coordinates[0].y);
+    SendInputEventACK(WebInputEvent::MouseWheel,
+                      INPUT_EVENT_ACK_STATE_CONSUMED);
+  }
+
+  {
+    WebMouseEvent event =
+        SyntheticWebMouseEventBuilder::Build(WebInputEvent::MouseMove);
+    event.x = 300;
+    event.y = 400;
+    host_->ForwardMouseEvent(event);
+    ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process_);
+    EXPECT_EQ(1u, latency_info.input_coordinates_size);
+    EXPECT_EQ(300, latency_info.input_coordinates[0].x);
+    EXPECT_EQ(400, latency_info.input_coordinates[0].y);
+    SendInputEventACK(WebInputEvent::MouseMove, INPUT_EVENT_ACK_STATE_CONSUMED);
+  }
+
+  {
+    WebGestureEvent event = SyntheticWebGestureEventBuilder::Build(
+        WebInputEvent::GestureScrollBegin, blink::WebGestureDeviceTouchscreen);
+    event.x = 500;
+    event.y = 600;
+    host_->ForwardGestureEvent(event);
+    ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process_);
+    EXPECT_EQ(1u, latency_info.input_coordinates_size);
+    EXPECT_EQ(500, latency_info.input_coordinates[0].x);
+    EXPECT_EQ(600, latency_info.input_coordinates[0].y);
+    SendInputEventACK(WebInputEvent::GestureScrollBegin,
+                      INPUT_EVENT_ACK_STATE_CONSUMED);
+  }
+
+  {
+    PressTouchPoint(700, 800);
+    PressTouchPoint(900, 1000);
+    PressTouchPoint(1100, 1200);  // LatencyInfo only holds two coordinates.
+    SendTouchEvent();
+    ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process_);
+    EXPECT_EQ(2u, latency_info.input_coordinates_size);
+    EXPECT_EQ(700, latency_info.input_coordinates[0].x);
+    EXPECT_EQ(800, latency_info.input_coordinates[0].y);
+    EXPECT_EQ(900, latency_info.input_coordinates[1].x);
+    EXPECT_EQ(1000, latency_info.input_coordinates[1].y);
+    SendInputEventACK(WebInputEvent::TouchStart,
+                      INPUT_EVENT_ACK_STATE_CONSUMED);
+  }
+
+  {
+    NativeWebKeyboardEvent event;
+    event.type = WebKeyboardEvent::KeyDown;
+    host_->ForwardKeyboardEvent(event);
+    ui::LatencyInfo latency_info = GetLatencyInfoFromInputEvent(process_);
+    EXPECT_EQ(0u, latency_info.input_coordinates_size);
+    SendInputEventACK(WebInputEvent::KeyDown, INPUT_EVENT_ACK_STATE_CONSUMED);
+  }
+}
+
 TEST_F(RenderWidgetHostTest, RendererExitedResetsInputRouter) {
   // RendererExited will delete the view.
   host_->SetView(new TestView(host_.get()));

ui/events/ipc/latency_info_param_traits.h

@@ -21,10 +21,18 @@ IPC_STRUCT_TRAITS_BEGIN(ui::LatencyInfo::LatencyComponent)
   IPC_STRUCT_TRAITS_MEMBER(event_count)
 IPC_STRUCT_TRAITS_END()
 
+IPC_STRUCT_TRAITS_BEGIN(ui::LatencyInfo::InputCoordinate)
+IPC_STRUCT_TRAITS_MEMBER(x)
+IPC_STRUCT_TRAITS_MEMBER(y)
+IPC_STRUCT_TRAITS_END()
+
 IPC_STRUCT_TRAITS_BEGIN(ui::LatencyInfo)
   IPC_STRUCT_TRAITS_MEMBER(latency_components)
   IPC_STRUCT_TRAITS_MEMBER(trace_id)
   IPC_STRUCT_TRAITS_MEMBER(terminated)
+  IPC_STRUCT_TRAITS_MEMBER(input_coordinates_size)
+  IPC_STRUCT_TRAITS_MEMBER(input_coordinates[0])
+  IPC_STRUCT_TRAITS_MEMBER(input_coordinates[1])
 IPC_STRUCT_TRAITS_END()
 
 #endif // UI_EVENTS_IPC_LATENCY_INFO_PARAM_TRAITS_H_

ui/events/latency_info.cc

@@ -117,6 +117,16 @@ scoped_refptr<base::debug::ConvertableToTraceFormat> AsTraceableData(
     record_data->Set(GetComponentName(it->first.first), component_info);
   }
   record_data->SetDouble("trace_id", latency.trace_id);
+
+  scoped_ptr<base::ListValue> coordinates(new base::ListValue());
+  for (size_t i = 0; i < latency.input_coordinates_size; i++) {
+    scoped_ptr<base::DictionaryValue> coordinate_pair(
+        new base::DictionaryValue());
+    coordinate_pair->SetDouble("x", latency.input_coordinates[i].x);
+    coordinate_pair->SetDouble("y", latency.input_coordinates[i].y);
+    coordinates->Append(coordinate_pair.release());
+  }
+  record_data->Set("coordinates", coordinates.release());
   return LatencyInfoTracedValue::FromValue(record_data.PassAs<base::Value>());
 }
 
@@ -124,7 +134,14 @@ scoped_refptr<base::debug::ConvertableToTraceFormat> AsTraceableData(
 
 namespace ui {
 
-LatencyInfo::LatencyInfo() : trace_id(-1), terminated(false) {
+LatencyInfo::InputCoordinate::InputCoordinate() : x(0), y(0) {
+}
+
+LatencyInfo::InputCoordinate::InputCoordinate(float x, float y) : x(x), y(y) {
+}
+
+LatencyInfo::LatencyInfo()
+    : input_coordinates_size(0), trace_id(-1), terminated(false) {
 }
 
 LatencyInfo::~LatencyInfo() {
@@ -137,6 +154,14 @@ bool LatencyInfo::Verify(const std::vector<LatencyInfo>& latency_info,
                << latency_info.size() << " is too big.";
     return false;
   }
+  for (size_t i = 0; i < latency_info.size(); i++) {
+    if (latency_info[i].input_coordinates_size > kMaxInputCoordinates) {
+      LOG(ERROR) << referring_msg << ", coordinate vector size "
+                 << latency_info[i].input_coordinates_size << " is too big.";
+      return false;
+    }
+  }
+
   return true;
 }
 

ui/events/latency_info.h

@@ -95,9 +95,19 @@ struct EVENTS_BASE_EXPORT LatencyInfo {
     uint32 event_count;
   };
 
+  struct EVENTS_BASE_EXPORT InputCoordinate {
+    InputCoordinate();
+    InputCoordinate(float x, float y);
+
+    float x;
+    float y;
+  };
+
   // Empirically determined constant based on a typical scroll sequence.
   enum { kTypicalMaxComponentsPerLatencyInfo = 6 };
 
+  enum { kMaxInputCoordinates = 2 };
+
   // Map a Latency Component (with a component-specific int64 id) to a
   // component info.
   typedef base::SmallMap<
@@ -153,6 +163,11 @@ struct EVENTS_BASE_EXPORT LatencyInfo {
   void TraceEventType(const char* event_type);
 
   LatencyMap latency_components;
+
+  // These coordinates represent window coordinates of the original input event.
+  uint32 input_coordinates_size;
+  InputCoordinate input_coordinates[kMaxInputCoordinates];
+
   // The unique id for matching the ASYNC_BEGIN/END trace event.
   int64 trace_id;
   // Whether a terminal component has been added.