[ChromeDriver] Set key modifiers with mouse actions

When we have keyboard events and mouse events send together, we should
keep the keys that we are pressing now and set them to the mouse events
that are in the same action sequence.

Bug: chromedriver:1897
Change-Id: I700d7a87d3d0a1b45c1588318ab21314aa284417
Reviewed-on: https://chromium-review.googlesource.com/c/1482009Reviewed-by: John Chen <johnchen@chromium.org>
Commit-Queue: Lan Wei <lanwei@chromium.org>
Cr-Commit-Position: refs/heads/master@{#634700}

chrome/test/chromedriver/window_commands.cc

@@ -174,6 +174,20 @@ int MouseButtonToButtons(MouseButton button) {
   }
 }
 
+int KeyToKeyModifiers(std::string key) {
+  if (key == "Shift") {
+    return kShiftKeyModifierMask;
+  } else if (key == "Control") {
+    return kControlKeyModifierMask;
+  } else if (key == "Alt") {
+    return kAltKeyModifierMask;
+  } else if (key == "Meta") {
+    return kMetaKeyModifierMask;
+  } else {
+    return 0;
+  }
+}
+
 PointerType StringToPointerType(std::string pointer_type) {
   CHECK(pointer_type == "pen" || pointer_type == "mouse");
   if (pointer_type == "pen")
@@ -1386,7 +1400,29 @@ Status ExecutePerformActions(Session* session,
       std::max({longest_mouse_list_size, longest_touch_list_size,
                 longest_key_list_size, tick_durations.size()});
   std::map<std::string, gfx::Point> element_center_point;
+  int key_modifiers = 0;
   for (size_t i = 0; i < max_list_length; i++) {
+    std::list<KeyEvent> dispatch_key_events;
+    for (size_t j = 0; j < key_events_list.size(); j++) {
+      if (i < key_events_list[j].size() &&
+          key_events_list[j][i].type != kPauseEventType) {
+        const KeyEvent& event = key_events_list[j][i];
+        dispatch_key_events.push_back(event);
+        if (event.type == kKeyDownEventType) {
+          session->input_cancel_list.emplace_back(key_input_states[j], nullptr,
+                                                  nullptr, &event);
+          key_modifiers |= KeyToKeyModifiers(event.key);
+        } else if (event.type == kKeyUpEventType) {
+          key_modifiers &= ~KeyToKeyModifiers(event.key);
+        }
+      }
+    }
+    if (dispatch_key_events.size() > 0) {
+      Status status = web_view->DispatchKeyEvents(dispatch_key_events);
+      if (status.IsError())
+        return status;
+    }
+
     std::list<MouseEvent> dispatch_mouse_events;
     for (size_t j = 0; j < mouse_events_list.size(); j++) {
       if (i < mouse_events_list[j].size() &&
@@ -1405,6 +1441,7 @@ Status ExecutePerformActions(Session* session,
           event.x += element_center_point[event.element_id].x();
           event.y += element_center_point[event.element_id].y();
         }
+        event.modifiers = key_modifiers;
         dispatch_mouse_events.push_back(event);
       }
     }
@@ -1442,23 +1479,6 @@ Status ExecutePerformActions(Session* session,
         return status;
     }
 
-    std::list<KeyEvent> dispatch_key_events;
-    for (size_t j = 0; j < key_events_list.size(); j++) {
-      if (i < key_events_list[j].size() &&
-          key_events_list[j][i].type != kPauseEventType) {
-        const KeyEvent& event = key_events_list[j][i];
-        dispatch_key_events.push_back(event);
-        if (event.type == kKeyDownEventType) {
-          session->input_cancel_list.emplace_back(key_input_states[j], nullptr,
-                                                  nullptr, &event);
-        }
-      }
-    }
-    if (dispatch_key_events.size() > 0) {
-      Status status = web_view->DispatchKeyEvents(dispatch_key_events);
-      if (status.IsError())
-        return status;
-    }
     if (i < tick_durations.size() && tick_durations[i] > 0) {
       base::PlatformThread::Sleep(
           base::TimeDelta::FromMilliseconds(tick_durations[i]));

third_party/blink/web_tests/TestExpectations

@@ -3197,6 +3197,7 @@ crbug.com/626703 [ Mac10.13 ] external/wpt/media-source/mediasource-detach.html
 crbug.com/626703 [ Retina ] external/wpt/media-source/mediasource-detach.html [ Crash ]
 crbug.com/626703 [ Mac ] external/wpt/infrastructure/testdriver/actions/pause.html [ Pass Timeout ]
 crbug.com/626703 external/wpt/infrastructure/testdriver/actions/eventOrder.html [ Timeout ]
+crbug.com/626703 external/wpt/infrastructure/testdriver/actions/actionsWithKeyPressed.html [ Timeout ]
 crbug.com/626703 external/wpt/css/css-text/writing-system/writing-system-segment-break-001.html [ Failure ]
 crbug.com/626703 external/wpt/css/css-text/writing-system/writing-system-line-break-001.html [ Failure ]
 crbug.com/626703 external/wpt/css/css-text/writing-system/writing-system-line-break-002.html [ Failure ]

third_party/blink/web_tests/external/wpt/infrastructure/metadata/infrastructure/testdriver/actions/actionsWithKeyPressed.html.ini

+[actionsWithKeyPressed.html]
+  expected:
+    if product == "safari": ERROR

third_party/blink/web_tests/external/wpt/infrastructure/testdriver/actions/actionsWithKeyPressed.html

+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>TestDriver actions: actions with key pressed</title>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script src="/resources/testdriver.js"></script>
+<script src="/resources/testdriver-actions.js"></script>
+<script src="/resources/testdriver-vendor.js"></script>
+
+<style>
+div#test1, div#test2 {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100px;
+  height: 100px;
+  background-color: blue;
+}
+
+div#test2 {
+  position: fixed;
+  top: 100px;
+  left: 0;
+  width: 100px;
+  height: 100px;
+  background-color: green;
+}
+</style>
+
+<div id="test1">
+</div>
+
+<div id="test2">
+</div>
+
+<script>
+let keys = [];
+
+async_test(t => {
+  let test1 = document.getElementById("test1");
+  let test2 = document.getElementById("test2");
+  document.getElementById("test1").addEventListener("click",
+    e => {keys.push(e.getModifierState("Control"))});
+  document.getElementById("test2").addEventListener("click",
+    e => {keys.push(e.getModifierState("Control"))});
+
+  let actions = new test_driver.Actions()
+    .keyDown("\uE009")
+    .addTick()
+    .pointerMove(0, 0, {origin: test1})
+    .pointerDown()
+    .pointerUp()
+    .pointerMove(0, 0, {origin: test2})
+    .pointerDown()
+    .pointerUp()
+    .addTick()
+    .keyUp("\uE009")
+    .addTick()
+    .pointerMove(0, 0, {origin: test1})
+    .pointerDown()
+    .pointerUp();
+
+  actions.send()
+    .then(t.step_func_done(() => assert_array_equals(keys, [true, true, false])))
+    .catch(e => t.step_func(() => assert_unreached("Actions sequence failed " + e)));
+});
+</script>