[ChromeDriver] Modify ExecutePerformActions to handle touch inputs

In ExecutePerformActions function, we add the code to handle touch
inputs. However, right now we are using EventTarget.dispatchEvent() to
send touch events, we should use another way to dispatch the touch
events.

Bug: chromedriver:1897
Change-Id: Idbecb39270d3abf80a30e5fd13fc0f5ff2ba5b3f
Reviewed-on: https://chromium-review.googlesource.com/c/1372663Reviewed-by: John Chen <johnchen@chromium.org>
Commit-Queue: Lan Wei <lanwei@chromium.org>
Cr-Commit-Position: refs/heads/master@{#615937}

chrome/test/chromedriver/test/run_py_tests.py

@@ -748,6 +748,28 @@ class ChromeDriverTest(ChromeDriverBaseTestWithWebServer):
     self._driver.PerformActions(actions)
     self.assertEquals(1, len(self._driver.FindElements('tag name', 'br')))
 
+  def testActionsTouchStart(self):
+    self._driver.Load(self.GetHttpUrlForFile('/chromedriver/empty.html'))
+    div = self._driver.ExecuteScript(
+        'document.body.innerHTML = "<div>old</div>";'
+        'var div = document.getElementsByTagName("div")[0];'
+        'div.style["width"] = "100px";'
+        'div.style["height"] = "100px";'
+        'div.addEventListener("touchstart", function() {'
+        '  var div = document.getElementsByTagName("div")[0];'
+        '  div.innerHTML="new<br>";'
+        '});'
+        'return div;')
+    actions = ({"actions": [{
+      "type":"pointer",
+      "actions":[{"type": "pointerMove", "x": 10, "y": 10},
+                 {"type": "pointerDown"},
+                 {"type": "pointerUp"}],
+      "parameters": {"pointerType": "touch"},
+      "id": "pointer1"}]})
+    self._driver.PerformActions(actions)
+    self.assertEquals(1, len(self._driver.FindElements('tag name', 'br')))
+
   def testPageLoadStrategyIsNormalByDefault(self):
     self.assertEquals('normal',
                       self._driver.capabilities['pageLoadStrategy'])

chrome/test/chromedriver/window_commands.cc

@@ -123,6 +123,17 @@ MouseButton StringToMouseButton(std::string button_type) {
     return kNoneMouseButton;
 }
 
+TouchEventType StringToTouchEventType(std::string action_type) {
+  if (action_type == "pointerDown")
+    return kTouchStart;
+  else if (action_type == "pointerUp")
+    return kTouchEnd;
+  else if (action_type == "pointerMove")
+    return kTouchMove;
+  else
+    return kTouchStart;
+}
+
 struct Cookie {
   Cookie(const std::string& name,
          const std::string& value,
@@ -1004,18 +1015,20 @@ Status ProcessInputActionSequence(
 
       action->SetString("type", subtype);
       if (subtype == "pointerDown" || subtype == "pointerUp") {
-        int button;
-        if (!action_item->GetInteger("button", &button) || button < 0 ||
-            button > 4) {
-          return Status(
-              kInvalidArgument,
-              "'button' must be a non-negative int and between 0 and 4");
+        if (pointer_type == "mouse") {
+          int button;
+          if (!action_item->GetInteger("button", &button) || button < 0 ||
+              button > 4) {
+            return Status(
+                kInvalidArgument,
+                "'button' must be a non-negative int and between 0 and 4");
+          }
+          std::string button_str;
+          Status status = IntToStringButton(button, button_str);
+          if (status.IsError())
+            return status;
+          action->SetString("button", button_str);
         }
-        std::string button_str;
-        Status status = IntToStringButton(button, button_str);
-        if (status.IsError())
-          return status;
-        action->SetString("button", button_str);
       } else if (subtype == "pointerMove") {
         int x;
         if (!action_item->GetInteger("x", &x))
@@ -1186,6 +1199,7 @@ Status ExecutePerformActions(Session* session,
     } else if (type == "pointer") {
       std::string pointer_type;
       action_sequence->GetString("pointerType", &pointer_type);
+
       if (input_pointer_type.empty())
         input_pointer_type = pointer_type;
 
@@ -1206,70 +1220,88 @@ Status ExecutePerformActions(Session* session,
         return Status(kInvalidArgument, "'id' already exists");
       pointer_id_set.insert(pointer_id);
 
-      std::list<MouseEvent> events;
-      if (pointer_type == "mouse") {
-        double x = 0;
-        double y = 0;
-        for (size_t j = 0; j < actions->GetSize(); j++) {
-          const base::DictionaryValue* mouse_action;
-          actions->GetDictionary(j, &mouse_action);
-          std::string action_type;
-          mouse_action->GetString("type", &action_type);
-          if (action_type == "pointerMove") {
-            mouse_action->GetDouble("x", &x);
-            mouse_action->GetDouble("y", &y);
-            const base::DictionaryValue* origin_dict;
-            if (mouse_action->HasKey("origin") &&
-                mouse_action->GetDictionary("origin", &origin_dict)) {
-              std::string element_id;
-              origin_dict->GetString(GetElementKey(), &element_id);
-              WebRect region;
-              Status status =
-                  GetElementRegion(session, web_view, element_id, &region);
-              if (status.IsError())
-                return status;
-              WebPoint region_offset;
-              status = ScrollElementRegionIntoView(
-                  session, web_view, element_id, region, true /* center */,
-                  std::string(), &region_offset);
-              if (status.IsError())
-                return status;
-              int innerWidth, innerHeight;
-              status = WindowViewportSize(session, web_view, &innerWidth,
-                                          &innerHeight);
-              if (status.IsError())
-                return status;
-              int left = std::max(
-                  0,
-                  std::min(region_offset.x, region_offset.x + region.Width()));
-              int right = std::min(
-                  innerWidth,
-                  std::max(region_offset.x, region_offset.x + region.Width()));
-              int top = std::max(
-                  0,
-                  std::min(region_offset.y, region_offset.y + region.Height()));
-              int bottom = std::min(
-                  innerHeight,
-                  std::max(region_offset.y, region_offset.y + region.Height()));
-              x += (left + right) / 2;
-              y += (top + bottom) / 2;
-            }
+      std::list<MouseEvent> mouse_events;
+      std::list<TouchEvent> touch_events;
+
+      double x = 0;
+      double y = 0;
+      bool has_touch_start = false;
+      for (size_t j = 0; j < actions->GetSize(); j++) {
+        const base::DictionaryValue* mouse_action;
+        actions->GetDictionary(j, &mouse_action);
+        std::string action_type;
+        mouse_action->GetString("type", &action_type);
+        if (action_type == "pointerMove") {
+          mouse_action->GetDouble("x", &x);
+          mouse_action->GetDouble("y", &y);
+          const base::DictionaryValue* origin_dict;
+          if (mouse_action->HasKey("origin") &&
+              mouse_action->GetDictionary("origin", &origin_dict)) {
+            std::string element_id;
+            origin_dict->GetString(GetElementKey(), &element_id);
+            WebRect region;
+            Status status =
+                GetElementRegion(session, web_view, element_id, &region);
+            if (status.IsError())
+              return status;
+            WebPoint region_offset;
+            status = ScrollElementRegionIntoView(session, web_view, element_id,
+                                                 region, true /* center */,
+                                                 std::string(), &region_offset);
+            if (status.IsError())
+              return status;
+            int innerWidth, innerHeight;
+            status = WindowViewportSize(session, web_view, &innerWidth,
+                                        &innerHeight);
+            if (status.IsError())
+              return status;
+            int left = std::max(
+                0, std::min(region_offset.x, region_offset.x + region.Width()));
+            int right = std::min(
+                innerWidth,
+                std::max(region_offset.x, region_offset.x + region.Width()));
+            int top = std::max(0, std::min(region_offset.y,
+                                           region_offset.y + region.Height()));
+            int bottom = std::min(
+                innerHeight,
+                std::max(region_offset.y, region_offset.y + region.Height()));
+            x += (left + right) / 2;
+            y += (top + bottom) / 2;
           }
+        }
 
+        if (pointer_type == "mouse") {
           std::string button_type;
           int click_count = 0;
           if (action_type == "pointerDown" || action_type == "pointerUp") {
             mouse_action->GetString("button", &button_type);
             click_count = 1;
           }
-          events.push_back(MouseEvent(StringToMouseEventType(action_type),
-                                      StringToMouseButton(button_type), x, y, 0,
-                                      click_count));
+          mouse_events.push_back(MouseEvent(StringToMouseEventType(action_type),
+                                            StringToMouseButton(button_type), x,
+                                            y, 0, click_count));
+        } else if (pointer_type == "touch") {
+          if (action_type == "pointerDown")
+            has_touch_start = true;
+          else if (action_type == "pointerUp")
+            has_touch_start = false;
+
+          if (action_type != "pointerMove" || has_touch_start) {
+            touch_events.push_back(
+                TouchEvent(StringToTouchEventType(action_type), x, y));
+          }
         }
-        Status status =
-            web_view->DispatchMouseEvents(events, session->GetCurrentFrameId());
-        return status;
       }
+
+      Status status(kOk);
+      if (pointer_type == "mouse") {
+        status = web_view->DispatchMouseEvents(mouse_events,
+                                               session->GetCurrentFrameId());
+      } else if (pointer_type == "touch") {
+        status = web_view->DispatchTouchEvents(touch_events);
+      }
+      if (status.IsError())
+        return status;
     }
   }
   return Status(kOk);

chrome/test/chromedriver/window_commands_unittest.cc

@@ -134,3 +134,67 @@ TEST(WindowCommandsTest, ProcessInputActionSequencePointerMouse) {
   ASSERT_EQ("pointerUp", action_type);
   ASSERT_EQ("left", button);
 }
+
+TEST(WindowCommandsTest, ProcessInputActionSequencePointerTouch) {
+  Session session("1");
+  std::unique_ptr<base::DictionaryValue> result(new base::DictionaryValue());
+  std::unique_ptr<base::DictionaryValue> action_sequence(
+      new base::DictionaryValue());
+  std::unique_ptr<base::ListValue> actions(new base::ListValue());
+  std::unique_ptr<base::DictionaryValue> action(new base::DictionaryValue());
+  std::unique_ptr<base::DictionaryValue> parameters(
+      new base::DictionaryValue());
+  parameters->SetString("pointerType", "touch");
+  action->SetString("type", "pointerMove");
+  action->SetInteger("x", 30);
+  action->SetInteger("y", 60);
+  actions->Append(std::move(action));
+  action = std::make_unique<base::DictionaryValue>();
+  action->SetString("type", "pointerDown");
+  actions->Append(std::move(action));
+  action = std::make_unique<base::DictionaryValue>();
+  action->SetString("type", "pointerUp");
+  actions->Append(std::move(action));
+
+  // pointer properties
+  action_sequence->SetString("type", "pointer");
+  action_sequence->SetString("id", "pointer1");
+  action_sequence->SetDictionary("parameters", std::move(parameters));
+  action_sequence->SetList("actions", std::move(actions));
+  const base::DictionaryValue* input_action_sequence = action_sequence.get();
+  Status status =
+      ProcessInputActionSequence(&session, input_action_sequence, &result);
+  ASSERT_TRUE(status.IsOk());
+
+  // check resulting action dictionary
+  const base::ListValue* actions_result;
+  const base::DictionaryValue* action_result;
+  std::string pointer_type;
+  std::string source_type;
+  std::string id;
+  std::string action_type;
+  int x, y;
+
+  result->GetString("sourceType", &source_type);
+  result->GetString("pointerType", &pointer_type);
+  result->GetString("id", &id);
+  ASSERT_EQ("pointer", source_type);
+  ASSERT_EQ("touch", pointer_type);
+  ASSERT_EQ("pointer1", id);
+
+  result->GetList("actions", &actions_result);
+  ASSERT_EQ(3U, actions_result->GetSize());
+  actions_result->GetDictionary(0, &action_result);
+  action_result->GetString("type", &action_type);
+  action_result->GetInteger("x", &x);
+  action_result->GetInteger("y", &y);
+  ASSERT_EQ("pointerMove", action_type);
+  ASSERT_EQ(30, x);
+  ASSERT_EQ(60, y);
+  actions_result->GetDictionary(1, &action_result);
+  action_result->GetString("type", &action_type);
+  ASSERT_EQ("pointerDown", action_type);
+  actions_result->GetDictionary(2, &action_result);
+  action_result->GetString("type", &action_type);
+  ASSERT_EQ("pointerUp", action_type);
+}