[ChromeDriver] Key type actions for the actions API

This CL contains the key type actions. Follow up CLs for tests and
pointer/pause actions

spec: https://w3c.github.io/webdriver/webdriver-spec.html#actions

Bug: chromedriver:1897
Change-Id: I135df58d97a45494e51a5ac09dfce021f7d094d3
Reviewed-on: https://chromium-review.googlesource.com/692599
Commit-Queue: Jonathon Kereliuk <kereliuk@chromium.org>
Reviewed-by: John Chen <johnchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#508958}

chrome/test/chromedriver/chrome/status.cc

@@ -32,6 +32,8 @@ const char* DefaultMessageForStatusCode(StatusCode code) {
       return "unknown error";
     case kInvalidArgument:
       return "invalid argument";
+    case kMoveTargetOutOfBounds:
+      return "move target out of bounds";
     case kElementNotInteractable:
       return "element not interactable";
     case kUnsupportedOperation:

chrome/test/chromedriver/chrome/status.h

@@ -22,6 +22,7 @@ enum StatusCode {
   kElementNotInteractable = 15,
   kUnsupportedOperation = 16,
   kJavaScriptError = 17,
+  kMoveTargetOutOfBounds = 18,
   kXPathLookupError = 19,
   kUnableToSetCookie = 20,
   kTimeout = 21,

chrome/test/chromedriver/server/http_handler.cc

@@ -493,9 +493,9 @@ HttpHandler::HttpHandler(
       CommandMapping(kGet, "session/:sessionId/log/types",
                      WrapToCommand("GetLogTypes",
                                    base::Bind(&ExecuteGetAvailableLogTypes))),
-      CommandMapping(kPost, "session/:sessionId/actions",
-                     WrapToCommand("PerformActions",
-                                   base::Bind(&ExecuteUnimplementedCommand))),
+      CommandMapping(
+          kPost, "session/:sessionId/actions",
+          WrapToCommand("PerformActions", base::Bind(&ExecutePerformActions))),
       CommandMapping(kDelete, "session/:sessionId/actions",
                      WrapToCommand("DeleteActions",
                                    base::Bind(&ExecuteUnimplementedCommand))),
@@ -724,6 +724,10 @@ HttpHandler::PrepareStandardResponse(
     case kJavaScriptError:
       response.reset(new net::HttpServerResponseInfo(net::HTTP_BAD_REQUEST));
       break;
+    case kMoveTargetOutOfBounds:
+      response.reset(
+          new net::HttpServerResponseInfo(net::HTTP_INTERNAL_SERVER_ERROR));
+      break;
     case kNoSuchCookie:
       response.reset(new net::HttpServerResponseInfo(net::HTTP_NOT_FOUND));
       break;

chrome/test/chromedriver/session.h

@@ -11,6 +11,7 @@
 #include <vector>
 
 #include "base/time/time.h"
+#include "base/values.h"
 #include "chrome/test/chromedriver/basic_types.h"
 #include "chrome/test/chromedriver/chrome/device_metrics.h"
 #include "chrome/test/chromedriver/chrome/geoposition.h"
@@ -68,6 +69,12 @@ struct Session {
   std::unique_ptr<Chrome> chrome;
   std::string window;
   int sticky_modifiers;
+  // List of input sources for each active input. Everytime a new input source
+  // is added, there must be a corresponding entry made in input_state_table.
+  std::unique_ptr<base::ListValue> active_input_sources;
+  // Map between input id and input source state for the corresponding input
+  // source. One entry for each item in active_input_sources
+  std::unique_ptr<base::DictionaryValue> input_state_table;
   // List of |FrameInfo|s for each frame to the current target frame from the
   // first frame element in the root document. If target frame is window.top,
   // this list will be empty.

chrome/test/chromedriver/window_commands.cc

@@ -14,6 +14,7 @@
 #include "base/memory/ptr_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/stringprintf.h"
+#include "base/strings/utf_string_conversions.h"
 #include "base/threading/platform_thread.h"
 #include "base/time/time.h"
 #include "base/values.h"
@@ -31,6 +32,8 @@
 #include "chrome/test/chromedriver/chrome/ui_events.h"
 #include "chrome/test/chromedriver/chrome/web_view.h"
 #include "chrome/test/chromedriver/element_util.h"
+#include "chrome/test/chromedriver/key_converter.h"
+#include "chrome/test/chromedriver/keycode_text_conversion.h"
 #include "chrome/test/chromedriver/net/timeout.h"
 #include "chrome/test/chromedriver/session.h"
 #include "chrome/test/chromedriver/util.h"
@@ -47,6 +50,9 @@ const char kDeprecatedUnreachableWebDataURL[] = "data:text/html,chromewebdata";
 // Defaults to 20 years into the future when adding a cookie.
 const double kDefaultCookieExpiryTime = 20*365*24*60*60;
 
+// for pointer actions
+enum class PointerActionType { NOT_INITIALIZED, PRESS, MOVE, RELEASE, IDLE };
+
 Status GetMouseButton(const base::DictionaryValue& params,
                       MouseButton* button) {
   int button_num;
@@ -745,6 +751,234 @@ Status ExecuteTouchPinch(Session* session,
 Status ProcessInputActionSequence(Session* session,
                                   const base::DictionaryValue* action_sequence,
                                   std::unique_ptr<base::ListValue>* result) {
+  std::string id;
+  std::string type;
+  const base::DictionaryValue* source;
+  const base::DictionaryValue* parameters;
+  std::string pointer_type = "mouse";
+
+  if (!action_sequence->GetString("type", &type) ||
+      ((type != "key") && (type != "pointer") && (type != "none"))) {
+    return Status(
+        kInvalidArgument,
+        "'type' must be one of the strings 'key', 'pointer' or 'none'");
+  }
+
+  if (!action_sequence->GetString("id", &id))
+    return Status(kInvalidArgument, "'id' must be a string");
+
+  if (type == "pointer") {
+    if (action_sequence->GetDictionary("parameters", &parameters)) {
+      // error check arguments
+      if (parameters->GetString("pointerType", &pointer_type) &&
+          (pointer_type != "mouse" && pointer_type != "pen" &&
+           pointer_type != "touch"))
+        return Status(kInvalidArgument,
+                      "'pointerType' must be one of mouse, pen or touch");
+    }
+  }
+
+  bool found = false;
+  for (size_t i = 0; i < session->active_input_sources->GetSize(); i++) {
+    session->active_input_sources->GetDictionary(i, &source);
+    DCHECK(source);
+
+    std::string source_id;
+    source->GetString("id", &source_id);
+    if (source_id == id) {
+      found = true;
+      if (type == "pointer") {
+        std::string source_pointer_type;
+        if (!source->GetString("pointerType", &source_pointer_type) ||
+            pointer_type != source_pointer_type) {
+          return Status(kInvalidArgument,
+                        "'pointerType' must be a string that matches sources "
+                        "pointer type");
+        }
+      }
+      std::string source_type;
+      source->GetString("type", &source_type);
+      if (source_type != type) {
+        return Status(kInvalidArgument,
+                      "input state with same id has a different type");
+      }
+      break;
+    }
+  }
+
+  // if we found no matching active input source
+  base::DictionaryValue tmp_source;
+  if (!found) {
+    // create input source
+    tmp_source.SetString("id", id);
+    tmp_source.SetString("type", type);
+    if (type == "pointer") {
+      tmp_source.SetString("pointerType", pointer_type);
+    }
+
+    session->active_input_sources->Append(
+        base::MakeUnique<base::DictionaryValue>(std::move(tmp_source)));
+
+    base::DictionaryValue tmp_state;
+    if (type == "key") {
+      std::unique_ptr<base::ListValue> pressed(new base::ListValue);
+      bool alt = false;
+      bool shift = false;
+      bool ctrl = false;
+      bool meta = false;
+
+      tmp_state.SetList("pressed", std::move(pressed));
+      tmp_state.SetBoolean("alt", alt);
+      tmp_state.SetBoolean("shift", shift);
+      tmp_state.SetBoolean("ctrl", ctrl);
+      tmp_state.SetBoolean("meta", meta);
+    } else if (type == "pointer") {
+      std::unique_ptr<base::ListValue> pressed(new base::ListValue);
+      int x = 0;
+      int y = 0;
+
+      tmp_state.SetList("pressed", std::move(pressed));
+      tmp_state.SetString("subtype", pointer_type);
+
+      tmp_state.SetInteger("x", x);
+      tmp_state.SetInteger("y", y);
+    }
+    session->input_state_table->SetDictionary(
+        id, base::MakeUnique<base::DictionaryValue>(std::move(tmp_state)));
+  }
+
+  const base::ListValue* actions;
+  if (!action_sequence->GetList("actions", &actions)) {
+    return Status(kInvalidArgument, "actions must be an array");
+  }
+
+  std::unique_ptr<base::ListValue> ret(new base::ListValue);
+  for (size_t i = 0; i < actions->GetSize(); i++) {
+    std::unique_ptr<base::DictionaryValue> action(new base::DictionaryValue());
+    const base::DictionaryValue* action_item;
+    if (!actions->GetDictionary(i, &action_item))
+      return Status(
+          kInvalidArgument,
+          "each argument in the action sequence must be a dictionary");
+
+    if (type == "none") {
+      // process null action
+      std::string subtype;
+      if (!action_item->GetString("type", &subtype) || subtype != "pause")
+        return Status(kInvalidArgument,
+                      "type of action must be the  string 'pause'");
+
+      action->SetString("id", id);
+      action->SetString("type", "none");
+      action->SetString("subtype", subtype);
+
+      int duration;
+      if (action_item->GetInteger("duration", &duration)) {
+        if (duration < 0)
+          return Status(kInvalidArgument,
+                        "duration must be a non-negative int");
+        action->SetInteger("duration", duration);
+      }
+    } else if (type == "key") {
+      // process key action
+      std::string subtype;
+      if (!action_item->GetString("type", &subtype) ||
+          (subtype != "keyUp" && subtype != "keyDown" && subtype != "pause"))
+        return Status(
+            kInvalidArgument,
+            "type of action must be the string 'keyUp', 'keyDown' or 'pause'");
+
+      action->SetString("id", id);
+      action->SetString("type", "key");
+      action->SetString("subtype", subtype);
+
+      if (subtype == "pause") {
+        int duration;
+        if (action_item->GetInteger("duration", &duration)) {
+          if (duration < 0)
+            return Status(kInvalidArgument,
+                          "duration must be a non-negative int");
+          action->SetInteger("duration", duration);
+        }
+      }
+      std::string key;
+      // TODO: check if key is a single unicode code point
+      if (!action_item->GetString("value", &key)) {
+        return Status(kInvalidArgument,
+                      "'value' must be a single unicode point");
+      }
+      action->SetString("value", key);
+    } else if (type == "pointer") {
+      std::string subtype;
+      if (!action_item->GetString("type", &subtype) ||
+          (subtype != "pointerUp" && subtype != "pointerDown" &&
+           subtype != "pointerMove" && subtype != "pointerCancel" &&
+           subtype != "pause"))
+        return Status(kInvalidArgument,
+                      "type of action must be the string 'pointerUp', "
+                      "'pointerDown', 'pointerMove' or 'pause'");
+
+      action->SetString("id", id);
+      action->SetString("type", "pointer");
+      action->SetString("subtype", subtype);
+
+      if (subtype == "pause") {
+        int duration;
+        if (action_item->GetInteger("duration", &duration)) {
+          if (duration < 0)
+            return Status(kInvalidArgument,
+                          "duration must be a non-negative int");
+          action->SetInteger("duration", duration);
+        }
+      }
+
+      action->SetString("pointerType", pointer_type);
+      if (subtype == "pointerUp" || subtype == "pointerDown") {
+        int button;
+        if (!action_item->GetInteger("button", &button) || button < 0)
+          return Status(kInvalidArgument,
+                        "'button' must be a non-negative int");
+        action->SetInteger("button", button);
+        if (subtype == "pointerDown") {
+          int x;
+          if (!action_item->GetInteger("x", &x))
+            return Status(kInvalidArgument, "'x' must be an integer");
+          int y;
+          if (!action_item->GetInteger("y", &y))
+            return Status(kInvalidArgument, "'y' must be an integer");
+
+          action->SetInteger("x", x);
+          action->SetInteger("y", y);
+        }
+      } else {
+        // pointerMove
+        int duration;
+        if (!action_item->GetInteger("duration", &duration) || duration < 0)
+          return Status(kInvalidArgument,
+                        "'duration' must be a non-negative int");
+
+        std::string origin;
+        if (!action_item->GetString("origin", &origin))
+          origin = "viewport";
+        if (origin != "viewport" && origin != "pointer")
+          return Status(kInvalidArgument, "'origin' must be a string");
+
+        action->SetString("origin", origin);
+
+        int x;
+        if (!action_item->GetInteger("x", &x))
+          return Status(kInvalidArgument, "'x' must be an integer");
+        int y;
+        if (!action_item->GetInteger("y", &y))
+          return Status(kInvalidArgument, "'y' must be an integer");
+
+        action->SetInteger("x", x);
+        action->SetInteger("y", y);
+      }
+    }
+    ret->Append(std::move(action));
+  }
+  *result = std::move(ret);
   return Status(kOk);
 }
 
@@ -763,23 +997,117 @@ Status ExecutePerformActions(Session* session,
   if (!params.GetList("actions", &actions))
     return Status(kInvalidArgument, "'actions' must be an array");
 
+  // the processed actions
   base::ListValue actions_by_tick;
-  std::unique_ptr<base::ListValue> input_source_actions(new base::ListValue());
+  // the type of each action list in actions_by_tick
+  std::list<std::string> action_list_types;
+
   for (size_t i = 0; i < actions->GetSize(); i++) {
+    std::unique_ptr<base::ListValue> input_source_actions(
+        new base::ListValue());
     // proccess input action sequence
     const base::DictionaryValue* action_sequence;
     if (!actions->GetDictionary(i, &action_sequence))
       return Status(kInvalidArgument, "each argument must be a dictionary");
 
+    std::string type;
+    if (!action_sequence->GetString("type", &type) ||
+        ((type != "key") && (type != "pointer") && (type != "none"))) {
+      return Status(
+          kInvalidArgument,
+          "'type' must be one of the strings 'key', 'pointer' or 'none'");
+    }
+    action_list_types.push_back(type);
+
     Status status = ProcessInputActionSequence(session, action_sequence,
                                                &input_source_actions);
-    actions_by_tick.Append(std::move(input_source_actions));
     if (status.IsError())
       return Status(kInvalidArgument, status);
+
+    actions_by_tick.Append(std::move(input_source_actions));
   }
 
-  // TODO(kereliuk): dispatch actions
+  for (size_t i = 0; i < actions_by_tick.GetSize(); i++) {
+    // compute duration
+    int max_duration = 0;
+    int duration;
+    base::ListValue* action_sequence;
+    actions_by_tick.GetList(i, &action_sequence);
+    DCHECK(action_sequence);
+    for (size_t j = 0; j < action_sequence->GetSize(); j++) {
+      base::DictionaryValue* action;
+      if (!action_sequence->GetDictionary(i, &action))
+        return Status(kInvalidArgument, "each argument must be a dictionary");
+      if (action->GetInteger("duration", &duration) &&
+          duration > max_duration) {
+        max_duration = duration;
+      }
+    }
 
+    // get the type of the actions so we can dispatch all at once for that type
+    std::string type = action_list_types.back();
+    action_list_types.pop_back();
+
+    // pause only
+
+    // key actions
+    if (type == "key") {
+      KeyEventBuilder builder;
+      std::list<KeyEvent> key_events;
+      for (size_t j = 0; j < action_sequence->GetSize(); j++) {
+        base::DictionaryValue* action;
+        if (!action_sequence->GetDictionary(j, &action))
+          return Status(kInvalidArgument, "each argument must be a dictionary");
+        std::string subtype;
+        if (!action->GetString("subtype", &subtype))
+          return Status(kInvalidArgument, "'type' must be a string");
+
+        std::string id;
+        if (!action->GetString("id", &id))
+          return Status(kInvalidArgument, "id");
+
+        if (subtype == "pause") {
+          // TODO: handle this
+        } else {
+          base::DictionaryValue dispatch_params;
+          base::string16 raw_key;
+
+          if (!action->GetString("value", &raw_key))
+            return Status(kInvalidArgument, "value");
+
+          base::char16 key = raw_key[0];
+          // TODO: understand necessary_modifiers
+          int necessary_modifiers = 0;
+          ui::KeyboardCode key_code = ui::VKEY_UNKNOWN;
+          std::string error_msg;
+          ConvertCharToKeyCode(key, &key_code, &necessary_modifiers,
+                               &error_msg);
+          if (!error_msg.empty())
+            return Status(kUnknownError, error_msg);
+
+          if (subtype == "keyDown")
+            key_events.push_back(builder.SetType(kKeyDownEventType)
+                                     ->SetText(base::UTF16ToUTF8(raw_key),
+                                               base::UTF16ToUTF8(raw_key))
+                                     ->SetKeyCode(key_code)
+                                     ->SetModifiers(0)
+                                     ->Build());
+          else if (subtype == "keyUp")
+            key_events.push_back(builder.SetType(kKeyUpEventType)
+                                     ->SetText(base::UTF16ToUTF8(raw_key),
+                                               base::UTF16ToUTF8(raw_key))
+                                     ->SetKeyCode(key_code)
+                                     ->SetModifiers(0)
+                                     ->Build());
+        }
+      }
+      Status status = web_view->DispatchKeyEvents(key_events);
+      if (status.IsError())
+        return status;
+    } else if (type == "pointer") {
+      // TODO:implement this
+    }
+  }
   return Status(kOk);
 }