[Autofill Assistant] Implement ClickElement

Bug: 806868
Change-Id: I401e37f930e5f729cd3e5f5977aca1472afa4eed
Reviewed-on: https://chromium-review.googlesource.com/1195431
Commit-Queue: Ganggui Tang <gogerald@chromium.org>
Reviewed-by: Rouslan Solomakhin <rouslan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#587310}

components/autofill_assistant/browser/assistant_web_controller.h

@@ -14,6 +14,8 @@
 #include "base/memory/weak_ptr.h"
 #include "components/autofill_assistant/browser/devtools/assistant_devtools_client.h"
 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_dom.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/types_input.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h"
 
 namespace content {
 class WebContents;
@@ -31,7 +33,7 @@ class AssistantWebController {
       std::unique_ptr<AssistantDevtoolsClient> devtools_client);
   virtual ~AssistantWebController();
 
-  // Perform a moust left button click on the element given by |selectors| and
+  // Perform a mouse left button click on the element given by |selectors| and
   // return the result through callback.
   // CSS selectors in |selectors| are ordered from top frame to the frame
   // contains the element and the element.
@@ -55,29 +57,47 @@ class AssistantWebController {
                             base::OnceCallback<void(bool)> callback);
 
  private:
-  void OnGetDocumentForElementExists(
-      const std::vector<std::string>& selectors,
-      base::OnceCallback<void(bool)> callback,
-      std::unique_ptr<dom::GetDocumentResult> result);
-  void RecursiveElementExists(int node_id,
-                              size_t index,
-                              const std::vector<std::string>& selectors,
-                              base::OnceCallback<void(bool)> callback);
-  void OnQuerySelectorAllForElementExists(
-      size_t index,
-      const std::vector<std::string>& selectors,
+  void OnFindElementForClick(base::OnceCallback<void(bool)> callback,
+                             std::string object_id);
+  void OnScrollIntoView(base::OnceCallback<void(bool)> callback,
+                        std::string object_id,
+                        std::unique_ptr<runtime::CallFunctionOnResult> result);
+  void OnGetBoxModel(base::OnceCallback<void(bool)> callback,
+                     std::unique_ptr<dom::GetBoxModelResult> result);
+  void OnDispatchPressMoustEvent(
       base::OnceCallback<void(bool)> callback,
-      std::unique_ptr<dom::QuerySelectorAllResult> result);
-  void OnDescribeNodeForElementExists(
-      int node_id,
-      size_t index,
-      const std::vector<std::string>& selectors,
+      double x,
+      double y,
+      std::unique_ptr<input::DispatchMouseEventResult> result);
+  void OnDispatchReleaseMoustEvent(
       base::OnceCallback<void(bool)> callback,
-      std::unique_ptr<dom::DescribeNodeResult> result);
-  void OnPushNodesByBackendIdsForElementExists(
+      std::unique_ptr<input::DispatchMouseEventResult> result);
+  void OnFindElementForExist(base::OnceCallback<void(bool)> callback,
+                             std::string object_id);
+  void FindElement(const std::vector<std::string>& selectors,
+                   base::OnceCallback<void(std::string)> callback);
+  void OnGetDocument(const std::vector<std::string>& selectors,
+                     base::OnceCallback<void(std::string)> callback,
+                     std::unique_ptr<dom::GetDocumentResult> result);
+  void RecursiveFindElement(int node_id,
+                            size_t index,
+                            const std::vector<std::string>& selectors,
+                            base::OnceCallback<void(std::string)> callback);
+  void OnQuerySelectorAll(size_t index,
+                          const std::vector<std::string>& selectors,
+                          base::OnceCallback<void(std::string)> callback,
+                          std::unique_ptr<dom::QuerySelectorAllResult> result);
+  void OnResolveNode(base::OnceCallback<void(std::string)> callback,
+                     std::unique_ptr<dom::ResolveNodeResult> result);
+  void OnDescribeNode(int node_id,
+                      size_t index,
+                      const std::vector<std::string>& selectors,
+                      base::OnceCallback<void(std::string)> callback,
+                      std::unique_ptr<dom::DescribeNodeResult> result);
+  void OnPushNodesByBackendIds(
       size_t index,
       const std::vector<std::string>& selectors,
-      base::OnceCallback<void(bool)> callback,
+      base::OnceCallback<void(std::string)> callback,
       std::unique_ptr<dom::PushNodesByBackendIdsToFrontendResult> result);
   void OnResult(bool result, base::OnceCallback<void(bool)> callback);
 

components/autofill_assistant/browser/assistant_web_controller_browsertest.cc

@@ -50,6 +50,39 @@ class AssistantWebControllerBrowserTest : public content::ContentBrowserTest {
     done_callback.Run();
   }
 
+  void ClickElement(const std::vector<std::string>& selectors) {
+    base::RunLoop run_loop;
+    assistant_web_controller_->ClickElement(
+        selectors,
+        base::BindOnce(&AssistantWebControllerBrowserTest::ClickElementCallback,
+                       base::Unretained(this), run_loop.QuitClosure()));
+    run_loop.Run();
+  }
+
+  void ClickElementCallback(const base::Closure& done_callback, bool result) {
+    ASSERT_TRUE(result);
+    done_callback.Run();
+  }
+
+  void WaitForElementRemove(const std::vector<std::string>& selectors) {
+    base::RunLoop run_loop;
+    assistant_web_controller_->ElementExists(
+        selectors,
+        base::BindOnce(
+            &AssistantWebControllerBrowserTest::OnWaitForElementRemove,
+            base::Unretained(this), run_loop.QuitClosure(), selectors));
+    run_loop.Run();
+  }
+
+  void OnWaitForElementRemove(const base::Closure& done_callback,
+                              const std::vector<std::string>& selectors,
+                              bool result) {
+    done_callback.Run();
+    if (result) {
+      WaitForElementRemove(selectors);
+    }
+  }
+
  private:
   std::unique_ptr<net::EmbeddedTestServer> http_server_;
   std::unique_ptr<autofill_assistant::AssistantWebController>
@@ -68,7 +101,7 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) {
   // IFrame.
   selectors.clear();
   selectors.emplace_back("#iframe");
-  selectors.emplace_back("#text");
+  selectors.emplace_back("#button");
   IsElementExists(selectors, true);
   selectors.emplace_back("#whatever");
   IsElementExists(selectors, false);
@@ -82,10 +115,31 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) {
   selectors.clear();
   selectors.emplace_back("#iframe");
   selectors.emplace_back("#shadowsection");
-  selectors.emplace_back("#button");
+  selectors.emplace_back("#shadowbutton");
   IsElementExists(selectors, true);
   selectors.emplace_back("#whatever");
   IsElementExists(selectors, false);
 }
 
+IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, ClickElement) {
+  std::vector<std::string> selectors;
+  selectors.emplace_back("#button");
+  ClickElement(selectors);
+
+  WaitForElementRemove(selectors);
+}
+
+IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest,
+                       ClickElementInIFrame) {
+  std::vector<std::string> selectors;
+  selectors.emplace_back("#iframe");
+  selectors.emplace_back("#shadowsection");
+  selectors.emplace_back("#shadowbutton");
+  ClickElement(selectors);
+
+  selectors.clear();
+  selectors.emplace_back("#iframe");
+  selectors.emplace_back("#button");
+  WaitForElementRemove(selectors);
+}
 }  // namespace

components/autofill_assistant/browser/devtools/assistant_devtools_client.cc

@@ -20,8 +20,9 @@ namespace autofill_assistant {
 AssistantDevtoolsClient::AssistantDevtoolsClient(
     scoped_refptr<content::DevToolsAgentHost> agent_host)
     : agent_host_(agent_host),
+      input_domain_(this),
       dom_domain_(this),
-      page_domain_(this),
+      runtime_domain_(this),
       renderer_crashed_(false),
       next_message_id_(0),
       weak_ptr_factory_(this) {
@@ -32,12 +33,16 @@ AssistantDevtoolsClient::AssistantDevtoolsClient(
 
 AssistantDevtoolsClient::~AssistantDevtoolsClient() {}
 
+input::Domain* AssistantDevtoolsClient::GetInput() {
+  return &input_domain_;
+}
+
 dom::Domain* AssistantDevtoolsClient::GetDOM() {
   return &dom_domain_;
 }
 
-page::Domain* AssistantDevtoolsClient::GetPage() {
-  return &page_domain_;
+runtime::Domain* AssistantDevtoolsClient::GetRuntime() {
+  return &runtime_domain_;
 }
 
 void AssistantDevtoolsClient::SendMessage(

components/autofill_assistant/browser/devtools/assistant_devtools_client.h

@@ -19,8 +19,9 @@
 #include "base/memory/scoped_refptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/sequenced_task_runner.h"
-#include "components/autofill_assistant/browser/devtools//devtools/domains/dom.h"
-#include "components/autofill_assistant/browser/devtools//devtools/domains/page.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/dom.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/input.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/runtime.h"
 #include "components/autofill_assistant/browser/devtools/message_dispatcher.h"
 #include "content/public/browser/devtools_agent_host.h"
 #include "content/public/browser/devtools_agent_host_client.h"
@@ -34,8 +35,9 @@ class AssistantDevtoolsClient : public MessageDispatcher,
       scoped_refptr<content::DevToolsAgentHost> agent_host);
   ~AssistantDevtoolsClient() override;
 
+  input::Domain* GetInput();
   dom::Domain* GetDOM();
-  page::Domain* GetPage();
+  runtime::Domain* GetRuntime();
 
   // MessageDispatcher implementation:
   void SendMessage(
@@ -92,8 +94,9 @@ class AssistantDevtoolsClient : public MessageDispatcher,
   scoped_refptr<content::DevToolsAgentHost> agent_host_;
   scoped_refptr<base::SequencedTaskRunner> browser_main_thread_;
 
+  input::ExperimentalDomain input_domain_;
   dom::ExperimentalDomain dom_domain_;
-  page::ExperimentalDomain page_domain_;
+  runtime::ExperimentalDomain runtime_domain_;
   std::unordered_map<int, Callback> pending_messages_;
   EventHandlerMap event_handlers_;
   bool renderer_crashed_;

components/test/data/autofill_assistant/autofill_assistant_target_website.html

@@ -6,16 +6,42 @@ Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 -->
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Autofill Assistant Test</title>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Autofill Assistant Test</title>
 
-<div>
-  <button id="button" type="button">Test Button</button><br>
-</div>
+    <script>
+      var removeButton = function() {
+        var button = document.getElementById("button");
+        button.parentNode.removeChild(button);
+      }
+    </script>
 
-<iframe id="iframe" width="100%" height="500" src=
-"autofill_assistant_target_website_iframe.html"></iframe>
-</body>
+    <style>
+      #full_height_section {
+        height: 100vh;
+      }
+    </style>
+  </head>
+
+  <body>
+    <!--
+    Intentionally make this section has the full height of the window
+    to force scroll when operating on the elements below not in this
+    section.
+    -->
+    <div id="full_height_section">
+      <p>Blank Section
+    </div>
+
+    <div>
+      <button id="button" type="button" onclick=
+          "removeButton()">Test Button</button>
+      <br>
+    </div>
+
+    <iframe id="iframe" width="100%" height="500" src=
+        "autofill_assistant_target_website_iframe.html"></iframe>
+  </body>
 </html>
\ No newline at end of file

components/test/data/autofill_assistant/autofill_assistant_target_website_iframe.html

@@ -6,52 +6,57 @@ Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 -->
 <html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>Autofill Assistant Test</title>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Autofill Assistant Test</title>
 
-<script>
-var CreateShadowDom = function() {
-  var shadowSection = document.getElementById("shadowsection");
-  var shadowRoot = shadowSection.attachShadow({mode: 'open'});
-  shadowRoot.innerHTML =
-  '<button id="button" type="button">Test Button</button>';
-}
-</script>
-</head>
+    <script>
+      var createShadowDom = function() {
+      var shadowSection = document.getElementById("shadowsection");
+      var shadowRoot = shadowSection.attachShadow({mode: 'open'});
+      shadowRoot.innerHTML =
+          '<button id="shadowbutton" type="button" onclick="removeButton()">\
+          Test Button</button>';
+      }
 
-<body onload="CreateShadowDom()">
-<div>
-  <form name="address" id="address_section">
-    <div id='billing'>
-      <h2>Billing Address</h2>
-      Name: <input type="text" name="name"><br>
-      Address: <input type="text" name="address"><br>
-      City: <input type="text" name="city"><br>
-      State: <select name="state">
-        <option value="CA">CA</option>
-        <option value="MA">MA</option>
-        <option value="NY">NY</option>
-        <option value="MD">MD</option>
-        <option value="OR">OR</option>
-        <option value="OH">OH</option>
-        <option value="IL">IL</option>
-        <option value="DC">DC</option>
-      </select> <br>
-      Zip: <input name="zip"> <br>
-      Country: <input name="country"> <br>
-      Email: <input name="email"> <br>
+      var removeButton = function() {
+        var button = document.getElementById("button");
+        button.parentNode.removeChild(button);
+      }
+    </script>
+  </head>
+
+  <body onload="createShadowDom()">
+    <div>
+      <form name="address" id="address_section">
+        <div id='billing'>
+          <h2>Billing Address</h2>
+          Name: <input type="text" name="name"><br>
+          Address: <input type="text" name="address"><br>
+          City: <input type="text" name="city"><br>
+          State: <select name="state">
+            <option value="CA">CA</option>
+            <option value="MA">MA</option>
+            <option value="NY">NY</option>
+            <option value="MD">MD</option>
+            <option value="OR">OR</option>
+            <option value="OH">OH</option>
+            <option value="IL">IL</option>
+            <option value="DC">DC</option>
+          </select> <br>
+          Zip: <input name="zip"> <br>
+          Country: <input name="country"> <br>
+          Email: <input name="email"> <br>
+        </div>
+      </form>
     </div>
-  </form>
-</div>
 
-<div>
-  <input id="text" type="text"><br>
-  <button id="button" type="button">Test Button</button><br>
-</div>
+    <div>
+      <button id="button" type="button">Test Button</button><br>
+    </div>
 
-<hr>
-<div id="shadowsection"></div>
-</body>
+    <hr>
+    <div id="shadowsection"></div>
+  </body>
 </html>
\ No newline at end of file