Skip to content

GitLab

T
tangled
Commit b66ee38d authored by gogerald's avatar gogerald Committed by Commit Bot
Browse files
Options

[Autofill Assistant] Implement ClickElement 

Bug: 806868
Change-Id: I401e37f930e5f729cd3e5f5977aca1472afa4eed
Reviewed-on: https://chromium-review.googlesource.com/1195431
Commit-Queue: Ganggui Tang <gogerald@chromium.org>
Reviewed-by: default avatarRouslan Solomakhin <rouslan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#587310}
parent 88f95127
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 368 additions and 114 deletions
components/autofill_assistant/browser/assistant_web_controller.h
View file @ b66ee38d
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
#include "base/memory/weak_ptr.h" #include "base/memory/weak_ptr.h"
#include "components/autofill_assistant/browser/devtools/assistant_devtools_client.h" #include "components/autofill_assistant/browser/devtools/assistant_devtools_client.h"
#include "components/autofill_assistant/browser/devtools/devtools/domains/types_dom.h" #include "components/autofill_assistant/browser/devtools/devtools/domains/types_dom.h"
#include "components/autofill_assistant/browser/devtools/devtools/domains/types_input.h"
#include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h"
namespace content { namespace content {
class WebContents; class WebContents;
...@@ -31,7 +33,7 @@ class AssistantWebController { ...@@ -31,7 +33,7 @@ class AssistantWebController {
std::unique_ptr<AssistantDevtoolsClient> devtools_client); std::unique_ptr<AssistantDevtoolsClient> devtools_client);
virtual ~AssistantWebController(); virtual ~AssistantWebController();
// Perform a moust left button click on the element given by |selectors| and // Perform a mouse left button click on the element given by |selectors| and
// return the result through callback. // return the result through callback.
// CSS selectors in |selectors| are ordered from top frame to the frame // CSS selectors in |selectors| are ordered from top frame to the frame
// contains the element and the element. // contains the element and the element.
...@@ -55,29 +57,47 @@ class AssistantWebController { ...@@ -55,29 +57,47 @@ class AssistantWebController {
base::OnceCallback<void(bool)> callback); base::OnceCallback<void(bool)> callback);
private: private:
void OnGetDocumentForElementExists( void OnFindElementForClick(base::OnceCallback<void(bool)> callback,
const std::vector<std::string>& selectors, std::string object_id);
base::OnceCallback<void(bool)> callback, void OnScrollIntoView(base::OnceCallback<void(bool)> callback,
std::unique_ptr<dom::GetDocumentResult> result); std::string object_id,
void RecursiveElementExists(int node_id, std::unique_ptr<runtime::CallFunctionOnResult> result);
size_t index, void OnGetBoxModel(base::OnceCallback<void(bool)> callback,
const std::vector<std::string>& selectors, std::unique_ptr<dom::GetBoxModelResult> result);
base::OnceCallback<void(bool)> callback); void OnDispatchPressMoustEvent(
void OnQuerySelectorAllForElementExists(
size_t index,
const std::vector<std::string>& selectors,
base::OnceCallback<void(bool)> callback, base::OnceCallback<void(bool)> callback,
std::unique_ptr<dom::QuerySelectorAllResult> result); double x,
void OnDescribeNodeForElementExists( double y,
int node_id, std::unique_ptr<input::DispatchMouseEventResult> result);
size_t index, void OnDispatchReleaseMoustEvent(
const std::vector<std::string>& selectors,
base::OnceCallback<void(bool)> callback, base::OnceCallback<void(bool)> callback,
std::unique_ptr<dom::DescribeNodeResult> result); std::unique_ptr<input::DispatchMouseEventResult> result);
void OnPushNodesByBackendIdsForElementExists( void OnFindElementForExist(base::OnceCallback<void(bool)> callback,
std::string object_id);
void FindElement(const std::vector<std::string>& selectors,
base::OnceCallback<void(std::string)> callback);
void OnGetDocument(const std::vector<std::string>& selectors,
base::OnceCallback<void(std::string)> callback,
std::unique_ptr<dom::GetDocumentResult> result);
void RecursiveFindElement(int node_id,
size_t index,
const std::vector<std::string>& selectors,
base::OnceCallback<void(std::string)> callback);
void OnQuerySelectorAll(size_t index,
const std::vector<std::string>& selectors,
base::OnceCallback<void(std::string)> callback,
std::unique_ptr<dom::QuerySelectorAllResult> result);
void OnResolveNode(base::OnceCallback<void(std::string)> callback,
std::unique_ptr<dom::ResolveNodeResult> result);
void OnDescribeNode(int node_id,
size_t index,
const std::vector<std::string>& selectors,
base::OnceCallback<void(std::string)> callback,
std::unique_ptr<dom::DescribeNodeResult> result);
void OnPushNodesByBackendIds(
size_t index, size_t index,
const std::vector<std::string>& selectors, const std::vector<std::string>& selectors,
base::OnceCallback<void(bool)> callback, base::OnceCallback<void(std::string)> callback,
std::unique_ptr<dom::PushNodesByBackendIdsToFrontendResult> result); std::unique_ptr<dom::PushNodesByBackendIdsToFrontendResult> result);
void OnResult(bool result, base::OnceCallback<void(bool)> callback); void OnResult(bool result, base::OnceCallback<void(bool)> callback);
......
components/autofill_assistant/browser/assistant_web_controller_browsertest.cc
View file @ b66ee38d
...@@ -50,6 +50,39 @@ class AssistantWebControllerBrowserTest : public content::ContentBrowserTest { ...@@ -50,6 +50,39 @@ class AssistantWebControllerBrowserTest : public content::ContentBrowserTest {
done_callback.Run(); done_callback.Run();
} }
void ClickElement(const std::vector<std::string>& selectors) {
base::RunLoop run_loop;
assistant_web_controller_->ClickElement(
selectors,
base::BindOnce(&AssistantWebControllerBrowserTest::ClickElementCallback,
base::Unretained(this), run_loop.QuitClosure()));
run_loop.Run();
}
void ClickElementCallback(const base::Closure& done_callback, bool result) {
ASSERT_TRUE(result);
done_callback.Run();
}
void WaitForElementRemove(const std::vector<std::string>& selectors) {
base::RunLoop run_loop;
assistant_web_controller_->ElementExists(
selectors,
base::BindOnce(
&AssistantWebControllerBrowserTest::OnWaitForElementRemove,
base::Unretained(this), run_loop.QuitClosure(), selectors));
run_loop.Run();
}
void OnWaitForElementRemove(const base::Closure& done_callback,
const std::vector<std::string>& selectors,
bool result) {
done_callback.Run();
if (result) {
WaitForElementRemove(selectors);
}
}
private: private:
std::unique_ptr<net::EmbeddedTestServer> http_server_; std::unique_ptr<net::EmbeddedTestServer> http_server_;
std::unique_ptr<autofill_assistant::AssistantWebController> std::unique_ptr<autofill_assistant::AssistantWebController>
...@@ -68,7 +101,7 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) { ...@@ -68,7 +101,7 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) {
// IFrame. // IFrame.
selectors.clear(); selectors.clear();
selectors.emplace_back("#iframe"); selectors.emplace_back("#iframe");
selectors.emplace_back("#text"); selectors.emplace_back("#button");
IsElementExists(selectors, true); IsElementExists(selectors, true);
selectors.emplace_back("#whatever"); selectors.emplace_back("#whatever");
IsElementExists(selectors, false); IsElementExists(selectors, false);
...@@ -82,10 +115,31 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) { ...@@ -82,10 +115,31 @@ IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, IsElementExists) {
selectors.clear(); selectors.clear();
selectors.emplace_back("#iframe"); selectors.emplace_back("#iframe");
selectors.emplace_back("#shadowsection"); selectors.emplace_back("#shadowsection");
selectors.emplace_back("#button"); selectors.emplace_back("#shadowbutton");
IsElementExists(selectors, true); IsElementExists(selectors, true);
selectors.emplace_back("#whatever"); selectors.emplace_back("#whatever");
IsElementExists(selectors, false); IsElementExists(selectors, false);
} }
IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest, ClickElement) {
std::vector<std::string> selectors;
selectors.emplace_back("#button");
ClickElement(selectors);
WaitForElementRemove(selectors);
}
IN_PROC_BROWSER_TEST_F(AssistantWebControllerBrowserTest,
ClickElementInIFrame) {
std::vector<std::string> selectors;
selectors.emplace_back("#iframe");
selectors.emplace_back("#shadowsection");
selectors.emplace_back("#shadowbutton");
ClickElement(selectors);
selectors.clear();
selectors.emplace_back("#iframe");
selectors.emplace_back("#button");
WaitForElementRemove(selectors);
}
} // namespace } // namespace
components/autofill_assistant/browser/devtools/assistant_devtools_client.cc
View file @ b66ee38d
...@@ -20,8 +20,9 @@ namespace autofill_assistant { ...@@ -20,8 +20,9 @@ namespace autofill_assistant {
AssistantDevtoolsClient::AssistantDevtoolsClient( AssistantDevtoolsClient::AssistantDevtoolsClient(
scoped_refptr<content::DevToolsAgentHost> agent_host) scoped_refptr<content::DevToolsAgentHost> agent_host)
: agent_host_(agent_host), : agent_host_(agent_host),
input_domain_(this),
dom_domain_(this), dom_domain_(this),
page_domain_(this), runtime_domain_(this),
renderer_crashed_(false), renderer_crashed_(false),
next_message_id_(0), next_message_id_(0),
weak_ptr_factory_(this) { weak_ptr_factory_(this) {
...@@ -32,12 +33,16 @@ AssistantDevtoolsClient::AssistantDevtoolsClient( ...@@ -32,12 +33,16 @@ AssistantDevtoolsClient::AssistantDevtoolsClient(
AssistantDevtoolsClient::~AssistantDevtoolsClient() {} AssistantDevtoolsClient::~AssistantDevtoolsClient() {}
input::Domain* AssistantDevtoolsClient::GetInput() {
return &input_domain_;
}
dom::Domain* AssistantDevtoolsClient::GetDOM() { dom::Domain* AssistantDevtoolsClient::GetDOM() {
return &dom_domain_; return &dom_domain_;
} }
page::Domain* AssistantDevtoolsClient::GetPage() { runtime::Domain* AssistantDevtoolsClient::GetRuntime() {
return &page_domain_; return &runtime_domain_;
} }
void AssistantDevtoolsClient::SendMessage( void AssistantDevtoolsClient::SendMessage(
......
components/autofill_assistant/browser/devtools/assistant_devtools_client.h
View file @ b66ee38d
...@@ -19,8 +19,9 @@ ...@@ -19,8 +19,9 @@
#include "base/memory/scoped_refptr.h" #include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h" #include "base/memory/weak_ptr.h"
#include "base/sequenced_task_runner.h" #include "base/sequenced_task_runner.h"
#include "components/autofill_assistant/browser/devtools//devtools/domains/dom.h" #include "components/autofill_assistant/browser/devtools/devtools/domains/dom.h"
#include "components/autofill_assistant/browser/devtools//devtools/domains/page.h" #include "components/autofill_assistant/browser/devtools/devtools/domains/input.h"
#include "components/autofill_assistant/browser/devtools/devtools/domains/runtime.h"
#include "components/autofill_assistant/browser/devtools/message_dispatcher.h" #include "components/autofill_assistant/browser/devtools/message_dispatcher.h"
#include "content/public/browser/devtools_agent_host.h" #include "content/public/browser/devtools_agent_host.h"
#include "content/public/browser/devtools_agent_host_client.h" #include "content/public/browser/devtools_agent_host_client.h"
...@@ -34,8 +35,9 @@ class AssistantDevtoolsClient : public MessageDispatcher, ...@@ -34,8 +35,9 @@ class AssistantDevtoolsClient : public MessageDispatcher,
scoped_refptr<content::DevToolsAgentHost> agent_host); scoped_refptr<content::DevToolsAgentHost> agent_host);
~AssistantDevtoolsClient() override; ~AssistantDevtoolsClient() override;
input::Domain* GetInput();
dom::Domain* GetDOM(); dom::Domain* GetDOM();
page::Domain* GetPage(); runtime::Domain* GetRuntime();
// MessageDispatcher implementation: // MessageDispatcher implementation:
void SendMessage( void SendMessage(
...@@ -92,8 +94,9 @@ class AssistantDevtoolsClient : public MessageDispatcher, ...@@ -92,8 +94,9 @@ class AssistantDevtoolsClient : public MessageDispatcher,
scoped_refptr<content::DevToolsAgentHost> agent_host_; scoped_refptr<content::DevToolsAgentHost> agent_host_;
scoped_refptr<base::SequencedTaskRunner> browser_main_thread_; scoped_refptr<base::SequencedTaskRunner> browser_main_thread_;
input::ExperimentalDomain input_domain_;
dom::ExperimentalDomain dom_domain_; dom::ExperimentalDomain dom_domain_;
page::ExperimentalDomain page_domain_; runtime::ExperimentalDomain runtime_domain_;
std::unordered_map<int, Callback> pending_messages_; std::unordered_map<int, Callback> pending_messages_;
EventHandlerMap event_handlers_; EventHandlerMap event_handlers_;
bool renderer_crashed_; bool renderer_crashed_;
......
components/test/data/autofill_assistant/autofill_assistant_target_website.html
View file @ b66ee38d
...@@ -6,16 +6,42 @@ Use of this source code is governed by a BSD-style license that can be ...@@ -6,16 +6,42 @@ Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file. found in the LICENSE file.
--> -->
<html> <html>
<head> <head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<title>Autofill Assistant Test</title> <title>Autofill Assistant Test</title>
<div> <script>
<button id="button" type="button">Test Button</button><br> var removeButton = function() {
</div> var button = document.getElementById("button");
button.parentNode.removeChild(button);
}
</script>
<iframe id="iframe" width="100%" height="500" src= <style>
"autofill_assistant_target_website_iframe.html"></iframe> #full_height_section {
</body> height: 100vh;
}
</style>
</head>
<body>
<!--
Intentionally make this section has the full height of the window
to force scroll when operating on the elements below not in this
section.
-->
<div id="full_height_section">
<p>Blank Section
</div>
<div>
<button id="button" type="button" onclick=
"removeButton()">Test Button</button>
<br>
</div>
<iframe id="iframe" width="100%" height="500" src=
"autofill_assistant_target_website_iframe.html"></iframe>
</body>
</html> </html>
\ No newline at end of file
components/test/data/autofill_assistant/autofill_assistant_target_website_iframe.html
View file @ b66ee38d
...@@ -6,52 +6,57 @@ Use of this source code is governed by a BSD-style license that can be ...@@ -6,52 +6,57 @@ Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file. found in the LICENSE file.
--> -->
<html> <html>
<head> <head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<title>Autofill Assistant Test</title> <title>Autofill Assistant Test</title>
<script> <script>
var CreateShadowDom = function() { var createShadowDom = function() {
var shadowSection = document.getElementById("shadowsection"); var shadowSection = document.getElementById("shadowsection");
var shadowRoot = shadowSection.attachShadow({mode: 'open'}); var shadowRoot = shadowSection.attachShadow({mode: 'open'});
shadowRoot.innerHTML = shadowRoot.innerHTML =
'<button id="button" type="button">Test Button</button>'; '<button id="shadowbutton" type="button" onclick="removeButton()">\
} Test Button</button>';
</script> }
</head>
<body onload="CreateShadowDom()"> var removeButton = function() {
<div> var button = document.getElementById("button");
<form name="address" id="address_section"> button.parentNode.removeChild(button);
<div id='billing'> }
<h2>Billing Address</h2> </script>
Name: <input type="text" name="name"><br> </head>
Address: <input type="text" name="address"><br>
City: <input type="text" name="city"><br> <body onload="createShadowDom()">
State: <select name="state"> <div>
<option value="CA">CA</option> <form name="address" id="address_section">
<option value="MA">MA</option> <div id='billing'>
<option value="NY">NY</option> <h2>Billing Address</h2>
<option value="MD">MD</option> Name: <input type="text" name="name"><br>
<option value="OR">OR</option> Address: <input type="text" name="address"><br>
<option value="OH">OH</option> City: <input type="text" name="city"><br>
<option value="IL">IL</option> State: <select name="state">
<option value="DC">DC</option> <option value="CA">CA</option>
</select> <br> <option value="MA">MA</option>
Zip: <input name="zip"> <br> <option value="NY">NY</option>
Country: <input name="country"> <br> <option value="MD">MD</option>
Email: <input name="email"> <br> <option value="OR">OR</option>
<option value="OH">OH</option>
<option value="IL">IL</option>
<option value="DC">DC</option>
</select> <br>
Zip: <input name="zip"> <br>
Country: <input name="country"> <br>
Email: <input name="email"> <br>
</div>
</form>
</div> </div>
</form>
</div>
<div> <div>
<input id="text" type="text"><br> <button id="button" type="button">Test Button</button><br>
<button id="button" type="button">Test Button</button><br> </div>
</div>
<hr> <hr>
<div id="shadowsection"></div> <div id="shadowsection"></div>
</body> </body>
</html> </html>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment