[Autofill Assistant] Stack for element position getting

Before this change, element coordinates were relative to their window,
i.e. frame. For elements in iFrame this meant that the "cutout" was
misplaced.
This change now keeps a frame-stack while retrieving elements and
then sums up the position to get a "global" position of the element
for using in a showcast.

Bug: b/143942385
Change-Id: I8f3827002df58874d7921e969954f911630cc4ae
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2074499
Commit-Queue: Sandro Maggi <sandromaggi@google.com>
Reviewed-by: Mathias Carlen <mcarlen@chromium.org>
Reviewed-by: Stephane Zermatten <szermatt@chromium.org>
Cr-Commit-Position: refs/heads/master@{#745566}

components/autofill_assistant/browser/BUILD.gn

@@ -154,6 +154,8 @@ jumbo_static_library("browser") {
     "web/element_finder.h",
     "web/element_position_getter.cc",
     "web/element_position_getter.h",
+    "web/element_rect_getter.cc",
+    "web/element_rect_getter.h",
     "web/web_controller.cc",
     "web/web_controller.h",
     "web/web_controller_util.cc",

components/autofill_assistant/browser/web/element_finder.cc

@@ -116,6 +116,12 @@ bool ConvertPseudoType(const PseudoType pseudo_type,
 }
 }  // namespace
 
+ElementFinder::Result::Result() = default;
+
+ElementFinder::Result::~Result() = default;
+
+ElementFinder::Result::Result(const Result& to_copy) = default;
+
 ElementFinder::ElementFinder(content::WebContents* web_contents,
                              DevtoolsClient* devtools_client,
                              const Selector& selector,
@@ -350,6 +356,13 @@ void ElementFinder::OnDescribeNode(
     element_result_->container_frame_host =
         FindCorrespondingRenderFrameHost(node->GetFrameId());
 
+    Result result_frame;
+    result_frame.container_frame_selector_index =
+        element_result_->container_frame_selector_index;
+    result_frame.container_frame_host = element_result_->container_frame_host;
+    result_frame.object_id = object_id;
+    element_result_->frame_stack.emplace_back(result_frame);
+
     if (!element_result_->container_frame_host) {
       VLOG(1) << __func__ << " Failed to find corresponding owner frame.";
       SendResult(ClientStatus(FRAME_HOST_NOT_FOUND));

components/autofill_assistant/browser/web/element_finder.h

@@ -30,8 +30,9 @@ class DevtoolsClient;
 class ElementFinder : public WebControllerWorker {
  public:
   struct Result {
-    Result() = default;
-    ~Result() = default;
+    Result();
+    ~Result();
+    Result(const Result& to_copy);
 
     // The render frame host contains the element.
     content::RenderFrameHost* container_frame_host;
@@ -47,6 +48,8 @@ class ElementFinder : public WebControllerWorker {
 
     // The id of the frame the element's node is in.
     std::string node_frame_id;
+
+    std::vector<Result> frame_stack;
   };
 
   // |web_contents| and |devtools_client| must be valid for the lifetime of the

components/autofill_assistant/browser/web/element_rect_getter.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/autofill_assistant/browser/web/element_rect_getter.h"
+
+#include "base/callback.h"
+#include "base/values.h"
+#include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h"
+#include "components/autofill_assistant/browser/devtools/devtools_client.h"
+#include "components/autofill_assistant/browser/rectf.h"
+#include "components/autofill_assistant/browser/web/element_finder.h"
+#include "components/autofill_assistant/browser/web/web_controller_util.h"
+
+namespace autofill_assistant {
+namespace {
+
+const char* const kGetBoundingClientRectAsList =
+    R"(function(node, addWindowScroll) {
+    const offsetX = addWindowScroll ? window.scrollX : 0;
+    const offsetY = addWindowScroll ? window.scrollY : 0;
+    const r = node.getBoundingClientRect();
+    return [offsetX + r.left,
+            offsetY + r.top,
+            offsetX + r.right,
+            offsetY + r.bottom];
+  })";
+
+}  // namespace
+
+ElementRectGetter::ElementRectGetter(DevtoolsClient* devtools_client)
+    : devtools_client_(devtools_client), weak_ptr_factory_(this) {}
+
+ElementRectGetter::~ElementRectGetter() = default;
+
+void ElementRectGetter::Start(std::unique_ptr<ElementFinder::Result> element,
+                              ElementRectCallback callback) {
+  GetBoundingClientRect(std::move(element), 0, RectF(), std::move(callback));
+}
+
+void ElementRectGetter::GetBoundingClientRect(
+    std::unique_ptr<ElementFinder::Result> element,
+    size_t index,
+    const RectF& stacked_rect,
+    ElementRectCallback callback) {
+  std::string object_id;
+  std::string node_frame_id;
+  if (index < element->frame_stack.size()) {
+    object_id = element->frame_stack[index].object_id;
+    node_frame_id = element->frame_stack[index].node_frame_id;
+  } else {
+    object_id = element->object_id;
+    node_frame_id = element->node_frame_id;
+  }
+
+  std::vector<std::unique_ptr<runtime::CallArgument>> arguments;
+  arguments.emplace_back(
+      runtime::CallArgument::Builder().SetObjectId(object_id).Build());
+  // Only the main frame should add window scrolling. Do not add scrolling from
+  // iFrame, those are already accounted for in the client bounding rect.
+  bool addWindowScroll = index == 0;
+  arguments.emplace_back(
+      runtime::CallArgument::Builder()
+          .SetValue(base::Value::ToUniquePtrValue(base::Value(addWindowScroll)))
+          .Build());
+  devtools_client_->GetRuntime()->CallFunctionOn(
+      runtime::CallFunctionOnParams::Builder()
+          .SetObjectId(object_id)
+          .SetArguments(std::move(arguments))
+          .SetFunctionDeclaration(std::string(kGetBoundingClientRectAsList))
+          .SetReturnByValue(true)
+          .Build(),
+      node_frame_id,
+      base::BindOnce(&ElementRectGetter::OnGetClientRectResult,
+                     weak_ptr_factory_.GetWeakPtr(), std::move(callback),
+                     std::move(element), index, stacked_rect));
+}
+
+void ElementRectGetter::OnGetClientRectResult(
+    ElementRectCallback callback,
+    std::unique_ptr<ElementFinder::Result> element,
+    size_t index,
+    const RectF& stacked_rect,
+    const DevtoolsClient::ReplyStatus& reply_status,
+    std::unique_ptr<runtime::CallFunctionOnResult> result) {
+  ClientStatus status =
+      CheckJavaScriptResult(reply_status, result.get(), __FILE__, __LINE__);
+  if (!status.ok() || !result->GetResult()->HasValue() ||
+      !result->GetResult()->GetValue()->is_list() ||
+      result->GetResult()->GetValue()->GetList().size() != 4u) {
+    VLOG(2) << __func__ << " Failed to get element rect: " << status;
+    std::move(callback).Run(false, RectF());
+    return;
+  }
+
+  const auto& list = result->GetResult()->GetValue()->GetList();
+  // Value::GetDouble() is safe to call without checking the value type; it'll
+  // return 0.0 if the value has the wrong type.
+
+  RectF rect;
+  rect.left = static_cast<float>(list[0].GetDouble());
+  rect.top = static_cast<float>(list[1].GetDouble());
+  rect.right = static_cast<float>(list[2].GetDouble());
+  rect.bottom = static_cast<float>(list[3].GetDouble());
+
+  if (index > 0) {
+    rect.left = std::min(stacked_rect.right, stacked_rect.left + rect.left);
+    rect.top = std::min(stacked_rect.bottom, stacked_rect.top + rect.top);
+    rect.right = std::min(stacked_rect.right, stacked_rect.left + rect.right);
+    rect.bottom = std::min(stacked_rect.bottom, stacked_rect.top + rect.bottom);
+  }
+
+  if (index >= element->frame_stack.size()) {
+    std::move(callback).Run(true, rect);
+  } else {
+    GetBoundingClientRect(std::move(element), index + 1, rect,
+                          std::move(callback));
+  }
+}
+
+}  // namespace autofill_assistant

components/autofill_assistant/browser/web/element_rect_getter.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_ELEMENT_RECT_GETTER_H_
+#define COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_ELEMENT_RECT_GETTER_H_
+
+#include "base/bind.h"
+#include "base/macros.h"
+#include "base/memory/weak_ptr.h"
+#include "components/autofill_assistant/browser/devtools/devtools_client.h"
+#include "components/autofill_assistant/browser/rectf.h"
+#include "components/autofill_assistant/browser/web/element_finder.h"
+#include "components/autofill_assistant/browser/web/web_controller_worker.h"
+
+namespace autofill_assistant {
+
+// Worker class to get an element's bounding rectangle in viewport coordinates.
+// This returns the global coordinates of the element rect, summing up (and
+// cutting) throughout the iframe stack.
+//
+// 0/0  _____________
+//     |   _______   |
+//     |  |       |  |
+//     |  |   X   |  |
+//     |  |_______|  |
+//     |_____________|
+//                    100/100
+//
+// X inside the iFrame has a local position of 20/20. This getter returns the
+// global position of 50/50.
+class ElementRectGetter : public WebControllerWorker {
+ public:
+  // |devtools_client| must be valid for the lifetime of the instance.
+  ElementRectGetter(DevtoolsClient* devtools_client);
+  ~ElementRectGetter() override;
+
+  // Callback that receives the bounding rect of the element.
+  //
+  // If the first element is false, the call failed. Otherwise, the second
+  // element contains the rect.
+  using ElementRectCallback = base::OnceCallback<void(bool, const RectF&)>;
+
+  void Start(std::unique_ptr<ElementFinder::Result> element,
+             ElementRectCallback callback);
+
+ private:
+  void GetBoundingClientRect(std::unique_ptr<ElementFinder::Result> element,
+                             size_t index,
+                             const RectF& stacked_rect,
+                             ElementRectCallback callback);
+
+  void OnGetClientRectResult(
+      ElementRectCallback callback,
+      std::unique_ptr<ElementFinder::Result> element,
+      size_t index,
+      const RectF& stacked_rect,
+      const DevtoolsClient::ReplyStatus& reply_status,
+      std::unique_ptr<runtime::CallFunctionOnResult> result);
+
+  DevtoolsClient* devtools_client_ = nullptr;
+  base::WeakPtrFactory<ElementRectGetter> weak_ptr_factory_;
+};
+
+}  // namespace autofill_assistant
+
+#endif  // COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_WEB_ELEMENT_POSITION_GETTER_H_

components/autofill_assistant/browser/web/web_controller.cc

@@ -40,14 +40,6 @@ namespace autofill_assistant {
 using autofill::ContentAutofillDriver;
 
 namespace {
-const char* const kGetBoundingClientRectAsList =
-    R"(function(node) {
-      const r = node.getBoundingClientRect();
-      return [window.scrollX + r.left,
-              window.scrollY + r.top,
-              window.scrollX + r.right,
-              window.scrollY + r.bottom];
-    })";
 
 const char* const kGetVisualViewport =
     R"({ const v = window.visualViewport;
@@ -1518,47 +1510,25 @@ void WebController::OnFindElementForPosition(
     std::move(callback).Run(false, empty);
     return;
   }
-
-  std::vector<std::unique_ptr<runtime::CallArgument>> argument;
-  argument.emplace_back(
-      runtime::CallArgument::Builder().SetObjectId(result->object_id).Build());
-  devtools_client_->GetRuntime()->CallFunctionOn(
-      runtime::CallFunctionOnParams::Builder()
-          .SetObjectId(result->object_id)
-          .SetArguments(std::move(argument))
-          .SetFunctionDeclaration(std::string(kGetBoundingClientRectAsList))
-          .SetReturnByValue(true)
-          .Build(),
-      result->node_frame_id,
-      base::BindOnce(&WebController::OnGetElementPositionResult,
-                     weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
+  std::unique_ptr<ElementRectGetter> getter =
+      std::make_unique<ElementRectGetter>(devtools_client_.get());
+  auto* ptr = getter.get();
+  pending_workers_.emplace_back(std::move(getter));
+  ptr->Start(
+      std::move(result),
+      base::BindOnce(&WebController::OnGetElementRectResult,
+                     weak_ptr_factory_.GetWeakPtr(), ptr, std::move(callback)));
 }
 
-void WebController::OnGetElementPositionResult(
+void WebController::OnGetElementRectResult(
+    ElementRectGetter* getter_to_release,
     base::OnceCallback<void(bool, const RectF&)> callback,
-    const DevtoolsClient::ReplyStatus& reply_status,
-    std::unique_ptr<runtime::CallFunctionOnResult> result) {
-  ClientStatus status =
-      CheckJavaScriptResult(reply_status, result.get(), __FILE__, __LINE__);
-  if (!status.ok() || !result->GetResult()->HasValue() ||
-      !result->GetResult()->GetValue()->is_list() ||
-      result->GetResult()->GetValue()->GetList().size() != 4u) {
-    VLOG(2) << __func__ << " Failed to get element position: " << status;
-    RectF empty;
-    std::move(callback).Run(false, empty);
-    return;
-  }
-  const auto& list = result->GetResult()->GetValue()->GetList();
-  // Value::GetDouble() is safe to call without checking the value type; it'll
-  // return 0.0 if the value has the wrong type.
-
-  RectF rect;
-  rect.left = static_cast<float>(list[0].GetDouble());
-  rect.top = static_cast<float>(list[1].GetDouble());
-  rect.right = static_cast<float>(list[2].GetDouble());
-  rect.bottom = static_cast<float>(list[3].GetDouble());
-
-  std::move(callback).Run(true, rect);
+    bool has_rect,
+    const RectF& element_rect) {
+  base::EraseIf(pending_workers_, [getter_to_release](const auto& worker) {
+    return worker.get() == getter_to_release;
+  });
+  std::move(callback).Run(has_rect, element_rect);
 }
 
 void WebController::GetOuterHtml(

components/autofill_assistant/browser/web/web_controller.h

@@ -25,6 +25,7 @@
 #include "components/autofill_assistant/browser/top_padding.h"
 #include "components/autofill_assistant/browser/web/element_finder.h"
 #include "components/autofill_assistant/browser/web/element_position_getter.h"
+#include "components/autofill_assistant/browser/web/element_rect_getter.h"
 #include "components/autofill_assistant/browser/web/web_controller_worker.h"
 #include "third_party/icu/source/common/unicode/umachine.h"
 #include "url/gurl.h"
@@ -342,6 +343,9 @@ class WebController {
       base::OnceCallback<void(const ClientStatus&)> callback,
       std::unique_ptr<ElementFinder::Result> target_element,
       bool result);
+  void OnFocusElement(base::OnceCallback<void(const ClientStatus&)> callback,
+                      const DevtoolsClient::ReplyStatus& reply_status,
+                      std::unique_ptr<runtime::CallFunctionOnResult> result);
   void OnFindElementForSelectOption(
       const std::string& value,
       DropdownSelectStrategy select_strategy,
@@ -351,9 +355,6 @@ class WebController {
   void OnSelectOption(base::OnceCallback<void(const ClientStatus&)> callback,
                       const DevtoolsClient::ReplyStatus& reply_status,
                       std::unique_ptr<runtime::CallFunctionOnResult> result);
-  void OnFocusElement(base::OnceCallback<void(const ClientStatus&)> callback,
-                      const DevtoolsClient::ReplyStatus& reply_status,
-                      std::unique_ptr<runtime::CallFunctionOnResult> result);
   void OnFindElementForHighlightElement(
       base::OnceCallback<void(const ClientStatus&)> callback,
       const ClientStatus& status,
@@ -473,10 +474,11 @@ class WebController {
       base::OnceCallback<void(bool, const RectF&)> callback,
       const DevtoolsClient::ReplyStatus& reply_status,
       std::unique_ptr<runtime::EvaluateResult> result);
-  void OnGetElementPositionResult(
+  void OnGetElementRectResult(
+      ElementRectGetter* getter_to_release,
       base::OnceCallback<void(bool, const RectF&)> callback,
-      const DevtoolsClient::ReplyStatus& reply_status,
-      std::unique_ptr<runtime::CallFunctionOnResult> result);
+      bool has_rect,
+      const RectF& element_rect);
 
   // Creates a new instance of DispatchKeyEventParams for the specified type and
   // unicode codepoint.

components/autofill_assistant/browser/web/web_controller_browsertest.cc

@@ -432,6 +432,30 @@ class WebControllerBrowserTest : public content::ContentBrowserTest,
     std::move(done_callback).Run();
   }
 
+  bool GetElementPosition(const Selector& selector, RectF* rect_output) {
+    base::RunLoop run_loop;
+    bool result;
+    web_controller_->GetElementPosition(
+        selector,
+        base::BindOnce(&WebControllerBrowserTest::OnGetElementPosition,
+                       base::Unretained(this), run_loop.QuitClosure(), &result,
+                       rect_output));
+    run_loop.Run();
+    return result;
+  }
+
+  void OnGetElementPosition(base::OnceClosure done_callback,
+                            bool* result_output,
+                            RectF* rect_output,
+                            bool non_empty,
+                            const RectF& rect) {
+    if (non_empty) {
+      *rect_output = rect;
+    }
+    *result_output = non_empty;
+    std::move(done_callback).Run();
+  }
+
   // Make sure scrolling is necessary for #scroll_container , no matter the
   // screen height
   void SetupScrollContainerHeights() {
@@ -1454,4 +1478,27 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest,
               AnyOf(DOCUMENT_LOADED, DOCUMENT_INTERACTIVE, DOCUMENT_COMPLETE));
 }
 
+IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, GetElementPosition) {
+  RectF document_element_rect;
+  Selector document_element({"#full_height_section"});
+  EXPECT_TRUE(GetElementPosition(document_element, &document_element_rect));
+
+  // This element must be after the full_height_section!
+  RectF iframe_element_rect;
+  Selector iframe_element({"#iframe", "#touch_area"});
+  EXPECT_TRUE(GetElementPosition(iframe_element, &iframe_element_rect));
+
+  EXPECT_GT(iframe_element_rect.top, document_element_rect.bottom);
+
+  // Make sure the element is within the iframe.
+  RectF iframe_rect;
+  Selector iframe({"#iframe"});
+  EXPECT_TRUE(GetElementPosition(iframe, &iframe_rect));
+
+  EXPECT_GT(iframe_element_rect.left, iframe_rect.left);
+  EXPECT_LT(iframe_element_rect.right, iframe_rect.right);
+  EXPECT_GT(iframe_element_rect.top, iframe_rect.top);
+  EXPECT_LT(iframe_element_rect.bottom, iframe_rect.bottom);
+}
+
 }  // namespace autofill_assistant