[Autofill Assistant] Scroll the element into view again if it was moved out of view

Bug: 806868
Change-Id: Ia2d3ab25ab95645db5215a2c6d8c4b0f1d9d3a1c
Reviewed-on: https://chromium-review.googlesource.com/c/1343561Reviewed-by: Mathias Carlen <mcarlen@chromium.org>
Commit-Queue: Ganggui Tang <gogerald@chromium.org>
Cr-Commit-Position: refs/heads/master@{#609712}

components/autofill_assistant/browser/web_controller.cc

@@ -57,6 +57,11 @@ const char* const kScrollIntoViewScript =
     node.scrollIntoViewIfNeeded();
   })";
 
+const char* const kScrollIntoViewIfNeededScript =
+    R"(function(node) {
+    node.scrollIntoViewIfNeeded();
+  })";
+
 const char* const kScrollByScript =
     R"(window.scrollBy(%f * window.visualViewport.width,
                        %f * window.visualViewport.height))";
@@ -208,15 +213,17 @@ void WebController::ElementPositionGetter::OnGetBoxModelForStableCheck(
   // Return the center of the element.
   const std::vector<double>* content_box = result->GetModel()->GetContent();
   DCHECK_EQ(content_box->size(), 8u);
-  int x = round((round((*content_box)[0]) + round((*content_box)[2])) * 0.5);
-  int y = round((round((*content_box)[3]) + round((*content_box)[5])) * 0.5);
+  int new_point_x =
+      round((round((*content_box)[0]) + round((*content_box)[2])) * 0.5);
+  int new_point_y =
+      round((round((*content_box)[3]) + round((*content_box)[5])) * 0.5);
 
   // Wait for at least three rounds (~600ms =
   // 3*kPeriodicBoxModelCheckInterval) for visual state update callback since
   // it might take longer time to return or never return if no updates.
   DCHECK(kPeriodicBoxModelCheckRounds > 2 &&
          kPeriodicBoxModelCheckRounds >= remaining_rounds);
-  if (x == point_x && y == point_y &&
+  if (new_point_x == point_x && new_point_y == point_y &&
       (visual_state_updated_ ||
        remaining_rounds + 2 < kPeriodicBoxModelCheckRounds)) {
     // Note that there is still a chance that the element's position has been
@@ -225,7 +232,7 @@ void WebController::ElementPositionGetter::OnGetBoxModelForStableCheck(
     // click or tap event after stable for kPeriodicBoxModelCheckInterval. In
     // addition, checking again after issuing click or tap event doesn't help
     // since the change may be expected.
-    OnResult(x, y);
+    OnResult(new_point_x, new_point_y);
     return;
   }
 
@@ -234,12 +241,54 @@ void WebController::ElementPositionGetter::OnGetBoxModelForStableCheck(
     return;
   }
 
+  // Scroll the element into view again if it was moved out of view.
+  // Check 'point_x' amd 'point_y' are greater or equal than zero to escape the
+  // first round.
+  if (point_x >= 0 && point_y >= 0) {
+    std::vector<std::unique_ptr<runtime::CallArgument>> argument;
+    argument.emplace_back(
+        runtime::CallArgument::Builder().SetObjectId(object_id).Build());
+    devtools_client->GetRuntime()->CallFunctionOn(
+        runtime::CallFunctionOnParams::Builder()
+            .SetObjectId(object_id)
+            .SetArguments(std::move(argument))
+            .SetFunctionDeclaration(std::string(kScrollIntoViewIfNeededScript))
+            .SetReturnByValue(true)
+            .Build(),
+        base::BindOnce(&WebController::ElementPositionGetter::OnScrollIntoView,
+                       weak_ptr_factory_.GetWeakPtr(), devtools_client,
+                       object_id, new_point_x, new_point_y, remaining_rounds));
+    return;
+  }
+
+  base::PostDelayedTaskWithTraits(
+      FROM_HERE, {content::BrowserThread::UI},
+      base::BindOnce(
+          &WebController::ElementPositionGetter::GetAndWaitBoxModelStable,
+          weak_ptr_factory_.GetWeakPtr(), devtools_client, object_id,
+          new_point_x, new_point_y, --remaining_rounds),
+      kPeriodicBoxModelCheckInterval);
+}
+
+void WebController::ElementPositionGetter::OnScrollIntoView(
+    DevtoolsClient* devtools_client,
+    std::string object_id,
+    int point_x,
+    int point_y,
+    int remaining_rounds,
+    std::unique_ptr<runtime::CallFunctionOnResult> result) {
+  if (!result || result->HasExceptionDetails()) {
+    DLOG(ERROR) << "Failed to scroll the element.";
+    OnResult(-1, -1);
+    return;
+  }
+
   base::PostDelayedTaskWithTraits(
       FROM_HERE, {content::BrowserThread::UI},
       base::BindOnce(
           &WebController::ElementPositionGetter::GetAndWaitBoxModelStable,
-          weak_ptr_factory_.GetWeakPtr(), devtools_client, object_id, x, y,
-          --remaining_rounds),
+          weak_ptr_factory_.GetWeakPtr(), devtools_client, object_id, point_x,
+          point_y, --remaining_rounds),
       kPeriodicBoxModelCheckInterval);
 }
 

components/autofill_assistant/browser/web_controller.h

@@ -193,6 +193,13 @@ class WebController {
         int point_y,
         int remaining_rounds,
         std::unique_ptr<dom::GetBoxModelResult> result);
+    void OnScrollIntoView(
+        DevtoolsClient* devtools_client,
+        std::string object_id,
+        int point_x,
+        int point_y,
+        int remaining_rounds,
+        std::unique_ptr<runtime::CallFunctionOnResult> result);
     void OnResult(int x, int y);
 
     base::OnceCallback<void(int, int)> callback_;

components/autofill_assistant/browser/web_controller_browsertest.cc

@@ -446,6 +446,13 @@ IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, TapElement) {
   WaitForElementRemove(selectors);
 }
 
+IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, TapElementMovingOutOfView) {
+  std::vector<std::string> selectors;
+  selectors.emplace_back("#touch_area_three");
+  TapElement(selectors);
+  WaitForElementRemove(selectors);
+}
+
 IN_PROC_BROWSER_TEST_F(WebControllerBrowserTest, TapElementAfterPageIsIdle) {
   // Set a very long timeout to make sure either the page is idle or the test
   // timeout.

components/test/data/autofill_assistant/autofill_assistant_target_website.html

@@ -22,14 +22,22 @@ found in the LICENSE file.
         touch_area.parentNode.removeChild(touch_area);
       }
 
-      var moveTouchAreaTwo = function() {
-        var touch_area = document.getElementById("touch_area_two");
-        setTimeout(function(){touch_area.style.left = "100px";}, 100);
-        setTimeout(function(){touch_area.style.left = "200px";}, 200);
-        setTimeout(function(){touch_area.style.left = "300px";}, 300);
-        setTimeout(function(){touch_area.style.left = "100px";}, 400);
-        setTimeout(function(){touch_area.style.left = "200px";}, 500);
-        setTimeout(function(){touch_area.style.left = "300px";}, 600);
+      var moveTouchAreaTwoAndThree = function() {
+        var touch_area_two = document.getElementById("touch_area_two");
+        setTimeout(function(){touch_area_two.style.left = "100px";}, 100);
+        setTimeout(function(){touch_area_two.style.left = "200px";}, 200);
+        setTimeout(function(){touch_area_two.style.left = "300px";}, 300);
+        setTimeout(function(){touch_area_two.style.left = "100px";}, 400);
+        setTimeout(function(){touch_area_two.style.left = "200px";}, 500);
+        setTimeout(function(){touch_area_two.style.left = "300px";}, 600);
+
+        var touch_area_three = document.getElementById("touch_area_three");
+        setTimeout(function(){touch_area_three.style.top += "100px";}, 100);
+        setTimeout(function(){touch_area_three.style.top += "200px";}, 200);
+        setTimeout(function(){touch_area_three.style.top += "300px";}, 300);
+        setTimeout(function(){touch_area_three.style.top += "400px";}, 400);
+        setTimeout(function(){touch_area_three.style.top += "500px";}, 500);
+        setTimeout(function(){touch_area_three.style.top += "600px";}, 600);
       }
     </script>
 
@@ -37,10 +45,14 @@ found in the LICENSE file.
       #full_height_section {
         height: 100vh;
       }
+
+      #touch_area_three {
+        position: absolute;
+      }
     </style>
   </head>
 
-  <body onload="moveTouchAreaTwo()">
+  <body onload="moveTouchAreaTwoAndThree()">
     <!-- Touch area can be accessed without needing scroll.-->
     <div>
       <p id="touch_area_one" ontouchend="removeTouchArea('touch_area_one')">
@@ -63,6 +75,12 @@ found in the LICENSE file.
       <br>
     </div>
 
+    <div>
+      <p id="touch_area_three" ontouchend="removeTouchArea('touch_area_three')">
+        Touchable Area Three</p>
+      <br>
+    </div>
+
     <div>
       <button id="button" type="button" onclick=
           "removeButton()">Test Button</button>

components/test/data/autofill_assistant/autofill_assistant_target_website_iframe_one.html

@@ -12,6 +12,11 @@ found in the LICENSE file.
     <title>Autofill Assistant Test</title>
 
     <script>
+      var load = function() {
+        createShadowDom();
+        moveTouchArea();
+      }
+
       var createShadowDom = function() {
       var shadowSection = document.getElementById("shadowsection");
       var shadowRoot = shadowSection.attachShadow({mode: 'open'});
@@ -29,16 +34,30 @@ found in the LICENSE file.
         var touch_area = document.getElementById("touch_area");
         touch_area.parentNode.removeChild(touch_area);
       }
+
+      var moveTouchArea = function() {
+        var touch_area = document.getElementById("touch_area");
+        setTimeout(function(){touch_area.style.top += "100px";}, 100);
+        setTimeout(function(){touch_area.style.top += "200px";}, 200);
+        setTimeout(function(){touch_area.style.top += "300px";}, 300);
+        setTimeout(function(){touch_area.style.top += "400px";}, 400);
+        setTimeout(function(){touch_area.style.top += "500px";}, 500);
+        setTimeout(function(){touch_area.style.top += "600px";}, 600);
+      }
     </script>
 
     <style>
       #full_height_section {
         height: 100vh;
       }
+
+      #touch_area {
+        position: absolute;
+      }
     </style>
   </head>
 
-  <body onload="createShadowDom()">
+  <body onload="load()">
     <div>
       <form name="address" id="address_section">
         <div id='billing'>