[Autofill Assistant] Highlight usable section of the page.

With this change, when TouchEventFilter restricts usage of the page to
some specified element, it also highlights these elements by graying out
the sections of the page that cannot be used. The color used for graying
out is the same as the one used for the overlay.
https://screenshot.googleplex.com/TX0MNV26coP.png

If the user taps 3 times within a 15s window, the message "Continue
Manually" appears in the snackbar, allowing 5s to decide to continue or
let Autofill Assistant shuts itself down.
https://screenshot.googleplex.com/zxYdR8i9JuO.png

This patch also lengthen the duration of the snackbar shown when closing
Autofill Assistant to 5s. And shortens the message shown after hitting
the back button, for example, to "I cannot help anymore, please continue
manually."

Bug: 806868
Change-Id: I6a31b31e6f8810b30b64e6a4060ca2fab117f5d1
Reviewed-on: https://chromium-review.googlesource.com/c/1328988
Commit-Queue: Stephane Zermatten <szermatt@chromium.org>
Reviewed-by: Mathias Carlen <mcarlen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#606980}

chrome/android/java/src/org/chromium/chrome/browser/autofill_assistant/AutofillAssistantUiController.java

@@ -5,6 +5,7 @@
 package org.chromium.chrome.browser.autofill_assistant;
 
 import android.accounts.Account;
+import android.graphics.RectF;
 import android.os.Bundle;
 import android.support.annotation.Nullable;
 
@@ -13,6 +14,7 @@ import org.chromium.base.ContextUtils;
 import org.chromium.base.ThreadUtils;
 import org.chromium.base.annotations.CalledByNative;
 import org.chromium.base.annotations.JNINamespace;
+import org.chromium.chrome.autofill_assistant.R;
 import org.chromium.chrome.browser.autofill.PersonalDataManager;
 import org.chromium.chrome.browser.customtabs.CustomTabActivity;
 import org.chromium.chrome.browser.preferences.autofill_assistant.AutofillAssistantPreferences;
@@ -34,6 +36,7 @@ import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Queue;
@@ -162,7 +165,13 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
 
     @Override
     public void onDismiss() {
-        mUiDelegateHolder.dismiss();
+        mUiDelegateHolder.dismiss(R.string.autofill_assistant_stopped,
+                AutofillAssistantUiDelegate.SNACKBAR_DELAY_MS / 1_000);
+    }
+
+    @Override
+    public void onUnexpectedTaps() {
+        mUiDelegateHolder.dismiss(R.string.autofill_assistant_maybe_give_up);
     }
 
     @Override
@@ -185,11 +194,6 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
         return nativeOnRequestDebugContext(mUiControllerAndroid);
     }
 
-    @Override
-    public boolean allowTouchEvent(float x, float y) {
-        return nativeAllowTouchEvent(mUiControllerAndroid, x, y);
-    }
-
     /** Return the value if the given boolean parameter from the extras. */
     private static boolean getBooleanParameter(Bundle extras, String parameterName) {
         return extras.getBoolean(INTENT_EXTRA_PREFIX + parameterName, false);
@@ -376,6 +380,17 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
         mUiDelegateHolder.performUiOperation(AutofillAssistantUiDelegate::hideProgressBar);
     }
 
+    @CalledByNative
+    private void updateTouchableArea(boolean enabled, float[] coords) {
+        List<RectF> boxes = new ArrayList<>();
+        for (int i = 0; i < coords.length; i += 4) {
+            boxes.add(new RectF(/* left= */ coords[i], /* top= */ coords[i + 1],
+                    /* right= */ coords[i + 2], /* bottom= */ coords[i + 3]));
+        }
+        mUiDelegateHolder.performUiOperation(
+                uiDelegate -> { uiDelegate.updateTouchableArea(enabled, boxes); });
+    }
+
     /**
      * Class holder for the AutofillAssistantUiDelegate to make sure we don't make UI changes when
      * we are in a pause state (i.e. few seconds before stopping completely).
@@ -419,7 +434,7 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
          * In normal mode, hides the UI, pauses UI operations and, unless undone within the time
          * delay, eventually destroy everything. In graceful shutdown mode, shutdown immediately.
          */
-        public void dismiss() {
+        public void dismiss(int stringResourceId, Object... formatArgs) {
             assert !mHasBeenShutdown;
 
             if (mIsShuttingDown) {
@@ -427,6 +442,11 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
                 return;
             }
 
+            if (mDismissSnackbar != null) {
+                // Remove duplicate calls.
+                return;
+            }
+
             pauseUiOperations();
             mUiDelegate.hide();
             mDismissSnackbar = new SnackbarManager.SnackbarController() {
@@ -443,7 +463,8 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
                     shutdown();
                 }
             };
-            mUiDelegate.showAutofillAssistantStoppedSnackbar(mDismissSnackbar);
+            mUiDelegate.showAutofillAssistantStoppedSnackbar(
+                    mDismissSnackbar, stringResourceId, formatArgs);
         }
 
         /** Enters graceful shutdown mode once we can again perform UI operations. */
@@ -601,5 +622,4 @@ public class AutofillAssistantUiController implements AutofillAssistantUiDelegat
             long nativeUiControllerAndroid, boolean success, String accessToken);
     private native String nativeGetPrimaryAccountName(long nativeUiControllerAndroid);
     private native String nativeOnRequestDebugContext(long nativeUiControllerAndroid);
-    private native boolean nativeAllowTouchEvent(long nativeUiControllerAndroid, float x, float y);
 }

chrome/android/java/src/org/chromium/chrome/browser/autofill_assistant/AutofillAssistantUiDelegate.java

@@ -10,6 +10,7 @@ import android.animation.ArgbEvaluator;
 import android.animation.ValueAnimator;
 import android.graphics.Bitmap;
 import android.graphics.Color;
+import android.graphics.RectF;
 import android.graphics.drawable.Drawable;
 import android.graphics.drawable.GradientDrawable;
 import android.media.ThumbnailUtils;
@@ -67,6 +68,9 @@ class AutofillAssistantUiDelegate {
     private static final int PROGRESS_BAR_INITIAL_PROGRESS = 10;
     private static final int DETAILS_PULSING_DURATION_MS = 1_000;
 
+    /** How long the snackbars created by {@link #showAutofillAssistantStoppedSnackbar} stay up. */
+    static final int SNACKBAR_DELAY_MS = 5_000;
+
     // TODO(crbug.com/806868): Use correct user locale.
     private static final SimpleDateFormat sDetailsTimeFormat =
             new SimpleDateFormat("H:mma", Locale.getDefault());
@@ -305,11 +309,15 @@ class AutofillAssistantUiDelegate {
                 "AutofillAssistantOverlay", "overlay_color");
         if (!overlayColor.isEmpty()) {
             try {
-                mOverlay.setBackgroundColor(Color.parseColor(overlayColor));
+                @ColorInt
+                int color = Color.parseColor(overlayColor);
+                mOverlay.setBackgroundColor(color);
+                mTouchEventFilter.setGrayOutColor(color);
             } catch (IllegalArgumentException exception) {
                 // ignore
             }
         }
+
         // TODO(crbug.com/806868): Listen for contextual search shown so as to hide this UI.
     }
 
@@ -450,16 +458,14 @@ class AutofillAssistantUiDelegate {
         mFullContainer.setVisibility(View.GONE);
     }
 
-    public void showAutofillAssistantStoppedSnackbar(
-            SnackbarManager.SnackbarController controller) {
-        int durationMs = SnackbarManager.DEFAULT_SNACKBAR_DURATION_MS;
+    public void showAutofillAssistantStoppedSnackbar(SnackbarManager.SnackbarController controller,
+            int stringResourceId, Object... formatArgs) {
         Snackbar snackBar =
-                Snackbar.make(mActivity.getString(
-                                      R.string.autofill_assistant_stopped, durationMs / 1_000),
-                                controller, Snackbar.TYPE_ACTION,
-                                Snackbar.UMA_AUTOFILL_ASSISTANT_STOP_UNDO)
+                Snackbar.make(mActivity.getString(stringResourceId, formatArgs), controller,
+                                Snackbar.TYPE_ACTION, Snackbar.UMA_AUTOFILL_ASSISTANT_STOP_UNDO)
                         .setAction(mActivity.getString(R.string.undo), /* actionData= */ null);
-        snackBar.setDuration(durationMs);
+        snackBar.setSingleLine(false);
+        snackBar.setDuration(SNACKBAR_DELAY_MS);
         mActivity.getSnackbarManager().showSnackbar(snackBar);
     }
 
@@ -630,6 +636,10 @@ class AutofillAssistantUiDelegate {
         mProgressBar.disablePulsing();
     }
 
+    public void updateTouchableArea(boolean enabled, List<RectF> boxes) {
+        mTouchEventFilter.updateTouchableArea(enabled, boxes);
+    }
+
     /**
      * Show profiles in the bar.
      *

chrome/browser/android/autofill_assistant/ui_controller_android.cc

@@ -23,6 +23,7 @@
 #include "components/autofill/core/browser/credit_card.h"
 #include "components/autofill_assistant/browser/access_token_fetcher.h"
 #include "components/autofill_assistant/browser/controller.h"
+#include "components/autofill_assistant/browser/rectf.h"
 #include "components/signin/core/browser/account_info.h"
 #include "components/variations/variations_associated_data.h"
 #include "components/version_info/channel.h"
@@ -247,14 +248,6 @@ UiControllerAndroid::OnRequestDebugContext(
   return base::android::ConvertUTF8ToJavaString(env, GetDebugContext());
 }
 
-jboolean UiControllerAndroid::AllowTouchEvent(
-    JNIEnv* env,
-    const base::android::JavaParamRef<jobject>& jcaller,
-    float x,
-    float y) {
-  return ui_delegate_->AllowTouchEvent(x, y);
-}
-
 void UiControllerAndroid::ChooseAddress(
     base::OnceCallback<void(const std::string&)> callback) {
   DCHECK(!address_or_card_callback_);
@@ -327,6 +320,21 @@ void UiControllerAndroid::HideProgressBar() {
       env, java_autofill_assistant_ui_controller_);
 }
 
+void UiControllerAndroid::UpdateTouchableArea(bool enabled,
+                                              const std::vector<RectF>& areas) {
+  JNIEnv* env = AttachCurrentThread();
+  std::vector<float> flattened;
+  for (const auto& rect : areas) {
+    flattened.emplace_back(rect.left);
+    flattened.emplace_back(rect.top);
+    flattened.emplace_back(rect.right);
+    flattened.emplace_back(rect.bottom);
+  }
+  Java_AutofillAssistantUiController_updateTouchableArea(
+      env, java_autofill_assistant_ui_controller_, enabled,
+      base::android::ToJavaFloatArray(env, flattened));
+}
+
 std::string UiControllerAndroid::GetDebugContext() const {
   return ui_delegate_->GetDebugContext();
 }

chrome/browser/android/autofill_assistant/ui_controller_android.h

@@ -54,6 +54,8 @@ class UiControllerAndroid : public UiController,
   void ShowDetails(const DetailsProto& details) override;
   void ShowProgressBar(int progress, const std::string& message) override;
   void HideProgressBar() override;
+  void UpdateTouchableArea(bool enabled,
+                           const std::vector<RectF>& areas) override;
   std::string GetDebugContext() const override;
 
   // Overrides Client:
@@ -105,10 +107,6 @@ class UiControllerAndroid : public UiController,
   base::android::ScopedJavaLocalRef<jstring> OnRequestDebugContext(
       JNIEnv* env,
       const base::android::JavaParamRef<jobject>& jcaller);
-  jboolean AllowTouchEvent(JNIEnv* env,
-                           const base::android::JavaParamRef<jobject>& jcaller,
-                           float x,
-                           float y);
 
  private:
   // Java-side AutofillAssistantUiController object.

components/autofill_assistant/browser/BUILD.gn

@@ -63,6 +63,8 @@ jumbo_static_library("browser") {
     "payment_information.h",
     "protocol_utils.cc",
     "protocol_utils.h",
+    "rectf.cc",
+    "rectf.h",
     "script.cc",
     "script.h",
     "script_executor.cc",

components/autofill_assistant/browser/controller.cc

@@ -141,6 +141,12 @@ void Controller::Start(const GURL& initialUrl) {
           base::UTF8ToUTF16(web_contents()->GetVisibleURL().host())));
     }
   }
+
+  touchable_element_area_.SetOnUpdate(base::BindRepeating(
+      &UiController::UpdateTouchableArea,
+      // Unretained is safe, since touchable_element_area_ is guaranteed to be
+      // deleted before the UI controller.
+      base::Unretained(GetUiController())));
 }
 
 void Controller::GetOrCheckScripts(const GURL& url) {
@@ -306,17 +312,6 @@ std::string Controller::GetDebugContext() {
   return output_js;
 }
 
-bool Controller::AllowTouchEvent(float x, float y) {
-  if (touchable_element_area_.IsEmpty()) {
-    // Allow all touch events if there is no touchable element area.
-    //
-    // TODO(crbug.com/806868): Enforce the use of touchable elements area once
-    // the feature is complete.
-    return true;
-  }
-  return touchable_element_area_.Contains(x, y);
-}
-
 void Controller::OnDestroy() {
   delete this;
 }

components/autofill_assistant/browser/controller.h

@@ -86,7 +86,6 @@ class Controller : public ScriptExecutorDelegate,
   void OnGiveUp() override;
   void OnScriptSelected(const std::string& script_path) override;
   std::string GetDebugContext() override;
-  bool AllowTouchEvent(float x, float y) override;
 
   // Overrides ScriptTracker::Listener:
   void OnNoRunnableScriptsAnymore() override;

components/autofill_assistant/browser/element_area.cc

@@ -31,13 +31,15 @@ ElementArea::~ElementArea() = default;
 void ElementArea::SetElements(
     const std::vector<std::vector<std::string>>& elements) {
   element_positions_.clear();
-  if (elements.empty())
-    return;
 
   for (const auto& selector : elements) {
     element_positions_.emplace_back();
     element_positions_.back().selector = selector;
   }
+  ReportUpdate();
+
+  if (element_positions_.empty())
+    return;
 
   if (!scheduled_update_) {
     // Check once and schedule regular updates.
@@ -64,15 +66,14 @@ void ElementArea::UpdatePositions() {
 
 bool ElementArea::IsEmpty() const {
   for (const auto& position : element_positions_) {
-    if (position.is_not_empty()) {
+    if (!position.rect.empty()) {
       return false;
     }
   }
   return true;
 }
 
-ElementArea::ElementPosition::ElementPosition()
-    : left(0.0), top(0.0), right(0.0), bottom(0.0) {}
+ElementArea::ElementPosition::ElementPosition() = default;
 ElementArea::ElementPosition::ElementPosition(const ElementPosition& orig) =
     default;
 ElementArea::ElementPosition::~ElementPosition() = default;
@@ -93,17 +94,19 @@ void ElementArea::KeepUpdatingPositions() {
 
 void ElementArea::OnGetElementPosition(const std::vector<std::string>& selector,
                                        bool found,
-                                       float left,
-                                       float top,
-                                       float right,
-                                       float bottom) {
+                                       const RectF& rect) {
   for (auto& position : element_positions_) {
     if (position.selector == selector) {
       // found == false, has all coordinates set to 0.0, which clears the area.
-      position.left = left;
-      position.top = top;
-      position.right = right;
-      position.bottom = bottom;
+      if (position.rect.left == rect.left && position.rect.top == rect.top &&
+          position.rect.right == rect.right &&
+          position.rect.bottom == rect.bottom) {
+        // Avoid reporting unnecessary updates
+        return;
+      }
+
+      position.rect = rect;
+      ReportUpdate();
       return;
     }
   }
@@ -113,12 +116,24 @@ void ElementArea::OnGetElementPosition(const std::vector<std::string>& selector,
 
 bool ElementArea::Contains(float x, float y) const {
   for (const auto& position : element_positions_) {
-    if (position.is_not_empty() && x >= position.left && x <= position.right &&
-        y >= position.top && y <= position.bottom) {
+    if (position.rect.Contains(x, y)) {
       return true;
     }
   }
   return false;
 }
 
+void ElementArea::ReportUpdate() {
+  if (!on_update_)
+    return;
+
+  std::vector<RectF> areas;
+  for (auto& position : element_positions_) {
+    if (!position.rect.empty()) {
+      areas.emplace_back(position.rect);
+    }
+  }
+  on_update_.Run(!element_positions_.empty(), areas);
+}
+
 }  // namespace autofill_assistant

components/autofill_assistant/browser/element_area.h

@@ -8,8 +8,10 @@
 #include <string>
 #include <vector>
 
+#include "base/callback.h"
 #include "base/macros.h"
 #include "base/memory/weak_ptr.h"
+#include "components/autofill_assistant/browser/rectf.h"
 
 namespace autofill_assistant {
 class WebController;
@@ -50,19 +52,27 @@ class ElementArea {
   // exist.
   bool IsEmpty() const;
 
+  // Returns true if there are elements to check.
+  bool HasElements() const { return !element_positions_.empty(); }
+
+  // Defines a callback that'll be run every time the set of element coordinates
+  // changes.
+  //
+  // The first argument is true if there are any elements in the area. The
+  // second reports the areas that corresponds to currently known elements,
+  // which might be empty.
+  void SetOnUpdate(
+      base::RepeatingCallback<void(bool, const std::vector<RectF>& areas)> cb) {
+    on_update_ = cb;
+  }
+
  private:
   // A rectangle that corresponds to the area of the visual viewport covered by
   // an element. Coordinates are values between 0 and 1, relative to the size of
   // the visible viewport.
   struct ElementPosition {
     std::vector<std::string> selector;
-
-    float left;
-    float top;
-    float right;
-    float bottom;
-
-    bool is_not_empty() const { return right > left && bottom > top; }
+    RectF rect;
 
     ElementPosition();
     ElementPosition(const ElementPosition& orig);
@@ -72,10 +82,8 @@ class ElementArea {
   void KeepUpdatingPositions();
   void OnGetElementPosition(const std::vector<std::string>& selector,
                             bool found,
-                            float left,
-                            float top,
-                            float right,
-                            float bottom);
+                            const RectF& rect);
+  void ReportUpdate();
 
   WebController* const web_controller_;
   std::vector<ElementPosition> element_positions_;
@@ -83,6 +91,9 @@ class ElementArea {
   // If true, regular updates are currently scheduled.
   bool scheduled_update_;
 
+  base::RepeatingCallback<void(bool, const std::vector<RectF>& areas)>
+      on_update_;
+
   base::WeakPtrFactory<ElementArea> weak_ptr_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(ElementArea);

components/autofill_assistant/browser/element_area_unittest.cc

@@ -24,7 +24,7 @@ class ElementAreaTest : public testing::Test {
             base::test::ScopedTaskEnvironment::MainThreadType::MOCK_TIME),
         element_area_(&mock_web_controller_) {
     ON_CALL(mock_web_controller_, OnGetElementPosition(_, _))
-        .WillByDefault(RunOnceCallback<1>(false, 0.0f, 0.0f, 0.0f, 0.0f));
+        .WillByDefault(RunOnceCallback<1>(false, RectF()));
   }
 
   // scoped_task_environment_ must be first to guarantee other field
@@ -49,7 +49,7 @@ TEST_F(ElementAreaTest, ElementNotFound) {
 TEST_F(ElementAreaTest, OneElement) {
   EXPECT_CALL(mock_web_controller_,
               OnGetElementPosition(ElementsAre("#found"), _))
-      .WillOnce(RunOnceCallback<1>(true, 0.25f, 0.25f, 0.75f, 0.75f));
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.25f, 0.25f, 0.75f, 0.75f)));
 
   element_area_.SetElements({{"#found"}});
   EXPECT_FALSE(element_area_.IsEmpty());
@@ -63,10 +63,10 @@ TEST_F(ElementAreaTest, OneElement) {
 TEST_F(ElementAreaTest, TwoElements) {
   EXPECT_CALL(mock_web_controller_,
               OnGetElementPosition(ElementsAre("#top_left"), _))
-      .WillOnce(RunOnceCallback<1>(true, 0.0f, 0.0f, 0.25f, 0.25f));
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.0f, 0.25f, 0.25f)));
   EXPECT_CALL(mock_web_controller_,
               OnGetElementPosition(ElementsAre("#bottom_right"), _))
-      .WillOnce(RunOnceCallback<1>(true, 0.25f, 0.25f, 1.0f, 1.0f));
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.25f, 0.25f, 1.0f, 1.0f)));
 
   element_area_.SetElements({{"#top_left"}, {"#bottom_right"}});
   EXPECT_FALSE(element_area_.IsEmpty());
@@ -80,8 +80,8 @@ TEST_F(ElementAreaTest, ElementMovesAfterUpdate) {
   testing::InSequence seq;
   EXPECT_CALL(mock_web_controller_,
               OnGetElementPosition(ElementsAre("#element"), _))
-      .WillOnce(RunOnceCallback<1>(true, 0.0f, 0.25f, 1.0f, 0.5f))
-      .WillOnce(RunOnceCallback<1>(true, 0.0f, 0.5f, 1.0f, 0.75f));
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.25f, 1.0f, 0.5f)))
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.5f, 1.0f, 0.75f)));
 
   element_area_.SetElements({{"#element"}});
 
@@ -102,8 +102,8 @@ TEST_F(ElementAreaTest, ElementMovesWithTime) {
   testing::InSequence seq;
   EXPECT_CALL(mock_web_controller_,
               OnGetElementPosition(ElementsAre("#element"), _))
-      .WillOnce(RunOnceCallback<1>(true, 0.0f, 0.25f, 1.0f, 0.5f))
-      .WillOnce(RunOnceCallback<1>(true, 0.0f, 0.5f, 1.0f, 0.75f));
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.25f, 1.0f, 0.5f)))
+      .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.5f, 1.0f, 0.75f)));
 
   element_area_.SetElements({{"#element"}});
 

components/autofill_assistant/browser/mock_ui_controller.h

@@ -53,6 +53,8 @@ class MockUiController : public UiController {
   MOCK_METHOD1(ShowDetails, void(const DetailsProto& details));
   MOCK_METHOD2(ShowProgressBar, void(int progress, const std::string& message));
   MOCK_METHOD0(HideProgressBar, void());
+  MOCK_METHOD2(UpdateTouchableArea,
+               void(bool enabled, const std::vector<RectF>& areas));
   MOCK_CONST_METHOD0(GetDebugContext, std::string());
 };
 

components/autofill_assistant/browser/mock_web_controller.h

@@ -13,6 +13,7 @@
 #include "testing/gmock/include/gmock/gmock.h"
 
 namespace autofill_assistant {
+struct RectF;
 
 class MockWebController : public WebController {
  public:
@@ -63,15 +64,13 @@ class MockWebController : public WebController {
 
   void GetElementPosition(
       const std::vector<std::string>& selectors,
-      base::OnceCallback<void(bool, float, float, float, float)> callback)
-      override {
+      base::OnceCallback<void(bool, const RectF&)> callback) override {
     OnGetElementPosition(selectors, callback);
   }
 
   MOCK_METHOD2(OnGetElementPosition,
                void(const std::vector<std::string>& selectors,
-                    base::OnceCallback<void(bool, float, float, float, float)>&
-                        callback));
+                    base::OnceCallback<void(bool, const RectF&)>& callback));
 };
 
 }  // namespace autofill_assistant

components/autofill_assistant/browser/rectf.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/autofill_assistant/browser/rectf.h"
+
+namespace autofill_assistant {
+
+RectF::RectF() : left(0.0f), top(0.0f), right(0.0f), bottom(0.0f) {}
+RectF::RectF(float l, float t, float r, float b)
+    : left(l), top(t), right(r), bottom(b) {}
+
+bool RectF::empty() const {
+  return right <= left || bottom <= top;
+}
+
+bool RectF::Contains(float x, float y) const {
+  if (empty())
+    return false;
+
+  return x >= left && x <= right && y >= top && y <= bottom;
+}
+
+}  // namespace autofill_assistant

components/autofill_assistant/browser/rectf.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_RECTF_H_
+#define COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_RECTF_H_
+
+namespace autofill_assistant {
+
+// A simple rectangle structure that uses float. Modelled on Android's RectF
+// class.
+struct RectF {
+  float left;
+  float top;
+  float right;
+  float bottom;
+
+  RectF();
+  RectF(float left, float top, float right, float bottom);
+
+  // Checks whether the rectangle is empty.
+  bool empty() const;
+
+  // Checks whether the rectangle contains the given point.
+  bool Contains(float x, float y) const;
+};
+
+}  // namespace autofill_assistant
+
+#endif  // COMPONENTS_AUTOFILL_ASSISTANT_BROWSER_RECTF_H_

components/autofill_assistant/browser/ui_controller.h

@@ -85,6 +85,15 @@ class UiController {
   // Hide the progress bar.
   virtual void HideProgressBar() = 0;
 
+  // Updates the area of the visible viewport that is accessible.
+  //
+  // If |enabled| is false, the visible viewport is accessible.
+  //
+  // |areas| is expressed in coordinates relative to the width or height of the
+  // visible viewport, as a number between 0 and 1. It can be empty.
+  virtual void UpdateTouchableArea(bool enabled,
+                                   const std::vector<RectF>& areas) = 0;
+
   // Returns a string describing the current execution context. This is useful
   // when analyzing feedback forms and for debugging in general.
   virtual std::string GetDebugContext() const = 0;

components/autofill_assistant/browser/ui_delegate.h

@@ -35,12 +35,6 @@ class UiDelegate {
   // when analyzing feedback forms and for debugging in general.
   virtual std::string GetDebugContext() = 0;
 
-  // Checks whether a touch event at the given position should be allowed.
-  //
-  // Coordinates are values between 0 and 1 relative to the size of the visible
-  // viewport.
-  virtual bool AllowTouchEvent(float x, float y) = 0;
-
  protected:
   UiDelegate() = default;
 };

components/autofill_assistant/browser/web_controller.cc

@@ -14,6 +14,7 @@
 #include "components/autofill/core/browser/credit_card.h"
 #include "components/autofill/core/common/autofill_constants.h"
 #include "components/autofill/core/common/form_data.h"
+#include "components/autofill_assistant/browser/rectf.h"
 #include "components/autofill_assistant/browser/service.pb.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/web_contents.h"
@@ -1055,7 +1056,7 @@ WebController::CreateBatchElementChecker() {
 
 void WebController::GetElementPosition(
     const std::vector<std::string>& selectors,
-    base::OnceCallback<void(bool, float, float, float, float)> callback) {
+    base::OnceCallback<void(bool, const RectF&)> callback) {
   FindElement(
       selectors, /* strict_mode= */ true,
       base::BindOnce(&WebController::OnFindElementForPosition,
@@ -1063,10 +1064,11 @@ void WebController::GetElementPosition(
 }
 
 void WebController::OnFindElementForPosition(
-    base::OnceCallback<void(bool, float, float, float, float)> callback,
+    base::OnceCallback<void(bool, const RectF&)> callback,
     std::unique_ptr<FindElementResult> result) {
   if (result->object_id.empty()) {
-    std::move(callback).Run(false, 0, 0, 0, 0);
+    RectF empty;
+    std::move(callback).Run(false, empty);
     return;
   }
 
@@ -1085,10 +1087,11 @@ void WebController::OnFindElementForPosition(
 }
 
 void WebController::OnGetElementPositionResult(
-    base::OnceCallback<void(bool, float, float, float, float)> callback,
+    base::OnceCallback<void(bool, const RectF&)> callback,
     std::unique_ptr<runtime::CallFunctionOnResult> result) {
   if (!result || result->HasExceptionDetails()) {
-    std::move(callback).Run(false, 0, 0, 0, 0);
+    RectF empty;
+    std::move(callback).Run(false, empty);
     return;
   }
   const auto* value = result->GetResult()->GetValue();
@@ -1108,12 +1111,13 @@ void WebController::OnGetElementPositionResult(
   float visual_w = static_cast<float>(list[6].GetDouble());
   float visual_h = static_cast<float>(list[7].GetDouble());
 
-  float left = std::max(0.0f, left_layout - visual_left_offset) / visual_w;
-  float top = std::max(0.0f, top_layout - visual_top_offset) / visual_h;
-  float right = std::max(0.0f, right_layout - visual_left_offset) / visual_w;
-  float bottom = std::max(0.0f, bottom_layout - visual_top_offset) / visual_h;
+  RectF rect;
+  rect.left = std::max(0.0f, left_layout - visual_left_offset) / visual_w;
+  rect.top = std::max(0.0f, top_layout - visual_top_offset) / visual_h;
+  rect.right = std::max(0.0f, right_layout - visual_left_offset) / visual_w;
+  rect.bottom = std::max(0.0f, bottom_layout - visual_top_offset) / visual_h;
 
-  std::move(callback).Run(true, left, top, right, bottom);
+  std::move(callback).Run(true, rect);
 }
 
 void WebController::OnFindElementForGetOuterHtml(

components/autofill_assistant/browser/web_controller.h

@@ -17,6 +17,7 @@
 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_input.h"
 #include "components/autofill_assistant/browser/devtools/devtools/domains/types_runtime.h"
 #include "components/autofill_assistant/browser/devtools/devtools_client.h"
+#include "components/autofill_assistant/browser/rectf.h"
 
 namespace autofill {
 class AutofillProfile;
@@ -134,7 +135,7 @@ class WebController {
   // height of the visible viewport.
   virtual void GetElementPosition(
       const std::vector<std::string>& selectors,
-      base::OnceCallback<void(bool, float, float, float, float)> callback);
+      base::OnceCallback<void(bool, const RectF&)> callback);
 
  protected:
   friend class BatchElementChecker;
@@ -355,11 +356,11 @@ class WebController {
       std::unique_ptr<runtime::CallFunctionOnResult> result);
 
   void OnFindElementForPosition(
-      base::OnceCallback<void(bool, float, float, float, float)> callback,
+      base::OnceCallback<void(bool, const RectF&)> callback,
       std::unique_ptr<FindElementResult> result);
 
   void OnGetElementPositionResult(
-      base::OnceCallback<void(bool, float, float, float, float)> callback,
+      base::OnceCallback<void(bool, const RectF&)> callback,
       std::unique_ptr<runtime::CallFunctionOnResult> result);
 
   // Weak pointer is fine here since it must outlive this web controller, which

components/autofill_assistant_strings.grdp

@@ -5,7 +5,10 @@
       Something went wrong. You can finish your order on the website.
     </message>
     <message name="IDS_AUTOFILL_ASSISTANT_GIVE_UP" desc="Text label that is shown when autofill assistant cannot help anymore, because of a user action." formatter_data="android_java">
-      It looks like you want to do this by yourself. It's ok to continue on without my help.
+      I cannot help anymore, please continue manually.
+    </message>
+    <message name="IDS_AUTOFILL_ASSISTANT_MAYBE_GIVE_UP" desc="Text label shown next to an UNDO button when user action indicate they want to continue on their own." formatter_data="android_java">
+      Continue manually?
     </message>
     <message name="IDS_AUTOFILL_ASSISTANT_LOADING" desc="Text label that is shown during the loading of the first page, right after being triggered.">
       Opening <ph name="SITE_NAME">$1<ex>google.com</ex></ph>…