Reanchor and resize Assistant UI when soft keyboard is present.

By limiting Assistant UI size not to exceed the usable work area:

-Fix Assistant UI goes offscreen if there is not enough screen space
 left when soft keyboard is present or rotate/zoom in the display.
-Fix Assistant UI disappears off the screen to the top during resizing
 animation with soft keyboard shows.
-Fix Assistant UI container resizes to the wrong size when dismissing
 soft keyboard.

Test: local compile and manually test.
Bug: b:111444849, b:112927824
Change-Id: I4a0c641386309f857ffd8ddc2b5a73187cc807c5
Reviewed-on: https://chromium-review.googlesource.com/1226369
Commit-Queue: Meilin Wang <meilinw@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Cr-Commit-Position: refs/heads/master@{#594802}

ash/assistant/assistant_ui_controller.cc

@@ -8,6 +8,7 @@
 #include "ash/assistant/assistant_interaction_controller.h"
 #include "ash/assistant/assistant_screen_context_controller.h"
 #include "ash/assistant/ui/assistant_container_view.h"
+#include "ash/assistant/ui/assistant_ui_constants.h"
 #include "ash/assistant/util/deep_link_util.h"
 #include "ash/shell.h"
 #include "ash/strings/grit/ash_strings.h"
@@ -93,8 +94,7 @@ void AssistantUiController::OnWidgetDestroying(views::Widget* widget) {
   model_.SetVisibility(AssistantVisibility::kClosed,
                        AssistantSource::kUnspecified);
 
-  container_view_->GetWidget()->RemoveObserver(this);
-  container_view_ = nullptr;
+  ResetContainerView();
 }
 
 void AssistantUiController::OnInputModalityChanged(
@@ -266,10 +266,8 @@ void AssistantUiController::ShowUi(AssistantSource source) {
     return;
   }
 
-  if (!container_view_) {
-    container_view_ = new AssistantContainerView(assistant_controller_);
-    container_view_->GetWidget()->AddObserver(this);
-  }
+  if (!container_view_)
+    CreateContainerView();
 
   // Note that we initially show the Assistant widget as inactive. This is
   // necessary due to limitations imposed by retrieving screen context. Once we
@@ -337,8 +335,105 @@ void AssistantUiController::UpdateUiMode(
                        : AssistantUiMode::kMainUi);
 }
 
+void AssistantUiController::OnKeyboardWorkspaceOccludedBoundsChanged(
+    const gfx::Rect& new_bounds) {
+  DCHECK(container_view_);
+
+  // Check the display for root window and where the keyboard shows to handle
+  // the case when there are multiple monitors and the virtual keyboard is shown
+  // on a different display other than Assistant UI.
+  aura::Window* root_window =
+      container_view_->GetWidget()->GetNativeWindow()->GetRootWindow();
+  display::Display keyboard_display =
+      display::Screen::GetScreen()->GetDisplayMatching(new_bounds);
+  if (!new_bounds.IsEmpty() &&
+      root_window !=
+          Shell::Get()->GetRootWindowForDisplayId(keyboard_display.id())) {
+    return;
+  }
+
+  // Cache the keyboard workspace occluded bounds.
+  keyboard_workspace_occluded_bounds_ = new_bounds;
+
+  // This keyboard event handles the Assistant UI change when:
+  // 1. accessibility keyboard or normal virtual keyboard pops up or
+  // dismisses. 2. display metrics change (zoom in/out or rotation) when
+  // keyboard shows.
+  UpdateUsableWorkArea(root_window);
+}
+
+void AssistantUiController::OnDisplayMetricsChanged(
+    const display::Display& display,
+    uint32_t changed_metrics) {
+  DCHECK(container_view_);
+
+  // Disable this display event when virtual keyboard shows for solving the
+  // inconsistency between normal virtual keyboard and accessibility keyboard in
+  // changing the work area (accessibility keyboard will change the display work
+  // area but virtual keyboard won't). Display metrics change with keyboard
+  // showing is instead handled by OnKeyboardWorkspaceOccludedBoundsChanged.
+  if (keyboard_workspace_occluded_bounds_.IsEmpty()) {
+    aura::Window* root_window =
+        container_view_->GetWidget()->GetNativeWindow()->GetRootWindow();
+    if (root_window == Shell::Get()->GetRootWindowForDisplayId(display.id())) {
+      UpdateUsableWorkArea(root_window);
+    }
+  }
+}
+
+void AssistantUiController::UpdateUsableWorkArea(aura::Window* root_window) {
+  gfx::Rect usable_work_area;
+  gfx::Rect screen_bounds = root_window->GetBoundsInScreen();
+
+  if (keyboard_workspace_occluded_bounds_.height() != 0) {
+    // When keyboard shows. Unlike accessibility keyboard, normal virtual
+    // keyboard won't change the display work area, so the new usable work
+    // area needs to be calculated manually by subtracting the keyboard
+    // occluded bounds from the screen bounds.
+    usable_work_area = gfx::Rect(
+        screen_bounds.x(), screen_bounds.y(), screen_bounds.width(),
+        screen_bounds.height() - keyboard_workspace_occluded_bounds_.height());
+  } else {
+    // When keyboard hides, the new usable display work area is the same
+    // as the whole display work area for the root window.
+    display::Display display =
+        display::Screen::GetScreen()->GetDisplayMatching(screen_bounds);
+    usable_work_area = display.work_area();
+  }
+
+  usable_work_area.Inset(kMarginDip, kMarginDip);
+  model_.SetUsableWorkArea(usable_work_area);
+}
+
 AssistantContainerView* AssistantUiController::GetViewForTest() {
   return container_view_;
 }
 
+void AssistantUiController::CreateContainerView() {
+  container_view_ = new AssistantContainerView(assistant_controller_);
+  container_view_->GetWidget()->AddObserver(this);
+
+  // To save resources, only watch these events while Assistant UI exists.
+  display::Screen::GetScreen()->AddObserver(this);
+  keyboard::KeyboardController::Get()->AddObserver(this);
+
+  // Retrieve the current keyboard occluded bounds.
+  keyboard_workspace_occluded_bounds_ =
+      keyboard::KeyboardController::Get()->GetWorkspaceOccludedBounds();
+
+  // Set the initial usable work area for Assistant views.
+  aura::Window* root_window =
+      container_view_->GetWidget()->GetNativeWindow()->GetRootWindow();
+  UpdateUsableWorkArea(root_window);
+}
+
+void AssistantUiController::ResetContainerView() {
+  // Remove observers when the Assistant UI is closed.
+  keyboard::KeyboardController::Get()->RemoveObserver(this);
+  display::Screen::GetScreen()->RemoveObserver(this);
+
+  container_view_->GetWidget()->RemoveObserver(this);
+  container_view_ = nullptr;
+}
+
 }  // namespace ash

ash/assistant/assistant_ui_controller.h

@@ -20,6 +20,9 @@
 #include "ash/highlighter/highlighter_controller.h"
 #include "base/macros.h"
 #include "base/timer/timer.h"
+#include "ui/display/display_observer.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/keyboard/keyboard_controller_observer.h"
 #include "ui/views/widget/widget_observer.h"
 
 namespace chromeos {
@@ -48,7 +51,9 @@ class ASH_EXPORT AssistantUiController
       public AssistantMiniViewDelegate,
       public CaptionBarDelegate,
       public DialogPlateObserver,
-      public HighlighterController::Observer {
+      public HighlighterController::Observer,
+      public keyboard::KeyboardControllerObserver,
+      public display::DisplayObserver {
  public:
   explicit AssistantUiController(AssistantController* assistant_controller);
   ~AssistantUiController() override;
@@ -102,6 +107,14 @@ class ASH_EXPORT AssistantUiController
                              AssistantVisibility old_visibility,
                              AssistantSource source) override;
 
+  // keyboard::KeyboardControllerObserver:
+  void OnKeyboardWorkspaceOccludedBoundsChanged(
+      const gfx::Rect& new_bounds) override;
+
+  // display::DisplayObserver:
+  void OnDisplayMetricsChanged(const display::Display& display,
+                               uint32_t changed_metrics) override;
+
   void ShowUi(AssistantSource source);
   void HideUi(AssistantSource source);
   void CloseUi(AssistantSource source);
@@ -114,6 +127,15 @@ class ASH_EXPORT AssistantUiController
   // the basis of interaction/widget visibility state.
   void UpdateUiMode(base::Optional<AssistantUiMode> ui_mode = base::nullopt);
 
+  // Calculate and update the usable work area.
+  void UpdateUsableWorkArea(aura::Window* root_window);
+
+  // Construct |container_view_| and add keyboard/display observers.
+  void CreateContainerView();
+
+  // Reset |container_view_| and remove keyboard/display observers.
+  void ResetContainerView();
+
   AssistantController* const assistant_controller_;  // Owned by Shell.
 
   // Owned by AssistantController.
@@ -124,6 +146,8 @@ class ASH_EXPORT AssistantUiController
   AssistantContainerView* container_view_ =
       nullptr;  // Owned by view hierarchy.
 
+  gfx::Rect keyboard_workspace_occluded_bounds_;
+
   // When hidden, Assistant automatically closes itself to finish the previous
   // session. We delay this behavior to allow the user an opportunity to resume.
   base::OneShotTimer auto_close_timer_;

ash/assistant/model/assistant_ui_model.cc

@@ -39,6 +39,14 @@ void AssistantUiModel::SetVisibility(AssistantVisibility visibility,
   NotifyUiVisibilityChanged(old_visibility, source);
 }
 
+void AssistantUiModel::SetUsableWorkArea(const gfx::Rect& usable_work_area) {
+  if (usable_work_area == usable_work_area_)
+    return;
+
+  usable_work_area_ = usable_work_area;
+  NotifyUsableWorkAreaChanged();
+}
+
 void AssistantUiModel::NotifyUiModeChanged() {
   for (AssistantUiModelObserver& observer : observers_)
     observer.OnUiModeChanged(ui_mode_);
@@ -51,4 +59,9 @@ void AssistantUiModel::NotifyUiVisibilityChanged(
     observer.OnUiVisibilityChanged(visibility_, old_visibility, source);
 }
 
+void AssistantUiModel::NotifyUsableWorkAreaChanged() {
+  for (AssistantUiModelObserver& observer : observers_)
+    observer.OnUsableWorkAreaChanged(usable_work_area_);
+}
+
 }  // namespace ash

ash/assistant/model/assistant_ui_model.h

@@ -7,6 +7,7 @@
 
 #include "base/macros.h"
 #include "base/observer_list.h"
+#include "ui/gfx/geometry/rect.h"
 
 namespace ash {
 
@@ -59,10 +60,17 @@ class AssistantUiModel {
 
   AssistantVisibility visibility() const { return visibility_; }
 
+  // Sets the current usable work area.
+  void SetUsableWorkArea(const gfx::Rect& usable_work_area);
+
+  // Returns the current usable work area.
+  const gfx::Rect& usable_work_area() const { return usable_work_area_; }
+
  private:
   void NotifyUiModeChanged();
   void NotifyUiVisibilityChanged(AssistantVisibility old_visibility,
                                  AssistantSource source);
+  void NotifyUsableWorkAreaChanged();
 
   AssistantUiMode ui_mode_ = AssistantUiMode::kMainUi;
 
@@ -70,6 +78,10 @@ class AssistantUiModel {
 
   base::ObserverList<AssistantUiModelObserver>::Unchecked observers_;
 
+  // Usable work area for Assistant. Value is only meaningful when Assistant
+  // UI exists.
+  gfx::Rect usable_work_area_;
+
   DISALLOW_COPY_AND_ASSIGN(AssistantUiModel);
 };
 

ash/assistant/model/assistant_ui_model_observer.h

@@ -6,6 +6,7 @@
 #define ASH_ASSISTANT_MODEL_ASSISTANT_UI_MODEL_OBSERVER_H_
 
 #include "base/macros.h"
+#include "ui/gfx/geometry/rect.h"
 
 namespace ash {
 
@@ -26,6 +27,11 @@ class AssistantUiModelObserver {
                                      AssistantVisibility old_visibility,
                                      AssistantSource source) {}
 
+  // Invoked when the usable display work area is changed. Observers should
+  // respond to this event by ensuring they are sized/positioned within the
+  // |usable_work_area|.
+  virtual void OnUsableWorkAreaChanged(const gfx::Rect& usable_work_area) {}
+
  protected:
   AssistantUiModelObserver() = default;
   virtual ~AssistantUiModelObserver() = default;

ash/assistant/ui/assistant_container_view.cc

@@ -164,7 +164,7 @@ AssistantContainerView::AssistantContainerView(
     : assistant_controller_(assistant_controller),
       animation_start_frame_number_(0) {
   set_accept_events(true);
-  SetAnchor(nullptr);
+  UpdateAnchor();
   set_close_on_deactivate(false);
   set_color(kBackgroundColor);
   set_margins(gfx::Insets());
@@ -192,13 +192,9 @@ AssistantContainerView::AssistantContainerView(
   // The AssistantController owns the view hierarchy to which
   // AssistantContainerView belongs so is guaranteed to outlive it.
   assistant_controller_->ui_controller()->AddModelObserver(this);
-  display::Screen::GetScreen()->AddObserver(this);
-  keyboard::KeyboardController::Get()->AddObserver(this);
 }
 
 AssistantContainerView::~AssistantContainerView() {
-  keyboard::KeyboardController::Get()->RemoveObserver(this);
-  display::Screen::GetScreen()->RemoveObserver(this);
   assistant_controller_->ui_controller()->RemoveModelObserver(this);
 }
 
@@ -326,22 +322,13 @@ void AssistantContainerView::RequestFocus() {
   }
 }
 
-void AssistantContainerView::SetAnchor(aura::Window* root_window) {
-  // If |root_window| is not specified, we'll use the root window corresponding
-  // to where new windows will be opened.
-  if (!root_window)
-    root_window = Shell::Get()->GetRootWindowForNewWindows();
-
-  // Anchor to the display matching |root_window|.
-  display::Display display = display::Screen::GetScreen()->GetDisplayMatching(
-      root_window->GetBoundsInScreen());
-
-  // Align to the bottom, horizontal center of the work area.
-  gfx::Rect work_area = display.work_area();
-  gfx::Rect anchor =
-      gfx::Rect(work_area.x(), work_area.bottom() - kVerticalMarginDip,
-                work_area.width(), 0);
-
+void AssistantContainerView::UpdateAnchor() {
+  // Align to the bottom, horizontal center of the current usable work area.
+  const gfx::Rect& usable_work_area =
+      assistant_controller_->ui_controller()->model()->usable_work_area();
+  const gfx::Rect anchor =
+      gfx::Rect(usable_work_area.x(), usable_work_area.bottom(),
+                usable_work_area.width(), 0);
   SetAnchorRect(anchor);
   SetArrow(views::BubbleBorder::Arrow::BOTTOM_CENTER);
 }
@@ -367,6 +354,16 @@ void AssistantContainerView::OnUiModeChanged(AssistantUiMode ui_mode) {
   RequestFocus();
 }
 
+void AssistantContainerView::OnUsableWorkAreaChanged(
+    const gfx::Rect& usable_work_area) {
+  UpdateAnchor();
+
+  // Call PreferredSizeChanged() to update animation params to avoid
+  // undesired effects (e.g., resize animation of Assistant UI when
+  // zooming in/out the screen).
+  PreferredSizeChanged();
+}
+
 // TODO(dmblack): Improve performance of this animation using transformations
 // for GPU acceleration. Lower spec hardware may struggle with numerous layouts.
 void AssistantContainerView::AnimationProgressed(
@@ -427,19 +424,6 @@ void AssistantContainerView::AnimationEnded(const gfx::Animation* animation) {
                            smoothness);
 }
 
-void AssistantContainerView::OnDisplayMetricsChanged(
-    const display::Display& display,
-    uint32_t changed_metrics) {
-  aura::Window* root_window = GetWidget()->GetNativeWindow()->GetRootWindow();
-  if (root_window == Shell::Get()->GetRootWindowForDisplayId(display.id()))
-    SetAnchor(root_window);
-}
-
-void AssistantContainerView::OnKeyboardWorkspaceDisplacingBoundsChanged(
-    const gfx::Rect& new_bounds) {
-  SetAnchor(GetWidget()->GetNativeWindow()->GetRootWindow());
-}
-
 void AssistantContainerView::UpdateShadow() {
   // Initialize shadow parameters in painting.
   gfx::ShadowValues shadow_values =

ash/assistant/ui/assistant_container_view.h

@@ -10,9 +10,7 @@
 #include "ash/assistant/model/assistant_ui_model_observer.h"
 #include "base/macros.h"
 #include "ui/compositor/layer.h"
-#include "ui/display/display_observer.h"
 #include "ui/gfx/animation/animation_delegate.h"
-#include "ui/keyboard/keyboard_controller_observer.h"
 #include "ui/views/animation/ink_drop_painted_layer_delegates.h"
 #include "ui/views/bubble/bubble_dialog_delegate_view.h"
 
@@ -33,9 +31,7 @@ class AssistantWebView;
 
 class AssistantContainerView : public views::BubbleDialogDelegateView,
                                public AssistantUiModelObserver,
-                               public display::DisplayObserver,
-                               public gfx::AnimationDelegate,
-                               public keyboard::KeyboardControllerObserver {
+                               public gfx::AnimationDelegate {
  public:
   explicit AssistantContainerView(AssistantController* assistant_controller);
   ~AssistantContainerView() override;
@@ -59,23 +55,15 @@ class AssistantContainerView : public views::BubbleDialogDelegateView,
 
   // AssistantUiModelObserver:
   void OnUiModeChanged(AssistantUiMode ui_mode) override;
+  void OnUsableWorkAreaChanged(const gfx::Rect& usable_work_area) override;
 
   // gfx::AnimationDelegate:
   void AnimationProgressed(const gfx::Animation* animation) override;
   void AnimationEnded(const gfx::Animation* animation) override;
 
-  // display::DisplayObserver:
-  void OnDisplayMetricsChanged(const display::Display& display,
-                               uint32_t changed_metrics) override;
-
-  // keyboard::KeyboardControllerObserver:
-  void OnKeyboardWorkspaceDisplacingBoundsChanged(
-      const gfx::Rect& new_bounds) override;
-
  private:
-  // Sets anchor rect to |root_window|. If it's null,
-  // result of GetRootWindowForNewWindows() will be used.
-  void SetAnchor(aura::Window* root_window);
+  // Update anchor rect with respect to the current usable work area.
+  void UpdateAnchor();
 
   // Update the shadow layer.
   void UpdateShadow();

ash/assistant/ui/assistant_main_view.cc

@@ -84,13 +84,10 @@ int AssistantMainView::GetHeightForWidth(int width) const {
   height = std::min(height, kMaxHeightDip);
   height = std::max(height, min_height_dip_);
 
-  // |height| should not exceed workspace height.
-  aura::Window* root_window =
-      parent()->GetWidget()->GetNativeWindow()->GetRootWindow();
-  display::Display display = display::Screen::GetScreen()->GetDisplayMatching(
-      root_window->GetBoundsInScreen());
-  gfx::Rect work_area = display.work_area();
-  height = std::min(height, work_area.height() - 2 * kVerticalMarginDip);
+  // |height| should not exceed the height of the usable work area.
+  gfx::Rect usable_work_area =
+      assistant_controller_->ui_controller()->model()->usable_work_area();
+  height = std::min(height, usable_work_area.height());
 
   return height;
 }

ash/assistant/ui/assistant_ui_constants.h

@@ -20,7 +20,7 @@ constexpr int kMaxHeightDip = 640;
 constexpr int kPaddingDip = 14;
 constexpr int kPreferredWidthDip = 640;
 constexpr int kSpacingDip = 8;
-constexpr int kVerticalMarginDip = 8;
+constexpr int kMarginDip = 8;
 
 // Typography.
 constexpr SkColor kTextColorHint = gfx::kGoogleGrey500;

ash/assistant/ui/assistant_web_view.cc

@@ -86,17 +86,11 @@ gfx::Size AssistantWebView::CalculatePreferredSize() const {
 
 int AssistantWebView::GetHeightForWidth(int width) const {
   // |height| <= |kMaxHeightDip|.
-  int height = kMaxHeightDip;
+  // |height| should not exceed the height of the usable work area.
+  gfx::Rect usable_work_area =
+      assistant_controller_->ui_controller()->model()->usable_work_area();
 
-  // |height| should not exceed workspace height.
-  aura::Window* root_window =
-      parent()->GetWidget()->GetNativeWindow()->GetRootWindow();
-  display::Display display = display::Screen::GetScreen()->GetDisplayMatching(
-      root_window->GetBoundsInScreen());
-  gfx::Rect work_area = display.work_area();
-  height = std::min(height, work_area.height() - 2 * kVerticalMarginDip);
-
-  return height;
+  return std::min(kMaxHeightDip, usable_work_area.height());
 }
 
 void AssistantWebView::ChildPreferredSizeChanged(views::View* child) {

ash/shell.cc

@@ -1189,10 +1189,6 @@ void Shell::Init(
   partial_magnification_controller_.reset(new PartialMagnificationController());
   highlighter_controller_.reset(new HighlighterController());
 
-  assistant_controller_ = chromeos::switches::IsAssistantEnabled()
-                              ? std::make_unique<AssistantController>()
-                              : nullptr;
-
   magnification_controller_ = std::make_unique<MagnificationController>();
   mru_window_tracker_ = std::make_unique<MruWindowTracker>();
 
@@ -1253,6 +1249,12 @@ void Shell::Init(
 
   window_tree_host_manager_->InitHosts();
 
+  // |assistant_controller_| needs to be created after InitHosts() since its
+  // keyboard observer function result has dependency on workspace change.
+  assistant_controller_ = chromeos::switches::IsAssistantEnabled()
+                              ? std::make_unique<AssistantController>()
+                              : nullptr;
+
   // Needs to be created after InitDisplays() since it may cause the virtual
   // keyboard to be deployed.
   virtual_keyboard_controller_ = std::make_unique<VirtualKeyboardController>();