[chromecast] Fullscreen magnficiation for cast.

  * Port of Ash's MagnificationController to cast. First patchset is
    the original source, with changes/removals for cast in patchset 2.
  * Enabled via triple-tap when that is enabled.
  * Two-finger pan & pinch-to-zoom.

Bug: internal b/110532452
Test: manual
Change-Id: Ia346fcc6d4ff70aaa0236f1d976c3b0de13a0f86
Reviewed-on: https://chromium-review.googlesource.com/1170987Reviewed-by: Alex Sakhartchouk <alexst@chromium.org>
Commit-Queue: Ryan Daum <rdaum@chromium.org>
Cr-Commit-Position: refs/heads/master@{#582308}

chromecast/browser/accessibility/accessibility_manager.cc

@@ -5,7 +5,7 @@
 #include "chromecast/browser/accessibility/accessibility_manager.h"
 
 #include "chromecast/graphics/accessibility/focus_ring_controller.h"
-#include "chromecast/graphics/accessibility/partial_magnification_controller.h"
+#include "chromecast/graphics/accessibility/fullscreen_magnification_controller.h"
 #include "ui/aura/window.h"
 #include "ui/aura/window_tree_host.h"
 #include "ui/wm/public/activation_client.h"
@@ -31,7 +31,7 @@ AccessibilityManager::AccessibilityManager(
       &accessibility_sound_proxy_);
   triple_tap_detector_ = std::make_unique<TripleTapDetector>(root_window, this);
   magnification_controller_ =
-      std::make_unique<PartialMagnificationController>(root_window);
+      std::make_unique<FullscreenMagnificationController>(root_window);
 }
 
 AccessibilityManager::~AccessibilityManager() {}

chromecast/graphics/BUILD.gn

@@ -38,6 +38,8 @@ cast_source_set("graphics") {
       "accessibility/focus_ring_controller.h",
       "accessibility/focus_ring_layer.cc",
       "accessibility/focus_ring_layer.h",
+      "accessibility/fullscreen_magnification_controller.cc",
+      "accessibility/fullscreen_magnification_controller.h",
       "accessibility/magnification_controller.h",
       "accessibility/partial_magnification_controller.cc",
       "accessibility/partial_magnification_controller.h",

chromecast/graphics/accessibility/fullscreen_magnification_controller.cc

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/graphics/accessibility/fullscreen_magnification_controller.h"
+
+#include "base/numerics/ranges.h"
+#include "ui/aura/window.h"
+#include "ui/aura/window_event_dispatcher.h"
+#include "ui/aura/window_tree_host.h"
+#include "ui/events/event.h"
+#include "ui/gfx/transform.h"
+#include "ui/gfx/transform_util.h"
+
+namespace chromecast {
+
+namespace {
+// Default ratio of magnifier scale.
+const float kDefaultMagnificationScale = 2.f;
+
+constexpr float kMaxMagnifiedScale = 20.0f;
+constexpr float kMinMagnifiedScaleThreshold = 1.1f;
+constexpr float kNonMagnifiedScale = 1.0f;
+
+constexpr float kZoomGestureLockThreshold = 0.1f;
+constexpr float kScrollGestureLockThreshold = 20000.0f;
+
+// Convert point locations to DIP by using the original transform, rather than
+// the one currently installed on the window tree host (which might be our
+// magnifier).
+gfx::Point ConvertPixelsToDIPWithOriginalTransform(
+    const gfx::Transform& transform,
+    const gfx::Point& point) {
+  gfx::Transform invert;
+  if (!transform.GetInverse(&invert)) {
+    // Some transforms can't be inverted, so we do the same as window tree
+    // host's DIP conversion and just use the transform as is in that case.
+    invert = transform;
+  }
+  gfx::PointF dip_point(point);
+  invert.TransformPoint(&dip_point);
+  return gfx::ToFlooredPoint(dip_point);
+}
+
+// Correct the given scale value if necessary.
+void ValidateScale(float* scale) {
+  *scale = base::ClampToRange(*scale, kNonMagnifiedScale, kMaxMagnifiedScale);
+  DCHECK(kNonMagnifiedScale <= *scale && *scale <= kMaxMagnifiedScale);
+}
+
+}  // namespace
+
+class FullscreenMagnificationController::GestureProviderClient
+    : public ui::GestureProviderAuraClient {
+ public:
+  GestureProviderClient() = default;
+  ~GestureProviderClient() override = default;
+
+  // ui::GestureProviderAuraClient overrides:
+  void OnGestureEvent(GestureConsumer* consumer,
+                      ui::GestureEvent* event) override {
+    // Do nothing. OnGestureEvent is for timer based gesture events, e.g. tap.
+    // MagnificationController is interested only in pinch and scroll
+    // gestures.
+    DCHECK_NE(ui::ET_GESTURE_SCROLL_BEGIN, event->type());
+    DCHECK_NE(ui::ET_GESTURE_SCROLL_END, event->type());
+    DCHECK_NE(ui::ET_GESTURE_SCROLL_UPDATE, event->type());
+    DCHECK_NE(ui::ET_GESTURE_PINCH_BEGIN, event->type());
+    DCHECK_NE(ui::ET_GESTURE_PINCH_END, event->type());
+    DCHECK_NE(ui::ET_GESTURE_PINCH_UPDATE, event->type());
+  }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(GestureProviderClient);
+};
+
+FullscreenMagnificationController::FullscreenMagnificationController(
+    aura::Window* root_window)
+    : root_window_(root_window),
+      magnification_scale_(kDefaultMagnificationScale) {
+  DCHECK(root_window);
+  root_window->GetHost()->GetEventSource()->AddEventRewriter(this);
+
+  gesture_provider_client_ = std::make_unique<GestureProviderClient>();
+  gesture_provider_ = std::make_unique<ui::GestureProviderAura>(
+      this, gesture_provider_client_.get());
+}
+
+FullscreenMagnificationController::~FullscreenMagnificationController() {}
+
+void FullscreenMagnificationController::SetEnabled(bool enabled) {
+  if (is_enabled_ == enabled)
+    return;
+  if (!is_enabled_) {
+    // Stash the original root window transform so we can restore it after we're
+    // done.
+    original_transform_ = root_window_->transform();
+  }
+  is_enabled_ = enabled;
+  root_window_->SetTransform(GetMagnifierTransform());
+}
+
+bool FullscreenMagnificationController::IsEnabled() const {
+  return is_enabled_;
+}
+
+void FullscreenMagnificationController::SetMagnificationScale(
+    float magnification_scale) {
+  magnification_scale_ = magnification_scale;
+  root_window_->SetTransform(GetMagnifierTransform());
+}
+
+gfx::Transform FullscreenMagnificationController::GetMagnifierTransform()
+    const {
+  gfx::Transform transform = original_transform_;
+  if (IsEnabled()) {
+    transform.Scale(magnification_scale_, magnification_scale_);
+
+    // Top corner of window.
+    gfx::Point offset = gfx::ToFlooredPoint(magnification_origin_);
+    transform.Translate(-offset.x(), -offset.y());
+  }
+
+  return transform;
+}
+
+// Overridden from ui::EventRewriter
+ui::EventRewriteStatus FullscreenMagnificationController::RewriteEvent(
+    const ui::Event& event,
+    std::unique_ptr<ui::Event>* rewritten_event) {
+  if (!IsEnabled()) {
+    return ui::EventRewriteStatus::EVENT_REWRITE_CONTINUE;
+  }
+  if (!event.IsTouchEvent())
+    return ui::EVENT_REWRITE_CONTINUE;
+
+  const ui::TouchEvent* touch_event = event.AsTouchEvent();
+
+  // Touch events come through in screen pixels, but untransformed. This is the
+  // raw coordinate not yet mapped to the root window's coordinate system or the
+  // screen. Convert it into the root window's coordinate system, in DIP which
+  // is what the rest of this class expects.
+  gfx::Point location = ConvertPixelsToDIPWithOriginalTransform(
+      original_transform_, touch_event->location());
+  gfx::Point root_location = ConvertPixelsToDIPWithOriginalTransform(
+      original_transform_, touch_event->root_location());
+
+  // We now need a TouchEvent that has its coordinates mapped into root window
+  // DIP.
+  ui::TouchEvent touch_event_dip = *touch_event;
+  touch_event_dip.set_location(location);
+  touch_event_dip.set_root_location(root_location);
+
+  // Track finger presses so we can look for our two-finger drag.
+  if (touch_event_dip.type() == ui::ET_TOUCH_PRESSED) {
+    touch_points_++;
+    press_event_map_[touch_event_dip.pointer_details().id] =
+        std::make_unique<ui::TouchEvent>(*touch_event);
+  } else if (touch_event_dip.type() == ui::ET_TOUCH_RELEASED) {
+    touch_points_--;
+    press_event_map_.erase(touch_event_dip.pointer_details().id);
+  }
+
+  if (gesture_provider_->OnTouchEvent(&touch_event_dip)) {
+    gesture_provider_->OnTouchEventAck(
+        touch_event_dip.unique_event_id(), false /* event_consumed */,
+        false /* is_source_touch_event_set_non_blocking */);
+  } else {
+    return ui::EVENT_REWRITE_DISCARD;
+  }
+
+  // The user can change the zoom level with two fingers pinch and pan around
+  // with two fingers scroll. Once we detect one of those two gestures, we start
+  // consuming all touch events by cancelling existing touches. If
+  // cancel_pressed_touches is set to true, ET_TOUCH_CANCELLED
+  // events are dispatched for existing touches after the next for-loop.
+  bool cancel_pressed_touches = ProcessGestures();
+
+  if (cancel_pressed_touches) {
+    // Start consuming all touch events after we cancel existing touches.
+    consume_touch_event_ = true;
+
+    if (!press_event_map_.empty()) {
+      auto it = press_event_map_.begin();
+
+      std::unique_ptr<ui::TouchEvent> rewritten_touch_event =
+          std::make_unique<ui::TouchEvent>(ui::ET_TOUCH_CANCELLED, gfx::Point(),
+                                           touch_event_dip.time_stamp(),
+                                           it->second->pointer_details());
+      rewritten_touch_event->set_location_f(it->second->location_f());
+      rewritten_touch_event->set_root_location_f(it->second->root_location_f());
+      rewritten_touch_event->set_flags(it->second->flags());
+      *rewritten_event = std::move(rewritten_touch_event);
+
+      // The other event is cancelled in NextDispatchEvent.
+      press_event_map_.erase(it);
+
+      return ui::EVENT_REWRITE_DISPATCH_ANOTHER;
+    }
+  }
+
+  bool discard = consume_touch_event_;
+
+  // Reset state once no point is touched on the screen.
+  if (touch_points_ == 0) {
+    consume_touch_event_ = false;
+    locked_gesture_ = NO_GESTURE;
+
+    // Jump back to exactly 1.0 if we are just a tiny bit zoomed in.
+    if (magnification_scale_ < kMinMagnifiedScaleThreshold)
+      SetMagnificationScale(kNonMagnifiedScale);
+  }
+
+  if (discard)
+    return ui::EVENT_REWRITE_DISCARD;
+
+  return ui::EVENT_REWRITE_CONTINUE;
+}
+
+ui::EventRewriteStatus FullscreenMagnificationController::NextDispatchEvent(
+    const ui::Event& last_event,
+    std::unique_ptr<ui::Event>* new_event) {
+  DCHECK_EQ(1u, press_event_map_.size());
+
+  auto it = press_event_map_.begin();
+
+  std::unique_ptr<ui::TouchEvent> event = std::make_unique<ui::TouchEvent>(
+      ui::ET_TOUCH_CANCELLED, gfx::Point(), last_event.time_stamp(),
+      it->second->pointer_details());
+  event->set_location_f(it->second->location_f());
+  event->set_root_location_f(it->second->root_location_f());
+  event->set_flags(it->second->flags());
+  *new_event = std::move(event);
+
+  press_event_map_.erase(it);
+
+  DCHECK_EQ(0u, press_event_map_.size());
+
+  return ui::EVENT_REWRITE_REWRITTEN;
+}
+
+bool FullscreenMagnificationController::RedrawDIP(
+    const gfx::PointF& position_in_dip,
+    float scale) {
+  DCHECK(root_window_);
+
+  float x = position_in_dip.x();
+  float y = position_in_dip.y();
+
+  ValidateScale(&scale);
+
+  if (x < 0)
+    x = 0;
+  if (y < 0)
+    y = 0;
+
+  const gfx::Size host_size_in_dip = root_window_->bounds().size();
+  const float scaled_width = host_size_in_dip.width() / scale;
+  const float scaled_height = host_size_in_dip.height() / scale;
+  float max_x = host_size_in_dip.width() - scaled_width;
+  float max_y = host_size_in_dip.height() - scaled_height;
+  if (x > max_x)
+    x = max_x;
+  if (y > max_y)
+    y = max_y;
+
+  // Does nothing if both the origin and the scale are not changed.
+  if (magnification_origin_.x() == x && magnification_origin_.y() == y &&
+      scale == magnification_scale_) {
+    return false;
+  }
+
+  magnification_origin_.set_x(x);
+  magnification_origin_.set_y(y);
+  magnification_scale_ = scale;
+
+  root_window_->SetTransform(GetMagnifierTransform());
+
+  return true;
+}
+
+bool FullscreenMagnificationController::ProcessGestures() {
+  bool cancel_pressed_touches = false;
+
+  std::vector<std::unique_ptr<ui::GestureEvent>> gestures =
+      gesture_provider_->GetAndResetPendingGestures();
+  for (const auto& gesture : gestures) {
+    const ui::GestureEventDetails& details = gesture->details();
+
+    if (gesture->type() == ui::ET_GESTURE_END ||
+        gesture->type() == ui::ET_GESTURE_BEGIN) {
+      locked_gesture_ = NO_GESTURE;
+      cancel_pressed_touches = false;
+    }
+
+    if (details.touch_points() != 2)
+      continue;
+
+    if (gesture->type() == ui::ET_GESTURE_PINCH_BEGIN) {
+      original_magnification_scale_ = magnification_scale_;
+
+      // Start consuming touch events with cancelling existing touches.
+      if (!consume_touch_event_)
+        cancel_pressed_touches = true;
+    } else if (gesture->type() == ui::ET_GESTURE_PINCH_UPDATE &&
+               (locked_gesture_ == NO_GESTURE || locked_gesture_ == ZOOM)) {
+      float scale = magnification_scale_ * details.scale();
+      ValidateScale(&scale);
+
+      // Lock to zoom mode if the difference between our new scale and old scale
+      // passes our zoom lock threshold.
+      if (locked_gesture_ == NO_GESTURE &&
+          std::abs(scale - original_magnification_scale_) >
+              kZoomGestureLockThreshold) {
+        locked_gesture_ = FullscreenMagnificationController::ZOOM;
+      }
+
+      // |details.bounding_box().CenterPoint()| return center of touch points
+      // of gesture in non-dip screen coordinate.
+      gfx::PointF gesture_center =
+          gfx::PointF(details.bounding_box().CenterPoint());
+
+      // Root transform does dip scaling, screen magnification scaling and
+      // translation. Apply inverse transform to convert non-dip screen
+      // coordinate to dip logical coordinate.
+      root_window_->GetHost()->GetInverseRootTransform().TransformPoint(
+          &gesture_center);
+
+      // Calculate new origin to keep the distance between |gesture_center|
+      // and |origin| same in screen coordinate. This means the following
+      // equation.
+      // (gesture_center.x - magnification_origin_.x) * magnification_scale_ =
+      //   (gesture_center.x - new_origin.x) * scale
+      // If you solve it for |new_origin|, you will get the following formula.
+      const gfx::PointF origin =
+          gfx::PointF(gesture_center.x() -
+                          (magnification_scale_ / scale) *
+                              (gesture_center.x() - magnification_origin_.x()),
+                      gesture_center.y() -
+                          (magnification_scale_ / scale) *
+                              (gesture_center.y() - magnification_origin_.y()));
+
+      RedrawDIP(origin, scale);
+    } else if (gesture->type() == ui::ET_GESTURE_SCROLL_BEGIN) {
+      original_magnification_origin_ = magnification_origin_;
+
+      // Start consuming all touch events with cancelling existing touches.
+      if (!consume_touch_event_)
+        cancel_pressed_touches = true;
+    } else if (gesture->type() == ui::ET_GESTURE_SCROLL_UPDATE &&
+               (locked_gesture_ == NO_GESTURE || locked_gesture_ == SCROLL)) {
+      // If we're not zoomed, scroll is a no-op.
+      if (magnification_scale_ <= kNonMagnifiedScale) {
+        continue;
+      }
+      float new_x = magnification_origin_.x() +
+                    (-1.0f * details.scroll_x() / magnification_scale_);
+      float new_y = magnification_origin_.y() +
+                    (-1.0f * details.scroll_y() / magnification_scale_);
+
+      // Lock to scroll mode if the squared distance from the old position
+      // passes our scroll lock threshold.
+      if (locked_gesture_ == NO_GESTURE) {
+        float diff_x =
+            (new_x - original_magnification_origin_.x()) * magnification_scale_;
+        float diff_y =
+            (new_y - original_magnification_origin_.y()) * magnification_scale_;
+        float squared_distance = (diff_x * diff_x) + (diff_y * diff_y);
+        if (squared_distance > kScrollGestureLockThreshold) {
+          locked_gesture_ = SCROLL;
+        }
+      }
+      RedrawDIP(gfx::PointF(new_x, new_y), magnification_scale_);
+    }
+  }
+
+  return cancel_pressed_touches;
+}
+
+}  // namespace chromecast

chromecast/graphics/accessibility/fullscreen_magnification_controller.h

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_GRAPHICS_ACCESSIBILITY_FULLSCREEN_MAGNIFICATION_CONTROLLER_H_
+#define CHROMECAST_GRAPHICS_ACCESSIBILITY_FULLSCREEN_MAGNIFICATION_CONTROLLER_H_
+
+#include "chromecast/graphics/accessibility/magnification_controller.h"
+#include "ui/events/event_handler.h"
+#include "ui/events/event_rewriter.h"
+#include "ui/events/gestures/gesture_provider_aura.h"
+#include "ui/events/gestures/gesture_types.h"
+#include "ui/gfx/geometry/point.h"
+#include "ui/gfx/transform.h"
+
+namespace aura {
+class Window;
+}  // namespace aura
+
+namespace ui {
+class GestureProviderAura;
+}  // namespace ui
+
+namespace chromecast {
+
+class FullscreenMagnificationController : public MagnificationController,
+                                          public ui::EventRewriter,
+                                          public ui::GestureConsumer {
+ public:
+  explicit FullscreenMagnificationController(aura::Window* root_window);
+  ~FullscreenMagnificationController() override;
+
+  void SetEnabled(bool enabled) override;
+  bool IsEnabled() const override;
+  void SetMagnificationScale(float magnification_scale) override;
+
+ private:
+  class GestureProviderClient;
+  enum LockedGestureType { NO_GESTURE, ZOOM, SCROLL };
+
+  // The current transform to apply to the root window to perform magnification.
+  gfx::Transform GetMagnifierTransform() const;
+
+  // Process pending gestures in |gesture_provider_|. This method returns true
+  // if the controller needs to cancel existing touches.
+  bool ProcessGestures();
+
+  // Redraws the magnification window with the given origin position in dip and
+  // the given scale. Returns true if the window is changed; otherwise, false.
+  bool RedrawDIP(const gfx::PointF& position_in_dip, float scale);
+
+  // Overridden from ui::EventRewriter
+  ui::EventRewriteStatus RewriteEvent(
+      const ui::Event& event,
+      std::unique_ptr<ui::Event>* rewritten_event) override;
+  ui::EventRewriteStatus NextDispatchEvent(
+      const ui::Event& last_event,
+      std::unique_ptr<ui::Event>* new_event) override;
+
+  aura::Window* root_window_;
+
+  bool is_enabled_ = false;
+  gfx::Transform original_transform_;
+
+  // Stores the last touched location. This value is used on zooming to keep
+  // this location visible.
+  gfx::Point point_of_interest_in_root_;
+
+  // Current scale, origin (left-top) position of the magnification window.
+  float magnification_scale_;
+  gfx::PointF magnification_origin_;
+
+  float original_magnification_scale_;
+  gfx::PointF original_magnification_origin_;
+
+  // We own our own GestureProvider to detect gestures with screen coordinates
+  // of touch events. As MagnificationController changes zoom level and moves
+  // viewport, logical coordinates of touches cannot be used for gesture
+  // detection as they are changed if the controller reacts to gestures.
+  std::unique_ptr<ui::GestureProviderAura> gesture_provider_;
+  std::unique_ptr<GestureProviderClient> gesture_provider_client_;
+
+  // We lock the gesture once user performs either scroll or pinch gesture above
+  // those thresholds.
+  LockedGestureType locked_gesture_ = NO_GESTURE;
+
+  // If true, consumes all touch events.
+  bool consume_touch_event_ = false;
+
+  // Number of touch points on the screen.
+  int32_t touch_points_ = 0;
+
+  // Map for holding ET_TOUCH_PRESS events. Those events are used to dispatch
+  // ET_TOUCH_CANCELLED events. Events will be removed from this map when press
+  // events are cancelled, i.e. size of this map can be different from number of
+  // touches on the screen. Key is pointer id.
+  std::map<int32_t, std::unique_ptr<ui::TouchEvent>> press_event_map_;
+};
+
+}  // namespace chromecast
+
+#endif  // CHROMECAST_GRAPHICS_ACCESSIBILITY_FULLSCREEN_MAGNIFICATION_CONTROLLER_H_