Commit 817178cd authored by Yuwei Huang's avatar Yuwei Huang Committed by Commit Bot

[CRD iOS] Improve viewport manipulation on iPhone X

This CL allows the iOS client to display desktop that is partially
obstructed by the notch while allowing user to pan the desktop out of
the obstructed area, similar to what we have done on the Android
client.

This CL also changes the previous logic that handles keyboard height.
We are now using DesktopViewport's SetSafeInsets() methods to adjust
for the keyboard rather than just changing the surface size.

Bug: 876014
Change-Id: I428cf2552acb3bc33cf932890a9395428d969a40
Reviewed-on: https://chromium-review.googlesource.com/1186027Reviewed-by: default avatarJoe Downing <joedow@chromium.org>
Commit-Queue: Yuwei Huang <yuweih@chromium.org>
Cr-Commit-Position: refs/heads/master@{#585666}
parent 42ad79bb
......@@ -182,6 +182,19 @@ void GestureInterpreter::OnSurfaceSizeChanged(int width, int height) {
void GestureInterpreter::OnDesktopSizeChanged(int width, int height) {
viewport_.SetDesktopSize(width, height);
if (viewport_.IsViewportReady()) {
input_strategy_->FocusViewportOnCursor(&viewport_);
}
}
void GestureInterpreter::OnSafeInsetsChanged(int left,
int top,
int right,
int bottom) {
viewport_.SetSafeInsets(left, top, right, bottom);
if (viewport_.IsViewportReady()) {
input_strategy_->FocusViewportOnCursor(&viewport_);
}
}
base::WeakPtr<GestureInterpreter> GestureInterpreter::GetWeakPtr() {
......
......@@ -82,6 +82,7 @@ class GestureInterpreter {
void OnSurfaceSizeChanged(int width, int height);
void OnDesktopSizeChanged(int width, int height);
void OnSafeInsetsChanged(int left, int top, int right, int bottom);
base::WeakPtr<GestureInterpreter> GetWeakPtr();
......
......@@ -35,25 +35,16 @@ void DesktopViewport::SetSurfaceSize(int surface_width, int surface_height) {
return;
}
// Only reset the viewport if both dimensions have changed, otherwise keep
// the offset and scale and just change the constraint. This is to cover these
// use cases:
// * Rotation => Reset
// * Keyboard => No reset
// * Settings menu => No reset
//
// TODO(yuweih): This is probably too much inferring. Let the caller to decide
// when to call ResizeToFit() if things don't work right.
bool need_to_reset =
surface_width != surface_size_.x && surface_height != surface_size_.y;
surface_size_.x = surface_width;
surface_size_.y = surface_height;
ResizeToFit();
}
if (need_to_reset) {
ResizeToFit();
return;
}
void DesktopViewport::SetSafeInsets(int left, int top, int right, int bottom) {
safe_insets_.left = left;
safe_insets_.top = top;
safe_insets_.right = right;
safe_insets_.bottom = bottom;
UpdateViewport();
}
......@@ -83,8 +74,12 @@ ViewMatrix::Point DesktopViewport::GetViewportCenter() const {
LOG(WARNING) << "Viewport is not ready before getting the viewport center";
return {0.f, 0.f};
}
float safe_area_center_x =
(surface_size_.x + safe_insets_.left - safe_insets_.right) / 2.f;
float safe_area_center_y =
(surface_size_.y + safe_insets_.top - safe_insets_.bottom) / 2.f;
return desktop_to_surface_transform_.Invert().MapPoint(
{surface_size_.x / 2.f, surface_size_.y / 2.f});
{safe_area_center_x, safe_area_center_y});
}
bool DesktopViewport::IsPointWithinDesktopBounds(
......@@ -109,7 +104,7 @@ ViewMatrix::Point DesktopViewport::ConstrainPointToDesktop(
return point;
}
return ConstrainPointToBounds({0.f, desktop_size_.x, 0.f, desktop_size_.y},
return ConstrainPointToBounds({0.f, 0.f, desktop_size_.x, desktop_size_.y},
point);
}
......@@ -149,10 +144,12 @@ void DesktopViewport::ResizeToFit() {
// +----------+ v
// resize the desktop such that it fits the viewport in one dimension.
float scale = std::max(surface_size_.x / desktop_size_.x,
surface_size_.y / desktop_size_.y);
ViewMatrix::Vector2D safe_area_size = GetSurfaceSafeAreaSize();
float scale = std::max(safe_area_size.x / desktop_size_.x,
safe_area_size.y / desktop_size_.y);
desktop_to_surface_transform_.SetScale(scale);
desktop_to_surface_transform_.SetOffset({0.f, 0.f});
desktop_to_surface_transform_.SetOffset(
{safe_insets_.left, safe_insets_.top});
UpdateViewport();
}
......@@ -173,10 +170,12 @@ void DesktopViewport::UpdateViewport() {
desktop_to_surface_transform_.SetScale(MAX_ZOOM_LEVEL);
}
ViewMatrix::Vector2D safe_area_size = GetSurfaceSafeAreaSize();
ViewMatrix::Vector2D desktop_size_on_surface_ =
desktop_to_surface_transform_.MapVector(desktop_size_);
if (desktop_size_on_surface_.x < surface_size_.x &&
desktop_size_on_surface_.y < surface_size_.y) {
if (desktop_size_on_surface_.x < safe_area_size.x &&
desktop_size_on_surface_.y < safe_area_size.y) {
// +==============+
// | VP | +==========+
// | | | VP |
......@@ -188,8 +187,8 @@ void DesktopViewport::UpdateViewport() {
// +==============+
// Displayed desktop is too small in both directions, so apply the minimum
// zoom level needed to fit either the width or height.
float scale = std::min(surface_size_.x / desktop_size_.x,
surface_size_.y / desktop_size_.y);
float scale = std::min(safe_area_size.x / desktop_size_.x,
safe_area_size.y / desktop_size_.y);
desktop_to_surface_transform_.SetScale(scale);
}
......@@ -239,13 +238,14 @@ DesktopViewport::Bounds DesktopViewport::GetViewportCenterBounds() const {
// Viewport size on the desktop space.
ViewMatrix::Vector2D viewport_size =
desktop_to_surface_transform_.Invert().MapVector(surface_size_);
desktop_to_surface_transform_.Invert().MapVector(
GetSurfaceSafeAreaSize());
// Scenario 1: If VP can fully fit inside the desktop, then VP's center can be
// anywhere inside the desktop as long as VP doesn't overlap with the border.
bounds.left = viewport_size.x / 2.f;
bounds.right = desktop_size_.x - viewport_size.x / 2.f;
bounds.top = viewport_size.y / 2.f;
bounds.right = desktop_size_.x - viewport_size.x / 2.f;
bounds.bottom = desktop_size_.y - viewport_size.y / 2.f;
// Scenario 2: If VP can't fully fit inside the desktop in dimension D, then
......@@ -266,6 +266,11 @@ DesktopViewport::Bounds DesktopViewport::GetViewportCenterBounds() const {
return bounds;
}
ViewMatrix::Vector2D DesktopViewport::GetSurfaceSafeAreaSize() const {
return {surface_size_.x - safe_insets_.left - safe_insets_.right,
surface_size_.y - safe_insets_.top - safe_insets_.bottom};
}
void DesktopViewport::MoveViewportWithoutUpdate(float dx, float dy) {
// <dx, dy> is defined on desktop's reference frame. Translation must be
// flipped and scaled.
......
......@@ -30,7 +30,8 @@ namespace remoting {
// reference frame.
class DesktopViewport {
public:
using TransformationCallback = base::Callback<void(const ViewMatrix&)>;
using TransformationCallback =
base::RepeatingCallback<void(const ViewMatrix&)>;
DesktopViewport();
~DesktopViewport();
......@@ -38,10 +39,17 @@ class DesktopViewport {
// Sets the |desktop_size_| and (re)initializes the viewport.
void SetDesktopSize(int desktop_width, int desktop_height);
// Sets the |surface_size_| and (re)initializes the viewport if both
// dimensions are changed.
// Sets the |surface_size_| and (re)initializes the viewport.
void SetSurfaceSize(int surface_width, int surface_height);
// Sets insets on the surface area to allow viewport to be panned out of them.
// Should be used to adjust for system UI like soft keyboard and screen
// notches/cutouts.
// This method effectively shrinks the size of the viewport on the surface.
// You may want to call this before SetSurfaceSize() so that safe insets are
// taken into account when initializing viewport.
void SetSafeInsets(int left, int top, int right, int bottom);
// Translates the desktop on the surface's reference frame by <dx, dy>.
void MoveDesktop(float dx, float dy);
......@@ -82,8 +90,8 @@ class DesktopViewport {
private:
struct Bounds {
float left;
float right;
float top;
float right;
float bottom;
};
......@@ -100,6 +108,9 @@ class DesktopViewport {
// locate.
Bounds GetViewportCenterBounds() const;
// Gets the size of |surface_size_| inset by |safe_insets_|.
ViewMatrix::Vector2D GetSurfaceSafeAreaSize() const;
// Translates the viewport on the desktop's reference frame by <dx, dy>,
// without calling UpdateViewport().
void MoveViewportWithoutUpdate(float dx, float dy);
......@@ -112,6 +123,7 @@ class DesktopViewport {
ViewMatrix::Vector2D desktop_size_{0.f, 0.f};
ViewMatrix::Vector2D surface_size_{0.f, 0.f};
Bounds safe_insets_{0, 0, 0, 0};
ViewMatrix desktop_to_surface_transform_;
......
......@@ -120,7 +120,7 @@ TEST_F(DesktopViewportTest, TestViewportInitialization3) {
// +========+----+
viewport_.SetDesktopSize(9, 3);
viewport_.SetSurfaceSize(2, 1);
AssertTransformationReceived(FROM_HERE, 0.333f, 0.f, 0.f);
AssertTransformationReceived(FROM_HERE, 1 / 3.f, 0.f, 0.f);
}
TEST_F(DesktopViewportTest, TestViewportInitialization4) {
......@@ -292,4 +292,115 @@ TEST_F(DesktopViewportTest, TestScaleDesktop) {
new_transformation.GetScale());
}
TEST_F(DesktopViewportTest, AsymmetricalSafeInsetsPanAndZoom) {
// Initialize with 6x5 desktop and 6x5 screen with this safe inset:
// left: 2, top: 2, right: 1, bottom: 1
viewport_.SetDesktopSize(6, 5);
viewport_.SetSafeInsets(2, 2, 1, 1);
viewport_.SetSurfaceSize(6, 5);
// Viewport is initialized to fit the inset area instead of the whole surface
// area.
AssertTransformationReceived(FROM_HERE, 0.5, 2, 2);
// Move the viewport all the way to the bottom right.
viewport_.MoveViewport(100, 100);
// The bottom right of the desktop is stuck with the bottom right of the
// safe area.
AssertTransformationReceived(FROM_HERE, 0.5, 2, 1.5);
// Zoom the viewport on the bottom right of the safe area to match the
// resolution of the surface.
viewport_.ScaleDesktop(5, 4, 2);
AssertTransformationReceived(FROM_HERE, 1, -1, -1);
// Move the desktop by <1, 1>. Now it perfectly fits the surface.
viewport_.MoveDesktop(1, 1);
AssertTransformationReceived(FROM_HERE, 1, 0, 0);
// Move the desktop all the way to the top left. Now it stucks with the top
// left corner of the safe area.
viewport_.MoveDesktop(100, 100);
AssertTransformationReceived(FROM_HERE, 1, 2, 2);
}
TEST_F(DesktopViewportTest, SingleNotchSafeInsetPanAndZoom) {
// Initialize with 6x5 desktop and 6x5 screen with this safe inset:
// left: 1, right: 1, top: 0, bottom: 0
viewport_.SetDesktopSize(6, 5);
viewport_.SetSafeInsets(1, 1, 0, 0);
viewport_.SetSurfaceSize(6, 5);
AssertTransformationReceived(FROM_HERE, 5 / 6.f, 1, 1);
viewport_.MoveViewport(100, 100);
AssertTransformationReceived(FROM_HERE, 5 / 6.f, 1, 5 / 6.f);
viewport_.ScaleDesktop(6, 5, 1.2);
AssertTransformationReceived(FROM_HERE, 1, 0, 0);
}
TEST_F(DesktopViewportTest, SymmetricSafeInsetPanAndZoom) {
// Initialize with 6x5 desktop and 6x5 screen with this safe inset:
// left: 1, right: 1, top: 1, bottom: 1
viewport_.SetDesktopSize(6, 5);
viewport_.SetSafeInsets(1, 1, 1, 1);
viewport_.SetSurfaceSize(6, 5);
AssertTransformationReceived(FROM_HERE, 2 / 3.f, 1, 1);
viewport_.MoveViewport(100, 100);
AssertTransformationReceived(FROM_HERE, 2 / 3.f, 1, 2 / 3.f);
viewport_.ScaleDesktop(5, 4, 1.5);
AssertTransformationReceived(FROM_HERE, 1, -1, -1);
viewport_.MoveDesktop(1, 1);
AssertTransformationReceived(FROM_HERE, 1, 0, 0);
}
TEST_F(DesktopViewportTest, RemoveSafeInsets) {
// Initialize with 6x5 desktop and 6x5 screen with this safe inset:
// left: 2, top: 2, right: 1, bottom: 1
viewport_.SetDesktopSize(6, 5);
viewport_.SetSafeInsets(2, 2, 1, 1);
viewport_.SetSurfaceSize(6, 5);
AssertTransformationReceived(FROM_HERE, 0.5, 2, 2);
// Move the viewport all the way to the bottom right.
viewport_.MoveViewport(100, 100);
AssertTransformationReceived(FROM_HERE, 0.5, 2, 1.5);
// Now remove the safe insets.
viewport_.SetSafeInsets(0, 0, 0, 0);
// Desktop is now stretched to fit the whole surface.
AssertTransformationReceived(FROM_HERE, 1, 0, 0);
}
TEST_F(DesktopViewportTest, AddAndRemoveSafeInsets) {
// This test case tests showing and hiding soft keyboard.
// Initialize with 12x9 desktop and screen with this safe inset:
// left: 2, top: 2, right: 1, bottom: 1
viewport_.SetDesktopSize(12, 9);
viewport_.SetSafeInsets(2, 2, 1, 1);
viewport_.SetSurfaceSize(12, 9);
AssertTransformationReceived(FROM_HERE, 0.75, 2, 2);
// Increase the bottom insets to simulate keyboard popup.
viewport_.SetSafeInsets(2, 2, 1, 4);
AssertTransformationReceived(FROM_HERE, 0.75, 2, 2);
// Move the viewport all the way down. (=moving the desktop all the way up.)
viewport_.MoveViewport(100, 100);
AssertTransformationReceived(FROM_HERE, 0.75, 2, -1.75);
// Now remove the extra insets.
viewport_.SetSafeInsets(2, 2, 1, 1);
// Viewport should bounce back.
AssertTransformationReceived(FROM_HERE, 0.75, 2, 1.25);
}
} // namespace remoting
......@@ -66,9 +66,9 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
// A placeholder view for anchoring views and calculating visible area.
UIView* _keyboardPlaceholderView;
// A display link for animating host surface size change. Use the paused
// A display link for animating keyboard height change. Use the paused
// property to start or stop the animation.
CADisplayLink* _surfaceSizeAnimationLink;
CADisplayLink* _keyboardHeightAnimationLink;
}
@end
......@@ -104,17 +104,11 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
_hostView.accessibilityTraits = UIAccessibilityTraitAllowsDirectInteraction;
[self.view addSubview:_hostView];
UILayoutGuide* safeAreaLayoutGuide =
remoting::SafeAreaLayoutGuideForView(self.view);
[NSLayoutConstraint activateConstraints:@[
[_hostView.topAnchor constraintEqualToAnchor:safeAreaLayoutGuide.topAnchor],
[_hostView.bottomAnchor
constraintEqualToAnchor:safeAreaLayoutGuide.bottomAnchor],
[_hostView.leadingAnchor
constraintEqualToAnchor:safeAreaLayoutGuide.leadingAnchor],
[_hostView.trailingAnchor
constraintEqualToAnchor:safeAreaLayoutGuide.trailingAnchor],
[_hostView.topAnchor constraintEqualToAnchor:self.view.topAnchor],
[_hostView.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor],
[_hostView.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor],
[_hostView.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor],
]];
_hostView.displayTaskRunner =
......@@ -243,12 +237,12 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
[self applicationWillResignActive:UIApplication.sharedApplication];
}
_surfaceSizeAnimationLink =
_keyboardHeightAnimationLink =
[CADisplayLink displayLinkWithTarget:self
selector:@selector(animateHostSurfaceSize:)];
_surfaceSizeAnimationLink.paused = YES;
[_surfaceSizeAnimationLink addToRunLoop:NSRunLoop.currentRunLoop
forMode:NSDefaultRunLoopMode];
selector:@selector(animateKeyboardHeight:)];
_keyboardHeightAnimationLink.paused = YES;
[_keyboardHeightAnimationLink addToRunLoop:NSRunLoop.currentRunLoop
forMode:NSDefaultRunLoopMode];
}
- (void)viewWillDisappear:(BOOL)animated {
......@@ -259,18 +253,23 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
[[NSNotificationCenter defaultCenter] removeObserver:self];
_surfaceSizeAnimationLink.paused = YES;
[_surfaceSizeAnimationLink invalidate];
_keyboardHeightAnimationLink.paused = YES;
[_keyboardHeightAnimationLink invalidate];
}
- (void)viewDidLayoutSubviews {
[super viewDidLayoutSubviews];
[self updateViewportSafeInsets];
// Pass the actual size of the view to the renderer.
[_client.displayHandler setSurfaceSize:_hostView.bounds];
// Start the animation on the host's visible area.
_surfaceSizeAnimationLink.paused = NO;
_client.gestureInterpreter->OnSurfaceSizeChanged(
_hostView.bounds.size.width, _hostView.bounds.size.height);
// Start the safe insets animation.
_keyboardHeightAnimationLink.paused = NO;
[self resizeHostToFitIfNeeded];
}
......@@ -402,24 +401,22 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
}
- (void)resizeHostToFitIfNeeded {
// Don't adjust the host resolution if the keyboard is active. That would end
// up with a very narrow desktop.
// Also don't adjust if it's the phone and in portrait orientation. This is
// the most used orientation on phones but the aspect ratio is uncommon on
// desktop devices.
// Don't adjust if it's the phone and in portrait orientation because UI looks
// too tight.
BOOL isPhonePortrait =
self.traitCollection.horizontalSizeClass ==
UIUserInterfaceSizeClassCompact &&
self.traitCollection.verticalSizeClass == UIUserInterfaceSizeClassRegular;
if (_settings.shouldResizeHostToFit && !isPhonePortrait &&
!_clientKeyboard.showsSoftKeyboard) {
[_client setHostResolution:_hostView.frame.size
if (_settings.shouldResizeHostToFit && !isPhonePortrait) {
UIEdgeInsets safeInsets = remoting::SafeAreaInsetsForView(_hostView);
CGRect safeRect = UIEdgeInsetsInsetRect(_hostView.frame, safeInsets);
[_client setHostResolution:safeRect.size
scale:_hostView.contentScaleFactor];
}
}
- (void)animateHostSurfaceSize:(CADisplayLink*)link {
- (void)animateKeyboardHeight:(CADisplayLink*)link {
// The method is called when the keyboard animation is in-progress. It
// calculates the intermediate visible area size during the animation and
// passes it to DesktopViewport.
......@@ -429,27 +426,36 @@ static NSString* const kFeedbackContext = @"InSessionFeedbackContext";
// done on the display thread asynchronously, so unfortunately the animation
// will not be perfectly synchronized with the keyboard animation.
CGSize viewSize = _hostView.frame.size;
CGFloat targetVisibleHeight =
viewSize.height - _keyboardPlaceholderView.frame.size.height;
[self updateViewportSafeInsets];
CALayer* kbPlaceholderLayer =
[_keyboardPlaceholderView.layer presentationLayer];
CGRect viewKeyboardIntersection =
CGRectIntersection(kbPlaceholderLayer.frame, _hostView.frame);
CGFloat currentVisibleHeight =
_hostView.frame.size.height - viewKeyboardIntersection.size.height;
_client.gestureInterpreter->OnSurfaceSizeChanged(viewSize.width,
currentVisibleHeight);
if (currentVisibleHeight == targetVisibleHeight) {
CGFloat currentKeyboardHeight = kbPlaceholderLayer.frame.size.height;
CGFloat targetKeyboardHeight = _keyboardPlaceholderView.frame.size.height;
if (currentKeyboardHeight == targetKeyboardHeight) {
// Animation is done.
_surfaceSizeAnimationLink.paused = YES;
_keyboardHeightAnimationLink.paused = YES;
}
}
- (void)updateViewportSafeInsets {
// The viewport safe insets consist of area that is (partially) obstructed by
// the notch and the soft keyboard.
CALayer* kbPlaceholderLayer =
[_keyboardPlaceholderView.layer presentationLayer];
CGRect viewKeyboardIntersection =
CGRectIntersection(kbPlaceholderLayer.frame, _hostView.frame);
UIEdgeInsets safeInsets = remoting::SafeAreaInsetsForView(_hostView);
safeInsets.bottom =
std::max(safeInsets.bottom, viewKeyboardIntersection.size.height);
_client.gestureInterpreter->OnSafeInsetsChanged(
safeInsets.left, safeInsets.top, safeInsets.right, safeInsets.bottom);
}
- (void)disconnectFromHost {
[_client disconnectFromHost];
[_surfaceSizeAnimationLink invalidate];
_surfaceSizeAnimationLink = nil;
[_keyboardHeightAnimationLink invalidate];
_keyboardHeightAnimationLink = nil;
}
- (void)applyInputMode {
......
......@@ -12,10 +12,14 @@ namespace remoting {
// Returns the current topmost presenting view controller of the app.
UIViewController* TopPresentingVC();
// Returns the proper safe area layout guide for iOS 11; returns a dumb layout
// Returns the proper safe area layout guide for iOS 11+; returns a dumb layout
// guide for older OS versions that exactly matches the anchors of the view.
UILayoutGuide* SafeAreaLayoutGuideForView(UIView* view);
// Returns the proper safe area insets for iOS 11+; returns empty insets for
// older OS versions.
UIEdgeInsets SafeAreaInsetsForView(UIView* view);
// Posts a delayed accessibility announcement so that it doesn't interrupt with
// the current announcing speech.
void PostDelayedAccessibilityNotification(NSString* announcement);
......
......@@ -46,6 +46,13 @@ UILayoutGuide* SafeAreaLayoutGuideForView(UIView* view) {
}
}
UIEdgeInsets SafeAreaInsetsForView(UIView* view) {
if (@available(iOS 11, *)) {
return view.safeAreaInsets;
}
return UIEdgeInsetsZero;
}
void PostDelayedAccessibilityNotification(NSString* announcement) {
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC),
dispatch_get_main_queue(), ^{
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment