[Remoting Android] Allow panning canvas out of the cutout area

Currently the Android app doesn't adjust the desktop canvas for cutouts
on a notched device, so part of the desktop will get obstructed by the
cutout.

This CL fixes this by introducing a concept of safe area to
DesktopCanvas. User can still see through and interact with the content
in the unsafe area and pan the canvas out of it, while we use the safe
area to calculate the minimum zoom level.

Bug: 831670
Change-Id: I6a42bab8f383fa682bd2d2363b41bc6bb7a6d1c3
Reviewed-on: https://chromium-review.googlesource.com/1174982Reviewed-by: Joe Downing <joedow@chromium.org>
Commit-Queue: Yuwei Huang <yuweih@chromium.org>
Cr-Commit-Position: refs/heads/master@{#584535}

remoting/android/java/src/org/chromium/chromoting/Desktop.java

@@ -7,6 +7,7 @@ package org.chromium.chromoting;
 import android.annotation.SuppressLint;
 import android.content.res.Configuration;
 import android.graphics.Color;
+import android.graphics.Rect;
 import android.graphics.drawable.ColorDrawable;
 import android.os.Build;
 import android.os.Bundle;
@@ -14,6 +15,7 @@ import android.os.Handler;
 import android.support.v7.app.ActionBar.OnMenuVisibilityListener;
 import android.support.v7.app.AppCompatActivity;
 import android.support.v7.widget.Toolbar;
+import android.view.DisplayCutout;
 import android.view.KeyEvent;
 import android.view.Menu;
 import android.view.MenuItem;
@@ -21,6 +23,8 @@ import android.view.MotionEvent;
 import android.view.View;
 import android.view.View.OnLayoutChangeListener;
 import android.view.View.OnTouchListener;
+import android.view.WindowManager;
+import android.view.WindowManager.LayoutParams;
 import android.view.animation.Animation;
 import android.view.animation.AnimationUtils;
 import android.view.inputmethod.InputMethodManager;
@@ -136,6 +140,15 @@ public class Desktop
         // background.  Setting the background color to match our canvas will prevent the flash.
         getWindow().setBackgroundDrawable(new ColorDrawable(Color.BLACK));
 
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+            // Short edges mode makes DesktopView stretch to the whole screen even if it gets
+            // obstructed by the cutouts.
+            WindowManager.LayoutParams layoutParams = getWindow().getAttributes();
+            layoutParams.layoutInDisplayCutoutMode =
+                    LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_SHORT_EDGES;
+            getWindow().setAttributes(layoutParams);
+        }
+
         mActivityLifecycleListener = mClient.getCapabilityManager().onActivityAcceptingListener(
                 this, Capabilities.CAST_CAPABILITY);
         mActivityLifecycleListener.onActivityCreated(this, savedInstanceState);
@@ -462,6 +475,24 @@ public class Desktop
         return flags;
     }
 
+    /**
+     * @return The insets from each edge on the screen that avoid display cutouts.
+     */
+    @SuppressLint("InlinedApi")
+    public Rect getSafeInsets() {
+        Rect insets = new Rect();
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
+            return insets;
+        }
+
+        DisplayCutout cutout = mRemoteHostDesktop.getRootWindowInsets().getDisplayCutout();
+        if (cutout != null) {
+            insets.set(cutout.getSafeInsetLeft(), cutout.getSafeInsetTop(),
+                    cutout.getSafeInsetRight(), cutout.getSafeInsetBottom());
+        }
+        return insets;
+    }
+
     /**
      * Shows the soft keyboard if no physical keyboard is attached.
      */

remoting/android/java/src/org/chromium/chromoting/DesktopCanvas.java

@@ -25,6 +25,12 @@ public class DesktopCanvas {
     private final RenderStub mRenderStub;
     private final RenderData mRenderData;
 
+    /**
+     * The insets from each edge of the screen which avoid display cutouts. Note that this is not
+     * a real rectangle as left > right and top > bottom can be true.
+     */
+    private final Rect mSafeInsets = new Rect();
+
     /**
      * Represents the desired center of the viewport in image space.  This value may not represent
      * the actual center of the viewport as adjustments are made to ensure as much of the desktop is
@@ -137,6 +143,20 @@ public class DesktopCanvas {
         }
     }
 
+    /**
+     *  Sets the insets from each edge on the screen that avoid display cutouts.
+     *
+     * @param insets The insets from each edge on the screen that avoid display cutouts.
+     */
+    public void setSafeInsets(Rect insets) {
+        mSafeInsets.set(insets);
+
+        if (mRenderData.initialized()) {
+            // The viewport center may have changed so update the position to reflect the new value.
+            repositionImage();
+        }
+    }
+
     public void adjustViewportForSystemUi(boolean adjustViewportForSystemUi) {
         mAdjustViewportSizeForSystemUi = adjustViewportForSystemUi;
 
@@ -153,8 +173,9 @@ public class DesktopCanvas {
             return;
         }
 
-        float widthRatio = (float) mRenderData.screenWidth / mRenderData.imageWidth;
-        float heightRatio = (float) mRenderData.screenHeight / mRenderData.imageHeight;
+        float widthRatio = getSafeScreenWidth() / mRenderData.imageWidth;
+        float heightRatio =
+                calculateSafeScreenHeight(mRenderData.screenHeight) / mRenderData.imageHeight;
         float screenToImageScale = Math.max(widthRatio, heightRatio);
 
         // If the image is smaller than the screen in either dimension, then we want to scale it
@@ -195,11 +216,12 @@ public class DesktopCanvas {
         float[] imageSize = {mRenderData.imageWidth, mRenderData.imageHeight};
         mRenderData.transform.mapVectors(imageSize);
 
-        if (imageSize[0] < mRenderData.screenWidth && imageSize[1] < mRenderData.screenHeight) {
+        if (imageSize[0] < getSafeScreenWidth()
+                && imageSize[1] < calculateSafeScreenHeight(mRenderData.screenHeight)) {
             // Displayed image is too small in both directions, so apply the minimum zoom
             // level needed to fit either the width or height.
-            float scale = Math.min((float) mRenderData.screenWidth / mRenderData.imageWidth,
-                                   (float) mRenderData.screenHeight / mRenderData.imageHeight);
+            float scale = Math.min(getSafeScreenWidth() / mRenderData.imageWidth,
+                    calculateSafeScreenHeight(mRenderData.screenHeight) / mRenderData.imageHeight);
             mRenderData.transform.setScale(scale, scale);
         }
 
@@ -208,8 +230,9 @@ public class DesktopCanvas {
         } else {
             // Find the new screen center (it probably changed during the zoom operation) and update
             // the viewport to smoothly track the zoom gesture.
-            float[] mappedPoints = {((float) mRenderData.screenWidth / 2) - mViewportOffset.x,
-                    ((float) mRenderData.screenHeight / 2) - mViewportOffset.y};
+            PointF safeScreenCenter = calculateSafeScreenCenterPoint(mRenderData.screenHeight);
+            float[] mappedPoints = {
+                    safeScreenCenter.x - mViewportOffset.x, safeScreenCenter.y - mViewportOffset.y};
             Matrix screenToImage = new Matrix();
             mRenderData.transform.invert(screenToImage);
             screenToImage.mapPoints(mappedPoints);
@@ -252,7 +275,7 @@ public class DesktopCanvas {
      * account.
      */
     private PointF getViewportScreenCenter() {
-        return new PointF((float) mRenderData.screenWidth / 2, getAdjustedScreenHeight() / 2);
+        return calculateSafeScreenCenterPoint(getAdjustedScreenHeight());
     }
 
     /**
@@ -313,15 +336,17 @@ public class DesktopCanvas {
         float[] screenVectors = {viewportCenter.x, viewportCenter.y};
         screenToImage.mapVectors(screenVectors);
 
-        PointF letterboxPadding = getLetterboxPadding();
-        float[] letterboxPaddingVectors = {letterboxPadding.x, letterboxPadding.y};
-        screenToImage.mapVectors(letterboxPaddingVectors);
+        RectF letterboxPadding = getLetterboxPadding();
+        float[] letterboxPaddingLeftTop = {letterboxPadding.left, letterboxPadding.top};
+        float[] letterboxPaddingRightBottom = {letterboxPadding.right, letterboxPadding.bottom};
+        screenToImage.mapVectors(letterboxPaddingLeftTop);
+        screenToImage.mapVectors(letterboxPaddingRightBottom);
 
-        // screenCenter values are 1/2 of a particular screen dimension mapped to image space.
-        float screenCenterX = screenVectors[0] - letterboxPaddingVectors[0];
-        float screenCenterY = screenVectors[1] - letterboxPaddingVectors[1];
         RectF imageBounds = getImageBounds();
-        imageBounds.inset(screenCenterX, screenCenterY);
+        imageBounds.left += screenVectors[0] - letterboxPaddingLeftTop[0];
+        imageBounds.top += screenVectors[1] - letterboxPaddingLeftTop[1];
+        imageBounds.right -= screenVectors[0] - letterboxPaddingRightBottom[0];
+        imageBounds.bottom -= screenVectors[1] - letterboxPaddingRightBottom[1];
         return imageBounds;
     }
 
@@ -348,18 +373,25 @@ public class DesktopCanvas {
     }
 
     /**
-     * Provides the amount of padding needed to center the image content on the screen.
+     * Provides the amount of padding needed to center the image content on the screen. Safe insets
+     * are fixed constant letterbox padding that exist even when the image is not smaller than the
+     * screen.
      */
-    private PointF getLetterboxPadding() {
+    private RectF getLetterboxPadding() {
         float[] imageVectors = {mRenderData.imageWidth, mRenderData.imageHeight};
         mRenderData.transform.mapVectors(imageVectors);
 
         // We want to letterbox when the image is smaller than the screen in a specific dimension.
         // Since we center the image, split the difference so it is equally distributed.
-        float widthAdjust = Math.max(((float) mRenderData.screenWidth - imageVectors[0]) / 2, 0);
-        float heightAdjust = Math.max((getAdjustedScreenHeight() - imageVectors[1]) / 2, 0);
+        float widthAdjust = Math.max((getSafeScreenWidth() - imageVectors[0]) / 2, 0);
+        float heightAdjust = Math.max(
+                (calculateSafeScreenHeight(getAdjustedScreenHeight()) - imageVectors[1]) / 2, 0);
 
-        return new PointF(widthAdjust, heightAdjust);
+        // Add the safe insets.
+        RectF padding = new RectF(widthAdjust + mSafeInsets.left, heightAdjust + mSafeInsets.top,
+                widthAdjust + mSafeInsets.right, heightAdjust + mSafeInsets.bottom);
+
+        return padding;
     }
 
     /**
@@ -376,12 +408,14 @@ public class DesktopCanvas {
         // between the System UI and the remote desktop image.
         // Note: Ignore negative padding (clamp to 0) since that means no overlap exists.
         float adjustedScreenHeight = getAdjustedScreenHeight();
-        PointF letterboxPadding = getLetterboxPadding();
-        return new RectF(Math.max(mSystemUiScreenRect.left - letterboxPadding.x, 0.0f),
-                Math.max(mSystemUiScreenRect.top - letterboxPadding.y, 0.0f),
-                Math.max(mRenderData.screenWidth - mSystemUiScreenRect.right - letterboxPadding.x,
+        RectF letterboxPadding = getLetterboxPadding();
+        return new RectF(Math.max(mSystemUiScreenRect.left - letterboxPadding.left, 0.0f),
+                Math.max(mSystemUiScreenRect.top - letterboxPadding.top, 0.0f),
+                Math.max(mRenderData.screenWidth - mSystemUiScreenRect.right
+                                - letterboxPadding.right,
                         0.0f),
-                Math.max(adjustedScreenHeight - mSystemUiScreenRect.bottom - letterboxPadding.y,
+                Math.max(
+                        adjustedScreenHeight - mSystemUiScreenRect.bottom - letterboxPadding.bottom,
                         0.0f));
     }
 
@@ -465,4 +499,31 @@ public class DesktopCanvas {
     private boolean arePointsEqual(PointF a, PointF b, float epsilon) {
         return Math.abs(a.x - b.x) < epsilon && Math.abs(a.y - b.y) < epsilon;
     }
+
+    /**
+     * @return Screen width inset by {@link #mSafeInsets}.
+     */
+    private float getSafeScreenWidth() {
+        return mRenderData.screenWidth - mSafeInsets.left - mSafeInsets.right;
+    }
+
+    /**
+     * @param height The height to be used to calculate the safe screen height. Generally either
+     * {@link #getAdjustedScreenHeight()} or mRenderData.screenHeight.
+     * @return Screen height inset by {@link #mSafeInsets}.
+     */
+    private float calculateSafeScreenHeight(float height) {
+        return height - mSafeInsets.top - mSafeInsets.bottom;
+    }
+
+    /**
+     * @param height The height to be used to calculate the safe screen height. Generally either
+     * {@link #getAdjustedScreenHeight()} or mRenderData.screenHeight.
+     * @return Center of the safe area on the screen inset by {@link #mSafeInsets}.
+     */
+    private PointF calculateSafeScreenCenterPoint(float height) {
+        return new PointF(
+                (float) mRenderData.screenWidth / 2 + mSafeInsets.left - mSafeInsets.right,
+                height / 2 + mSafeInsets.top - mSafeInsets.bottom);
+    }
 }

remoting/android/java/src/org/chromium/chromoting/TouchInputHandler.java

@@ -384,6 +384,8 @@ public class TouchInputHandler {
         mRenderData.screenWidth = width;
         mRenderData.screenHeight = height;
 
+        mDesktopCanvas.setSafeInsets(mDesktop.getSafeInsets());
+
         mPanGestureBounds = new Rect(
                 mEdgeSlopInPx, mEdgeSlopInPx, width - mEdgeSlopInPx, height - mEdgeSlopInPx);
         resizeImageToFitScreen();