[Paint Preview] Compress Bitmaps outside of viewport

This CL compresses out-of-viewport bitmaps to reduce memory usage.
This leads to additional pop-in time for tiles outside of the current
viewport, but is reasonably fast and might be necessary to reduce
memory usage to something reasonable.

Bug: TODO
Change-Id: I7dcd050b58fb18b4f79eebc64f55abd04091531b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2445853
Commit-Queue: Calder Kitagawa <ckitagawa@chromium.org>
Reviewed-by: Mehran Mahmoudi <mahmoudi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#815275}

components/paint_preview/player/android/BUILD.gn

@@ -61,6 +61,7 @@ android_library("java") {
     "java/src/org/chromium/components/paintpreview/player/PlayerSwipeRefreshHandler.java",
     "java/src/org/chromium/components/paintpreview/player/PlayerUserActionRecorder.java",
     "java/src/org/chromium/components/paintpreview/player/PlayerUserFrustrationDetector.java",
+    "java/src/org/chromium/components/paintpreview/player/frame/CompressibleBitmap.java",
     "java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapPainter.java",
     "java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapState.java",
     "java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapStateController.java",
@@ -168,6 +169,7 @@ junit_binary("paint_preview_junit_tests") {
   sources = [
     "junit/src/org/chromium/components/paintpreview/player/PlayerManagerTest.java",
     "junit/src/org/chromium/components/paintpreview/player/PlayerUserFrustrationDetectorTest.java",
+    "junit/src/org/chromium/components/paintpreview/player/frame/CompressibleBitmapTest.java",
     "junit/src/org/chromium/components/paintpreview/player/frame/PaintPreviewCustomFlingingShadowScroller.java",
     "junit/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapPainterTest.java",
     "junit/src/org/chromium/components/paintpreview/player/frame/PlayerFrameCoordinatorTest.java",

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/CompressibleBitmap.java

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.components.paintpreview.player.frame;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+
+import org.chromium.base.Callback;
+import org.chromium.base.ThreadUtils;
+import org.chromium.base.task.SequencedTaskRunner;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A class representing a {@link Bitmap} that can be compressed into the associated byte array.
+ * When compressed the {@link Bitmap} can safely be discarded and restored from the compressed
+ * version. Compressing the bitmap is preferred for all bitmaps outside the current viewport.
+ */
+class CompressibleBitmap {
+    private static final int IN_USE_BACKOFF_MS = 50;
+
+    private Bitmap mBitmap;
+    private byte[] mCompressedData;
+    private SequencedTaskRunner mTaskRunner;
+    private AtomicBoolean mInUse = new AtomicBoolean();
+    private ThreadUtils.ThreadChecker mThreadChecker;
+
+    /**
+     * Creates a new compressible bitmap which starts to compress immediately.
+     * @param bitmap The bitmap to store.
+     * @param taskRunner The task runner to compress/inflate the bitmap on.
+     * @param visible Whether the bitmap is currently visible. If visible, the bitmap won't be
+     *     immediately discarded.
+     */
+    CompressibleBitmap(Bitmap bitmap, SequencedTaskRunner taskRunner, boolean visible) {
+        mBitmap = bitmap;
+        mTaskRunner = taskRunner;
+        mTaskRunner.postTask(() -> { mThreadChecker = new ThreadUtils.ThreadChecker(); });
+        compressInBackground(visible);
+    }
+
+    /**
+     * Locks modifying {@link mBitmap} to prevent use/discard from happening in parallel.
+     */
+    boolean lock() {
+        return mInUse.compareAndSet(false, true);
+    }
+
+    /**
+     * Unlocks modifying of {@link mBitmap} so that it is available for use/discard by the next
+     * thread that calls {@link lock()}.
+     */
+    boolean unlock() {
+        return mInUse.compareAndSet(true, false);
+    }
+
+    /**
+     * Gets the bitmap if one is inflated.
+     * @return the bitmap or null if not inflated.
+     */
+    Bitmap getBitmap() {
+        return mBitmap;
+    }
+
+    /**
+     * Destroys the data associated with this bitmap.
+     */
+    void destroy() {
+        mTaskRunner.postTask(this::destroyInternal);
+    }
+
+    /**
+     * Discards the inflated bitmap if it has been successfully compressed.
+     */
+    void discardBitmap() {
+        mTaskRunner.postTask(this::discardBitmapInternal);
+    }
+
+    /**
+     * Inflates the compressed bitmap in the background. Call from the UI thread.
+     * @param onInflated Callback that is called when inflation is completed on the UI Thread.
+     *     Callers should check that the bitmap was actually inflated via {@link getBitmap()}.
+     */
+    void inflateInBackground(Callback<CompressibleBitmap> onInflated) {
+        mTaskRunner.postTask(() -> {
+            inflate();
+            if (onInflated != null) {
+                onInflated.onResult(this);
+            }
+        });
+    }
+
+    private boolean inflate() {
+        mThreadChecker.assertOnValidThread();
+        if (mBitmap != null) return true;
+
+        if (mCompressedData == null) return false;
+
+        mBitmap = BitmapFactory.decodeByteArray(mCompressedData, 0, mCompressedData.length);
+        return mBitmap != null;
+    }
+
+    private void compress() {
+        mThreadChecker.assertOnValidThread();
+        if (mBitmap == null) return;
+
+        ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream();
+        boolean success = mBitmap.compress(Bitmap.CompressFormat.JPEG, 100, byteArrayStream);
+        if (success) {
+            mCompressedData = byteArrayStream.toByteArray();
+        }
+    }
+
+    private void compressInBackground(boolean visible) {
+        mTaskRunner.postTask(() -> {
+            compress();
+            if (visible) return;
+
+            discardBitmapInternal();
+        });
+    }
+
+    private void discardBitmapInternal() {
+        mThreadChecker.assertOnValidThread();
+        if (!lock()) {
+            mTaskRunner.postDelayedTask(this::discardBitmapInternal, IN_USE_BACKOFF_MS);
+            return;
+        }
+
+        if (mBitmap != null && mCompressedData != null) {
+            mBitmap.recycle();
+            mBitmap = null;
+        }
+        unlock();
+    }
+
+    private void destroyInternal() {
+        mThreadChecker.assertOnValidThread();
+        if (!lock()) {
+            mTaskRunner.postDelayedTask(this::destroyInternal, IN_USE_BACKOFF_MS);
+            return;
+        }
+        if (mBitmap != null) {
+            mBitmap.recycle();
+            mBitmap = null;
+        }
+        mCompressedData = null;
+        unlock();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+
+        if (!(o instanceof CompressibleBitmap)) return false;
+
+        CompressibleBitmap od = (CompressibleBitmap) o;
+
+        if (mCompressedData != null && od.mCompressedData != null) {
+            return Arrays.equals(mCompressedData, od.mCompressedData);
+        }
+        if (mBitmap != null && od.mBitmap != null) {
+            return mBitmap.equals(od.mBitmap);
+        }
+        return false;
+    }
+}

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapPainter.java

@@ -7,23 +7,48 @@ package org.chromium.components.paintpreview.player.frame;
 import android.graphics.Bitmap;
 import android.graphics.Canvas;
 import android.graphics.Rect;
+import android.os.Handler;
 import android.util.Size;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Given a viewport {@link Rect} and a matrix of {@link Bitmap} tiles, this class draws the bitmaps
  * on a {@link Canvas}.
  */
 class PlayerFrameBitmapPainter {
     private Size mTileSize;
-    private Bitmap[][] mBitmapMatrix;
+    private CompressibleBitmap[][] mBitmapMatrix;
     private Rect mViewPort = new Rect();
     private Rect mDrawBitmapSrc = new Rect();
     private Rect mDrawBitmapDst = new Rect();
     private Runnable mInvalidateCallback;
     private Runnable mFirstPaintListener;
+    private Handler mHandler = new Handler();
+
+    // The following sets should only be modified on {@link mHandler} or UI thread.
+
+    /**
+     * Tracks which bitmaps are used in each {@link onDraw(Canvas)} call. Bitmaps in this set were
+     * in the viewport for the last draw. Bitmaps that are not in this set but are in
+     * {@link mInflatedBitmaps} are discarded at the end of {@link onDraw(Canvas)}.
+     */
+    private Set<CompressibleBitmap> mBitmapsToKeep = new HashSet<>();
+    /**
+     * Keeps track of which bitmaps are queued for inflation. Each bitmap in this list will be
+     * inflated. Although if the bitmap leaves the viewport before being added to this set it
+     * will be discarded in the next {@link onDraw(Canvas)}.
+     */
+    private Set<CompressibleBitmap> mInflatingBitmaps = new HashSet<>();
+    /**
+     * Keeps track of which bitmaps are inflated. Bitmaps in this set are cached in inflated form
+     * to keep {@link onDraw(Canvas)} performant.
+     */
+    private Set<CompressibleBitmap> mInflatedBitmaps = new HashSet<>();
 
     PlayerFrameBitmapPainter(@NonNull Runnable invalidateCallback,
             @Nullable Runnable firstPaintListener) {
@@ -40,7 +65,7 @@ class PlayerFrameBitmapPainter {
         mInvalidateCallback.run();
     }
 
-    void updateBitmapMatrix(Bitmap[][] bitmapMatrix) {
+    void updateBitmapMatrix(CompressibleBitmap[][] bitmapMatrix) {
         mBitmapMatrix = bitmapMatrix;
         mInvalidateCallback.run();
     }
@@ -63,11 +88,42 @@ class PlayerFrameBitmapPainter {
         rowEnd = Math.min(rowEnd, mBitmapMatrix.length);
         colEnd = Math.min(colEnd, rowEnd >= 1 ? mBitmapMatrix[rowEnd - 1].length : 0);
 
+        mInflatingBitmaps.clear();
+        mBitmapsToKeep.clear();
         for (int row = rowStart; row < rowEnd; row++) {
             for (int col = colStart; col < colEnd; col++) {
-                Bitmap tileBitmap = mBitmapMatrix[row][col];
+                CompressibleBitmap compressibleBitmap = mBitmapMatrix[row][col];
+                if (compressibleBitmap == null) continue;
+                mBitmapsToKeep.add(compressibleBitmap);
+
+                if (!compressibleBitmap.lock()) {
+                    // Re-issue an invalidation on the chance access was blocked due to being
+                    // discarded.
+                    mHandler.post(mInvalidateCallback);
+                    continue;
+                }
+
+                Bitmap tileBitmap = compressibleBitmap.getBitmap();
                 if (tileBitmap == null) {
+                    compressibleBitmap.unlock();
+                    mInflatingBitmaps.add(compressibleBitmap);
+                    compressibleBitmap.inflateInBackground(inflatedBitmap -> {
+                        final boolean inflated = inflatedBitmap.getBitmap() != null;
+                        // Handler is on the UI thread so the needed bitmaps will be the last
+                        // set of bitmaps requested.
+                        mHandler.post(() -> {
+                            if (inflated) {
+                                mInflatedBitmaps.add(inflatedBitmap);
+                            }
+                            mInflatingBitmaps.remove(inflatedBitmap);
+                            if (mInflatingBitmaps.isEmpty()) {
+                                mInvalidateCallback.run();
+                            }
+                        });
+                    });
                     continue;
+                } else {
+                    mInflatedBitmaps.add(compressibleBitmap);
                 }
 
                 // Calculate the portion of this tileBitmap that is visible in mViewPort.
@@ -87,11 +143,19 @@ class PlayerFrameBitmapPainter {
                 mDrawBitmapDst.set(canvasLeft, canvasTop, canvasRight, canvasBottom);
 
                 canvas.drawBitmap(tileBitmap, mDrawBitmapSrc, mDrawBitmapDst, null);
+                compressibleBitmap.unlock();
                 if (mFirstPaintListener != null) {
                     mFirstPaintListener.run();
                     mFirstPaintListener = null;
                 }
             }
         }
+        for (CompressibleBitmap inflatedBitmap : mInflatedBitmaps) {
+            if (mBitmapsToKeep.contains(inflatedBitmap)) continue;
+
+            inflatedBitmap.discardBitmap();
+        }
+        mInflatedBitmaps.clear();
+        mInflatedBitmaps.addAll(mBitmapsToKeep);
     }
 }

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapState.java

@@ -12,6 +12,7 @@ import androidx.annotation.VisibleForTesting;
 
 import org.chromium.base.Callback;
 import org.chromium.base.UnguessableToken;
+import org.chromium.base.task.SequencedTaskRunner;
 import org.chromium.components.paintpreview.player.PlayerCompositorDelegate;
 
 import java.util.HashSet;
@@ -27,36 +28,43 @@ public class PlayerFrameBitmapState {
     /** The scale factor of bitmaps. */
     private float mScaleFactor;
     /** Bitmaps that make up the contents. */
-    private Bitmap[][] mBitmapMatrix;
+    private CompressibleBitmap[][] mBitmapMatrix;
     /** Whether a request for a bitmap tile is pending. */
-    private boolean[][] mPendingBitmapRequests;
+    private BitmapRequestHandler[][] mPendingBitmapRequests;
     /**
      * Whether we currently need a bitmap tile. This is used for deleting bitmaps that we don't
      * need and freeing up memory.
      */
     private boolean[][] mRequiredBitmaps;
+    /**
+     * Whether a bitmap is visible for a given request.
+     */
+    private boolean[][] mVisibleBitmaps;
     /** Delegate for accessing native to request bitmaps. */
     private final PlayerCompositorDelegate mCompositorDelegate;
     private final PlayerFrameBitmapStateController mStateController;
     private Set<Integer> mInitialMissingVisibleBitmaps = new HashSet<>();
+    private final SequencedTaskRunner mTaskRunner;
 
     PlayerFrameBitmapState(UnguessableToken guid, int tileWidth, int tileHeight, float scaleFactor,
             Size contentSize, PlayerCompositorDelegate compositorDelegate,
-            PlayerFrameBitmapStateController stateController) {
+            PlayerFrameBitmapStateController stateController, SequencedTaskRunner taskRunner) {
         mGuid = guid;
         mTileSize = new Size(tileWidth, tileHeight);
         mScaleFactor = scaleFactor;
         mCompositorDelegate = compositorDelegate;
         mStateController = stateController;
+        mTaskRunner = taskRunner;
 
         // Each tile is as big as the initial view port. Here we determine the number of
         // columns and rows for the current scale factor.
         int rows = (int) Math.ceil((contentSize.getHeight() * scaleFactor) / tileHeight);
         int cols = (int) Math.ceil((contentSize.getWidth() * scaleFactor) / tileWidth);
 
-        mBitmapMatrix = new Bitmap[rows][cols];
-        mPendingBitmapRequests = new boolean[rows][cols];
+        mBitmapMatrix = new CompressibleBitmap[rows][cols];
+        mPendingBitmapRequests = new BitmapRequestHandler[rows][cols];
         mRequiredBitmaps = new boolean[rows][cols];
+        mVisibleBitmaps = new boolean[rows][cols];
     }
 
     @VisibleForTesting
@@ -64,7 +72,7 @@ public class PlayerFrameBitmapState {
         return mRequiredBitmaps;
     }
 
-    Bitmap[][] getMatrix() {
+    CompressibleBitmap[][] getMatrix() {
         return mBitmapMatrix;
     }
 
@@ -129,6 +137,7 @@ public class PlayerFrameBitmapState {
      */
     void requestBitmapForRect(Rect viewportRect) {
         if (mRequiredBitmaps == null || mBitmapMatrix == null) return;
+        clearVisibleBitmaps();
 
         final int rowStart =
                 Math.max(0, (int) Math.floor((double) viewportRect.top / mTileSize.getHeight()));
@@ -142,6 +151,7 @@ public class PlayerFrameBitmapState {
 
         for (int col = colStart; col < colEnd; col++) {
             for (int row = rowStart; row < rowEnd; row++) {
+                mVisibleBitmaps[row][col] = true;
                 if (requestBitmapForTile(row, col) && mInitialMissingVisibleBitmaps != null) {
                     mInitialMissingVisibleBitmaps.add(row * mBitmapMatrix.length + col);
                 }
@@ -179,8 +189,12 @@ public class PlayerFrameBitmapState {
         if (mRequiredBitmaps == null) return false;
 
         mRequiredBitmaps[row][col] = true;
+        if (mPendingBitmapRequests != null && mPendingBitmapRequests[row][col] != null) {
+            mPendingBitmapRequests[row][col].setVisible(mVisibleBitmaps[row][col]);
+            return false;
+        }
         if (mBitmapMatrix == null || mPendingBitmapRequests == null
-                || mBitmapMatrix[row][col] != null || mPendingBitmapRequests[row][col]) {
+                || mBitmapMatrix[row][col] != null || mPendingBitmapRequests[row][col] != null) {
             return false;
         }
 
@@ -188,8 +202,8 @@ public class PlayerFrameBitmapState {
         final int x = col * mTileSize.getWidth();
 
         BitmapRequestHandler bitmapRequestHandler =
-                new BitmapRequestHandler(row, col, mScaleFactor);
-        mPendingBitmapRequests[row][col] = true;
+                new BitmapRequestHandler(row, col, mScaleFactor, mVisibleBitmaps[row][col]);
+        mPendingBitmapRequests[row][col] = bitmapRequestHandler;
         mCompositorDelegate.requestBitmap(mGuid,
                 new Rect(x, y, x + mTileSize.getWidth(), y + mTileSize.getHeight()), mScaleFactor,
                 bitmapRequestHandler, bitmapRequestHandler::onError);
@@ -205,9 +219,9 @@ public class PlayerFrameBitmapState {
 
         for (int row = 0; row < mBitmapMatrix.length; row++) {
             for (int col = 0; col < mBitmapMatrix[row].length; col++) {
-                Bitmap bitmap = mBitmapMatrix[row][col];
+                CompressibleBitmap bitmap = mBitmapMatrix[row][col];
                 if (!mRequiredBitmaps[row][col] && bitmap != null) {
-                    bitmap.recycle();
+                    bitmap.destroy();
                     mBitmapMatrix[row][col] = null;
                 }
             }
@@ -234,6 +248,16 @@ public class PlayerFrameBitmapState {
         mStateController.stateUpdated(this);
     }
 
+    private void clearVisibleBitmaps() {
+        if (mVisibleBitmaps == null) return;
+
+        for (int row = 0; row < mVisibleBitmaps.length; row++) {
+            for (int col = 0; col < mVisibleBitmaps[row].length; col++) {
+                mVisibleBitmaps[row][col] = false;
+            }
+        }
+    }
+
     /**
      * Used as the callback for bitmap requests from the Paint Preview compositor.
      */
@@ -241,11 +265,18 @@ public class PlayerFrameBitmapState {
         int mRequestRow;
         int mRequestCol;
         float mRequestScaleFactor;
+        boolean mVisible;
 
-        private BitmapRequestHandler(int requestRow, int requestCol, float requestScaleFactor) {
+        private BitmapRequestHandler(
+                int requestRow, int requestCol, float requestScaleFactor, boolean visible) {
             mRequestRow = requestRow;
             mRequestCol = requestCol;
             mRequestScaleFactor = requestScaleFactor;
+            mVisible = visible;
+        }
+
+        private void setVisible(boolean visible) {
+            mVisible = visible;
         }
 
         /**
@@ -259,18 +290,22 @@ public class PlayerFrameBitmapState {
                 return;
             }
             if (mBitmapMatrix == null || mPendingBitmapRequests == null || mRequiredBitmaps == null
-                    || !mPendingBitmapRequests[mRequestRow][mRequestCol]
+                    || mPendingBitmapRequests[mRequestRow][mRequestCol] == null
                     || !mRequiredBitmaps[mRequestRow][mRequestCol]) {
-                markBitmapReceived(mRequestRow, mRequestCol);
                 result.recycle();
                 deleteUnrequiredBitmaps();
+                markBitmapReceived(mRequestRow, mRequestCol);
+                if (mPendingBitmapRequests != null) {
+                    mPendingBitmapRequests[mRequestRow][mRequestCol] = null;
+                }
                 return;
             }
 
-            mPendingBitmapRequests[mRequestRow][mRequestCol] = false;
-            mBitmapMatrix[mRequestRow][mRequestCol] = result;
-            markBitmapReceived(mRequestRow, mRequestCol);
+            mBitmapMatrix[mRequestRow][mRequestCol] =
+                    new CompressibleBitmap(result, mTaskRunner, mVisible);
             deleteUnrequiredBitmaps();
+            markBitmapReceived(mRequestRow, mRequestCol);
+            mPendingBitmapRequests[mRequestRow][mRequestCol] = null;
         }
 
         /**
@@ -284,9 +319,9 @@ public class PlayerFrameBitmapState {
             // TODO(crbug.com/1021590): Handle errors.
             assert mBitmapMatrix != null;
             assert mBitmapMatrix[mRequestRow][mRequestCol] == null;
-            assert mPendingBitmapRequests[mRequestRow][mRequestCol];
+            assert mPendingBitmapRequests[mRequestRow][mRequestCol] != null;
 
-            mPendingBitmapRequests[mRequestRow][mRequestCol] = false;
+            mPendingBitmapRequests[mRequestRow][mRequestCol] = null;
         }
     }
 

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameBitmapStateController.java

@@ -9,6 +9,7 @@ import android.util.Size;
 import androidx.annotation.VisibleForTesting;
 
 import org.chromium.base.UnguessableToken;
+import org.chromium.base.task.SequencedTaskRunner;
 import org.chromium.components.paintpreview.player.PlayerCompositorDelegate;
 
 /**
@@ -23,15 +24,17 @@ public class PlayerFrameBitmapStateController {
     private final Size mContentSize;
     private final PlayerCompositorDelegate mCompositorDelegate;
     private final PlayerFrameMediatorDelegate mMediatorDelegate;
+    private final SequencedTaskRunner mTaskRunner;
 
     PlayerFrameBitmapStateController(UnguessableToken guid, PlayerFrameViewport viewport,
             Size contentSize, PlayerCompositorDelegate compositorDelegate,
-            PlayerFrameMediatorDelegate mediatorDelegate) {
+            PlayerFrameMediatorDelegate mediatorDelegate, SequencedTaskRunner taskRunner) {
         mGuid = guid;
         mViewport = viewport;
         mContentSize = contentSize;
         mCompositorDelegate = compositorDelegate;
         mMediatorDelegate = mediatorDelegate;
+        mTaskRunner = taskRunner;
     }
 
     @VisibleForTesting
@@ -50,9 +53,10 @@ public class PlayerFrameBitmapStateController {
                 (mLoadingBitmapState == null) ? mVisibleBitmapState : mLoadingBitmapState;
         if (scaleUpdated || activeLoadingState == null) {
             invalidateLoadingBitmaps();
-            mLoadingBitmapState = new PlayerFrameBitmapState(mGuid, mViewport.getWidth() / 2,
-                    mViewport.getHeight() / 2, mViewport.getScale(), mContentSize,
-                    mCompositorDelegate, this);
+            mLoadingBitmapState =
+                    new PlayerFrameBitmapState(mGuid, Math.round(mViewport.getWidth() / 2.0f),
+                            Math.round(mViewport.getHeight() / 2.0f), mViewport.getScale(),
+                            mContentSize, mCompositorDelegate, this, mTaskRunner);
             if (mVisibleBitmapState == null) {
                 mLoadingBitmapState.skipWaitingForVisibleBitmaps();
                 swap(mLoadingBitmapState);

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameMediator.java

@@ -4,7 +4,6 @@
 
 package org.chromium.components.paintpreview.player.frame;
 
-import android.graphics.Bitmap;
 import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.util.Size;
@@ -13,6 +12,9 @@ import android.view.View;
 import androidx.annotation.VisibleForTesting;
 
 import org.chromium.base.UnguessableToken;
+import org.chromium.base.task.PostTask;
+import org.chromium.base.task.SequencedTaskRunner;
+import org.chromium.base.task.TaskTraits;
 import org.chromium.components.paintpreview.player.PlayerCompositorDelegate;
 import org.chromium.components.paintpreview.player.PlayerGestureListener;
 import org.chromium.ui.modelutil.PropertyModel;
@@ -28,8 +30,8 @@ import java.util.List;
  * <li>Maintaining a viewport {@link Rect} that represents the current user-visible section of this
  * frame. The dimension of the viewport is constant and is equal to the initial values received on
  * {@link #setLayoutDimensions}.</li>
- * <li>Constructing a matrix of {@link Bitmap} tiles that represents the content of this frame for a
- * given scale factor. Each tile is as big as the view port.</li>
+ * <li>Constructing a matrix of {@link CompressibleBitmap} tiles that represents the content of this
+ * frame for a given scale factor. Each tile is as big as the view port.</li>
  * <li>Requesting bitmaps from Paint Preview compositor.</li>
  * <li>Updating the viewport on touch gesture notifications (scrolling and scaling).<li/>
  * <li>Determining which sub-frames are visible given the current viewport and showing them.<li/>
@@ -85,8 +87,10 @@ class PlayerFrameMediator implements PlayerFrameViewDelegate, PlayerFrameMediato
         mInitialScaleFactor = 0f;
         mGuid = frameGuid;
         mContentSize = contentSize;
+        SequencedTaskRunner taskRunner =
+                PostTask.createSequencedTaskRunner(TaskTraits.USER_VISIBLE);
         mBitmapStateController = new PlayerFrameBitmapStateController(
-                mGuid, mViewport, mContentSize, mCompositorDelegate, this);
+                mGuid, mViewport, mContentSize, mCompositorDelegate, this, taskRunner);
         mViewport.offset(initialScrollX, initialScrollY);
         mViewport.setScale(0f);
     }
@@ -271,7 +275,7 @@ class PlayerFrameMediator implements PlayerFrameViewDelegate, PlayerFrameMediato
     }
 
     @Override
-    public void updateBitmapMatrix(Bitmap[][] bitmapMatrix) {
+    public void updateBitmapMatrix(CompressibleBitmap[][] bitmapMatrix) {
         mModel.set(PlayerFrameProperties.BITMAP_MATRIX, bitmapMatrix);
     }
 

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameMediatorDelegate.java

@@ -4,7 +4,6 @@
 
 package org.chromium.components.paintpreview.player.frame;
 
-import android.graphics.Bitmap;
 import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.util.Size;
@@ -67,7 +66,7 @@ public interface PlayerFrameMediatorDelegate {
     /**
      * Updates the bitmap matrix in the model.
      */
-    void updateBitmapMatrix(Bitmap[][] bitmapMatrix);
+    void updateBitmapMatrix(CompressibleBitmap[][] bitmapMatrix);
 
     /**
      * Update the model when the bitmap state is swapped.

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameProperties.java

@@ -4,7 +4,6 @@
 
 package org.chromium.components.paintpreview.player.frame;
 
-import android.graphics.Bitmap;
 import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.util.Size;
@@ -20,7 +19,7 @@ import java.util.List;
  */
 class PlayerFrameProperties {
     /** A matrix of bitmap tiles that collectively make the entire content. */
-    static final PropertyModel.WritableObjectPropertyKey<Bitmap[][]> BITMAP_MATRIX =
+    static final PropertyModel.WritableObjectPropertyKey<CompressibleBitmap[][]> BITMAP_MATRIX =
             new PropertyModel.WritableObjectPropertyKey<>(true);
     /** The dimensions of each bitmap tile in the current bitmap matrix. */
     static final PropertyModel.WritableObjectPropertyKey<Size> TILE_DIMENSIONS =

components/paint_preview/player/android/java/src/org/chromium/components/paintpreview/player/frame/PlayerFrameView.java

@@ -5,7 +5,6 @@
 package org.chromium.components.paintpreview.player.frame;
 
 import android.content.Context;
-import android.graphics.Bitmap;
 import android.graphics.Canvas;
 import android.graphics.Matrix;
 import android.graphics.Rect;
@@ -45,7 +44,7 @@ class PlayerFrameView extends FrameLayout {
         super(context);
         setWillNotDraw(false);
         mDelegate = playerFrameViewDelegate;
-        mBitmapPainter = new PlayerFrameBitmapPainter(this::invalidate, firstPaintListener);
+        mBitmapPainter = new PlayerFrameBitmapPainter(this::postInvalidate, firstPaintListener);
         mGestureDetector =
                 new PlayerFrameGestureDetector(context, canDetectZoom, gestureDetectorDelegate);
     }
@@ -80,7 +79,7 @@ class PlayerFrameView extends FrameLayout {
         layoutSubFrames();
     }
 
-    void updateBitmapMatrix(Bitmap[][] bitmapMatrix) {
+    void updateBitmapMatrix(CompressibleBitmap[][] bitmapMatrix) {
         mBitmapPainter.updateBitmapMatrix(bitmapMatrix);
     }
 

components/paint_preview/player/android/junit/src/org/chromium/components/paintpreview/player/frame/CompressibleBitmapTest.java

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.components.paintpreview.player.frame;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.robolectric.annotation.Config;
+import org.robolectric.annotation.Implementation;
+import org.robolectric.annotation.Implements;
+
+import org.chromium.base.task.SequencedTaskRunner;
+import org.chromium.base.test.BaseRobolectricTestRunner;
+import org.chromium.base.test.util.CallbackHelper;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests for the {@link CompressibleBitmap} class.
+ */
+@RunWith(BaseRobolectricTestRunner.class)
+@Config(shadows = {CompressibleBitmapTest.FakeShadowBitmapFactory.class})
+public class CompressibleBitmapTest {
+    /**
+     * A fake {@link BitmapFactory} used to avoid native for decoding.
+     */
+    @Implements(BitmapFactory.class)
+    public static class FakeShadowBitmapFactory {
+        private static Bitmap sBitmap;
+
+        public static void setBitmap(Bitmap bitmap) {
+            sBitmap = bitmap;
+        }
+
+        @Implementation
+        public static Bitmap decodeByteArray(byte[] array, int offset, int length) {
+            return sBitmap;
+        }
+    }
+
+    @Test
+    public void testCompressAndDiscard() {
+        Bitmap bitmap = Mockito.mock(Bitmap.class);
+        when(bitmap.compress(any(), anyInt(), any())).thenReturn(true);
+
+        SequencedTaskRunner taskRunner = Mockito.mock(SequencedTaskRunner.class);
+        doAnswer(invocation -> {
+            ((Runnable) invocation.getArgument(0)).run();
+            return null;
+        })
+                .when(taskRunner)
+                .postTask(any());
+
+        CompressibleBitmap compressibleBitmap = new CompressibleBitmap(bitmap, taskRunner, false);
+        verify(bitmap, times(1)).compress(any(), eq(100), any());
+
+        Assert.assertNull(compressibleBitmap.getBitmap());
+    }
+
+    @Test
+    public void testCompressAndKeep() {
+        Bitmap bitmap = Mockito.mock(Bitmap.class);
+        when(bitmap.compress(any(), anyInt(), any())).thenReturn(true);
+
+        SequencedTaskRunner taskRunner = Mockito.mock(SequencedTaskRunner.class);
+        doAnswer(invocation -> {
+            ((Runnable) invocation.getArgument(0)).run();
+            return null;
+        })
+                .when(taskRunner)
+                .postTask(any());
+
+        CompressibleBitmap compressibleBitmap = new CompressibleBitmap(bitmap, taskRunner, true);
+        verify(bitmap, times(1)).compress(any(), eq(100), any());
+
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+        compressibleBitmap.discardBitmap();
+        Assert.assertNull(compressibleBitmap.getBitmap());
+
+        // Ensure doing this again doesn't crash.
+        compressibleBitmap.discardBitmap();
+    }
+
+    @Test
+    public void testNoDiscardIfCompressFails() {
+        Bitmap bitmap = Mockito.mock(Bitmap.class);
+        when(bitmap.compress(any(), anyInt(), any())).thenReturn(false);
+
+        SequencedTaskRunner taskRunner = Mockito.mock(SequencedTaskRunner.class);
+        doAnswer(invocation -> {
+            ((Runnable) invocation.getArgument(0)).run();
+            return null;
+        })
+                .when(taskRunner)
+                .postTask(any());
+
+        CompressibleBitmap compressibleBitmap = new CompressibleBitmap(bitmap, taskRunner, false);
+        verify(bitmap, times(1)).compress(any(), eq(100), any());
+
+        // Discarding should fail.
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+        compressibleBitmap.discardBitmap();
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+    }
+
+    @Test
+    public void testInflate() throws TimeoutException {
+        Bitmap bitmap = Mockito.mock(Bitmap.class);
+        when(bitmap.compress(any(), anyInt(), any())).thenReturn(true);
+
+        SequencedTaskRunner taskRunner = Mockito.mock(SequencedTaskRunner.class);
+        doAnswer(invocation -> {
+            ((Runnable) invocation.getArgument(0)).run();
+            return null;
+        })
+                .when(taskRunner)
+                .postTask(any());
+
+        CompressibleBitmap compressibleBitmap = new CompressibleBitmap(bitmap, taskRunner, false);
+        verify(bitmap, times(1)).compress(any(), eq(100), any());
+        Assert.assertNull(compressibleBitmap.getBitmap());
+
+        FakeShadowBitmapFactory.setBitmap(bitmap);
+
+        CallbackHelper helper = new CallbackHelper();
+        compressibleBitmap.inflateInBackground(compressible -> { helper.notifyCalled(); });
+        helper.waitForFirst();
+
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+        compressibleBitmap.destroy();
+        Assert.assertNull(compressibleBitmap.getBitmap());
+
+        // Inflation should fail if the CompressibleBitmap is destroyed.
+        CallbackHelper inflatedNoBitmap = new CallbackHelper();
+        compressibleBitmap.inflateInBackground(compressible -> {
+            Assert.assertNull(compressible.getBitmap());
+            inflatedNoBitmap.notifyCalled();
+        });
+        inflatedNoBitmap.waitForFirst();
+    }
+
+    @Test
+    public void testLocking() throws TimeoutException {
+        Bitmap bitmap = Mockito.mock(Bitmap.class);
+        when(bitmap.compress(any(), anyInt(), any())).thenReturn(true);
+        SequencedTaskRunner taskRunner = Mockito.mock(SequencedTaskRunner.class);
+        doAnswer(invocation -> {
+            ((Runnable) invocation.getArgument(0)).run();
+            return null;
+        })
+                .when(taskRunner)
+                .postTask(any());
+
+        CompressibleBitmap compressibleBitmap = new CompressibleBitmap(bitmap, taskRunner, true);
+        verify(bitmap, times(1)).compress(any(), eq(100), any());
+        Assert.assertTrue(compressibleBitmap.lock());
+        Assert.assertFalse(compressibleBitmap.lock());
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+
+        compressibleBitmap.discardBitmap();
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+
+        compressibleBitmap.destroy();
+        Assert.assertEquals(compressibleBitmap.getBitmap(), bitmap);
+        Assert.assertTrue(compressibleBitmap.unlock());
+        Assert.assertFalse(compressibleBitmap.unlock());
+
+        compressibleBitmap.discardBitmap();
+        Assert.assertTrue(compressibleBitmap.lock());
+        Assert.assertNull(compressibleBitmap.getBitmap());
+        Assert.assertTrue(compressibleBitmap.unlock());
+
+        CallbackHelper helper = new CallbackHelper();
+        compressibleBitmap.inflateInBackground(compressible -> { helper.notifyCalled(); });
+        helper.waitForFirst();
+
+        compressibleBitmap.destroy();
+        Assert.assertTrue(compressibleBitmap.lock());
+        Assert.assertNull(compressibleBitmap.getBitmap());
+        Assert.assertTrue(compressibleBitmap.unlock());
+        verify(taskRunner, times(2)).postDelayedTask(any(), anyLong());
+    }
+}

components/paint_preview/player/android/junit/src/org/chromium/components/paintpreview/player/frame/PlayerFrameMediatorTest.java

@@ -36,6 +36,7 @@ import org.robolectric.shadows.ShadowView;
 import org.chromium.base.Callback;
 import org.chromium.base.ContextUtils;
 import org.chromium.base.UnguessableToken;
+import org.chromium.base.task.SequencedTaskRunner;
 import org.chromium.base.test.BaseRobolectricTestRunner;
 import org.chromium.components.paintpreview.player.PlayerCompositorDelegate;
 import org.chromium.components.paintpreview.player.PlayerGestureListener;
@@ -270,8 +271,8 @@ public class PlayerFrameMediatorTest {
         // The bitmap matrix should be empty, but initialized with the correct number of rows and
         // columns. Because we set the initial scale factor to view port width over content width,
         // we should have only one column.
-        Bitmap[][] bitmapMatrix = mModel.get(PlayerFrameProperties.BITMAP_MATRIX);
-        Assert.assertTrue(Arrays.deepEquals(bitmapMatrix, new Bitmap[4][2]));
+        CompressibleBitmap[][] bitmapMatrix = mModel.get(PlayerFrameProperties.BITMAP_MATRIX);
+        Assert.assertTrue(Arrays.deepEquals(bitmapMatrix, new CompressibleBitmap[4][2]));
         Assert.assertEquals(new ArrayList<Pair<View, Rect>>(),
                 mModel.get(PlayerFrameProperties.SUBFRAME_VIEWS));
     }
@@ -558,27 +559,64 @@ public class PlayerFrameMediatorTest {
         Bitmap bitmap03 = Mockito.mock(Bitmap.class);
         Bitmap bitmap13 = Mockito.mock(Bitmap.class);
         Bitmap bitmap23 = Mockito.mock(Bitmap.class);
-
-        Bitmap[][] expectedBitmapMatrix = new Bitmap[12][8];
-        expectedBitmapMatrix[0][0] = bitmap00;
-        expectedBitmapMatrix[0][1] = bitmap01;
-        expectedBitmapMatrix[0][2] = bitmap02;
-        expectedBitmapMatrix[1][0] = bitmap10;
-        expectedBitmapMatrix[1][1] = bitmap11;
-        expectedBitmapMatrix[1][2] = bitmap12;
-        expectedBitmapMatrix[2][0] = bitmap20;
-        expectedBitmapMatrix[2][1] = bitmap21;
+        SequencedTaskRunner mockTaskRunner = Mockito.mock(SequencedTaskRunner.class);
+        CompressibleBitmap compressibleBitmap00 =
+                new CompressibleBitmap(bitmap00, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap10 =
+                new CompressibleBitmap(bitmap10, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap20 =
+                new CompressibleBitmap(bitmap20, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap01 =
+                new CompressibleBitmap(bitmap01, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap11 =
+                new CompressibleBitmap(bitmap11, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap21 =
+                new CompressibleBitmap(bitmap21, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap31 =
+                new CompressibleBitmap(bitmap31, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap02 =
+                new CompressibleBitmap(bitmap02, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap12 =
+                new CompressibleBitmap(bitmap12, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap22 =
+                new CompressibleBitmap(bitmap22, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap32 =
+                new CompressibleBitmap(bitmap32, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap03 =
+                new CompressibleBitmap(bitmap03, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap13 =
+                new CompressibleBitmap(bitmap13, mockTaskRunner, true);
+        CompressibleBitmap compressibleBitmap23 =
+                new CompressibleBitmap(bitmap23, mockTaskRunner, true);
+
+        CompressibleBitmap[][] expectedBitmapMatrix = new CompressibleBitmap[12][8];
+        expectedBitmapMatrix[0][0] = compressibleBitmap00;
+        expectedBitmapMatrix[0][1] = compressibleBitmap01;
+        expectedBitmapMatrix[0][2] = compressibleBitmap02;
+        expectedBitmapMatrix[1][0] = compressibleBitmap10;
+        expectedBitmapMatrix[1][1] = compressibleBitmap11;
+        expectedBitmapMatrix[1][2] = compressibleBitmap12;
+        expectedBitmapMatrix[2][0] = compressibleBitmap20;
+        expectedBitmapMatrix[2][1] = compressibleBitmap21;
 
         // Call the request callback with mock bitmaps and assert they're added to the model.
-        mCompositorDelegate.mRequestedBitmap.get(0).mBitmapCallback.onResult(bitmap00);
-        mCompositorDelegate.mRequestedBitmap.get(1).mBitmapCallback.onResult(bitmap10);
-        mCompositorDelegate.mRequestedBitmap.get(2).mBitmapCallback.onResult(bitmap01);
-        mCompositorDelegate.mRequestedBitmap.get(3).mBitmapCallback.onResult(bitmap11);
-        mCompositorDelegate.mRequestedBitmap.get(4).mBitmapCallback.onResult(bitmap20);
-        mCompositorDelegate.mRequestedBitmap.get(5).mBitmapCallback.onResult(bitmap02);
-        mCompositorDelegate.mRequestedBitmap.get(6).mBitmapCallback.onResult(bitmap21);
-        mCompositorDelegate.mRequestedBitmap.get(7).mBitmapCallback.onResult(bitmap12);
-        Bitmap[][] mat = mModel.get(PlayerFrameProperties.BITMAP_MATRIX);
+        mCompositorDelegate.mRequestedBitmap.get(0).mBitmapCallback.onResult(
+                compressibleBitmap00.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(1).mBitmapCallback.onResult(
+                compressibleBitmap10.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(2).mBitmapCallback.onResult(
+                compressibleBitmap01.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(3).mBitmapCallback.onResult(
+                compressibleBitmap11.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(4).mBitmapCallback.onResult(
+                compressibleBitmap20.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(5).mBitmapCallback.onResult(
+                compressibleBitmap02.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(6).mBitmapCallback.onResult(
+                compressibleBitmap21.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(7).mBitmapCallback.onResult(
+                compressibleBitmap12.getBitmap());
+        CompressibleBitmap[][] mat = mModel.get(PlayerFrameProperties.BITMAP_MATRIX);
         Assert.assertTrue(Arrays.deepEquals(
                 expectedBitmapMatrix, mModel.get(PlayerFrameProperties.BITMAP_MATRIX)));
 
@@ -589,20 +627,26 @@ public class PlayerFrameMediatorTest {
         // tiles. See comments on {@link #testBitmapRequest} for details on which tiles will be
         // requested.
         // Call the request callback with mock bitmaps and assert they're added to the model.
-        expectedBitmapMatrix[2][2] = bitmap22;
-        expectedBitmapMatrix[0][3] = bitmap03;
-        expectedBitmapMatrix[3][1] = bitmap31;
-        expectedBitmapMatrix[1][3] = bitmap13;
-        expectedBitmapMatrix[3][2] = bitmap32;
-        expectedBitmapMatrix[2][3] = bitmap23;
-        mCompositorDelegate.mRequestedBitmap.get(8).mBitmapCallback.onResult(bitmap22);
+        expectedBitmapMatrix[2][2] = compressibleBitmap22;
+        expectedBitmapMatrix[0][3] = compressibleBitmap03;
+        expectedBitmapMatrix[3][1] = compressibleBitmap31;
+        expectedBitmapMatrix[1][3] = compressibleBitmap13;
+        expectedBitmapMatrix[3][2] = compressibleBitmap32;
+        expectedBitmapMatrix[2][3] = compressibleBitmap23;
+        mCompositorDelegate.mRequestedBitmap.get(8).mBitmapCallback.onResult(
+                compressibleBitmap22.getBitmap());
         // Mock a compositing failure for this tile. No bitmaps should be added.
         mCompositorDelegate.mRequestedBitmap.get(9).mErrorCallback.run();
-        mCompositorDelegate.mRequestedBitmap.get(10).mBitmapCallback.onResult(bitmap31);
-        mCompositorDelegate.mRequestedBitmap.get(11).mBitmapCallback.onResult(bitmap03);
-        mCompositorDelegate.mRequestedBitmap.get(12).mBitmapCallback.onResult(bitmap13);
-        mCompositorDelegate.mRequestedBitmap.get(13).mBitmapCallback.onResult(bitmap32);
-        mCompositorDelegate.mRequestedBitmap.get(14).mBitmapCallback.onResult(bitmap23);
+        mCompositorDelegate.mRequestedBitmap.get(10).mBitmapCallback.onResult(
+                compressibleBitmap31.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(11).mBitmapCallback.onResult(
+                compressibleBitmap03.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(12).mBitmapCallback.onResult(
+                compressibleBitmap13.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(13).mBitmapCallback.onResult(
+                compressibleBitmap32.getBitmap());
+        mCompositorDelegate.mRequestedBitmap.get(14).mBitmapCallback.onResult(
+                compressibleBitmap23.getBitmap());
         Assert.assertTrue(Arrays.deepEquals(
                 expectedBitmapMatrix, mModel.get(PlayerFrameProperties.BITMAP_MATRIX)));