Capture entire webpage for thumbnails.

ThumbnailTabHelper now captures the entire contents of a web page
(within the bounds of the window) as the raw thumbnail instead of an
image specifically cropped for tab hover cards.

The image is scaled down to a size that can be used by both the touch-
only tabstrip and hover cards, even when it must be cropped. Observers
can now specify a size hint for uncompressed images; so that when the
image is received by the observer it is pre-cropped appropriately.

We use this new system to ensure that tab hover cards are still getting
appropriately-sized thumbnails, but the touch-only tabstrip gets the
entire compressed image of the webpage, at approximately the original
aspect ratio.

Bug: 1013646
Change-Id: If63dc7e13f8ddef09e991fcb20f5bef8cacad302
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1887734
Commit-Queue: Dana Fried <dfried@chromium.org>
Reviewed-by: Collin Baker <collinbaker@chromium.org>
Cr-Commit-Position: refs/heads/master@{#710974}

chrome/browser/ui/BUILD.gn

@@ -1151,8 +1151,6 @@ jumbo_static_library("ui") {
       "thumbnails/thumbnail_page_observer.h",
       "thumbnails/thumbnail_tab_helper.cc",
       "thumbnails/thumbnail_tab_helper.h",
-      "thumbnails/thumbnail_utils.cc",
-      "thumbnails/thumbnail_utils.h",
       "toolbar/app_menu_icon_controller.cc",
       "toolbar/app_menu_icon_controller.h",
       "toolbar/app_menu_model.cc",

chrome/browser/ui/thumbnails/thumbnail_image.cc

@@ -9,6 +9,7 @@
 #include "base/task/post_task.h"
 #include "base/task/task_traits.h"
 #include "ui/gfx/codec/jpeg_codec.h"
+#include "ui/gfx/skia_util.h"
 
 void ThumbnailImage::Observer::OnThumbnailImageAvailable(
     gfx::ImageSkia thumbnail_image) {}
@@ -16,6 +17,11 @@ void ThumbnailImage::Observer::OnThumbnailImageAvailable(
 void ThumbnailImage::Observer::OnCompressedThumbnailDataAvailable(
     CompressedThumbnailData thumbnail_data) {}
 
+base::Optional<gfx::Size> ThumbnailImage::Observer::GetThumbnailSizeHint()
+    const {
+  return base::nullopt;
+}
+
 ThumbnailImage::Delegate::~Delegate() {
   if (thumbnail_)
     thumbnail_->delegate_ = nullptr;
@@ -105,8 +111,11 @@ void ThumbnailImage::NotifyUncompressedDataObservers(gfx::ImageSkia image) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
   if (async_operation_finished_callback_)
     async_operation_finished_callback_.Run();
-  for (auto& observer : observers_)
-    observer.OnThumbnailImageAvailable(image);
+  for (auto& observer : observers_) {
+    auto size_hint = observer.GetThumbnailSizeHint();
+    observer.OnThumbnailImageAvailable(
+        size_hint ? CropPreviewImage(image, *size_hint) : image);
+  }
 }
 
 void ThumbnailImage::NotifyCompressedDataObservers(
@@ -135,3 +144,39 @@ gfx::ImageSkia ThumbnailImage::UncompressImage(
   result.MakeThreadSafe();
   return result;
 }
+
+// static
+gfx::ImageSkia ThumbnailImage::CropPreviewImage(
+    const gfx::ImageSkia& source_image,
+    const gfx::Size& minimum_size) {
+  DCHECK(!source_image.isNull());
+  DCHECK(!source_image.size().IsEmpty());
+  DCHECK(!minimum_size.IsEmpty());
+  const float desired_aspect =
+      float{minimum_size.width()} / minimum_size.height();
+  const float source_aspect =
+      float{source_image.width()} / float{source_image.height()};
+
+  if (source_aspect == desired_aspect ||
+      source_image.width() < minimum_size.width() ||
+      source_image.height() < minimum_size.height()) {
+    return source_image;
+  }
+
+  gfx::Rect clip_rect;
+  if (source_aspect > desired_aspect) {
+    // Wider than tall, clip horizontally: we center the smaller
+    // thumbnail in the wider screen.
+    const int new_width = source_image.height() * desired_aspect;
+    const int x_offset = (source_image.width() - new_width) / 2;
+    clip_rect = {x_offset, 0, new_width, source_image.height()};
+  } else {
+    // Taller than wide; clip vertically.
+    const int new_height = source_image.width() / desired_aspect;
+    clip_rect = {0, 0, source_image.width(), new_height};
+  }
+
+  SkBitmap cropped;
+  source_image.bitmap()->extractSubset(&cropped, gfx::RectToSkIRect(clip_rect));
+  return gfx::ImageSkia::CreateFrom1xBitmap(cropped);
+}

chrome/browser/ui/thumbnails/thumbnail_image.h

@@ -12,6 +12,7 @@
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/observer_list.h"
+#include "base/optional.h"
 #include "base/sequence_checker.h"
 #include "ui/gfx/image/image_skia.h"
 
@@ -34,6 +35,22 @@ class ThumbnailImage : public base::RefCounted<ThumbnailImage> {
     // Receives compressed thumbnail image data. Default is no-op.
     virtual void OnCompressedThumbnailDataAvailable(
         CompressedThumbnailData thumbnail_data);
+
+    // Provides a desired aspect ratio and minimum size that the observer will
+    // accept. If not specified, or if available thumbnail data is smaller in
+    // either dimension than this value, OnThumbnailImageAvailable will be
+    // called with an uncropped image. If this value is specified, and the
+    // available image is larger, the image passed to OnThumbnailImageAvailable
+    // will be cropped to the same aspect ratio (but otherwise unchanged,
+    // including scale).
+    //
+    // OnCompressedThumbnailDataAvailable is not affected by this value.
+    //
+    // This method is used to ensure that except for very small thumbnails, the
+    // image passed to OnThumbnailImageAvailable fits the needs of the observer
+    // for display purposes, without the observer having to further crop the
+    // image. The default is unspecified.
+    virtual base::Optional<gfx::Size> GetThumbnailSizeHint() const;
   };
 
   // Represents the endpoint
@@ -91,6 +108,11 @@ class ThumbnailImage : public base::RefCounted<ThumbnailImage> {
   static std::vector<uint8_t> CompressBitmap(SkBitmap bitmap);
   static gfx::ImageSkia UncompressImage(CompressedThumbnailData compressed);
 
+  // Crops and returns a preview from a thumbnail of an entire web page. Uses
+  // logic appropriate for fixed-aspect previews (e.g. hover cards).
+  static gfx::ImageSkia CropPreviewImage(const gfx::ImageSkia& source_image,
+                                         const gfx::Size& minimum_size);
+
   Delegate* delegate_;
 
   CompressedThumbnailData data_;

chrome/browser/ui/thumbnails/thumbnail_tab_helper.h

@@ -35,6 +35,23 @@ class ThumbnailTabHelper
   friend class content::WebContentsUserData<ThumbnailTabHelper>;
   friend class ThumanailImageImpl;
 
+  // Describes how a thumbnail bitmap should be generated from a target surface.
+  // All sizes are in pixels, not DIPs.
+  struct ThumbnailCaptureInfo {
+    // The total source size (including scrollbars).
+    gfx::Size source_size;
+
+    // Insets for scrollbars in the source image that should probably be
+    // ignored for thumbnailing purposes.
+    gfx::Insets scrollbar_insets;
+
+    // Cropping rectangle for the source canvas, in pixels.
+    gfx::Rect copy_rect;
+
+    // Size of the target bitmap in pixels.
+    gfx::Size target_size;
+  };
+
   explicit ThumbnailTabHelper(content::WebContents* contents);
 
   // ThumbnailImage::Delegate:
@@ -49,8 +66,6 @@ class ThumbnailTabHelper
 
   content::RenderWidgetHostView* GetView();
 
-  gfx::Size GetThumbnailSize() const;
-
   // content::WebContentsObserver:
   void OnVisibilityChanged(content::Visibility visibility) override;
   void DidFinishNavigation(
@@ -65,6 +80,31 @@ class ThumbnailTabHelper
           callbacks) override;
   void OnStopped() override;
 
+  // Returns the dimensions of the multipurpose thumbnail that should be
+  // captured from an entire webpage. Can be cropped or compressed later.
+  // If |include_scrollbars_in_capture| is false, the area which is likely to
+  // contain scrollbars will be removed from both the result's |copy_rect| and
+  // |target_size|. In both cases, |scrollbar_insets| is calculated. This
+  // function always returns a result with |clip_result| = kSourceNotClipped.
+  static ThumbnailCaptureInfo GetInitialCaptureInfo(
+      const gfx::Size& source_size,
+      float scale_factor,
+      bool include_scrollbars_in_capture);
+
+  // The implementation of the 'classic' thumbnail cropping algorithm. It is not
+  // content-driven in any meaningful way. Rather, the choice of a cropping
+  // region is based on relation between source and target sizes. The selected
+  // source region is then rescaled into the target thumbnail image.
+  //
+  // Provides information necessary to crop-and-resize image data from a source
+  // canvas of |source_size|. Auxiliary |scale_factor| helps compute the target
+  // thumbnail size to be copied from the backing store, in pixels. The return
+  // value contains the type of clip and the clip parameters.
+  // static ThumbnailCaptureInfo GetThumbnailCropInfo(const gfx::Size&
+  // source_size,
+  //  float scale_factor,
+  //  const gfx::Size& unscaled_target_size);
+
   // The last known visibility WebContents visibility.
   content::Visibility last_visibility_;
 
@@ -72,9 +112,8 @@ class ThumbnailTabHelper
   // was last hidden.
   bool captured_loaded_thumbnail_since_tab_hidden_ = false;
 
-  // The estimated percentage of the most recent frame being captured that
-  // consists of possible right and bottom scroll bars.
-  float last_frame_scrollbar_percent_ = 0.0f;
+  // Copy info from the most recent frame we have captured.
+  ThumbnailCaptureInfo last_frame_capture_info_;
 
   // Captures frames from the WebContents while it's hidden. The capturer count
   // of the WebContents is incremented/decremented when a capturer is set/unset.

chrome/browser/ui/thumbnails/thumbnail_utils.cc

-// Copyright (c) 2013 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/ui/thumbnails/thumbnail_utils.h"
-
-#include <algorithm>
-
-#include "components/history/core/common/thumbnail_score.h"
-#include "content/public/browser/browser_thread.h"
-#include "ui/gfx/geometry/size_conversions.h"
-
-namespace thumbnails {
-
-bool IsGoodClipping(ClipResult clip_result) {
-  return clip_result == ClipResult::kSourceWiderThanTall ||
-         clip_result == ClipResult::kSourceTallerThanWide ||
-         clip_result == ClipResult::kSourceNotClipped;
-}
-
-CanvasCopyInfo GetCanvasCopyInfo(const gfx::Size& source_size,
-                                 float scale_factor,
-                                 const gfx::Size& unscaled_target_size) {
-  DCHECK(!source_size.IsEmpty());
-  DCHECK(!unscaled_target_size.IsEmpty());
-  DCHECK_GT(scale_factor, 0.0f);
-
-  const float desired_aspect =
-      float{unscaled_target_size.width()} / unscaled_target_size.height();
-
-  CanvasCopyInfo copy_info;
-  copy_info.target_size =
-      gfx::ScaleToFlooredSize(unscaled_target_size, scale_factor);
-
-  // Get the clipping rect so that we can preserve the aspect ratio while
-  // filling the destination.
-  if (source_size.width() < unscaled_target_size.width() ||
-      source_size.height() < unscaled_target_size.height()) {
-    // Source image is smaller: we clip the part of source image within the
-    // dest rect, and then stretch it to fill the dest rect. We don't respect
-    // the aspect ratio in this case.
-    gfx::Size copy_size = source_size;
-    copy_size.SetToMin(copy_info.target_size);
-
-    copy_info.copy_rect = gfx::Rect(copy_size);
-    copy_info.clip_result = ClipResult::kSourceSmallerThanTarget;
-
-  } else {
-    const float src_aspect =
-        float{source_size.width()} / float{source_size.height()};
-
-    if (src_aspect > desired_aspect) {
-      // Wider than tall, clip horizontally: we center the smaller
-      // thumbnail in the wider screen.
-      const int new_width = source_size.height() * desired_aspect;
-      const int x_offset = (source_size.width() - new_width) / 2;
-      copy_info.clip_result =
-          (src_aspect >= history::ThumbnailScore::kTooWideAspectRatio)
-              ? ClipResult::kSourceMuchWiderThanTall
-              : ClipResult::kSourceWiderThanTall;
-      copy_info.copy_rect.SetRect(x_offset, 0, new_width, source_size.height());
-
-    } else if (src_aspect < desired_aspect) {
-      copy_info.clip_result = ClipResult::kSourceTallerThanWide;
-      copy_info.copy_rect =
-          gfx::Rect(source_size.width(), source_size.width() / desired_aspect);
-
-    } else {
-      copy_info.clip_result = ClipResult::kSourceNotClipped;
-      copy_info.copy_rect = gfx::Rect(source_size);
-    }
-  }
-
-  return copy_info;
-}
-
-}  // namespace thumbnails

chrome/browser/ui/thumbnails/thumbnail_utils.h

-// Copyright (c) 2013 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef CHROME_BROWSER_UI_THUMBNAILS_THUMBNAIL_UTILS_H_
-#define CHROME_BROWSER_UI_THUMBNAILS_THUMBNAIL_UTILS_H_
-
-#include "base/macros.h"
-#include "ui/base/resource/scale_factor.h"
-#include "ui/gfx/geometry/rect.h"
-#include "ui/gfx/geometry/size.h"
-
-namespace thumbnails {
-
-// The result of clipping. This can be used to determine if the generated
-// thumbnail is good or not.
-enum class ClipResult {
-  kSourceNotClipped,         // Source and target aspect ratios are identical.
-  kSourceSmallerThanTarget,  // Source image is smaller than target.
-  kSourceMuchWiderThanTall,  // Wider than tall by 2x+, clip horizontally.
-  kSourceWiderThanTall,      // Wider than tall, clip horizontally.
-  kSourceTallerThanWide,     // Taller than wide, clip vertically.
-};
-
-// Describes how a thumbnail bitmap should be generated from a target surface.
-struct CanvasCopyInfo {
-  // How the source canvas is clipped to achieve the target size.
-  ClipResult clip_result = ClipResult::kSourceNotClipped;
-
-  // Cropping rectangle for the source canvas, in pixels (not DIPs).
-  gfx::Rect copy_rect;
-
-  // Size of the target bitmap in pixels.
-  gfx::Size target_size;
-};
-
-bool IsGoodClipping(ClipResult clip_result);
-
-// The implementation of the 'classic' thumbnail cropping algorithm. It is not
-// content-driven in any meaningful way. Rather, the choice of a cropping region
-// is based on relation between source and target sizes. The selected source
-// region is then rescaled into the target thumbnail image.
-//
-// Provides information necessary to crop-and-resize image data from a source
-// canvas of |source_size|. Auxiliary |scale_factor| helps compute the target
-// thumbnail size to be copied from the backing store, in pixels. The return
-// value contains the type of clip and the clip parameters.
-CanvasCopyInfo GetCanvasCopyInfo(const gfx::Size& source_size,
-                                 float scale_factor,
-                                 const gfx::Size& unscaled_target_size);
-
-}  // namespace thumbnails
-
-#endif  // CHROME_BROWSER_UI_THUMBNAILS_THUMBNAIL_UTILS_H_

chrome/browser/ui/thumbnails/thumbnail_utils_unittest.cc

-// Copyright (c) 2012 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/ui/thumbnails/thumbnail_utils.h"
-
-#include "content/public/browser/notification_service.h"
-#include "content/public/browser/notification_types.h"
-#include "content/public/test/mock_render_process_host.h"
-#include "content/public/test/test_renderer_host.h"
-#include "skia/ext/platform_canvas.h"
-#include "testing/gtest/include/gtest/gtest.h"
-#include "third_party/skia/include/core/SkColorPriv.h"
-#include "ui/gfx/canvas.h"
-#include "ui/gfx/geometry/size_conversions.h"
-#include "ui/surface/transport_dib.h"
-
-using content::WebContents;
-
-typedef testing::Test SimpleThumbnailCropTest;
-
-namespace thumbnails {
-
-// Test which generates thumbnails from various source scales.
-class ThumbnailUtilsTest : public testing::TestWithParam<float> {
- public:
-  ThumbnailUtilsTest() = default;
-
- private:
-  DISALLOW_COPY_AND_ASSIGN(ThumbnailUtilsTest);
-};
-
-static float GetAspect(const gfx::Size& size) {
-  return float{size.width()} / float{size.height()};
-}
-
-INSTANTIATE_TEST_SUITE_P(,
-                         ThumbnailUtilsTest,
-                         testing::ValuesIn({1.0f, 1.25f, 1.62f, 2.0f}));
-
-TEST_P(ThumbnailUtilsTest, GetCanvasCopyInfo) {
-  constexpr gfx::Size kThumbnailSize(200, 120);
-  const float scale_factor = GetParam();
-  const gfx::Size expected_size =
-      gfx::ScaleToFlooredSize(kThumbnailSize, scale_factor);
-  const float desired_aspect = GetAspect(kThumbnailSize);
-  const gfx::Size wider_than_tall_source(400, 210);
-  const gfx::Size much_wider_than_tall_source(600, 200);
-  const gfx::Size taller_than_wide_source(300, 600);
-  const gfx::Size small_source(200, 100);
-
-  CanvasCopyInfo result =
-      GetCanvasCopyInfo(wider_than_tall_source, scale_factor, kThumbnailSize);
-  EXPECT_EQ(ClipResult::kSourceWiderThanTall, result.clip_result);
-  EXPECT_EQ(expected_size, result.target_size);
-  EXPECT_NEAR(desired_aspect, GetAspect(result.copy_rect.size()), 0.01);
-
-  result = GetCanvasCopyInfo(much_wider_than_tall_source, scale_factor,
-                             kThumbnailSize);
-  EXPECT_EQ(ClipResult::kSourceMuchWiderThanTall, result.clip_result);
-  EXPECT_EQ(expected_size, result.target_size);
-  EXPECT_NEAR(desired_aspect, GetAspect(result.copy_rect.size()), 0.01);
-
-  result =
-      GetCanvasCopyInfo(taller_than_wide_source, scale_factor, kThumbnailSize);
-  EXPECT_EQ(ClipResult::kSourceTallerThanWide, result.clip_result);
-  EXPECT_EQ(expected_size, result.target_size);
-  EXPECT_NEAR(desired_aspect, GetAspect(result.copy_rect.size()), 0.01);
-
-  result = GetCanvasCopyInfo(small_source, scale_factor, kThumbnailSize);
-  EXPECT_EQ(ClipResult::kSourceSmallerThanTarget, result.clip_result);
-  EXPECT_EQ(expected_size, result.target_size);
-}
-
-}  // namespace thumbnails

chrome/browser/ui/views/tabs/tab_hover_card_bubble_view.cc

@@ -51,17 +51,6 @@ namespace {
 // Maximum number of lines that a title label occupies.
 int kTitleMaxLines = 2;
 
-// Hover card and preview image dimensions.
-int GetPreferredTabHoverCardWidth() {
-  return TabStyle::GetStandardWidth();
-}
-
-gfx::Size GetTabHoverCardPreviewImageSize() {
-  constexpr float kTabHoverCardPreviewImageAspectRatio = 16.0f / 9.0f;
-  const int width = GetPreferredTabHoverCardWidth();
-  return gfx::Size(width, width / kTabHoverCardPreviewImageAspectRatio);
-}
-
 bool AreHoverCardImagesEnabled() {
   return base::FeatureList::IsEnabled(features::kTabHoverCardImages);
 }
@@ -325,12 +314,13 @@ TabHoverCardBubbleView::TabHoverCardBubbleView(Tab* tab)
 
   if (AreHoverCardImagesEnabled()) {
     using Alignment = views::ImageView::Alignment;
+    const gfx::Size preview_size = TabStyle::GetPreviewImageSize();
     preview_image_ = new views::ImageView();
     preview_image_->SetVisible(AreHoverCardImagesEnabled());
     preview_image_->SetHorizontalAlignment(Alignment::kCenter);
     preview_image_->SetVerticalAlignment(Alignment::kCenter);
-    preview_image_->SetImageSize(GetTabHoverCardPreviewImageSize());
-    preview_image_->SetPreferredSize(GetTabHoverCardPreviewImageSize());
+    preview_image_->SetImageSize(preview_size);
+    preview_image_->SetPreferredSize(preview_size);
     AddChildView(preview_image_);
   }
 
@@ -645,7 +635,7 @@ void TabHoverCardBubbleView::ClearPreviewImage() {
       kGlobeIcon, kNoPreviewImageSize.width(), foreground_color);
   preview_image_->SetImage(no_preview_image);
   preview_image_->SetImageSize(kNoPreviewImageSize);
-  preview_image_->SetPreferredSize(GetTabHoverCardPreviewImageSize());
+  preview_image_->SetPreferredSize(TabStyle::GetPreviewImageSize());
 
   // Also possibly regenerate the background if it has changed.
   const SkColor background_color = GetThemeProvider()->GetColor(
@@ -659,15 +649,20 @@ void TabHoverCardBubbleView::ClearPreviewImage() {
 
 void TabHoverCardBubbleView::OnThumbnailImageAvailable(
     gfx::ImageSkia preview_image) {
+  const gfx::Size preview_size = TabStyle::GetPreviewImageSize();
   preview_image_->SetImage(preview_image);
-  preview_image_->SetImageSize(GetTabHoverCardPreviewImageSize());
-  preview_image_->SetPreferredSize(GetTabHoverCardPreviewImageSize());
+  preview_image_->SetImageSize(preview_size);
+  preview_image_->SetPreferredSize(preview_size);
   preview_image_->SetBackground(nullptr);
 }
 
+base::Optional<gfx::Size> TabHoverCardBubbleView::GetThumbnailSizeHint() const {
+  return TabStyle::GetPreviewImageSize();
+}
+
 gfx::Size TabHoverCardBubbleView::CalculatePreferredSize() const {
   gfx::Size preferred_size = GetLayoutManager()->GetPreferredSize(this);
-  preferred_size.set_width(GetPreferredTabHoverCardWidth());
+  preferred_size.set_width(TabStyle::GetPreviewImageSize().width());
   DCHECK(!preferred_size.IsEmpty());
   return preferred_size;
 }

chrome/browser/ui/views/tabs/tab_hover_card_bubble_view.h

@@ -87,6 +87,7 @@ class TabHoverCardBubbleView : public views::BubbleDialogDelegateView,
 
   // ThumbnailImage::Observer:
   void OnThumbnailImageAvailable(gfx::ImageSkia thumbnail_image) override;
+  base::Optional<gfx::Size> GetThumbnailSizeHint() const override;
 
   // views::BubbleDialogDelegateView:
   gfx::Size CalculatePreferredSize() const override;

chrome/test/BUILD.gn

@@ -3393,7 +3393,6 @@ test("unit_tests") {
       "../browser/ui/bookmarks/recently_used_folders_combo_model_unittest.cc",
       "../browser/ui/sync/profile_signin_confirmation_helper_unittest.cc",
       "../browser/ui/sync/sync_promo_ui_unittest.cc",
-      "../browser/ui/thumbnails/thumbnail_utils_unittest.cc",
       "../browser/ui/toolbar/app_menu_icon_controller_unittest.cc",
       "../browser/ui/webui/devtools_ui_data_source_unittest.cc",
       "../browser/ui/webui/discards/graph_dump_impl_unittest.cc",