Add support for classification of image sprites in dark mode.

To generate the bitmap for sprite images the location of the
images cannot be assumed to be (0,0) which is the current
scenario. Instead the source rectangle of the image should be
used for getting the correct bitmap of the image.

In the case of image sprites, a single image object has to
hold the classifications of all the images that it contains.
Having a single DarkModeClassification object will wrongly
give the classification of the first image that is classified
to all the images of the sprite.

So instead use the location of source rect of the image as a
key for a map which stores the classifications.

Bug: None
Change-Id: I52487bb511bd5ed9c847cb381e537610f7a420f6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1514437Reviewed-by: Stephen Chenney <schenney@chromium.org>
Reviewed-by: Prashant Nevase <prashant.n@samsung.com>
Commit-Queue: Prashant Nevase <prashant.n@samsung.com>
Cr-Commit-Position: refs/heads/master@{#641702}

third_party/blink/renderer/platform/graphics/dark_mode_image_classifier.cc

@@ -64,23 +64,28 @@ int DarkModeImageClassifier::GetRandomInt(const int min, const int max) {
   return base::RandInt(min, max - 1);
 }
 
-bool DarkModeImageClassifier::ShouldApplyDarkModeFilterToImage(Image& image) {
-  DarkModeClassification result = image.GetDarkModeClassification();
+bool DarkModeImageClassifier::ShouldApplyDarkModeFilterToImage(
+    Image& image,
+    const FloatRect& src_rect) {
+  DarkModeClassification result = image.GetDarkModeClassification(src_rect);
+  // Check if the image has already been classified.
   if (result != DarkModeClassification::kNotClassified)
     return result == DarkModeClassification::kApplyDarkModeFilter;
 
-  if (image.width() < kMinImageSizeForClassification1D ||
-      image.height() < kMinImageSizeForClassification1D) {
+  if (src_rect.Width() < kMinImageSizeForClassification1D ||
+      src_rect.Height() < kMinImageSizeForClassification1D) {
     result = DarkModeClassification::kApplyDarkModeFilter;
   } else {
     std::vector<float> features;
-    if (!ComputeImageFeatures(image, &features))
+    if (!ComputeImageFeatures(image, src_rect, &features))
       result = DarkModeClassification::kDoNotApplyDarkModeFilter;
     else
       result = ClassifyImage(features);
   }
 
-  image.SetDarkModeClassification(result);
+  // Store the classification result in the image object using
+  // src_rect's location as a key for the map.
+  image.AddDarkModeClassification(src_rect, result);
   return result == DarkModeClassification::kApplyDarkModeFilter;
 }
 
@@ -89,9 +94,10 @@ bool DarkModeImageClassifier::ShouldApplyDarkModeFilterToImage(Image& image) {
 // method, and |GetFeatures| function for description of the features.
 bool DarkModeImageClassifier::ComputeImageFeatures(
     Image& image,
+    const FloatRect& src_rect,
     std::vector<float>* features) {
   SkBitmap bitmap;
-  if (!GetBitmap(image, &bitmap))
+  if (!GetBitmap(image, src_rect, &bitmap))
     return false;
 
   if (use_testing_random_generator_)
@@ -106,16 +112,25 @@ bool DarkModeImageClassifier::ComputeImageFeatures(
   return true;
 }
 
-bool DarkModeImageClassifier::GetBitmap(Image& image, SkBitmap* bitmap) {
-  if (!image.IsBitmapImage() || !image.width() || !image.height())
+bool DarkModeImageClassifier::GetBitmap(Image& image,
+                                        const FloatRect& src_rect,
+                                        SkBitmap* bitmap) {
+  if (!image.IsBitmapImage() || !src_rect.Width() || !src_rect.Height())
     return false;
 
-  bitmap->allocPixels(
-      SkImageInfo::MakeN32(image.width(), image.height(), kPremul_SkAlphaType));
+  SkScalar sx = SkFloatToScalar(src_rect.X());
+  SkScalar sy = SkFloatToScalar(src_rect.Y());
+  SkScalar sw = SkFloatToScalar(src_rect.Width());
+  SkScalar sh = SkFloatToScalar(src_rect.Height());
+  SkRect src = {sx, sy, sx + sw, sy + sh};
+  SkRect dest = {0, 0, sw, sh};
+  bitmap->allocPixels(SkImageInfo::MakeN32(static_cast<int>(src_rect.Width()),
+                                           static_cast<int>(src_rect.Height()),
+                                           kPremul_SkAlphaType));
   SkCanvas canvas(*bitmap);
   canvas.clear(SK_ColorTRANSPARENT);
-  canvas.drawImageRect(image.PaintImageForCurrentFrame().GetSkImage(),
-                       SkRect::MakeIWH(image.width(), image.height()), nullptr);
+  canvas.drawImageRect(image.PaintImageForCurrentFrame().GetSkImage(), src,
+                       dest, nullptr);
   return true;
 }
 

third_party/blink/renderer/platform/graphics/dark_mode_image_classifier.h

@@ -7,6 +7,7 @@
 
 #include <vector>
 
+#include "third_party/blink/renderer/platform/geometry/float_rect.h"
 #include "third_party/blink/renderer/platform/graphics/graphics_types.h"
 #include "third_party/blink/renderer/platform/graphics/image.h"
 #include "third_party/blink/renderer/platform/wtf/allocator.h"
@@ -23,11 +24,20 @@ class PLATFORM_EXPORT DarkModeImageClassifier {
   ~DarkModeImageClassifier() = default;
 
   // Decides if a dark mode filter should be applied to the image or not.
-  bool ShouldApplyDarkModeFilterToImage(Image&);
+  // |src_rect| is needed in case of image sprites for the location and
+  // size of the smaller images that the sprite holds.
+  // For images that come from sprites the |src_rect.X| and |src_rect.Y|
+  // can be non-zero. But for normal images they are both zero.
+  bool ShouldApplyDarkModeFilterToImage(Image& image,
+                                        const FloatRect& src_rect);
 
   bool ComputeImageFeaturesForTesting(Image& image,
                                       std::vector<float>* features) {
-    return ComputeImageFeatures(image, features);
+    return ComputeImageFeatures(
+        image,
+        FloatRect(0, 0, static_cast<float>(image.width()),
+                  static_cast<float>(image.height())),
+        features);
   }
 
   void SetRandomGeneratorForTesting() { use_testing_random_generator_ = true; }
@@ -41,10 +51,10 @@ class PLATFORM_EXPORT DarkModeImageClassifier {
   enum class ColorMode { kColor = 0, kGrayscale = 1 };
 
   // Computes the features vector for a given image.
-  bool ComputeImageFeatures(Image&, std::vector<float>*);
+  bool ComputeImageFeatures(Image&, const FloatRect&, std::vector<float>*);
 
   // Converts image to SkBitmap and returns true if successful.
-  bool GetBitmap(Image&, SkBitmap*);
+  bool GetBitmap(Image&, const FloatRect&, SkBitmap*);
 
   // Given a SkBitmap, extracts a sample set of pixels (|sampled_pixels|),
   // |transparency_ratio|, and |background_ratio|.

third_party/blink/renderer/platform/graphics/dark_mode_image_classifier_test.cc

@@ -6,6 +6,7 @@
 
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/renderer/platform/graphics/bitmap_image.h"
+#include "third_party/blink/renderer/platform/graphics/paint/paint_image.h"
 #include "third_party/blink/renderer/platform/shared_buffer.h"
 #include "third_party/blink/renderer/platform/testing/testing_platform_support_with_mock_scheduler.h"
 #include "third_party/blink/renderer/platform/testing/unit_test_helpers.h"
@@ -17,6 +18,28 @@ const float kEpsilon = 0.00001;
 
 namespace blink {
 
+class FakeImageForCacheTest : public Image {
+ public:
+  static scoped_refptr<FakeImageForCacheTest> Create() {
+    return base::AdoptRef(new FakeImageForCacheTest());
+  }
+
+  int GetMapSize() { return dark_mode_classifications_.size(); }
+
+  // Pure virtual functions that have to be overridden.
+  bool CurrentFrameKnownToBeOpaque() override { return false; }
+  IntSize Size() const override { return IntSize(0, 0); }
+  void DestroyDecodedData() override {}
+  PaintImage PaintImageForCurrentFrame() override { return PaintImage(); }
+  void Draw(cc::PaintCanvas*,
+            const cc::PaintFlags&,
+            const FloatRect& dst_rect,
+            const FloatRect& src_rect,
+            RespectImageOrientationEnum,
+            ImageClampingMode,
+            ImageDecodingMode) override {}
+};
+
 class DarkModeImageClassifierTest : public testing::Test {
  public:
   // Loads the image from |file_name|, computes features vector into |features|,
@@ -27,7 +50,8 @@ class DarkModeImageClassifierTest : public testing::Test {
     scoped_refptr<BitmapImage> image = LoadImage(file_name);
     classifier_.SetRandomGeneratorForTesting();
     classifier_.ComputeImageFeaturesForTesting(*image.get(), features);
-    return classifier_.ShouldApplyDarkModeFilterToImage(*image.get());
+    return classifier_.ShouldApplyDarkModeFilterToImage(
+        *image.get(), FloatRect(0, 0, image->width(), image->height()));
   }
 
   void AssertFeaturesEqual(const std::vector<float>& features,
@@ -117,4 +141,34 @@ TEST_F(DarkModeImageClassifierTest, FeaturesAndClassification) {
   AssertFeaturesEqual(features, {1.0f, 0.0166016f, 0.0f, 0.59f});
 }
 
+TEST_F(DarkModeImageClassifierTest, Caching) {
+  scoped_refptr<FakeImageForCacheTest> image = FakeImageForCacheTest::Create();
+  FloatRect src_rect1(0, 0, 50, 50);
+  FloatRect src_rect2(5, 20, 100, 100);
+  FloatRect src_rect3(6, -9, 50, 50);
+
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect1),
+            DarkModeClassification::kNotClassified);
+  image->AddDarkModeClassification(
+      src_rect1, DarkModeClassification::kApplyDarkModeFilter);
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect1),
+            DarkModeClassification::kApplyDarkModeFilter);
+
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect2),
+            DarkModeClassification::kNotClassified);
+  image->AddDarkModeClassification(
+      src_rect2, DarkModeClassification::kDoNotApplyDarkModeFilter);
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect2),
+            DarkModeClassification::kDoNotApplyDarkModeFilter);
+
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect3),
+            DarkModeClassification::kNotClassified);
+  image->AddDarkModeClassification(
+      src_rect3, DarkModeClassification::kApplyDarkModeFilter);
+  EXPECT_EQ(image->GetDarkModeClassification(src_rect3),
+            DarkModeClassification::kApplyDarkModeFilter);
+
+  EXPECT_EQ(image->GetMapSize(), 3);
+}
+
 }  // namespace blink

third_party/blink/renderer/platform/graphics/graphics_context.cc

@@ -916,7 +916,7 @@ void GraphicsContext::DrawImage(
   image_flags.setBlendMode(op);
   image_flags.setColor(SK_ColorBLACK);
   image_flags.setFilterQuality(ComputeFilterQuality(image, dest, src));
-  if (ShouldApplyDarkModeFilterToImage(*image))
+  if (ShouldApplyDarkModeFilterToImage(*image, src))
     image_flags.setColorFilter(dark_mode_filter_);
   image->Draw(canvas_, image_flags, dest, src, should_respect_image_orientation,
               Image::kClampImageToSourceRect, decode_mode);
@@ -1414,14 +1414,16 @@ sk_sp<SkColorFilter> GraphicsContext::WebCoreColorFilterToSkiaColorFilter(
   return nullptr;
 }
 
-bool GraphicsContext::ShouldApplyDarkModeFilterToImage(Image& image) {
+bool GraphicsContext::ShouldApplyDarkModeFilterToImage(
+    Image& image,
+    const FloatRect& src_rect) {
   if (!dark_mode_filter_)
     return false;
 
   switch (dark_mode_settings_.image_policy) {
     case DarkModeImagePolicy::kFilterSmart:
       return dark_mode_image_classifier_.ShouldApplyDarkModeFilterToImage(
-          image);
+          image, src_rect);
     case DarkModeImagePolicy::kFilterAll:
       return true;
     default:

third_party/blink/renderer/platform/graphics/graphics_context.h

@@ -469,7 +469,8 @@ class PLATFORM_EXPORT GraphicsContext {
                                const Color&);
 
   class DarkModeFlags;
-  bool ShouldApplyDarkModeFilterToImage(Image&);
+  bool ShouldApplyDarkModeFilterToImage(Image& image,
+                                        const FloatRect& src_rect);
   Color ApplyDarkModeFilter(const Color& input) const;
 
   // null indicates painting is contextDisabled. Never delete this object.

third_party/blink/renderer/platform/graphics/image.cc

@@ -92,8 +92,7 @@ Image::Image(ImageObserver* observer, bool is_multipart)
     : image_observer_disabled_(false),
       image_observer_(observer),
       stable_image_id_(PaintImage::GetNextId()),
-      is_multipart_(is_multipart),
-      dark_mode_classification_(DarkModeClassification::kNotClassified) {}
+      is_multipart_(is_multipart) {}
 
 Image::~Image() = default;
 
@@ -375,4 +374,28 @@ SkBitmap Image::AsSkBitmapForCurrentFrame(
   return bitmap;
 }
 
+DarkModeClassification Image::GetDarkModeClassification(
+    const FloatRect& src_rect) {
+  // Assuming that multiple uses of the same sprite region all have the same
+  // size, only the top left corner coordinates of the src_rect are used to
+  // generate the key for caching and retrieving the classification.
+  ClassificationKey key(src_rect.X(), src_rect.Y());
+  std::map<ClassificationKey, DarkModeClassification>::iterator result =
+      dark_mode_classifications_.find(key);
+  if (result == dark_mode_classifications_.end())
+    return DarkModeClassification::kNotClassified;
+
+  return result->second;
+}
+
+void Image::AddDarkModeClassification(
+    const FloatRect& src_rect,
+    DarkModeClassification dark_mode_classification) {
+  // Add the classification in the map only if the image is not classified yet.
+  DCHECK(GetDarkModeClassification(src_rect) ==
+         DarkModeClassification::kNotClassified);
+  ClassificationKey key(src_rect.X(), src_rect.Y());
+  dark_mode_classifications_[key] = dark_mode_classification;
+}
+
 }  // namespace blink

third_party/blink/renderer/platform/graphics/image.h

@@ -236,16 +236,13 @@ class PLATFORM_EXPORT Image : public ThreadSafeRefCounted<Image> {
     return nullptr;
   }
 
-  DarkModeClassification GetDarkModeClassification() {
-    return dark_mode_classification_;
-  }
+  DarkModeClassification GetDarkModeClassification(const FloatRect& src_rect);
 
   // Dark mode classification result is cached to be consistent and have
   // higher performance for future paints.
-  void SetDarkModeClassification(
-      const DarkModeClassification dark_mode_classification) {
-    dark_mode_classification_ = dark_mode_classification;
-  }
+  void AddDarkModeClassification(
+      const FloatRect& src_rect,
+      const DarkModeClassification dark_mode_classification);
 
   PaintImage::Id paint_image_id() const { return stable_image_id_; }
 
@@ -270,6 +267,10 @@ class PLATFORM_EXPORT Image : public ThreadSafeRefCounted<Image> {
   // Whether or not size is available yet.
   virtual bool IsSizeAvailable() { return true; }
 
+  typedef std::pair<float, float> ClassificationKey;
+  std::map<ClassificationKey, DarkModeClassification>
+      dark_mode_classifications_;
+
  private:
   bool image_observer_disabled_;
   scoped_refptr<SharedBuffer> encoded_image_data_;
@@ -283,8 +284,6 @@ class PLATFORM_EXPORT Image : public ThreadSafeRefCounted<Image> {
   WeakPersistent<ImageObserver> image_observer_;
   PaintImage::Id stable_image_id_;
   const bool is_multipart_;
-  DarkModeClassification dark_mode_classification_;
-
   DISALLOW_COPY_AND_ASSIGN(Image);
 };