Neural Network added to High Contrast classifier.

A tfNative generated neural network is added to High Contrast classifier
to make a better decision for images which are not decidable using the
decision tree.

Bug: 685242
Cq-Include-Trybots: master.tryserver.chromium.linux:linux_layout_tests_slimming_paint_v2
Change-Id: I4c30f785d7ca65d129fb09902b748978dd6de4f6
Reviewed-on: https://chromium-review.googlesource.com/753588Reviewed-by: Chris Harrelson <chrishtr@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Commit-Queue: Ramin Halavati <rhalavati@chromium.org>
Cr-Commit-Position: refs/heads/master@{#517387}

third_party/WebKit/Source/platform/BUILD.gn

@@ -1086,6 +1086,8 @@ jumbo_component("platform") {
     "graphics/gpu/SharedGpuContext.h",
     "graphics/gpu/WebGLImageConversion.cpp",
     "graphics/gpu/WebGLImageConversion.h",
+    "graphics/highcontrast/highcontrast_classifier.cc",
+    "graphics/highcontrast/highcontrast_classifier.h",
     "graphics/paint/ClipDisplayItem.cpp",
     "graphics/paint/ClipDisplayItem.h",
     "graphics/paint/ClipPaintPropertyNode.cpp",

third_party/WebKit/Source/platform/graphics/HighContrastImageClassifier.cpp

@@ -5,6 +5,7 @@
 #include "platform/graphics/HighContrastImageClassifier.h"
 
 #include "base/rand_util.h"
+#include "platform/graphics/highcontrast/highcontrast_classifier.h"
 #include "third_party/WebKit/Source/platform/geometry/IntRect.h"
 #include "third_party/skia/include/utils/SkNullCanvas.h"
 
@@ -41,6 +42,10 @@ const int kPixelsToSample = 1000;
 const int kBlocksCount1D = 10;
 const int kMinImageSizeForClassification1D = 24;
 
+// Decision tree lower and upper thresholds for grayscale and color images.
+const float kLowColorCountThreshold[2] = {0.8125, 0.015137};
+const float kHighColorCountThreshold[2] = {1, 0.025635};
+
 }  // namespace
 
 namespace blink {
@@ -253,7 +258,7 @@ bool HighContrastImageClassifier::IsBlockBackground(
 // 1: Ratio of the number of bucketed colors used in the image to all
 //    possiblities. Color buckets are represented with 4 bits per color channel.
 // 2: Ratio of transparent area to the whole image.
-// 3: Ratio of the brackground area to the whole image.
+// 3: Ratio of the background area to the whole image.
 void HighContrastImageClassifier::GetFeatures(
     const std::vector<SkColor>& sampled_pixels,
     const float transparency_ratio,
@@ -316,18 +321,46 @@ float HighContrastImageClassifier::ComputeColorBucketsRatio(
          max_buckets[color_mode == ColorMode::kColor];
 }
 
-HighContrastClassification HighContrastImageClassifier::ClassifyImage(
+HighContrastClassification
+HighContrastImageClassifier::ClassifyImageUsingDecisionTree(
     const std::vector<float>& features) {
-  bool result = false;
+  DCHECK_EQ(features.size(), 4u);
 
-  // Shallow decision tree trained by C4.5.
-  if (features.size() >= 2) {
-    float threshold = (features[0] == 0) ? 0.8125 : 0.0166;
-    result = features[1] < threshold;
-  }
+  int is_color = features[0] > 0;
+  float color_count_ratio = features[1];
+  float low_color_count_threshold = kLowColorCountThreshold[is_color];
+  float high_color_count_threshold = kHighColorCountThreshold[is_color];
 
-  return result ? HighContrastClassification::kApplyHighContrastFilter
+  // Very few colors means it's not a photo, apply the filter.
+  if (color_count_ratio < low_color_count_threshold)
+    return HighContrastClassification::kApplyHighContrastFilter;
+
+  // Too many colors means it's probably photorealistic, do not apply it.
+  if (color_count_ratio > high_color_count_threshold)
+    return HighContrastClassification::kDoNotApplyHighContrastFilter;
+
+  // In-between, decision tree cannot give a precise result.
+  return HighContrastClassification::kNotClassified;
+}
+
+HighContrastClassification HighContrastImageClassifier::ClassifyImage(
+    const std::vector<float>& features) {
+  DCHECK_EQ(features.size(), 4u);
+
+  HighContrastClassification result = ClassifyImageUsingDecisionTree(features);
+
+  // If decision tree cannot decide, we use a neural network to decide whether
+  // to filter or not based on all the features.
+  if (result == HighContrastClassification::kNotClassified) {
+    highcontrast_tfnative_model::FixedAllocations nn_temp;
+    float nn_out;
+    highcontrast_tfnative_model::Inference(&features[0], &nn_out, &nn_temp);
+    result = nn_out > 0
+                 ? HighContrastClassification::kApplyHighContrastFilter
                  : HighContrastClassification::kDoNotApplyHighContrastFilter;
+  }
+
+  return result;
 }
 
 }  // namespace blink

third_party/WebKit/Source/platform/graphics/HighContrastImageClassifier.h

@@ -29,6 +29,11 @@ class PLATFORM_EXPORT HighContrastImageClassifier {
 
   void SetRandomGeneratorForTesting() { use_testing_random_generator_ = true; }
 
+  HighContrastClassification ClassifyImageUsingDecisionTreeForTesting(
+      const std::vector<float>& features) {
+    return ClassifyImageUsingDecisionTree(features);
+  }
+
  private:
   enum class ColorMode { kColor = 0, kGrayscale = 1 };
 
@@ -76,6 +81,12 @@ class PLATFORM_EXPORT HighContrastImageClassifier {
   // Returns a random number in range [min, max).
   int GetRandomInt(const int min, const int max);
 
+  // Decides if the filter should be applied to the image or not, only using the
+  // decision tree. Returns 'kNotClassified' if decision tree cannot give a
+  // trustable answer.
+  HighContrastClassification ClassifyImageUsingDecisionTree(
+      const std::vector<float>&);
+
   bool use_testing_random_generator_;
   int testing_random_generator_seed_;
 };

third_party/WebKit/Source/platform/graphics/HighContrastImageClassifierTest.cpp

@@ -39,12 +39,14 @@ class HighContrastImageClassifierTest : public ::testing::Test {
     }
   }
 
+  HighContrastImageClassifier* classifier() { return &classifier_; }
+
  protected:
   scoped_refptr<BitmapImage> LoadImage(const std::string& file_name) {
     String file_path = testing::BlinkRootDir();
     file_path.append(file_name.c_str());
     scoped_refptr<SharedBuffer> image_data = testing::ReadFromFile(file_path);
-    EXPECT_TRUE(image_data.get());
+    EXPECT_TRUE(image_data.get() && image_data.get()->size());
 
     scoped_refptr<BitmapImage> image = BitmapImage::Create();
     image->SetData(image_data, true);
@@ -60,15 +62,60 @@ TEST_F(HighContrastImageClassifierTest, FeaturesAndClassification) {
   std::vector<float> features;
 
   // Test Case 1:
+  // Grayscale
+  // Color Buckets Ratio: Low
+  // Decision Tree: Apply
+  // Neural Network: NA
+  EXPECT_TRUE(GetFeaturesAndClassification(
+      "/LayoutTests/images/resources/grid-large.png", &features));
+  EXPECT_EQ(classifier()->ClassifyImageUsingDecisionTreeForTesting(features),
+            HighContrastClassification::kApplyHighContrastFilter);
+  AssertFeaturesEqual(features, {0.0f, 0.1875f, 0.0f, 0.1f});
+
+  // Test Case 2:
+  // Grayscale
+  // Color Buckets Ratio: Medium
+  // Decision Tree: Can't Decide
+  // Neural Network: Apply
+  EXPECT_TRUE(GetFeaturesAndClassification(
+      "/LayoutTests/images/resources/apng08-ref.png", &features));
+  EXPECT_EQ(classifier()->ClassifyImageUsingDecisionTreeForTesting(features),
+            HighContrastClassification::kNotClassified);
+  AssertFeaturesEqual(features, {0.0f, 0.8125f, 0.409f, 0.59f});
+
+  // Test Case 3:
+  // Color
+  // Color Buckets Ratio: Low
+  // Decision Tree: Apply
+  // Neural Network: NA.
+  EXPECT_TRUE(GetFeaturesAndClassification(
+      "/LayoutTests/images/resources/count-down-color-test.png", &features));
+  EXPECT_EQ(classifier()->ClassifyImageUsingDecisionTreeForTesting(features),
+            HighContrastClassification::kApplyHighContrastFilter);
+  AssertFeaturesEqual(features, {1.0f, 0.0134277f, 0.0f, 0.43f});
+
+  // Test Case 4:
+  // Color
+  // Color Buckets Ratio: High
+  // Decision Tree: Do Not Apply
+  // Neural Network: NA.
   EXPECT_FALSE(GetFeaturesAndClassification(
       "/LayoutTests/images/resources/blue-wheel-srgb-color-profile.png",
       &features));
+  EXPECT_EQ(classifier()->ClassifyImageUsingDecisionTreeForTesting(features),
+            HighContrastClassification::kDoNotApplyHighContrastFilter);
   AssertFeaturesEqual(features, {1.0f, 0.03027f, 0.0f, 0.24f});
 
-  // Test Case 2:
+  // Test Case 5:
+  // Color
+  // Color Buckets Ratio: Medium
+  // Decision Tree: Can't Decide
+  // Neural Network: Apply.
   EXPECT_TRUE(GetFeaturesAndClassification(
-      "/LayoutTests/images/resources/grid-large.png", &features));
-  AssertFeaturesEqual(features, {0.0f, 0.1875f, 0.0f, 0.1f});
+      "/LayoutTests/images/resources/ycbcr-444-float.jpg", &features));
+  EXPECT_EQ(classifier()->ClassifyImageUsingDecisionTreeForTesting(features),
+            HighContrastClassification::kNotClassified);
+  AssertFeaturesEqual(features, {1.0f, 0.0166016f, 0.0f, 0.59f});
 }
 
 }  // namespace blink

third_party/WebKit/Source/platform/graphics/highcontrast/README.md

+# HighContrast Classifier
+
+These files are automatically generated using tfNative from a neural network
+trained by TensorFlow, to provide a classifier for High Contrast mode in cases
+that decision making is not possible based on color counts.
\ No newline at end of file

third_party/WebKit/Source/platform/graphics/highcontrast/highcontrast_classifier.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file is automatically generated using tfNative from a neural network,
+// trained by TensorFlow. Please do not edit.
+
+#include <cstdint>
+
+namespace highcontrast_tfnative_model {
+
+struct alignas(16) FixedAllocations {
+  float alloc0[10];
+  int32_t alloc0_shape[2];
+  float alloc1[10];
+  int32_t alloc1_shape[2];
+};
+
+extern int32_t input0Shape[2];
+extern int32_t logits_MatMul_merged_with_dnn_logits_BiasAdd0Shape[2];
+
+void Inference(
+    const float* __restrict input0 /* shape: 1,4 */,
+    float* __restrict logits_MatMul_merged_with_dnn_logits_BiasAdd0 /* shape:
+                                                                       1,1 */
+    ,
+    FixedAllocations* __restrict fixed);
+
+}  // namespace highcontrast_tfnative_model