Add memory pressure listener to Blob storage

This CL adds a memory pressure listener to blob storage memory
controller and evicts the blob items to disk on pressure. The evictions
are throttled to 30seconds to avoid thrashing disk. Records histograms
of sizes evicted with reasons.

BUG=715859

Review-Url: https://codereview.chromium.org/2857283002
Cr-Commit-Position: refs/heads/master@{#471189}

storage/browser/blob/blob_memory_controller.cc

@@ -16,6 +16,7 @@
 #include "base/guid.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
+#include "base/metrics/histogram_functions.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/numerics/safe_math.h"
@@ -41,6 +42,7 @@ namespace storage {
 namespace {
 constexpr int64_t kUnknownDiskAvailability = -1ll;
 constexpr uint64_t kMegabyte = 1024ull * 1024;
+const int64_t kMinSecondsForPressureEvictions = 30;
 
 using FileCreationInfo = BlobMemoryController::FileCreationInfo;
 using MemoryAllocation = BlobMemoryController::MemoryAllocation;
@@ -505,6 +507,9 @@ BlobMemoryController::BlobMemoryController(
       disk_space_function_(&base::SysInfo::AmountOfFreeDiskSpace),
       populated_memory_items_(
           base::MRUCache<uint64_t, ShareableBlobDataItem*>::NO_AUTO_EVICT),
+      memory_pressure_listener_(
+          base::Bind(&BlobMemoryController::OnMemoryPressure,
+                     base::Unretained(this))),
       weak_factory_(this) {}
 
 BlobMemoryController::~BlobMemoryController() {}
@@ -604,7 +609,8 @@ base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveMemoryQuota(
   if (total_bytes_needed <= GetAvailableMemoryForBlobs()) {
     GrantMemoryAllocations(&unreserved_memory_items,
                            static_cast<size_t>(total_bytes_needed));
-    MaybeScheduleEvictionUntilSystemHealthy();
+    MaybeScheduleEvictionUntilSystemHealthy(
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
     done_callback.Run(true);
     return base::WeakPtr<QuotaAllocationTask>();
   }
@@ -615,7 +621,8 @@ base::WeakPtr<QuotaAllocationTask> BlobMemoryController::ReserveMemoryQuota(
 
   auto weak_ptr = AppendMemoryTask(
       total_bytes_needed, std::move(unreserved_memory_items), done_callback);
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
   return weak_ptr;
 }
 
@@ -649,7 +656,8 @@ void BlobMemoryController::NotifyMemoryItemsUsed(
       populated_memory_items_.Put(item->item_id(), item.get());
     }
   }
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
 }
 
 void BlobMemoryController::CalculateBlobStorageLimits() {
@@ -771,11 +779,12 @@ void BlobMemoryController::MaybeGrantPendingMemoryRequests() {
 }
 
 size_t BlobMemoryController::CollectItemsForEviction(
-    std::vector<scoped_refptr<ShareableBlobDataItem>>* output) {
+    std::vector<scoped_refptr<ShareableBlobDataItem>>* output,
+    uint64_t min_page_file_size) {
   base::CheckedNumeric<size_t> total_items_size = 0;
   // Process the recent item list and remove items until we have at least a
   // minimum file size or we're at the end of our items to page to disk.
-  while (total_items_size.ValueOrDie() < limits_.min_page_file_size &&
+  while (total_items_size.ValueOrDie() < min_page_file_size &&
          !populated_memory_items_.empty()) {
     auto iterator = --populated_memory_items_.end();
     ShareableBlobDataItem* item = iterator->second;
@@ -789,7 +798,8 @@ size_t BlobMemoryController::CollectItemsForEviction(
   return total_items_size.ValueOrDie();
 }
 
-void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
+void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
   // Don't do eviction when others are happening, as we don't change our
   // pending_memory_quota_total_size_ value until after the paging files have
   // been written.
@@ -800,20 +810,42 @@ void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
       static_cast<uint64_t>(pending_memory_quota_total_size_) +
       blob_memory_used_;
 
+  size_t in_memory_limit = limits_.memory_limit_before_paging();
+  uint64_t min_page_file_size = limits_.min_page_file_size;
+  if (memory_pressure_level !=
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+    in_memory_limit = 0;
+    // Use lower page file size to reduce using more memory for writing under
+    // pressure.
+    min_page_file_size = limits_.max_blob_in_memory_space *
+                         limits_.max_blob_in_memory_space_under_pressure_ratio;
+  }
+
   // We try to page items to disk until our current system size + requested
   // memory is below our size limit.
   // Size limit is a lower |memory_limit_before_paging()| if we have disk space.
   while (total_memory_usage > limits_.effective_max_disk_space ||
          (disk_used_ < limits_.effective_max_disk_space &&
-          total_memory_usage > limits_.memory_limit_before_paging())) {
+          total_memory_usage > in_memory_limit)) {
+    const char* reason = nullptr;
+    if (memory_pressure_level !=
+        base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
+      reason = "OnMemoryPressure";
+    } else if (total_memory_usage > limits_.effective_max_disk_space) {
+      reason = "SizeExceededMaxDiskSpace";
+    } else {
+      reason = "SizeExceededInMemoryLimit";
+    }
+
     // We only page when we have enough items to fill a whole page file.
-    if (populated_memory_items_bytes_ < limits_.min_page_file_size)
+    if (populated_memory_items_bytes_ < min_page_file_size)
       break;
-    DCHECK_LE(limits_.min_page_file_size,
-              static_cast<uint64_t>(blob_memory_used_));
+    DCHECK_LE(min_page_file_size, static_cast<uint64_t>(blob_memory_used_));
 
     std::vector<scoped_refptr<ShareableBlobDataItem>> items_to_swap;
-    size_t total_items_size = CollectItemsForEviction(&items_to_swap);
+
+    size_t total_items_size =
+        CollectItemsForEviction(&items_to_swap, min_page_file_size);
     if (total_items_size == 0)
       break;
 
@@ -849,7 +881,10 @@ void BlobMemoryController::MaybeScheduleEvictionUntilSystemHealthy() {
                    total_items_size),
         base::Bind(&BlobMemoryController::OnEvictionComplete,
                    weak_factory_.GetWeakPtr(), base::Passed(&file_reference),
-                   base::Passed(&items_to_swap), total_items_size));
+                   base::Passed(&items_to_swap), total_items_size, reason,
+                   total_memory_usage));
+
+    last_eviction_time_ = base::TimeTicks::Now();
   }
   RecordTracingCounters();
 }
@@ -858,6 +893,8 @@ void BlobMemoryController::OnEvictionComplete(
     scoped_refptr<ShareableFileReference> file_reference,
     std::vector<scoped_refptr<ShareableBlobDataItem>> items,
     size_t total_items_size,
+    const char* evict_reason,
+    size_t memory_usage_before_eviction,
     std::pair<FileCreationInfo, int64_t /* avail_disk */> result) {
   if (!file_paging_enabled_)
     return;
@@ -893,12 +930,32 @@ void BlobMemoryController::OnEvictionComplete(
   }
   in_flight_memory_used_ -= total_items_size;
 
+  // Record change in memory usage at the last eviction reply.
+  size_t total_usage = blob_memory_used_ + pending_memory_quota_total_size_;
+  if (!pending_evictions_ && memory_usage_before_eviction >= total_usage) {
+    std::string full_histogram_name =
+        std::string("Storage.Blob.SizeEvictedToDiskInKB.") + evict_reason;
+    base::UmaHistogramCounts100000(
+        full_histogram_name,
+        (memory_usage_before_eviction - total_usage) / 1024);
+  }
+
   // We want callback on blobs up to the amount we've freed.
   MaybeGrantPendingMemoryRequests();
 
   // If we still have more blobs waiting and we're not waiting on more paging
   // operations, schedule more.
-  MaybeScheduleEvictionUntilSystemHealthy();
+  MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE);
+}
+
+void BlobMemoryController::OnMemoryPressure(
+    base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level) {
+  auto time_from_last_evicion = base::TimeTicks::Now() - last_eviction_time_;
+  if (time_from_last_evicion.InSeconds() < kMinSecondsForPressureEvictions)
+    return;
+
+  MaybeScheduleEvictionUntilSystemHealthy(memory_pressure_level);
 }
 
 FilePath BlobMemoryController::GenerateNextPageFileName() {

storage/browser/blob/blob_memory_controller.h

@@ -21,7 +21,9 @@
 #include "base/containers/mru_cache.h"
 #include "base/files/file.h"
 #include "base/files/file_path.h"
+#include "base/gtest_prod_util.h"
 #include "base/macros.h"
+#include "base/memory/memory_pressure_listener.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/optional.h"
@@ -178,6 +180,7 @@ class STORAGE_EXPORT BlobMemoryController {
   class FileQuotaAllocationTask;
   class MemoryQuotaAllocationTask;
 
+  FRIEND_TEST_ALL_PREFIXES(BlobMemoryControllerTest, OnMemoryPressure);
   // So this (and only this) class can call CalculateBlobStorageLimits().
   friend class content::ChromeBlobStorageContext;
 
@@ -205,10 +208,12 @@ class STORAGE_EXPORT BlobMemoryController {
   void MaybeGrantPendingMemoryRequests();
 
   size_t CollectItemsForEviction(
-      std::vector<scoped_refptr<ShareableBlobDataItem>>* output);
+      std::vector<scoped_refptr<ShareableBlobDataItem>>* output,
+      uint64_t min_page_file_size);
 
   // Schedule paging until our memory usage is below our memory limit.
-  void MaybeScheduleEvictionUntilSystemHealthy();
+  void MaybeScheduleEvictionUntilSystemHealthy(
+      base::MemoryPressureListener::MemoryPressureLevel level);
 
   // Called when we've completed evicting a list of items to disk. This is where
   // we swap the bytes items for file items, and update our bookkeeping.
@@ -216,7 +221,12 @@ class STORAGE_EXPORT BlobMemoryController {
       scoped_refptr<ShareableFileReference> file_reference,
       std::vector<scoped_refptr<ShareableBlobDataItem>> items,
       size_t total_items_size,
-      std::pair<FileCreationInfo, int64_t /* disk_avail */> result);
+      const char* evict_reason,
+      size_t memory_usage_before_eviction,
+      std::pair<FileCreationInfo, int64_t /* avail_disk */> result);
+
+  void OnMemoryPressure(
+      base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level);
 
   size_t GetAvailableMemoryForBlobs() const;
   uint64_t GetAvailableFileSpaceForBlobs() const;
@@ -265,6 +275,7 @@ class STORAGE_EXPORT BlobMemoryController {
   scoped_refptr<base::TaskRunner> file_runner_;
   // This defaults to calling base::SysInfo::AmountOfFreeDiskSpace.
   DiskSpaceFuncPtr disk_space_function_;
+  base::TimeTicks last_eviction_time_;
 
   // Lifetime of the ShareableBlobDataItem objects is handled externally in the
   // BlobStorageContext class.
@@ -275,6 +286,8 @@ class STORAGE_EXPORT BlobMemoryController {
   // item to the recent_item_cache_ above.
   std::unordered_set<uint64_t> items_paging_to_file_;
 
+  base::MemoryPressureListener memory_pressure_listener_;
+
   base::WeakPtrFactory<BlobMemoryController> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(BlobMemoryController);

storage/browser/blob/blob_memory_controller_unittest.cc

@@ -31,6 +31,7 @@ const std::string kBlobStorageDirectory = "blob_storage";
 const size_t kTestBlobStorageIPCThresholdBytes = 20;
 const size_t kTestBlobStorageMaxSharedMemoryBytes = 50;
 const size_t kTestBlobStorageMaxBlobMemorySize = 500;
+const float kTestMaxBlobInMemorySpaceUnderPressureRatio = 0.004f;
 const uint64_t kTestBlobStorageMaxDiskSpace = 1000;
 const uint64_t kTestBlobStorageMinFileSizeBytes = 10;
 const uint64_t kTestBlobStorageMaxFileSizeBytes = 100;
@@ -88,6 +89,8 @@ class BlobMemoryControllerTest : public testing::Test {
     limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
     limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
     limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.max_blob_in_memory_space_under_pressure_ratio =
+        kTestMaxBlobInMemorySpaceUnderPressureRatio;
     limits.desired_max_disk_space = kTestBlobStorageMaxDiskSpace;
     limits.effective_max_disk_space = kTestBlobStorageMaxDiskSpace;
     limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
@@ -100,6 +103,8 @@ class BlobMemoryControllerTest : public testing::Test {
     limits.max_ipc_memory_size = kTestBlobStorageIPCThresholdBytes;
     limits.max_shared_memory_size = kTestBlobStorageMaxSharedMemoryBytes;
     limits.max_blob_in_memory_space = kTestBlobStorageMaxBlobMemorySize;
+    limits.max_blob_in_memory_space_under_pressure_ratio =
+        kTestMaxBlobInMemorySpaceUnderPressureRatio;
     limits.desired_max_disk_space = kTestSmallBlobStorageMaxDiskSpace;
     limits.effective_max_disk_space = kTestSmallBlobStorageMaxDiskSpace;
     limits.min_page_file_size = kTestBlobStorageMinFileSizeBytes;
@@ -1120,4 +1125,48 @@ TEST_F(BlobMemoryControllerTest, DiskSpaceUnknown) {
   EXPECT_FALSE(controller.limits().IsDiskSpaceConstrained());
 }
 
+TEST_F(BlobMemoryControllerTest, OnMemoryPressure) {
+  BlobMemoryController controller(temp_dir_.GetPath(), file_runner_);
+  SetTestMemoryLimits(&controller);
+  AssertEnoughDiskSpace();
+
+  char kData[1];
+  kData[0] = 'e';
+
+  std::vector<scoped_refptr<ShareableBlobDataItem>> small_items;
+  size_t size_to_load = 2 * kTestBlobStorageMaxBlobMemorySize *
+                            kTestMaxBlobInMemorySpaceUnderPressureRatio +
+                        1;
+  for (size_t i = 0; i < size_to_load; i++) {
+    BlobDataBuilder builder("fake");
+    builder.AppendData(kData, 1);
+    std::vector<scoped_refptr<ShareableBlobDataItem>> items =
+        CreateSharedDataItems(builder);
+    base::WeakPtr<QuotaAllocationTask> memory_task =
+        controller.ReserveMemoryQuota(items, GetMemoryRequestCallback());
+    EXPECT_FALSE(memory_task);
+    items[0]->set_state(ItemState::POPULATED_WITH_QUOTA);
+    small_items.insert(small_items.end(), items.begin(), items.end());
+  }
+  controller.NotifyMemoryItemsUsed(small_items);
+  EXPECT_FALSE(file_runner_->HasPendingTask());
+  EXPECT_EQ(size_to_load, controller.memory_usage());
+
+  controller.OnMemoryPressure(
+      base::MemoryPressureListener::MemoryPressureLevel::
+          MEMORY_PRESSURE_LEVEL_CRITICAL);
+
+  EXPECT_TRUE(file_runner_->HasPendingTask());
+  RunFileThreadTasks();
+
+  base::RunLoop().RunUntilIdle();
+
+  // 2 page files of size |kTestBlobStorageMaxBlobMemorySize *
+  // kTestMaxBlobInMemorySpaceUnderPressureRatio| should be evicted with 1 byte
+  // left in-memory.
+  EXPECT_EQ(1u, controller.memory_usage());
+  EXPECT_EQ(size_to_load - 1, controller.disk_usage());
+  return;
+}
+
 }  // namespace storage

storage/common/blob_storage/blob_storage_constants.h

@@ -23,8 +23,10 @@ constexpr uint64_t kDefaultMaxPageFileSize = 100ull * 1024 * 1024;
 #if defined(OS_ANDROID)
 // On minimal Android maximum in-memory space can be as low as 5MB.
 constexpr uint64_t kDefaultMinPageFileSize = 5ull * 1024 * 1024 / 2;
+const float kDefaultMaxBlobInMemorySpaceUnderPressureRatio = 0.02f;
 #else
 constexpr uint64_t kDefaultMinPageFileSize = 5ull * 1024 * 1024;
+const float kDefaultMaxBlobInMemorySpaceUnderPressureRatio = 0.002f;
 #endif
 
 // All sizes are in bytes.
@@ -52,6 +54,12 @@ struct STORAGE_COMMON_EXPORT BlobStorageLimits {
 
   // This is the maximum amount of memory we can use to store blobs.
   size_t max_blob_in_memory_space = kDefaultMaxBlobInMemorySpace;
+  // The ratio applied to |max_blob_in_memory_space| to reduce memory usage
+  // under memory pressure. Note: Under pressure we modify the
+  // |min_page_file_size| to ensure we can evict items until we get below the
+  // reduced memory limit.
+  float max_blob_in_memory_space_under_pressure_ratio =
+      kDefaultMaxBlobInMemorySpaceUnderPressureRatio;
 
   // This is the maximum amount of disk space we can use.
   uint64_t desired_max_disk_space = kDefaultMaxBlobDiskSpace;

tools/metrics/histograms/histograms.xml

@@ -71425,6 +71425,11 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries.
   </summary>
 </histogram>
 
+<histogram base="true" name="Storage.Blob.SizeEvictedToDiskInKB" units="KB">
+  <owner>ssid@chromium.org</owner>
+  <summary>Records the total size of in-memory blobs evicted to disk.</summary>
+</histogram>
+
 <histogram name="Storage.Blob.StorageSizeAfterAppend" units="KB">
   <owner>dmurph@chromium.org</owner>
   <summary>
@@ -91903,6 +91908,18 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries.
   <affected-histogram name="Storage.BlobItemSize.BlobSlice"/>
 </histogram_suffixes>
 
+<histogram_suffixes name="Storage.BlobEvictReason" separator=".">
+  <suffix name="OnMemoryPressure"
+      label="Eviction was triggered because of memory pressure signal."/>
+  <suffix name="SizeExceededMaxDiskSpace"
+      label="Eviction was triggered because the total size exceeded maximum
+             available disk space"/>
+  <suffix name="SizeExceededInMemoryLimit"
+      label="Eviction was triggered because the total size exceeded maximum
+             in memory size allwed"/>
+  <affected-histogram name="Storage.Blob.SizeEvictedToDiskInKB"/>
+</histogram_suffixes>
+
 <histogram_suffixes name="StunPingInternal" separator=".">
   <suffix name="0ms" label="0ms between requests"/>
   <suffix name="5ms" label="5ms between requests"/>