Introduce a request throttler for limiting requests in mobile NTP.

This CL adds a generic throttler of requests that
 - checks new requests against daily quota (configurable by Finch),
 - reports to UMA status of each request w.r.t. the quota,
 - reports to UMA the total count of request per day.

As an example application, it adds throttler for NTPSnippetsFetcher. This throttler will be used in another CL.

BUG=627073

Review-Url: https://codereview.chromium.org/2158843002
Cr-Commit-Position: refs/heads/master@{#407124}

chrome/browser/prefs/browser_prefs.cc

@@ -79,6 +79,7 @@
 #include "components/metrics/metrics_service.h"
 #include "components/network_time/network_time_tracker.h"
 #include "components/ntp_snippets/ntp_snippets_service.h"
+#include "components/ntp_snippets/request_throttler.h"
 #include "components/omnibox/browser/zero_suggest_provider.h"
 #include "components/password_manager/core/browser/password_bubble_experiment.h"
 #include "components/password_manager/core/browser/password_manager.h"
@@ -461,6 +462,7 @@ void RegisterProfilePrefs(user_prefs::PrefRegistrySyncable* registry) {
   MediaStreamDevicesController::RegisterProfilePrefs(registry);
   NetPrefObserver::RegisterProfilePrefs(registry);
   ntp_snippets::NTPSnippetsService::RegisterProfilePrefs(registry);
+  ntp_snippets::RequestThrottler::RegisterProfilePrefs(registry);
   password_bubble_experiment::RegisterPrefs(registry);
   password_manager::PasswordManager::RegisterProfilePrefs(registry);
   PrefProxyConfigTrackerImpl::RegisterProfilePrefs(registry);

components/ntp_snippets.gypi

@@ -57,6 +57,8 @@
         'ntp_snippets/pref_names.cc',
         'ntp_snippets/pref_names.h',
         'ntp_snippets/proto/ntp_snippets.proto',
+        'ntp_snippets/request_throttler.cc',
+        'ntp_snippets/request_throttler.h',
         'ntp_snippets/switches.cc',
         'ntp_snippets/switches.h',
       ],

components/ntp_snippets/BUILD.gn

@@ -38,6 +38,8 @@ static_library("ntp_snippets") {
     "offline_pages/offline_page_suggestions_provider.h",
     "pref_names.cc",
     "pref_names.h",
+    "request_throttler.cc",
+    "request_throttler.h",
     "switches.cc",
     "switches.h",
   ]
@@ -87,6 +89,7 @@ source_set("unit_tests") {
     "ntp_snippets_status_service_unittest.cc",
     "ntp_snippets_test_utils.cc",
     "ntp_snippets_test_utils.h",
+    "request_throttler_unittest.cc",
   ]
 
   deps = [

components/ntp_snippets/pref_names.cc

@@ -11,5 +11,10 @@ const char kEnableSnippets[] = "ntp_snippets.enable";
 
 const char kSnippetHosts[] = "ntp_snippets.hosts";
 
+const char kSnippetFetcherQuotaDay[] =
+    "ntp.request_throttler.suggestion_fetcher.day";
+const char kSnippetFetcherQuotaCount[] =
+    "ntp.request_throttler.suggestion_fetcher.count";
+
 }  // namespace prefs
 }  // namespace ntp_snippets

components/ntp_snippets/pref_names.h

@@ -12,7 +12,13 @@ extern const char kEnableSnippets[];
 
 extern const char kSnippetHosts[];
 
+// The pref name for today's count of NTPSnippetsFetcher requests, so far.
+extern const char kSnippetFetcherQuotaCount[];
+// The pref name for the current day for the counter of NTPSnippetsFetcher
+// requests.
+extern const char kSnippetFetcherQuotaDay[];
+
 }  // namespace prefs
 }  // namespace ntp_snippets
 
-#endif
+#endif  // COMPONENTS_NTP_SNIPPETS_PREF_NAMES_H_

components/ntp_snippets/request_throttler.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/ntp_snippets/request_throttler.h"
+
+#include <vector>
+
+#include "base/metrics/histogram.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+#include "base/time/time.h"
+#include "components/ntp_snippets/ntp_snippets_constants.h"
+#include "components/ntp_snippets/pref_names.h"
+#include "components/prefs/pref_registry_simple.h"
+#include "components/prefs/pref_service.h"
+#include "components/variations/variations_associated_data.h"
+
+namespace ntp_snippets {
+
+// Used internally for working with a RequestType.
+struct RequestTypeInfo {
+  const char* name;
+  const char* count_pref;
+  const char* day_pref;
+};
+
+namespace {
+
+// Enumeration listing all possible outcomes for fetch attempts. Used for UMA
+// histogram, so do not change existing values. Insert new values at the end,
+// and update the histogram definition.
+enum class RequestStatus {
+  FORCED,
+  QUOTA_GRANTED,
+  QUOTA_EXCEEDED,
+  REQUEST_STATUS_COUNT
+};
+
+// When adding a new type here, extend also the "RequestCounterTypes"
+// <histogram_suffixes> in histograms.xml with the |name| string.
+const RequestTypeInfo kRequestTypeInfo[] = {
+    // RequestCounter::RequestType::CONTENT_SUGGESTION_FETCHER,
+    {"SuggestionFetcher", prefs::kSnippetFetcherQuotaCount,
+     prefs::kSnippetFetcherQuotaDay}};
+
+}  // namespace
+
+RequestThrottler::RequestThrottler(PrefService* pref_service,
+                               RequestType type,
+                               int default_quota)
+    : pref_service_(pref_service),
+      type_info_(kRequestTypeInfo[static_cast<int>(type)]) {
+  DCHECK(pref_service);
+
+  std::string quota = variations::GetVariationParamValue(
+      ntp_snippets::kStudyName,
+      base::StringPrintf("quota_%s", GetRequestTypeAsString()));
+  if (!base::StringToInt(quota, &quota_)) {
+    LOG_IF(WARNING, !quota.empty())
+        << "Invalid variation parameter for quota for "
+        << GetRequestTypeAsString();
+    quota_ = default_quota;
+  }
+
+  // Since the histogram names are dynamic, we cannot use the standard macros
+  // and we need to lookup the histograms, instead.
+  int status_count = static_cast<int>(RequestStatus::REQUEST_STATUS_COUNT);
+  // Corresponds to UMA_HISTOGRAM_ENUMERATION(name, sample, |status_count|).
+  histogram_request_status_ = base::LinearHistogram::FactoryGet(
+      base::StringPrintf("NewTabPage.RequestThrottler.RequestStatus_%s",
+                         GetRequestTypeAsString()),
+      1, status_count, status_count + 1,
+      base::HistogramBase::kUmaTargetedHistogramFlag);
+  // Corresponds to UMA_HISTOGRAM_COUNTS_100(name, sample).
+  histogram_per_day_ = base::Histogram::FactoryGet(
+      base::StringPrintf("NewTabPage.RequestThrottler.PerDay_%s",
+                         GetRequestTypeAsString()),
+      1, 100, 50, base::HistogramBase::kUmaTargetedHistogramFlag);
+}
+
+// static
+void RequestThrottler::RegisterProfilePrefs(PrefRegistrySimple* registry) {
+  for (const RequestTypeInfo& info : kRequestTypeInfo) {
+    registry->RegisterIntegerPref(info.count_pref, 0);
+    registry->RegisterIntegerPref(info.day_pref, 0);
+  }
+}
+
+bool RequestThrottler::DemandQuotaForRequest(bool forced_request) {
+  ResetCounterIfDayChanged();
+
+  if (forced_request) {
+    histogram_request_status_->Add(static_cast<int>(RequestStatus::FORCED));
+    return true;
+  }
+
+  int new_count = GetCount() + 1;
+  SetCount(new_count);
+  bool available = (new_count <= quota_);
+
+  histogram_request_status_->Add(
+      static_cast<int>(available ? RequestStatus::QUOTA_GRANTED
+                                 : RequestStatus::QUOTA_EXCEEDED));
+  return available;
+}
+
+void RequestThrottler::ResetCounterIfDayChanged() {
+  // The count of days since a fixed reference (the Unix epoch).
+  int now_day = (base::Time::Now() - base::Time::UnixEpoch()).InDays();
+
+  if (!HasDay()) {
+    // The counter is used for the first time in this profile.
+    SetDay(now_day);
+  } else if (now_day != GetDay()) {
+    // Day has changed - report the number of requests from the previous day.
+    histogram_per_day_->Add(GetCount());
+    // Reset the counter.
+    SetCount(0);
+    SetDay(now_day);
+  }
+}
+
+const char* RequestThrottler::GetRequestTypeAsString() const {
+  return type_info_.name;
+}
+
+int RequestThrottler::GetCount() const {
+  return pref_service_->GetInteger(type_info_.count_pref);
+}
+
+void RequestThrottler::SetCount(int count) {
+  pref_service_->SetInteger(type_info_.count_pref, count);
+}
+
+int RequestThrottler::GetDay() const {
+  return pref_service_->GetInteger(type_info_.day_pref);
+}
+
+void RequestThrottler::SetDay(int day) {
+  pref_service_->SetInteger(type_info_.day_pref, day);
+}
+
+bool RequestThrottler::HasDay() const {
+  return pref_service_->HasPrefPath(type_info_.day_pref);
+}
+
+}  // namespace ntp_snippets

components/ntp_snippets/request_throttler.h

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_NTP_SNIPPETS_REQUEST_THROTTLER_H_
+#define COMPONENTS_NTP_SNIPPETS_REQUEST_THROTTLER_H_
+
+#include <string>
+
+#include "base/macros.h"
+
+class PrefRegistrySimple;
+class PrefService;
+
+namespace base {
+class HistogramBase;
+}  // namespace base
+
+namespace ntp_snippets {
+
+struct RequestTypeInfo;
+
+// Counts requests to external services, compares them to a daily quota, reports
+// them to UMA. In the application code, create one local instance for each type
+// of requests, identified by the RequestType. The request counter is based on:
+//  - daily quota from a variation param "quota_|type|" in the NTPSnippets trial
+//  - pref "ntp.request_throttler.|type|.count" to store the current counter,
+//  - pref "ntp.request_throttler.|type|.day" to store current day to which the
+//    current counter value applies.
+// Furthermore the counter reports to histograms:
+//  - "NewTabPage.RequestThrottler.RequestStatus_|type|" - status of each
+//  request;
+//  - "NewTabPage.RequestThrottler.PerDay_|type|" - the daily count of requests.
+//
+// Implementation notes: When extending this class for a new RequestType, please
+//  1) define in request_counter.cc in kRequestTypeInfo
+//     a) the string value for your |type| and
+//     b) constants for day/count prefs;
+//  2) define a new RequestThrottlerTypes histogram suffix in histogram.xml
+//     (with the same string value as in 1a)).
+class RequestThrottler {
+ public:
+  // Enumeration listing all current applications of the request counter.
+  enum class RequestType {
+    CONTENT_SUGGESTION_FETCHER
+  };
+
+  RequestThrottler(PrefService* pref_service,
+                 RequestType type,
+                 int default_quota);
+
+  // Registers profile prefs for all RequestTypes. Called from browser_prefs.cc.
+  static void RegisterProfilePrefs(PrefRegistrySimple* registry);
+
+  // Returns whether quota is available for another request and reports this
+  // information to UMA. Forced requests always return true -- should be only
+  // used for requests initiated by the user (if it is safe to assume that all
+  // users cannot generate an amount of requests we cannot handle).
+  bool DemandQuotaForRequest(bool force_request);
+
+ private:
+  // Also emits the PerDay histogram if the day changed.
+  void ResetCounterIfDayChanged();
+
+  const char* GetRequestTypeAsString() const;
+
+  int GetCount() const;
+  void SetCount(int count);
+  int GetDay() const;
+  void SetDay(int day);
+  bool HasDay() const;
+
+  PrefService* pref_service_;
+  const RequestTypeInfo& type_info_;
+
+  // It comes from a variation parameter or |default_quota| as a fallback.
+  int quota_;
+
+  // The histograms for reporting the requests of the given |type_|.
+  base::HistogramBase* histogram_request_status_;
+  base::HistogramBase* histogram_per_day_;
+
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottler);
+};
+
+}  // namespace ntp_snippets
+
+#endif  // COMPONENTS_NTP_SNIPPETS_REQUEST_THROTTLER_H_

components/ntp_snippets/request_throttler_unittest.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/ntp_snippets/request_throttler.h"
+
+#include <memory>
+
+#include "base/strings/stringprintf.h"
+#include "components/ntp_snippets/pref_names.h"
+#include "components/prefs/pref_registry_simple.h"
+#include "components/prefs/testing_pref_service.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace {
+const int kCounterQuota = 2;
+}  // namespace
+
+namespace ntp_snippets {
+
+class RequestThrottlerTest : public testing::Test {
+ public:
+  RequestThrottlerTest() {
+    RequestThrottler::RegisterProfilePrefs(test_prefs_.registry());
+    // Use any arbitrary RequestType for this unittest.
+    throttler_.reset(new RequestThrottler(
+        &test_prefs_, RequestThrottler::RequestType::CONTENT_SUGGESTION_FETCHER,
+        kCounterQuota));
+  }
+
+ protected:
+  TestingPrefServiceSimple test_prefs_;
+  std::unique_ptr<RequestThrottler> throttler_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottlerTest);
+};
+
+TEST_F(RequestThrottlerTest, QuotaExceeded) {
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_FALSE(throttler_->DemandQuotaForRequest(false));
+}
+
+TEST_F(RequestThrottlerTest, ForcedDoesNotCountInQuota) {
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(true));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+}
+
+TEST_F(RequestThrottlerTest, ForcedWorksRegardlessOfQuota) {
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(true));
+}
+
+TEST_F(RequestThrottlerTest, QuotaIsPerDay) {
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+
+  // Now fake the day pref so that the counter believes the count comes from
+  // yesterday.
+  int now_day = (base::Time::Now() - base::Time::UnixEpoch()).InDays();
+  test_prefs_.SetInteger(ntp_snippets::prefs::kSnippetFetcherQuotaDay,
+                         now_day - 1);
+
+  // The quota should get reset as the day has changed.
+  EXPECT_TRUE(throttler_->DemandQuotaForRequest(false));
+}
+
+}  // namespace ntp_snippets

tools/metrics/histograms/histograms.xml

@@ -35051,6 +35051,28 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries.
   </summary>
 </histogram>
 
+<histogram name="NewTabPage.RequestThrottler.PerDay" units="requests">
+  <owner>jkrcal@chromium.org</owner>
+  <summary>
+    Records how many requests of the given type the browser tried to perform
+    each day where the type is specified by the _suffix of the histogram. The
+    histogram is emitted only after midnight of the given day passes - right
+    before the first following request (which can be several days later if the
+    user does not use Chrome in the meantime). The histogram counts requests
+    with both QUOTA_GRANTED and QUOTA_EXCEEDED status, i.e. the count can easily
+    exceed the daily quota. Forced requests are not counted here.
+  </summary>
+</histogram>
+
+<histogram name="NewTabPage.RequestThrottler.RequestStatus"
+    enum="NtpRequestThrottlerStatus">
+  <owner>jkrcal@chromium.org</owner>
+  <summary>
+    Records the status w.r.t. the quota for all requests of the given type. The
+    type of request is specified by the _suffix of the histogram.
+  </summary>
+</histogram>
+
 <histogram name="NewTabPage.SearchURLs.Total">
   <obsolete>
     Deprecated 2016-02 (and not recorded for some time before that).
@@ -85055,6 +85077,12 @@ To add a new entry, add it with any value and run test to compute valid value.
   <int value="2" label="NTP Promo link clicked"/>
 </enum>
 
+<enum name="NtpRequestThrottlerStatus" type="int">
+  <int value="0" label="Forced request"/>
+  <int value="1" label="Quota granted"/>
+  <int value="2" label="Quota exceeded"/>
+</enum>
+
 <enum name="NtpSnippetsFetchResult" type="int">
   <int value="0" label="Success"/>
   <int value="1" label="Empty hosts"/>
@@ -100219,6 +100247,13 @@ To add a new entry, add it with any value and run test to compute valid value.
   <affected-histogram name="Media.EME.RequestMediaKeySystemAccess"/>
 </histogram_suffixes>
 
+<histogram_suffixes name="RequestThrottlerTypes">
+  <suffix name="SuggestionFetcher"
+      label="Fetcher for content suggestions on mobile NTP"/>
+  <affected-histogram name="NewTabPage.RequestThrottler.PerDay"/>
+  <affected-histogram name="NewTabPage.RequestThrottler.RequestStatus"/>
+</histogram_suffixes>
+
 <histogram_suffixes name="ResourceLoaderSizeSlice" separator=".">
   <suffix name="LT_2kB" label="Sliced for resources smaller than 2kB."/>
   <suffix name="LT_32kB"