Implement throttling logic for fetching affiliation information.

BUG=437865

Review URL: https://codereview.chromium.org/807503002

Cr-Commit-Position: refs/heads/master@{#314127}

base/test/test_mock_time_task_runner.cc

@@ -102,6 +102,11 @@ bool TestMockTimeTaskRunner::HasPendingTask() const {
   return !tasks_.empty();
 }
 
+size_t TestMockTimeTaskRunner::GetPendingTaskCount() const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  return tasks_.size();
+}
+
 TimeDelta TestMockTimeTaskRunner::NextPendingTaskDelay() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return tasks_.empty() ? TimeDelta::Max() : tasks_.top().GetTimeToRun() - now_;

base/test/test_mock_time_task_runner.h

@@ -60,6 +60,7 @@ class TestMockTimeTaskRunner : public base::SingleThreadTaskRunner {
   scoped_ptr<TickClock> GetMockTickClock() const;
 
   bool HasPendingTask() const;
+  size_t GetPendingTaskCount() const;
   TimeDelta NextPendingTaskDelay() const;
 
   // SingleThreadTaskRunner:

components/components_tests.gyp

@@ -192,6 +192,7 @@
             'packed_ct_ev_whitelist/bit_stream_reader_unittest.cc',
             'packed_ct_ev_whitelist/packed_ct_ev_whitelist_unittest.cc',
             'password_manager/core/browser/affiliation_database_unittest.cc',
+            'password_manager/core/browser/affiliation_fetch_throttler_unittest.cc',
             'password_manager/core/browser/affiliation_fetcher_unittest.cc',
             'password_manager/core/browser/affiliation_utils_unittest.cc',
             'password_manager/core/browser/browser_save_password_progress_logger_unittest.cc',

components/password_manager.gypi

@@ -32,6 +32,9 @@
         'password_manager/core/browser/affiliation_fetcher_delegate.h',
         'password_manager/core/browser/affiliation_database.cc',
         'password_manager/core/browser/affiliation_database.h',
+        'password_manager/core/browser/affiliation_fetch_throttler.cc',
+        'password_manager/core/browser/affiliation_fetch_throttler.h',
+        'password_manager/core/browser/affiliation_fetch_throttler_delegate.h',
         'password_manager/core/browser/affiliation_fetcher.cc',
         'password_manager/core/browser/affiliation_fetcher.h',
         'password_manager/core/browser/affiliation_service.cc',

components/password_manager/core/browser/BUILD.gn

@@ -21,6 +21,9 @@ static_library("browser") {
     "affiliation_backend.h",
     "affiliation_database.cc",
     "affiliation_database.h",
+    "affiliation_fetch_throttler.cc",
+    "affiliation_fetch_throttler.h",
+    "affiliation_fetch_throttler_delegate.h",
     "affiliation_fetcher_delegate.h",
     "affiliation_fetcher.cc",
     "affiliation_fetcher.h",

components/password_manager/core/browser/affiliation_fetch_throttler.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/password_manager/core/browser/affiliation_fetch_throttler.h"
+
+#include <stdint.h>
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/thread_task_runner_handle.h"
+#include "base/time/tick_clock.h"
+#include "base/time/time.h"
+#include "components/password_manager/core/browser/affiliation_fetch_throttler_delegate.h"
+
+namespace password_manager {
+
+namespace {
+
+// Implementation of net::BackoffEntry that allows mocking its tick source.
+class BackoffEntryImpl : public net::BackoffEntry {
+ public:
+  // |tick_clock| must outlive this instance.
+  explicit BackoffEntryImpl(const net::BackoffEntry::Policy* const policy,
+                            base::TickClock* tick_clock)
+      : BackoffEntry(policy), tick_clock_(tick_clock) {}
+  ~BackoffEntryImpl() override {}
+
+ private:
+  // net::BackoffEntry:
+  base::TimeTicks ImplGetTimeNow() const override {
+    return tick_clock_->NowTicks();
+  }
+
+  base::TickClock* tick_clock_;
+
+  DISALLOW_COPY_AND_ASSIGN(BackoffEntryImpl);
+};
+
+}  // namespace
+
+// static
+const net::BackoffEntry::Policy AffiliationFetchThrottler::kBackoffPolicy = {
+    // Number of initial errors (in sequence) to ignore before going into
+    // exponential backoff.
+    0,
+
+    // Initial delay (in ms) once backoff starts.
+    10 * 1000,  // 10 seconds
+
+    // Factor by which the delay will be multiplied on each subsequent failure.
+    4,
+
+    // Fuzzing percentage: 50% will spread delays randomly between 50%--100% of
+    // the nominal time.
+    .5,  // 50%
+
+    // Maximum delay (in ms) during exponential backoff.
+    6 * 3600 * 1000,  // 6 hours
+
+    // Time to keep an entry from being discarded even when it has no
+    // significant state, -1 to never discard. (Not applicable.)
+    -1,
+
+    // False means that initial_delay_ms is the first delay once we start
+    // exponential backoff, i.e., there is no delay after subsequent successful
+    // requests.
+    false,
+};
+
+// static
+const int64_t AffiliationFetchThrottler::kGracePeriodAfterReconnectMs =
+    10 * 1000;  // 10 seconds
+
+AffiliationFetchThrottler::AffiliationFetchThrottler(
+    AffiliationFetchThrottlerDelegate* delegate,
+    const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
+    scoped_ptr<base::TickClock> tick_clock)
+    : delegate_(delegate),
+      task_runner_(task_runner),
+      state_(IDLE),
+      has_network_connectivity_(false),
+      is_fetch_scheduled_(false),
+      tick_clock_(tick_clock.Pass()),
+      exponential_backoff_(
+          new BackoffEntryImpl(&kBackoffPolicy, tick_clock_.get())),
+      weak_ptr_factory_(this) {
+  DCHECK(delegate);
+  // Start observing before querying the current connectivity state, so that if
+  // the state changes concurrently in-between, it will not go unnoticed.
+  net::NetworkChangeNotifier::AddConnectionTypeObserver(this);
+  has_network_connectivity_ = !net::NetworkChangeNotifier::IsOffline();
+}
+
+AffiliationFetchThrottler::~AffiliationFetchThrottler() {
+  net::NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
+}
+
+void AffiliationFetchThrottler::SignalNetworkRequestNeeded() {
+  if (state_ != IDLE)
+    return;
+
+  state_ = FETCH_NEEDED;
+  if (has_network_connectivity_)
+    EnsureCallbackIsScheduled();
+}
+
+void AffiliationFetchThrottler::InformOfNetworkRequestComplete(bool success) {
+  DCHECK_EQ(state_, FETCH_IN_FLIGHT);
+  state_ = IDLE;
+  exponential_backoff_->InformOfRequest(success);
+}
+
+void AffiliationFetchThrottler::EnsureCallbackIsScheduled() {
+  DCHECK_EQ(state_, FETCH_NEEDED);
+  DCHECK(has_network_connectivity_);
+
+  if (is_fetch_scheduled_)
+    return;
+
+  is_fetch_scheduled_ = true;
+  task_runner_->PostDelayedTask(
+      FROM_HERE,
+      base::Bind(&AffiliationFetchThrottler::OnBackoffDelayExpiredCallback,
+                 weak_ptr_factory_.GetWeakPtr()),
+      exponential_backoff_->GetTimeUntilRelease());
+}
+
+void AffiliationFetchThrottler::OnBackoffDelayExpiredCallback() {
+  DCHECK_EQ(state_, FETCH_NEEDED);
+  DCHECK(is_fetch_scheduled_);
+  is_fetch_scheduled_ = false;
+
+  // Do nothing if network connectivity was lost while this callback was in the
+  // task queue. The callback will be posted in the OnConnectionTypeChanged
+  // handler once again.
+  if (!has_network_connectivity_)
+    return;
+
+  // The release time might have been increased if network connectivity was lost
+  // and restored while this callback was in the task queue. If so, reschedule.
+  if (exponential_backoff_->ShouldRejectRequest())
+    EnsureCallbackIsScheduled();
+  else
+    state_ = delegate_->OnCanSendNetworkRequest() ? FETCH_IN_FLIGHT : IDLE;
+}
+
+void AffiliationFetchThrottler::OnConnectionTypeChanged(
+    net::NetworkChangeNotifier::ConnectionType type) {
+  bool old_has_network_connectivity = has_network_connectivity_;
+  has_network_connectivity_ =
+      (type != net::NetworkChangeNotifier::CONNECTION_NONE);
+
+  // Only react when network connectivity has been reestablished.
+  if (!has_network_connectivity_ || old_has_network_connectivity)
+    return;
+
+  double grace_ms = kGracePeriodAfterReconnectMs *
+                    (1 - base::RandDouble() * kBackoffPolicy.jitter_factor);
+  exponential_backoff_->SetCustomReleaseTime(std::max(
+      exponential_backoff_->GetReleaseTime(),
+      tick_clock_->NowTicks() + base::TimeDelta::FromMillisecondsD(grace_ms)));
+
+  if (state_ == FETCH_NEEDED)
+    EnsureCallbackIsScheduled();
+}
+
+}  // namespace password_manager

components/password_manager/core/browser/affiliation_fetch_throttler.h

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_H_
+#define COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_H_
+
+#include "base/gtest_prod_util.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/weak_ptr.h"
+#include "net/base/backoff_entry.h"
+#include "net/base/network_change_notifier.h"
+
+namespace base {
+class TickClock;
+class SingleThreadTaskRunner;
+}  // namespace base
+
+namespace password_manager {
+
+class AffiliationFetchThrottlerDelegate;
+
+// Implements the throttling logic that the AffiliationBackend will use when it
+// needs to issue requests over the network to fetch affiliation information.
+//
+// This class manages only the scheduling of the requests. It is up to the
+// consumer (the AffiliationBackend) to actually assemble and send the requests,
+// to report back about their success or failure, and to retry them if desired.
+// The process goes like this:
+//   1.) The consumer calls SignalNetworkRequestNeeded().
+//   2.) Once appropriate, OnCanSendNetworkRequest() is called on the delegate.
+//   3.) The consumer sends the request, and waits until it completes.
+//   4.) The consumer calls InformOfNetworkRequestComplete().
+// Note that only a single request at a time is supported.
+//
+// If the request fails in Step 3, the consumer should not automatically retry
+// it. Instead it should always proceed to Step 4, and then -- if retrying the
+// request is desired -- proceed immediately to Step 1. That is, it should act
+// as if another request was needed right away.
+//
+// Essentially, this class implements exponential backoff in case of network and
+// server errors with the additional constraint that no requests will be issued
+// in the first place while there is known to be no network connectivity. This
+// prevents the exponential backoff delay from growing huge during long offline
+// periods, so that requests will not be held back for too long after
+// connectivity is restored.
+class AffiliationFetchThrottler
+    : public net::NetworkChangeNotifier::ConnectionTypeObserver {
+ public:
+  // Creates an instance that will use |tick_clock| as its tick source, and will
+  // post to |task_runner| to call the |delegate|'s OnSendNetworkRequest(). The
+  // |delegate| should outlive the throttler.
+  AffiliationFetchThrottler(
+      AffiliationFetchThrottlerDelegate* delegate,
+      const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
+      scoped_ptr<base::TickClock> tick_clock);
+  ~AffiliationFetchThrottler() override;
+
+  // Signals to the throttling logic that a network request is needed, and that
+  // OnCanSendNetworkRequest() should be called as soon as the request can be
+  // sent. OnCanSendNetworkRequest() will always be called asynchronously.
+  //
+  // Calls to this method will be ignored when a request is already known to be
+  // needed or while a request is in flight. To signal that another request will
+  // be needed right away after the current one, call this method after calling
+  // InformOfNetworkRequestComplete().
+  void SignalNetworkRequestNeeded();
+
+  // Informs the back-off logic that the in-flight network request has been
+  // completed, either with |success| or not.
+  void InformOfNetworkRequestComplete(bool success);
+
+ private:
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest, FailedRequests);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           GracePeriodAfterConnectivityIsRestored);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           GracePeriodAfterConnectivityIsRestored2);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           GracePeriodAfterConnectivityIsRestored3);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           ConnectivityLostDuringBackoff);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           ConnectivityLostAndRestoredDuringBackoff);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest, FlakyConnectivity);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           ConnectivityLostDuringRequest);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           ConnectivityLostAndRestoredDuringRequest);
+  FRIEND_TEST_ALL_PREFIXES(AffiliationFetchThrottlerTest,
+                           ConnectivityLostAndRestoredDuringRequest2);
+
+  enum State { IDLE, FETCH_NEEDED, FETCH_IN_FLIGHT };
+
+  // Exponential backoff parameters in case of network and server errors
+  static const net::BackoffEntry::Policy kBackoffPolicy;
+
+  // Minimum delay before sending the first request once network connectivity is
+  // restored. The fuzzing factor in |kBackoffParameters.jitter_factor| applies.
+  static const int64_t kGracePeriodAfterReconnectMs;
+
+  // Ensures that OnBackoffDelayExpiredCallback() is scheduled to be called back
+  // once the |exponential_backoff_| delay expires.
+  void EnsureCallbackIsScheduled();
+
+  // Called back when the |exponential_backoff_| delay expires.
+  void OnBackoffDelayExpiredCallback();
+
+  // net::NetworkChangeNotifier::ConnectionTypeObserver:
+  void OnConnectionTypeChanged(
+      net::NetworkChangeNotifier::ConnectionType type) override;
+
+  AffiliationFetchThrottlerDelegate* delegate_;
+  scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
+  State state_;
+  bool has_network_connectivity_;
+  bool is_fetch_scheduled_;
+  scoped_ptr<base::TickClock> tick_clock_;
+  scoped_ptr<net::BackoffEntry> exponential_backoff_;
+
+  base::WeakPtrFactory<AffiliationFetchThrottler> weak_ptr_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(AffiliationFetchThrottler);
+};
+
+}  // namespace password_manager
+
+#endif  // COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_H_

components/password_manager/core/browser/affiliation_fetch_throttler_delegate.h

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_DELEGATE_H_
+#define COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_DELEGATE_H_
+
+namespace password_manager {
+
+// An interface that users of AffiliationFetchThrottler need to implement to get
+// notified once it is okay to issue the next network request.
+class AffiliationFetchThrottlerDelegate {
+ public:
+  // Will be called once the throttling policy allows issuing a network request,
+  // provided SignalNetworkRequestNeeded() has been called at least once since
+  // the last request.
+  //
+  // The implementation must return true if a request was actually issued in
+  // response to this call, and then call InformOfNetworkRequestComplete() once
+  // the request is complete. Otherwise, the implementation must return false.
+  virtual bool OnCanSendNetworkRequest() = 0;
+
+ protected:
+  virtual ~AffiliationFetchThrottlerDelegate() {}
+};
+
+}  // namespace password_manager
+
+#endif  // COMPONENTS_PASSWORD_MANAGER_CORE_BROWSER_AFFILIATION_FETCH_THROTTLER_DELEGATE_H_

components/password_manager/core/browser/affiliation_fetch_throttler_unittest.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/password_manager/core/browser/affiliation_fetch_throttler.h"
+
+#include <cmath>
+
+#include "base/callback.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/message_loop/message_loop.h"
+#include "base/numerics/safe_math.h"
+#include "base/run_loop.h"
+#include "base/test/test_mock_time_task_runner.h"
+#include "base/thread_task_runner_handle.h"
+#include "base/time/tick_clock.h"
+#include "base/time/time.h"
+#include "components/password_manager/core/browser/affiliation_fetch_throttler_delegate.h"
+#include "net/base/network_change_notifier.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace password_manager {
+namespace {
+
+class MockAffiliationFetchThrottlerDelegate
+    : public AffiliationFetchThrottlerDelegate {
+ public:
+  explicit MockAffiliationFetchThrottlerDelegate(
+      scoped_ptr<base::TickClock> tick_clock)
+      : tick_clock_(tick_clock.Pass()),
+        emulated_return_value_(true),
+        can_send_count_(0u) {}
+  ~MockAffiliationFetchThrottlerDelegate() override {
+    EXPECT_EQ(0u, can_send_count_);
+  }
+
+  void set_emulated_return_value(bool value) { emulated_return_value_ = value; }
+  void reset_can_send_count() { can_send_count_ = 0u; }
+  size_t can_send_count() const { return can_send_count_; }
+  base::TimeTicks last_can_send_time() const { return last_can_send_time_; }
+
+  // AffiliationFetchThrottlerDelegate:
+  bool OnCanSendNetworkRequest() override {
+    ++can_send_count_;
+    last_can_send_time_ = tick_clock_->NowTicks();
+    return emulated_return_value_;
+  }
+
+ private:
+  scoped_ptr<base::TickClock> tick_clock_;
+  bool emulated_return_value_;
+  size_t can_send_count_;
+  base::TimeTicks last_can_send_time_;
+
+  DISALLOW_COPY_AND_ASSIGN(MockAffiliationFetchThrottlerDelegate);
+};
+
+}  // namespace
+
+class AffiliationFetchThrottlerTest : public testing::Test {
+ public:
+  AffiliationFetchThrottlerTest()
+      : network_change_notifier_(net::NetworkChangeNotifier::CreateMock()),
+        task_runner_(new base::TestMockTimeTaskRunner),
+        mock_delegate_(task_runner_->GetMockTickClock()) {}
+  ~AffiliationFetchThrottlerTest() override {}
+
+  scoped_ptr<AffiliationFetchThrottler> CreateThrottler() {
+    return make_scoped_ptr(new AffiliationFetchThrottler(
+        &mock_delegate_, task_runner_, task_runner_->GetMockTickClock()));
+  }
+
+  void SimulateHasNetworkConnectivity(bool has_connectivity) {
+    net::NetworkChangeNotifier::NotifyObserversOfConnectionTypeChangeForTests(
+        has_connectivity ? net::NetworkChangeNotifier::CONNECTION_UNKNOWN
+                         : net::NetworkChangeNotifier::CONNECTION_NONE);
+    base::RunLoop().RunUntilIdle();
+  }
+
+  // Runs the task runner until no tasks remain, and asserts that by this time,
+  // OnCanSendNetworkRequest() will have been called exactly once, with a delay
+  // between |min_delay_ms| and |max_delay_ms|, modulo 0.5 ms to allow for
+  // floating point errors. When OnCanSendNetworkRequest() is called, the mock
+  // will return |emulated_return_value|. This value normally indicates whether
+  // or not a request was actually issued in response to the call.
+  void AssertReleaseInBetween(bool emulated_return_value,
+                              double min_delay_ms,
+                              double max_delay_ms) {
+    ASSERT_EQ(0u, mock_delegate_.can_send_count());
+    base::TimeTicks ticks_at_start = task_runner_->GetCurrentMockTime();
+    mock_delegate_.set_emulated_return_value(emulated_return_value);
+    task_runner_->FastForwardUntilNoTasksRemain();
+    ASSERT_EQ(1u, mock_delegate_.can_send_count());
+    base::TimeDelta delay =
+        mock_delegate_.last_can_send_time() - ticks_at_start;
+    EXPECT_LE(min_delay_ms - 1, delay.InMillisecondsF());
+    EXPECT_GE(max_delay_ms + 1, delay.InMillisecondsF());
+    mock_delegate_.reset_can_send_count();
+  }
+
+  // Runs the task runner for |secs| and asserts that OnCanSendNetworkRequest()
+  // will not have been called by the end of this period.
+  void AssertNoReleaseForSecs(int64_t secs) {
+    task_runner_->FastForwardBy(base::TimeDelta::FromSeconds(secs));
+    ASSERT_EQ(0u, mock_delegate_.can_send_count());
+  }
+
+  // Runs the task runner until no tasks remain, and asserts that
+  // OnCanSendNetworkRequest() will not have been called.
+  void AssertNoReleaseUntilNoTasksRemain() {
+    task_runner_->FastForwardUntilNoTasksRemain();
+    ASSERT_EQ(0u, mock_delegate_.can_send_count());
+  }
+
+  size_t GetPendingTaskCount() const {
+    return task_runner_->GetPendingTaskCount();
+  }
+
+ private:
+  // Needed because NetworkChangeNotifier uses ObserverList, which notifies
+  // observers on the MessageLoop that belongs to the thread from which they
+  // have registered.
+  base::MessageLoop message_loop_;
+  scoped_ptr<net::NetworkChangeNotifier> network_change_notifier_;
+  scoped_refptr<base::TestMockTimeTaskRunner> task_runner_;
+  MockAffiliationFetchThrottlerDelegate mock_delegate_;
+
+  DISALLOW_COPY_AND_ASSIGN(AffiliationFetchThrottlerTest);
+};
+
+TEST_F(AffiliationFetchThrottlerTest, SuccessfulRequests) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+
+  // Signal while request is in flight should be ignored.
+  throttler->SignalNetworkRequestNeeded();
+  AssertNoReleaseUntilNoTasksRemain();
+  throttler->InformOfNetworkRequestComplete(true);
+  AssertNoReleaseUntilNoTasksRemain();
+
+  // Duplicate the second signal 3 times: still only 1 callback should arrive.
+  throttler->SignalNetworkRequestNeeded();
+  throttler->SignalNetworkRequestNeeded();
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+}
+
+TEST_F(AffiliationFetchThrottlerTest, FailedRequests) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  // The request after the first failure should be delayed by |initial_delay_ms|
+  // spread out over Uniform(1 - |jitter_factor|, 1).
+  throttler->SignalNetworkRequestNeeded();
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kPolicy.initial_delay_ms * (1 - kPolicy.jitter_factor),
+      kPolicy.initial_delay_ms));
+  throttler->InformOfNetworkRequestComplete(true);
+
+  // After a successful request, the next one should be released immediately.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  // In general, the request after the n-th failure should be delayed by
+  //   |multiply_factor| ^ (n-1) * |initial_delay_ms|,
+  // spread out over Uniform(1 - |jitter_factor|, 1), up until
+  //   |maximum_backoff_ms|
+  // is reached.
+  for (int num_failures = 1; num_failures < 100; ++num_failures) {
+    throttler->SignalNetworkRequestNeeded();
+    double max_delay_ms = kPolicy.initial_delay_ms *
+                          pow(kPolicy.multiply_factor, num_failures - 1);
+    double min_delay_ms = max_delay_ms * (1 - kPolicy.jitter_factor);
+    if (max_delay_ms > kPolicy.maximum_backoff_ms)
+      max_delay_ms = kPolicy.maximum_backoff_ms;
+    if (min_delay_ms > kPolicy.maximum_backoff_ms)
+      min_delay_ms = kPolicy.maximum_backoff_ms;
+    ASSERT_NO_FATAL_FAILURE(
+        AssertReleaseInBetween(true, min_delay_ms, max_delay_ms));
+    throttler->InformOfNetworkRequestComplete(false);
+  }
+}
+
+TEST_F(AffiliationFetchThrottlerTest, OnCanSendNetworkRequestReturnsFalse) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  // A need for a network request is signaled, but as OnCanSendNetworkRequest()
+  // is called, the implementation returns false to indicate that the request
+  // will not be needed after all. InformOfNetworkRequestComplete() must not be
+  // called in this case.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(false, 0, 0));
+
+  // A subsequent signaling, however, should result in OnCanSendNetworkRequest()
+  // being called immediately.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+}
+
+TEST_F(AffiliationFetchThrottlerTest, GracePeriodAfterConnectivityIsRestored) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+  SimulateHasNetworkConnectivity(false);
+
+  // After connectivity is restored, the first request should be delayed by the
+  // grace period, spread out over Uniform(1 - |jitter_factor|, 1).
+  throttler->SignalNetworkRequestNeeded();
+  AssertNoReleaseUntilNoTasksRemain();
+
+  SimulateHasNetworkConnectivity(true);
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+  throttler->InformOfNetworkRequestComplete(true);
+
+  // The next request should not be delayed.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+}
+
+// Same as GracePeriodAfterConnectivityIsRestored, but the network comes back
+// just before SignalNetworkRequestNeeded() is called.
+TEST_F(AffiliationFetchThrottlerTest, GracePeriodAfterConnectivityIsRestored2) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+  SimulateHasNetworkConnectivity(false);
+
+  SimulateHasNetworkConnectivity(true);
+  throttler->SignalNetworkRequestNeeded();
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+  throttler->InformOfNetworkRequestComplete(true);
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+}
+
+TEST_F(AffiliationFetchThrottlerTest, ConnectivityLostDuringBackoff) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  throttler->SignalNetworkRequestNeeded();
+  SimulateHasNetworkConnectivity(false);
+
+  // Let the exponential backoff delay expire, and verify nothing happens.
+  AssertNoReleaseUntilNoTasksRemain();
+
+  // Verify that the request is, however, sent after the normal grace period
+  // once connectivity is restored.
+  SimulateHasNetworkConnectivity(true);
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+  throttler->InformOfNetworkRequestComplete(true);
+}
+
+TEST_F(AffiliationFetchThrottlerTest,
+       ConnectivityLostAndRestoredDuringBackoff) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  throttler->SignalNetworkRequestNeeded();
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kPolicy.initial_delay_ms * (1 - kPolicy.jitter_factor),
+      kPolicy.initial_delay_ms));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  SimulateHasNetworkConnectivity(false);
+  SimulateHasNetworkConnectivity(true);
+
+  // This test expects that the exponential backoff interval after the 2nd error
+  // is larger than the normal grace period after connectivity is restored.
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  EXPECT_PRED_FORMAT2(testing::DoubleLE, kGraceMs,
+                      kPolicy.initial_delay_ms * kPolicy.multiply_factor);
+
+  // The release should come after the longest of the two intervals expire.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kPolicy.initial_delay_ms * kPolicy.multiply_factor *
+                (1 - kPolicy.jitter_factor),
+      kPolicy.initial_delay_ms * kPolicy.multiply_factor));
+  throttler->InformOfNetworkRequestComplete(false);
+}
+
+TEST_F(AffiliationFetchThrottlerTest, FlakyConnectivity) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  // Run for a total of 5 grace periods and simulate connectivity being lost and
+  // restored every second. This verifies that a flaky connection will not flood
+  // the task queue with lots of of tasks and also that release will not happen
+  // while the connection is flaky even once the first grace period has expired.
+  throttler->SignalNetworkRequestNeeded();
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  int64_t five_grace_periods_secs =
+      kGraceMs * 5 / base::Time::kMillisecondsPerSecond;
+  for (int64_t t = 0; t < five_grace_periods_secs; ++t) {
+    SimulateHasNetworkConnectivity(false);
+    AssertNoReleaseForSecs(1);
+    SimulateHasNetworkConnectivity(true);
+    EXPECT_EQ(1u, GetPendingTaskCount());
+  }
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+}
+
+TEST_F(AffiliationFetchThrottlerTest, ConnectivityLostDuringRequest) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+
+  SimulateHasNetworkConnectivity(false);
+  AssertNoReleaseUntilNoTasksRemain();
+  throttler->InformOfNetworkRequestComplete(false);
+  AssertNoReleaseUntilNoTasksRemain();
+  throttler->SignalNetworkRequestNeeded();
+  AssertNoReleaseUntilNoTasksRemain();
+
+  SimulateHasNetworkConnectivity(true);
+
+  // Verify that the next request is released after the normal grace period.
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+  throttler->InformOfNetworkRequestComplete(true);
+}
+
+TEST_F(AffiliationFetchThrottlerTest,
+       ConnectivityLostAndRestoredDuringRequest) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+
+  SimulateHasNetworkConnectivity(false);
+  AssertNoReleaseUntilNoTasksRemain();
+  SimulateHasNetworkConnectivity(true);
+  AssertNoReleaseUntilNoTasksRemain();
+  throttler->InformOfNetworkRequestComplete(true);
+
+  // Even though the previous request succeeded, the next request should still
+  // be held back for the normal grace period after connection is restored.
+  throttler->SignalNetworkRequestNeeded();
+  const auto& kPolicy = AffiliationFetchThrottler::kBackoffPolicy;
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(
+      true, kGraceMs * (1 - kPolicy.jitter_factor), kGraceMs));
+  throttler->InformOfNetworkRequestComplete(true);
+}
+
+TEST_F(AffiliationFetchThrottlerTest,
+       ConnectivityLostAndRestoredDuringRequest2) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+
+  SimulateHasNetworkConnectivity(false);
+  AssertNoReleaseUntilNoTasksRemain();
+  SimulateHasNetworkConnectivity(true);
+
+  const int64_t& kGraceMs =
+      AffiliationFetchThrottler::kGracePeriodAfterReconnectMs;
+  AssertNoReleaseForSecs(kGraceMs / base::Time::kMillisecondsPerSecond);
+  throttler->InformOfNetworkRequestComplete(true);
+
+  // The next request should not be held back.
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+}
+
+TEST_F(AffiliationFetchThrottlerTest, InstanceDestroyedWhileInBackoff) {
+  scoped_ptr<AffiliationFetchThrottler> throttler(CreateThrottler());
+
+  throttler->SignalNetworkRequestNeeded();
+  ASSERT_NO_FATAL_FAILURE(AssertReleaseInBetween(true, 0, 0));
+  throttler->InformOfNetworkRequestComplete(false);
+
+  throttler->SignalNetworkRequestNeeded();
+  throttler.reset();
+  EXPECT_EQ(1u, GetPendingTaskCount());
+  AssertNoReleaseUntilNoTasksRemain();
+}
+
+}  // namespace password_manager