Add exponential backoff for managed Kerberos account addition

Add retry logic with exponential backoff to KerberosCredentialsManager
for addition of managed accounts. This will cover the cases when the
first attempts fail because network is not ready yet, for example.
Network instabilities are common on first user login because
OpenNetworkConfigurations often changes network config on ChromeOS.

Bug: 1049331
Change-Id: I74b91cfd6fda3fc923061050a97fcb791761815e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2039453
Commit-Queue: Felipe Andrade <fsandrade@chromium.org>
Reviewed-by: Maksim Ivanov <emaxx@chromium.org>
Cr-Commit-Position: refs/heads/master@{#746481}

chrome/browser/chromeos/kerberos/kerberos_credentials_manager.cc

@@ -56,6 +56,17 @@ constexpr char kDefaultKerberosConfig[] = R"([libdefaults]
   permitted_enctypes = aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96
   forwardable = true)";
 
+// Backoff policy used to control managed accounts addition retries.
+const net::BackoffEntry::Policy kBackoffPolicyForManagedAccounts = {
+    0,               // Number of initial errors to ignore without backoff.
+    1 * 1000,        // Initial delay for backoff in ms: 1 second.
+    2,               // Factor to multiply for exponential backoff.
+    0,               // Fuzzing percentage.
+    10 * 60 * 1000,  // Maximum time to delay requests in ms: 10 minutes.
+    -1,              // Don't discard entry even if unused.
+    false            // Don't use initial delay unless the last was an error.
+};
+
 // If |principal_name| is "UsEr@realm.com", sets |principal_name| to
 // "user@REALM.COM". Returns false if the given name has no @ or one of the
 // parts is empty.
@@ -93,6 +104,13 @@ bool Succeeded(kerberos::ErrorType error) {
   return error == kerberos::ERROR_NONE;
 }
 
+bool ShouldRetry(kerberos::ErrorType error) {
+  // The error types that should trigger a managed accounts addition retry.
+  return error == kerberos::ERROR_NETWORK_PROBLEM ||
+         error == kerberos::ERROR_CONTACTING_KDC_FAILED ||
+         error == kerberos::ERROR_IN_PROGRESS;
+}
+
 }  // namespace
 
 // Encapsulates the steps to add a Kerberos account. Overview of the flow:
@@ -278,7 +296,8 @@ KerberosCredentialsManager::KerberosCredentialsManager(PrefService* local_state,
       primary_profile_(primary_profile),
       kerberos_files_handler_(std::make_unique<KerberosFilesHandler>(
           base::BindRepeating(&KerberosCredentialsManager::GetKerberosFiles,
-                              base::Unretained(this)))) {
+                              base::Unretained(this)))),
+      backoff_entry_for_managed_accounts_(&kBackoffPolicyForManagedAccounts) {
   DCHECK(primary_profile_);
   const user_manager::User* primary_user =
       chromeos::ProfileHelper::Get()->GetUserByProfile(primary_profile);
@@ -326,7 +345,7 @@ KerberosCredentialsManager::KerberosCredentialsManager(PrefService* local_state,
   pref_change_registrar_->Add(
       prefs::kKerberosAccounts,
       base::BindRepeating(&KerberosCredentialsManager::UpdateAccountsFromPref,
-                          weak_factory_.GetWeakPtr()));
+                          weak_factory_.GetWeakPtr(), false /* is_retry */));
 
   // Update accounts if policy is already available or start observing.
   policy_service_ =
@@ -335,7 +354,7 @@ KerberosCredentialsManager::KerberosCredentialsManager(PrefService* local_state,
       policy_service_->IsInitializationComplete(policy::POLICY_DOMAIN_CHROME);
   VLOG(1) << "Policy service initialized at startup: " << policy_initialized;
   if (policy_initialized)
-    UpdateAccountsFromPref();
+    UpdateAccountsFromPref(false /* is_retry */);
   else
     policy_service_->AddObserver(policy::POLICY_DOMAIN_CHROME, this);
 
@@ -398,7 +417,7 @@ void KerberosCredentialsManager::OnPolicyServiceInitialized(
   if (policy_service_->IsInitializationComplete(policy::POLICY_DOMAIN_CHROME)) {
     VLOG(1) << "Policy service initialized";
     policy_service_->RemoveObserver(policy::POLICY_DOMAIN_CHROME, this);
-    UpdateAccountsFromPref();
+    UpdateAccountsFromPref(false /* is_retry */);
   }
 }
 
@@ -472,6 +491,18 @@ void KerberosCredentialsManager::OnAddAccountRunnerDone(
 
 void KerberosCredentialsManager::OnAddManagedAccountRunnerDone(
     kerberos::ErrorType error) {
+  if (!managed_accounts_retry_timer_.IsRunning() && ShouldRetry(error)) {
+    backoff_entry_for_managed_accounts_.InformOfRequest(false);
+
+    if (backoff_entry_for_managed_accounts_.failure_count() <
+        kMaxFailureCountForManagedAccounts) {
+      managed_accounts_retry_timer_.Start(
+          FROM_HERE, backoff_entry_for_managed_accounts_.GetTimeUntilRelease(),
+          base::BindOnce(&KerberosCredentialsManager::UpdateAccountsFromPref,
+                         weak_factory_.GetWeakPtr(), true /* is_retry */));
+    }
+  }
+
   if (add_managed_account_callback_for_testing_) {
     add_managed_account_callback_for_testing_.Run(error);
   }
@@ -746,7 +777,7 @@ void KerberosCredentialsManager::UpdateEnabledFromPref() {
   if (IsKerberosEnabled()) {
     // Kerberos got enabled, re-populate managed accounts.
     VLOG(1) << "Kerberos got enabled, populating managed accounts";
-    UpdateAccountsFromPref();
+    UpdateAccountsFromPref(false /* is_retry */);
     return;
   }
 
@@ -781,7 +812,14 @@ void KerberosCredentialsManager::UpdateAddAccountsAllowedFromPref() {
                               EmptyResultCallback()));
 }
 
-void KerberosCredentialsManager::UpdateAccountsFromPref() {
+void KerberosCredentialsManager::UpdateAccountsFromPref(bool is_retry) {
+  if (is_retry) {
+    VLOG(1) << "Retrying to update KerberosAccounts from Prefs";
+  } else {
+    // Refreshing backoff entry, since this call was triggered by prefs change.
+    backoff_entry_for_managed_accounts_.Reset();
+  }
+
   if (!IsKerberosEnabled()) {
     VLOG(1) << "Kerberos disabled";
     NotifyRequiresLoginPassword(false);

chrome/browser/chromeos/kerberos/kerberos_credentials_manager.h

@@ -14,11 +14,13 @@
 #include "base/observer_list.h"
 #include "base/observer_list_types.h"
 #include "base/optional.h"
+#include "base/timer/timer.h"
 #include "chrome/browser/chromeos/authpolicy/kerberos_files_handler.h"
 #include "chromeos/dbus/kerberos/kerberos_service.pb.h"
 #include "components/keyed_service/core/keyed_service.h"
 #include "components/policy/core/common/policy_namespace.h"
 #include "components/policy/core/common/policy_service.h"
+#include "net/base/backoff_entry.h"
 
 class PrefRegistrySimple;
 class PrefService;
@@ -56,6 +58,9 @@ class KerberosCredentialsManager : public KeyedService,
     DISALLOW_COPY_AND_ASSIGN(Observer);
   };
 
+  // Maximum number of managed accounts addition retries per prefs change.
+  static constexpr int kMaxFailureCountForManagedAccounts = 10;
+
   KerberosCredentialsManager(PrefService* local_state,
                              Profile* primary_profile);
   ~KerberosCredentialsManager() override;
@@ -236,7 +241,7 @@ class KerberosCredentialsManager : public KeyedService,
   void UpdateEnabledFromPref();
   void UpdateRememberPasswordEnabledFromPref();
   void UpdateAddAccountsAllowedFromPref();
-  void UpdateAccountsFromPref();
+  void UpdateAccountsFromPref(bool is_retry);
 
   // Does the main work for UpdateAccountsFromPref(). To clean up stale managed
   // accounts, an up-to-date accounts list is needed. UpdateAccountsFromPref()
@@ -277,6 +282,12 @@ class KerberosCredentialsManager : public KeyedService,
   // List of objects that observe this instance.
   base::ObserverList<Observer, true /* check_empty */> observers_;
 
+  // Backoff entry used to control managed accounts addition retries.
+  net::BackoffEntry backoff_entry_for_managed_accounts_;
+
+  // Timer for keeping track of managed accounts addition retries.
+  base::OneShotTimer managed_accounts_retry_timer_;
+
   // Callback optionally used for testing.
   base::RepeatingCallback<void(kerberos::ErrorType)>
       add_managed_account_callback_for_testing_;

chrome/browser/chromeos/kerberos/kerberos_credentials_manager_test.cc

@@ -15,6 +15,7 @@
 #include "base/run_loop.h"
 #include "base/stl_util.h"
 #include "base/test/bind_test_util.h"
+#include "base/test/task_environment.h"
 #include "chrome/browser/chromeos/authpolicy/kerberos_files_handler.h"
 #include "chrome/browser/chromeos/login/session/user_session_manager.h"
 #include "chrome/browser/chromeos/login/users/mock_user_manager.h"
@@ -73,6 +74,10 @@ const int kOneAccount = 1;
 const int kTwoAccounts = 2;
 const int kThreeAccounts = 3;
 
+const int kOneFailure = 1;
+const int kThreeFailures = 3;
+const int kLotsOfFailures = 1000000;
+
 // Account keys for the kerberos.accounts pref.
 constexpr char kKeyPrincipal[] = "principal";
 constexpr char kKeyPassword[] = "password";
@@ -89,6 +94,11 @@ constexpr char kDefaultConfig[] = R"([libdefaults]
   permitted_enctypes = aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96
   forwardable = true)";
 
+// A long time delta, used to fast forward the task environment until all
+// pending operations are completed. This value should be equal to the maximum
+// time to delay requests on |kBackoffPolicyForManagedAccounts|.
+const base::TimeDelta kLongTimeDelay = base::TimeDelta::FromMinutes(10);
+
 // Fake observer used to test notifications sent by KerberosCredentialsManager
 // on accounts changes.
 class FakeKerberosCredentialsManagerObserver
@@ -192,23 +202,23 @@ class KerberosCredentialsManagerTest : public testing::Test {
     return KerberosClient::Get()->GetTestInterface();
   }
 
-  void SetupResultCallback(int account_count) {
+  void SetupResultCallback(int operation_count) {
     // If this is the first account addition, sets |result_run_loop_|.
-    if (accounts_addition_count_ == 0) {
+    if (remaining_operation_count_ == 0) {
       EXPECT_TRUE(result_errors_.empty());
       EXPECT_FALSE(result_run_loop_);
       result_run_loop_ = std::make_unique<base::RunLoop>();
     }
 
-    accounts_addition_count_ += account_count;
+    remaining_operation_count_ += operation_count;
   }
 
   // Gets a callback that adds the passed-in error to |result_errors_|.
-  // |account_count| is the number of times the callback should be called
+  // |operation_count| is the number of times the callback should be called
   // before stopping |result_run_loop_|.
   base::RepeatingCallback<void(kerberos::ErrorType)> GetRepeatingCallback(
-      int account_count) {
-    SetupResultCallback(account_count);
+      int operation_count) {
+    SetupResultCallback(operation_count);
     return base::BindRepeating(&KerberosCredentialsManagerTest::OnResult,
                                weak_ptr_factory_.GetWeakPtr());
   }
@@ -221,34 +231,45 @@ class KerberosCredentialsManagerTest : public testing::Test {
   }
 
   void OnResult(kerberos::ErrorType error) {
-    DCHECK_LT(0, accounts_addition_count_);
-    accounts_addition_count_--;
+    // Fails if the test tries to execute more operations than expected.
+    ASSERT_LT(0, remaining_operation_count_);
+    remaining_operation_count_--;
     result_errors_.insert(error);
 
     // Stops |result_run_loop_| if all additions are finished.
-    if (accounts_addition_count_ == 0) {
+    if (remaining_operation_count_ == 0) {
       result_run_loop_->Quit();
     }
   }
 
-  void WaitAndVerifyResult(std::multiset<kerberos::ErrorType> expected_errors_,
+  void WaitAndVerifyResult(std::multiset<kerberos::ErrorType> expected_errors,
                            int expected_notifications_count,
                            int expected_accounts_count) {
-    EXPECT_LT(0, accounts_addition_count_);
+    EXPECT_LT(0, remaining_operation_count_);
     ASSERT_TRUE(result_run_loop_);
     result_run_loop_->Run();
 
-    EXPECT_EQ(expected_errors_, result_errors_);
+    EXPECT_EQ(expected_errors, result_errors_);
     EXPECT_EQ(expected_notifications_count, observer_.notifications_count());
     EXPECT_EQ(expected_accounts_count,
               observer_.accounts_count_at_last_notification());
 
-    EXPECT_EQ(0, accounts_addition_count_);
+    EXPECT_EQ(0, remaining_operation_count_);
     result_run_loop_.reset();
     result_errors_.clear();
     observer_.Reset();
   }
 
+  std::multiset<kerberos::ErrorType> GetRepeatedError(kerberos::ErrorType error,
+                                                      int repetitions) {
+    std::multiset<kerberos::ErrorType> result;
+    for (int i = 0; i < repetitions; i++) {
+      result.insert(error);
+    }
+
+    return result;
+  }
+
   // Calls |mgr_->AddAccountAndAuthenticate()| with |principal_name|,
   // |is_managed| and some default parameters, waits for the result and checks
   // expectations.
@@ -338,7 +359,8 @@ class KerberosCredentialsManagerTest : public testing::Test {
     EXPECT_TRUE(user_context.GetPasswordKey()->GetSecret().empty());
   }
 
-  content::BrowserTaskEnvironment task_environment_;
+  content::BrowserTaskEnvironment task_environment_{
+      base::test::TaskEnvironment::TimeSource::MOCK_TIME};
   user_manager::ScopedUserManager scoped_user_manager_;
   ScopedTestingLocalState local_state_;
   std::unique_ptr<TestingProfile> profile_;
@@ -346,7 +368,7 @@ class KerberosCredentialsManagerTest : public testing::Test {
   std::unique_ptr<KerberosCredentialsManager> mgr_;
   FakeKerberosCredentialsManagerObserver observer_;
 
-  int accounts_addition_count_ = 0;
+  int remaining_operation_count_ = 0;
   std::unique_ptr<base::RunLoop> result_run_loop_;
   std::multiset<kerberos::ErrorType> result_errors_;
 
@@ -587,8 +609,7 @@ TEST_F(KerberosCredentialsManagerTest,
                                   kConfig, kAllowExisting, GetResultCallback());
 
   WaitAndVerifyResult(
-      {kerberos::ERROR_BAD_PASSWORD, kerberos::ERROR_BAD_PASSWORD,
-       kerberos::ERROR_BAD_PASSWORD},
+      GetRepeatedError(kerberos::ERROR_BAD_PASSWORD, kThreeAccounts),
       kOneNotification, kThreeAccounts);
   EXPECT_TRUE(mgr_->GetActiveAccount().empty());
 }
@@ -608,9 +629,8 @@ TEST_F(KerberosCredentialsManagerTest,
                                   kDontRememberPassword, kConfig,
                                   kAllowExisting, GetResultCallback());
 
-  WaitAndVerifyResult(
-      {kerberos::ERROR_NONE, kerberos::ERROR_NONE, kerberos::ERROR_NONE},
-      kOneNotification, kThreeAccounts);
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kThreeAccounts),
+                      kOneNotification, kThreeAccounts);
   EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
 }
 
@@ -780,7 +800,7 @@ TEST_F(KerberosCredentialsManagerTest, UpdateEnabledFromPrefKerberosEnabled) {
 
   // Two notifications are expected: one from AddAccountRunner and another from
   // RemoveAllManagedAccountsExcept.
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kTwoNotifications, kTwoAccounts);
 
   EXPECT_TRUE(mgr_->IsKerberosEnabled());
@@ -813,7 +833,7 @@ TEST_F(KerberosCredentialsManagerTest,
                                   kRememberPassword, kConfig, kAllowExisting,
                                   GetResultCallback());
 
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kOneNotification, kTwoAccounts);
 
   SetPref(prefs::kKerberosRememberPasswordEnabled, base::Value(false));
@@ -966,7 +986,7 @@ TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefConfig) {
 
   // Two notifications are expected: one from AddAccountRunner and another from
   // RemoveAllManagedAccountsExcept().
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kTwoNotifications, kTwoAccounts);
 
   VerifyVotedForSavingLoginPassword(kDontSaveLoginPassword);
@@ -1005,7 +1025,7 @@ TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefPassword) {
 
   // Two notifications are expected: one from AddAccountRunner and another from
   // RemoveAllManagedAccountsExcept().
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kTwoNotifications, kTwoAccounts);
 
   VerifyVotedForSavingLoginPassword(kSaveLoginPassword);
@@ -1046,7 +1066,7 @@ TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefRememberPassword) {
 
   // Two notifications are expected: one from AddAccountRunner and another from
   // RemoveAllManagedAccountsExcept().
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kTwoNotifications, kTwoAccounts);
 
   VerifyVotedForSavingLoginPassword(kSaveLoginPassword);
@@ -1089,7 +1109,7 @@ TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefClearAccounts) {
 
   // Two notifications are expected: one from AddAccountRunner and another from
   // RemoveAllManagedAccountsExcept().
-  WaitAndVerifyResult({kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+  WaitAndVerifyResult(GetRepeatedError(kerberos::ERROR_NONE, kTwoAccounts),
                       kTwoNotifications, kTwoAccounts);
 
   VerifyVotedForSavingLoginPassword(kDontSaveLoginPassword);
@@ -1101,6 +1121,178 @@ TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefClearAccounts) {
   EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
 }
 
+// UpdateAccountsFromPref retries to add account if addition fails for network
+// related errors.
+TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefRetry) {
+  // Starting with Kerberos enabled.
+  SetPref(prefs::kKerberosEnabled, base::Value(true));
+
+  client_test_interface()->SetSimulatedNumberOfNetworkFailures(kOneFailure *
+                                                               kOneAccount);
+
+  mgr_->SetAddManagedAccountCallbackForTesting(
+      GetRepeatingCallback((kOneFailure + 1) * kOneAccount));
+
+  base::Value managed_account_1(base::Value::Type::DICTIONARY);
+
+  managed_account_1.SetStringKey(kKeyPrincipal, kPrincipal);
+  managed_account_1.SetStringKey(kKeyPassword, kPassword);
+
+  base::Value managed_accounts(base::Value::Type::LIST);
+  managed_accounts.Append(std::move(managed_account_1));
+
+  SetPref(prefs::kKerberosAccounts, std::move(managed_accounts));
+
+  // Two notifications are expected for each attempt: one from AddAccountRunner
+  // and another from RemoveAllManagedAccountsExcept().
+  WaitAndVerifyResult({kerberos::ERROR_NETWORK_PROBLEM, kerberos::ERROR_NONE},
+                      (kOneFailure + 1) * kTwoNotifications, kOneAccount);
+
+  // Fast forwarding the task environment to force all pending tasks to be
+  // executed. Makes sure no retry is scheduled after a successful attempt.
+  task_environment_.FastForwardBy(kLongTimeDelay);
+
+  Accounts accounts = ListAccounts();
+  ASSERT_EQ(1u, accounts.size());
+  EXPECT_EQ(kNormalizedPrincipal, accounts[0].principal_name());
+  EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
+}
+
+// UpdateAccountsFromPref retries multiple times to add account if addition
+// fails multiple times for network related errors.
+TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefMultipleRetries) {
+  // Starting with Kerberos enabled.
+  SetPref(prefs::kKerberosEnabled, base::Value(true));
+
+  client_test_interface()->SetSimulatedNumberOfNetworkFailures(kThreeFailures *
+                                                               kOneAccount);
+
+  mgr_->SetAddManagedAccountCallbackForTesting(
+      GetRepeatingCallback((kThreeFailures + 1) * kOneAccount));
+
+  base::Value managed_account_1(base::Value::Type::DICTIONARY);
+
+  managed_account_1.SetStringKey(kKeyPrincipal, kPrincipal);
+  managed_account_1.SetStringKey(kKeyPassword, kPassword);
+
+  base::Value managed_accounts(base::Value::Type::LIST);
+  managed_accounts.Append(std::move(managed_account_1));
+
+  SetPref(prefs::kKerberosAccounts, std::move(managed_accounts));
+
+  // Two notifications are expected for each attempt: one from AddAccountRunner
+  // and another from RemoveAllManagedAccountsExcept().
+  WaitAndVerifyResult(
+      {kerberos::ERROR_NETWORK_PROBLEM, kerberos::ERROR_NETWORK_PROBLEM,
+       kerberos::ERROR_NETWORK_PROBLEM, kerberos::ERROR_NONE},
+      (kThreeFailures + 1) * kTwoNotifications, kOneAccount);
+
+  // Fast forwarding the task environment to force all pending tasks to be
+  // executed. This will make sure no retry is scheduled after a successful
+  // attempt.
+  task_environment_.FastForwardBy(kLongTimeDelay);
+
+  Accounts accounts = ListAccounts();
+  ASSERT_EQ(1u, accounts.size());
+  EXPECT_EQ(kNormalizedPrincipal, accounts[0].principal_name());
+  EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
+}
+
+// UpdateAccountsFromPref retries to add multiple accounts if addition fails for
+// network related errors.
+TEST_F(KerberosCredentialsManagerTest,
+       UpdateAccountsFromPrefRetryMultipleAccounts) {
+  // Starting with Kerberos enabled.
+  SetPref(prefs::kKerberosEnabled, base::Value(true));
+
+  client_test_interface()->SetSimulatedNumberOfNetworkFailures(kOneFailure *
+                                                               kTwoAccounts);
+
+  mgr_->SetAddManagedAccountCallbackForTesting(
+      GetRepeatingCallback((kOneFailure + 1) * kTwoAccounts));
+
+  base::Value managed_account_1(base::Value::Type::DICTIONARY);
+  base::Value managed_account_2(base::Value::Type::DICTIONARY);
+
+  managed_account_1.SetStringKey(kKeyPrincipal, kPrincipal);
+  managed_account_1.SetStringKey(kKeyPassword, kPassword);
+  managed_account_2.SetStringKey(kKeyPrincipal, kOtherPrincipal);
+  managed_account_2.SetStringKey(kKeyPassword, kPassword);
+
+  base::Value managed_accounts(base::Value::Type::LIST);
+  managed_accounts.Append(std::move(managed_account_1));
+  managed_accounts.Append(std::move(managed_account_2));
+
+  SetPref(prefs::kKerberosAccounts, std::move(managed_accounts));
+
+  // Two notifications are expected for each attempt: one from AddAccountRunner
+  // and another from RemoveAllManagedAccountsExcept().
+  WaitAndVerifyResult(
+      {kerberos::ERROR_NETWORK_PROBLEM, kerberos::ERROR_NETWORK_PROBLEM,
+       kerberos::ERROR_NONE, kerberos::ERROR_NONE},
+      (kOneFailure + 1) * kTwoNotifications, kTwoAccounts);
+
+  // Fast forwarding the task environment to force all pending tasks to be
+  // executed. This will make sure no retry is scheduled after a successful
+  // attempt.
+  task_environment_.FastForwardBy(kLongTimeDelay);
+
+  Accounts accounts = ListAccounts();
+  ASSERT_EQ(2u, accounts.size());
+  EXPECT_EQ(kNormalizedPrincipal, accounts[0].principal_name());
+  EXPECT_EQ(kNormalizedOtherPrincipal, accounts[1].principal_name());
+  EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
+}
+
+// UpdateAccountsFromPref stops retrying after a certain number of network
+// related errors.
+TEST_F(KerberosCredentialsManagerTest, UpdateAccountsFromPrefStopsRetrying) {
+  // Starting with Kerberos enabled.
+  SetPref(prefs::kKerberosEnabled, base::Value(true));
+
+  client_test_interface()->SetSimulatedNumberOfNetworkFailures(kLotsOfFailures);
+
+  mgr_->SetAddManagedAccountCallbackForTesting(GetRepeatingCallback(
+      KerberosCredentialsManager::kMaxFailureCountForManagedAccounts *
+      kTwoAccounts));
+
+  base::Value managed_account_1(base::Value::Type::DICTIONARY);
+  base::Value managed_account_2(base::Value::Type::DICTIONARY);
+
+  managed_account_1.SetStringKey(kKeyPrincipal, kPrincipal);
+  managed_account_1.SetStringKey(kKeyPassword, kPassword);
+  managed_account_2.SetStringKey(kKeyPrincipal, kOtherPrincipal);
+  managed_account_2.SetStringKey(kKeyPassword, kPassword);
+
+  base::Value managed_accounts(base::Value::Type::LIST);
+  managed_accounts.Append(std::move(managed_account_1));
+  managed_accounts.Append(std::move(managed_account_2));
+
+  SetPref(prefs::kKerberosAccounts, std::move(managed_accounts));
+
+  // Two notifications are expected for each attempt: one from AddAccountRunner
+  // and another from RemoveAllManagedAccountsExcept().
+  WaitAndVerifyResult(
+      GetRepeatedError(
+          kerberos::ERROR_NETWORK_PROBLEM,
+          KerberosCredentialsManager::kMaxFailureCountForManagedAccounts *
+              kTwoAccounts),
+      KerberosCredentialsManager::kMaxFailureCountForManagedAccounts *
+          kTwoNotifications,
+      kTwoAccounts);
+
+  // Fast forwarding the task environment to force all pending tasks to be
+  // executed. This will make sure no retry is scheduled after a certain number
+  // of network related errors.
+  task_environment_.FastForwardBy(kLongTimeDelay);
+
+  Accounts accounts = ListAccounts();
+  ASSERT_EQ(2u, accounts.size());
+  EXPECT_EQ(kNormalizedPrincipal, accounts[0].principal_name());
+  EXPECT_EQ(kNormalizedOtherPrincipal, accounts[1].principal_name());
+  EXPECT_EQ(kNormalizedPrincipal, mgr_->GetActiveAccount());
+}
+
 // TODO(https://crbug.com/952251): Add more tests
 // - ClearAccounts
 //     + Normalization like in AddAccountAndAuthenticate

chromeos/dbus/kerberos/fake_kerberos_client.cc

@@ -121,14 +121,14 @@ void FakeKerberosClient::AddAccount(const kerberos::AddAccountRequest& request,
   if (it != accounts_.end()) {
     it->is_managed |= request.is_managed();
     PostResponse(std::move(callback), kerberos::ERROR_DUPLICATE_PRINCIPAL_NAME,
-                 mTaskDelay);
+                 task_delay_);
     return;
   }
 
   AccountData data(request.principal_name());
   data.is_managed = request.is_managed();
   accounts_.push_back(data);
-  PostResponse(std::move(callback), kerberos::ERROR_NONE, mTaskDelay);
+  PostResponse(std::move(callback), kerberos::ERROR_NONE, task_delay_);
 }
 
 void FakeKerberosClient::RemoveAccount(
@@ -146,7 +146,7 @@ void FakeKerberosClient::RemoveAccount(
   }
 
   MapAccountData(response.mutable_accounts());
-  PostProtoResponse(std::move(callback), response, mTaskDelay);
+  PostProtoResponse(std::move(callback), response, task_delay_);
 }
 
 void FakeKerberosClient::ClearAccounts(
@@ -183,7 +183,7 @@ void FakeKerberosClient::ClearAccounts(
   kerberos::ClearAccountsResponse response;
   MapAccountData(response.mutable_accounts());
   response.set_error(kerberos::ERROR_NONE);
-  PostProtoResponse(std::move(callback), response, mTaskDelay);
+  PostProtoResponse(std::move(callback), response, task_delay_);
 }
 
 void FakeKerberosClient::ListAccounts(
@@ -193,7 +193,7 @@ void FakeKerberosClient::ListAccounts(
   kerberos::ListAccountsResponse response;
   MapAccountData(response.mutable_accounts());
   response.set_error(kerberos::ERROR_NONE);
-  PostProtoResponse(std::move(callback), response, mTaskDelay);
+  PostProtoResponse(std::move(callback), response, task_delay_);
 }
 
 void FakeKerberosClient::SetConfig(const kerberos::SetConfigRequest& request,
@@ -202,19 +202,19 @@ void FakeKerberosClient::SetConfig(const kerberos::SetConfigRequest& request,
   AccountData* data = GetAccountData(request.principal_name());
   if (!data) {
     PostResponse(std::move(callback), kerberos::ERROR_UNKNOWN_PRINCIPAL_NAME,
-                 mTaskDelay);
+                 task_delay_);
     return;
   }
 
   kerberos::ConfigErrorInfo error_info =
       ValidateConfigLines(request.krb5conf());
   if (error_info.code() != kerberos::CONFIG_ERROR_NONE) {
-    PostResponse(std::move(callback), kerberos::ERROR_BAD_CONFIG, mTaskDelay);
+    PostResponse(std::move(callback), kerberos::ERROR_BAD_CONFIG, task_delay_);
     return;
   }
 
   data->krb5conf = request.krb5conf();
-  PostResponse(std::move(callback), kerberos::ERROR_NONE, mTaskDelay);
+  PostResponse(std::move(callback), kerberos::ERROR_NONE, task_delay_);
 }
 
 void FakeKerberosClient::ValidateConfig(
@@ -229,7 +229,7 @@ void FakeKerberosClient::ValidateConfig(
                          ? kerberos::ERROR_BAD_CONFIG
                          : kerberos::ERROR_NONE);
   *response.mutable_error_info() = std::move(error_info);
-  PostProtoResponse(std::move(callback), response, mTaskDelay);
+  PostProtoResponse(std::move(callback), response, task_delay_);
 }
 
 void FakeKerberosClient::AcquireKerberosTgt(
@@ -240,7 +240,7 @@ void FakeKerberosClient::AcquireKerberosTgt(
   AccountData* data = GetAccountData(request.principal_name());
   if (!data) {
     PostResponse(std::move(callback), kerberos::ERROR_UNKNOWN_PRINCIPAL_NAME,
-                 mTaskDelay);
+                 task_delay_);
     return;
   }
 
@@ -271,13 +271,21 @@ void FakeKerberosClient::AcquireKerberosTgt(
 
   // Reject empty passwords.
   if (password.empty()) {
-    PostResponse(std::move(callback), kerberos::ERROR_BAD_PASSWORD, mTaskDelay);
+    PostResponse(std::move(callback), kerberos::ERROR_BAD_PASSWORD,
+                 task_delay_);
+    return;
+  }
+
+  if (simulated_number_of_network_failures_ > 0) {
+    simulated_number_of_network_failures_--;
+    PostResponse(std::move(callback), kerberos::ERROR_NETWORK_PROBLEM,
+                 task_delay_);
     return;
   }
 
   // It worked! Magic!
   data->has_tgt = true;
-  PostResponse(std::move(callback), kerberos::ERROR_NONE, mTaskDelay);
+  PostResponse(std::move(callback), kerberos::ERROR_NONE, task_delay_);
 }
 
 void FakeKerberosClient::GetKerberosFiles(
@@ -287,7 +295,7 @@ void FakeKerberosClient::GetKerberosFiles(
   AccountData* data = GetAccountData(request.principal_name());
   if (!data) {
     PostResponse(std::move(callback), kerberos::ERROR_UNKNOWN_PRINCIPAL_NAME,
-                 mTaskDelay);
+                 task_delay_);
     return;
   }
 
@@ -297,7 +305,7 @@ void FakeKerberosClient::GetKerberosFiles(
     response.mutable_files()->set_krb5conf("Fake Kerberos configuration");
   }
   response.set_error(kerberos::ERROR_NONE);
-  PostProtoResponse(std::move(callback), response, mTaskDelay);
+  PostProtoResponse(std::move(callback), response, task_delay_);
 }
 
 void FakeKerberosClient::ConnectToKerberosFileChangedSignal(
@@ -315,7 +323,7 @@ void FakeKerberosClient::ConnectToKerberosTicketExpiringSignal(
 }
 
 void FakeKerberosClient::SetTaskDelay(base::TimeDelta delay) {
-  mTaskDelay = delay;
+  task_delay_ = delay;
 }
 
 void FakeKerberosClient::StartRecordingFunctionCalls() {
@@ -335,6 +343,11 @@ std::size_t FakeKerberosClient::GetNumberOfAccounts() const {
   return accounts_.size();
 }
 
+void FakeKerberosClient::SetSimulatedNumberOfNetworkFailures(
+    int number_of_failures) {
+  simulated_number_of_network_failures_ = number_of_failures;
+}
+
 void FakeKerberosClient::MaybeRecordFunctionCallForTesting(
     const char* function_name) {
   if (!recorded_function_calls_)

chromeos/dbus/kerberos/fake_kerberos_client.h

@@ -53,6 +53,7 @@ class COMPONENT_EXPORT(CHROMEOS_DBUS) FakeKerberosClient
   void StartRecordingFunctionCalls() override;
   std::string StopRecordingAndGetRecordedFunctionCalls() override;
   std::size_t GetNumberOfAccounts() const override;
+  void SetSimulatedNumberOfNetworkFailures(int number_of_failures) override;
 
  private:
   using RepeatedAccountField =
@@ -108,7 +109,11 @@ class COMPONENT_EXPORT(CHROMEOS_DBUS) FakeKerberosClient
   base::Optional<std::string> recorded_function_calls_;
 
   // Fake delay for any asynchronous operation.
-  base::TimeDelta mTaskDelay = base::TimeDelta::FromMilliseconds(100);
+  base::TimeDelta task_delay_ = base::TimeDelta::FromMilliseconds(100);
+
+  // The simulated number of network failures on |AcquireKerberosTgt()| (for
+  // testing).
+  int simulated_number_of_network_failures_ = 0;
 
   KerberosFilesChangedCallback kerberos_files_changed_callback_;
   KerberosTicketExpiringCallback kerberos_ticket_expiring_callback_;

chromeos/dbus/kerberos/kerberos_client.h

@@ -61,6 +61,12 @@ class COMPONENT_EXPORT(KERBEROS) KerberosClient {
     // Returns the number of accounts currently saved.
     virtual std::size_t GetNumberOfAccounts() const = 0;
 
+    // Sets the simulated number of network failures for |AcquireKerberosTgt()|.
+    // The default value is zero. This value should be set when testing the
+    // exponential backoff retry for adding managed accounts.
+    virtual void SetSimulatedNumberOfNetworkFailures(
+        int number_of_failures) = 0;
+
    protected:
     virtual ~TestInterface() {}
   };