Add a short random delay to scheduled policy fetches

This change adds a small (0 to 5 minutes) delay to scheduled policy
fetches in order to alleviate server-side traffic spikes. The delay is
randomly set at policy initialization time in the
CloudPolicyRefreshScheduler constructor.

To avoid causing issues with ChromeOS' synchronous policy fetches, we
skip adding the delay in some cases.

Bug: 963910
TEST: updated logic of component_unittest and one unit_tests
Change-Id: Ic02ac2ef18e7e7b03ac95bdfab19e946c2f46324
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1998789Reviewed-by: Pavol Marko <pmarko@chromium.org>
Reviewed-by: Owen Min <zmin@chromium.org>
Commit-Queue: proberge <proberge@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732976}

chrome/browser/chromeos/policy/user_cloud_policy_manager_chromeos_unittest.cc

@@ -1248,7 +1248,8 @@ TEST_P(UserCloudPolicyManagerChromeOSChildTest, RefreshScheduler) {
   // of the test will work incorrectly and should be updated.
   const int iterations = 3;
   base::TimeDelta refresh_delay = base::TimeDelta::FromMilliseconds(
-      manager_->core()->refresh_scheduler()->GetActualRefreshDelay());
+      manager_->core()->refresh_scheduler()->GetActualRefreshDelay() +
+      manager_->core()->refresh_scheduler()->GetSaltDelayForTesting());
   ASSERT_GT(refresh_delay, iterations * token_lifetime);
 
   // Advancing the clock will trigger delivery of new tokens. It should not

components/policy/core/common/cloud/cloud_policy_refresh_scheduler.cc

@@ -11,6 +11,7 @@
 #include "base/bind_helpers.h"
 #include "base/logging.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/rand_util.h"
 #include "base/sequenced_task_runner.h"
 #include "build/build_config.h"
 #include "components/policy/core/common/cloud/cloud_policy_constants.h"
@@ -36,6 +37,8 @@ const int64_t CloudPolicyRefreshScheduler::kRefreshDelayMinMs =
     30 * 60 * 1000;  // 30 minutes.
 const int64_t CloudPolicyRefreshScheduler::kRefreshDelayMaxMs =
     7 * 24 * 60 * 60 * 1000;  // 1 week.
+const int64_t CloudPolicyRefreshScheduler::kRandomSaltDelayMaxValueMs =
+    5 * 60 * 1000;  // 5 minutes.
 
 #else
 
@@ -53,6 +56,8 @@ const int64_t CloudPolicyRefreshScheduler::kRefreshDelayMinMs =
     30 * 60 * 1000;  // 30 minutes.
 const int64_t CloudPolicyRefreshScheduler::kRefreshDelayMaxMs =
     24 * 60 * 60 * 1000;  // 1 day.
+const int64_t CloudPolicyRefreshScheduler::kRandomSaltDelayMaxValueMs =
+    5 * 60 * 1000;  // 5 minutes.
 
 #endif
 
@@ -69,6 +74,8 @@ CloudPolicyRefreshScheduler::CloudPolicyRefreshScheduler(
       network_connection_tracker_(network_connection_tracker_getter.Run()),
       error_retry_delay_ms_(kInitialErrorRetryDelayMs),
       refresh_delay_ms_(kDefaultRefreshDelayMs),
+      refresh_delay_salt_ms_(static_cast<int64_t>(
+          base::RandGenerator(kRandomSaltDelayMaxValueMs))),
       invalidations_available_(false),
       creation_time_(base::Time::NowFromSystemTime()) {
   client_->AddObserver(this);
@@ -364,7 +371,13 @@ void CloudPolicyRefreshScheduler::RefreshAfter(int delta_ms) {
   const base::TimeDelta time_ticks_delay =
       std::max((last_refresh_ticks_ + delta) - base::TimeTicks::Now(),
                base::TimeDelta());
-  const base::TimeDelta delay = std::min(system_delay, time_ticks_delay);
+  base::TimeDelta delay = std::min(system_delay, time_ticks_delay);
+
+  // Unless requesting an immediate refresh, add a delay to the scheduled policy
+  // refresh in order to spread out server load.
+  if (!delay.is_zero())
+    delay += base::TimeDelta::FromMilliseconds(refresh_delay_salt_ms_);
+
   refresh_callback_.Reset(base::BindOnce(
       &CloudPolicyRefreshScheduler::PerformRefresh, base::Unretained(this)));
   task_runner_->PostDelayedTask(FROM_HERE, refresh_callback_.callback(), delay);

components/policy/core/common/cloud/cloud_policy_refresh_scheduler.h

@@ -36,6 +36,7 @@ class POLICY_EXPORT CloudPolicyRefreshScheduler
   static const int64_t kUnmanagedRefreshDelayMs;
   static const int64_t kWithInvalidationsRefreshDelayMs;
   static const int64_t kInitialErrorRetryDelayMs;
+  static const int64_t kRandomSaltDelayMaxValueMs;
 
   // Refresh delay bounds.
   static const int64_t kRefreshDelayMinMs;
@@ -62,6 +63,9 @@ class POLICY_EXPORT CloudPolicyRefreshScheduler
   // invalidations are available or not).
   int64_t GetActualRefreshDelay() const;
 
+  // For testing: get the value randomly assigned to refresh_delay_salt_ms_.
+  int64_t GetSaltDelayForTesting() const { return refresh_delay_salt_ms_; }
+
   // Schedules a refresh to be performed immediately.
   void RefreshSoon();
 
@@ -106,8 +110,9 @@ class POLICY_EXPORT CloudPolicyRefreshScheduler
   // Triggers a policy refresh.
   void PerformRefresh();
 
-  // Schedules a policy refresh to happen no later than |delta_ms| msecs after
-  // |last_refresh_| or |last_refresh_ticks_| whichever is sooner.
+  // Schedules a policy refresh to happen no later than |delta_ms| +
+  // |refresh_delay_salt_ms_| msecs after |last_refresh_| or
+  // |last_refresh_ticks_| whichever is sooner.
   void RefreshAfter(int delta_ms);
 
   // Cancels the scheduled policy refresh.
@@ -153,6 +158,10 @@ class POLICY_EXPORT CloudPolicyRefreshScheduler
   // The refresh delay.
   int64_t refresh_delay_ms_;
 
+  // A randomly-generated (between 0 and |kRandomSaltDelayMaxValueMs|) delay
+  // added to all non-immediately scheduled refresh requests.
+  const int64_t refresh_delay_salt_ms_;
+
   // Whether the invalidations service is available and receiving notifications
   // of policy updates.
   bool invalidations_available_;