QUIC - Persist 1000 MRU alternate protocols to preferences file.

+ Flip a coin and if we have persisted 1000 MRU servers, then load only
  either 200 or 1000 servers from disk.

+ Persist only one server (or don't duplicate server names) that
  spoke QUIC for canonical suffixes like:
  + .c.youtube.com
  + .googlevideo.com

+ Bumped up the version number of http_server_properties.

+ Added two histograms to measure the impact of persisting 1000
  alternate protocol servers.
  ++ Net.AlternateProtocolUsage.200Truncated
  ++ Net.AlternateProtocolUsage.1000Truncated

R=jar@chromium.org, rch@chromium.org

Review URL: https://codereview.chromium.org/298683010

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@272836 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/net/http_server_properties_manager.cc

@@ -7,6 +7,7 @@
 #include "base/bind.h"
 #include "base/metrics/histogram.h"
 #include "base/prefs/pref_service.h"
+#include "base/rand_util.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/stringprintf.h"
@@ -39,12 +40,15 @@ const int64 kUpdatePrefsDelayMs = 5000;
 const int kMissingVersion = 0;
 
 // The version number of persisted http_server_properties.
-const int kVersionNumber = 2;
+const int kVersionNumber = 3;
 
 typedef std::vector<std::string> StringVector;
 
-// Persist 200 MRU AlternateProtocolHostPortPairs.
-const int kMaxAlternateProtocolHostsToPersist = 200;
+// Load either 200 or 1000 servers based on a coin flip.
+const int k200AlternateProtocolHostsToLoad = 200;
+const int k1000AlternateProtocolHostsToLoad = 1000;
+// Persist 1000 MRU AlternateProtocolHostPortPairs.
+const int kMaxAlternateProtocolHostsToPersist = 1000;
 
 // Persist 200 MRU SpdySettingsHostPortPairs.
 const int kMaxSpdySettingsHostsToPersist = 200;
@@ -213,6 +217,16 @@ HttpServerPropertiesManager::alternate_protocol_map() const {
   return http_server_properties_impl_->alternate_protocol_map();
 }
 
+void HttpServerPropertiesManager::SetAlternateProtocolExperiment(
+    net::AlternateProtocolExperiment experiment) {
+  http_server_properties_impl_->SetAlternateProtocolExperiment(experiment);
+}
+
+net::AlternateProtocolExperiment
+HttpServerPropertiesManager::GetAlternateProtocolExperiment() const {
+  return http_server_properties_impl_->GetAlternateProtocolExperiment();
+}
+
 const net::SettingsMap&
 HttpServerPropertiesManager::GetSpdySettings(
     const net::HostPortPair& host_port_pair) {
@@ -345,7 +359,24 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnUI() {
       new net::PipelineCapabilityMap);
   scoped_ptr<net::AlternateProtocolMap> alternate_protocol_map(
       new net::AlternateProtocolMap(kMaxAlternateProtocolHostsToPersist));
+  // TODO(rtenneti): Delete the following code after the experiment.
+  int alternate_protocols_to_load = k200AlternateProtocolHostsToLoad;
+  net::AlternateProtocolExperiment alternate_protocol_experiment =
+      net::ALTERNATE_PROTOCOL_NOT_PART_OF_EXPERIMENT;
+  if (version == kVersionNumber) {
+    if (base::RandInt(0, 99) == 0) {
+      alternate_protocol_experiment =
+          net::ALTERNATE_PROTOCOL_TRUNCATED_200_SERVERS;
+    } else {
+      alternate_protocols_to_load = k1000AlternateProtocolHostsToLoad;
+      alternate_protocol_experiment =
+          net::ALTERNATE_PROTOCOL_TRUNCATED_1000_SERVERS;
+    }
+    DVLOG(1) << "# of servers that support alternate_protocol: "
+             << alternate_protocols_to_load;
+  }
 
+  int count = 0;
   for (base::DictionaryValue::Iterator it(*servers_dict); !it.IsAtEnd();
        it.Advance()) {
     // Get server's host/pair.
@@ -418,6 +449,8 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnUI() {
       continue;
     }
 
+    if (count >= alternate_protocols_to_load)
+      continue;
     do {
       int port = 0;
       if (!port_alternate_protocol_dict->GetIntegerWithoutPathExpansion(
@@ -446,6 +479,7 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnUI() {
       port_alternate_protocol.protocol = protocol;
 
       alternate_protocol_map->Put(server, port_alternate_protocol);
+      ++count;
     } while (false);
   }
 
@@ -459,6 +493,7 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnUI() {
                  base::Owned(spdy_settings_map.release()),
                  base::Owned(alternate_protocol_map.release()),
                  base::Owned(pipeline_capability_map.release()),
+                 alternate_protocol_experiment,
                  detected_corrupted_prefs));
 }
 
@@ -467,6 +502,7 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnIO(
     net::SpdySettingsMap* spdy_settings_map,
     net::AlternateProtocolMap* alternate_protocol_map,
     net::PipelineCapabilityMap* pipeline_capability_map,
+    net::AlternateProtocolExperiment alternate_protocol_experiment,
     bool detected_corrupted_prefs) {
   // Preferences have the master data because admins might have pushed new
   // preferences. Update the cached data with new data from preferences.
@@ -475,17 +511,19 @@ void HttpServerPropertiesManager::UpdateCacheFromPrefsOnIO(
   UMA_HISTOGRAM_COUNTS("Net.CountOfSpdyServers", spdy_servers->size());
   http_server_properties_impl_->InitializeSpdyServers(spdy_servers, true);
 
-  // Clear the cached data and use the new spdy_settings from preferences.
+  // Update the cached data and use the new spdy_settings from preferences.
   UMA_HISTOGRAM_COUNTS("Net.CountOfSpdySettings", spdy_settings_map->size());
   http_server_properties_impl_->InitializeSpdySettingsServers(
       spdy_settings_map);
 
-  // Clear the cached data and use the new Alternate-Protocol server list from
+  // Update the cached data and use the new Alternate-Protocol server list from
   // preferences.
   UMA_HISTOGRAM_COUNTS("Net.CountOfAlternateProtocolServers",
                        alternate_protocol_map->size());
   http_server_properties_impl_->InitializeAlternateProtocolServers(
       alternate_protocol_map);
+  http_server_properties_impl_->SetAlternateProtocolExperiment(
+      alternate_protocol_experiment);
 
   UMA_HISTOGRAM_COUNTS("Net.CountOfPipelineCapableServers",
                        pipeline_capability_map->size());
@@ -547,10 +585,21 @@ void HttpServerPropertiesManager::UpdatePrefsFromCacheOnIO(
   const net::AlternateProtocolMap& map =
       http_server_properties_impl_->alternate_protocol_map();
   count = 0;
+  typedef std::map<std::string, bool> CanonicalHostPersistedMap;
+  CanonicalHostPersistedMap persisted_map;
   for (net::AlternateProtocolMap::const_iterator it = map.begin();
        it != map.end() && count < kMaxAlternateProtocolHostsToPersist;
-       ++it, ++count) {
-    alternate_protocol_map->Put(it->first, it->second);
+       ++it) {
+    const net::HostPortPair& server = it->first;
+    std::string canonical_suffix =
+        http_server_properties_impl_->GetCanonicalSuffix(server);
+    if (!canonical_suffix.empty()) {
+      if (persisted_map.find(canonical_suffix) != persisted_map.end())
+        continue;
+      persisted_map[canonical_suffix] = true;
+    }
+    alternate_protocol_map->Put(server, it->second);
+    ++count;
   }
 
   net::PipelineCapabilityMap* pipeline_capability_map =

chrome/browser/net/http_server_properties_manager.h

@@ -129,6 +129,12 @@ class HttpServerPropertiesManager
   virtual const net::AlternateProtocolMap&
       alternate_protocol_map() const OVERRIDE;
 
+  virtual void SetAlternateProtocolExperiment(
+      net::AlternateProtocolExperiment experiment) OVERRIDE;
+
+  virtual net::AlternateProtocolExperiment GetAlternateProtocolExperiment()
+      const OVERRIDE;
+
   // Gets a reference to the SettingsMap stored for a host.
   // If no settings are stored, returns an empty SettingsMap.
   virtual const net::SettingsMap& GetSpdySettings(
@@ -193,6 +199,7 @@ class HttpServerPropertiesManager
       net::SpdySettingsMap* spdy_settings_map,
       net::AlternateProtocolMap* alternate_protocol_map,
       net::PipelineCapabilityMap* pipeline_capability_map,
+      net::AlternateProtocolExperiment alternate_protocol_experiment,
       bool detected_corrupted_prefs);
 
   // These are used to delay updating the preferences when cached data in

chrome/browser/net/http_server_properties_manager_unittest.cc

@@ -61,11 +61,12 @@ class TestingHttpServerPropertiesManager : public HttpServerPropertiesManager {
 
   MOCK_METHOD0(UpdateCacheFromPrefsOnUI, void());
   MOCK_METHOD1(UpdatePrefsFromCacheOnIO, void(const base::Closure&));
-  MOCK_METHOD5(UpdateCacheFromPrefsOnIO,
+  MOCK_METHOD6(UpdateCacheFromPrefsOnIO,
                void(std::vector<std::string>* spdy_servers,
                     net::SpdySettingsMap* spdy_settings_map,
                     net::AlternateProtocolMap* alternate_protocol_map,
                     net::PipelineCapabilityMap* pipeline_capability_map,
+                    net::AlternateProtocolExperiment experiment,
                     bool detected_corrupted_prefs));
   MOCK_METHOD4(UpdatePrefsOnUI,
                void(base::ListValue* spdy_server_list,

net/http/http_server_properties.cc

@@ -31,9 +31,20 @@ COMPILE_ASSERT(
 
 }  // namespace
 
-void HistogramAlternateProtocolUsage(AlternateProtocolUsage usage) {
+void HistogramAlternateProtocolUsage(
+    AlternateProtocolUsage usage,
+    AlternateProtocolExperiment alternate_protocol_experiment) {
   UMA_HISTOGRAM_ENUMERATION("Net.AlternateProtocolUsage", usage,
                             ALTERNATE_PROTOCOL_USAGE_MAX);
+  if (alternate_protocol_experiment ==
+      ALTERNATE_PROTOCOL_TRUNCATED_200_SERVERS) {
+    UMA_HISTOGRAM_ENUMERATION("Net.AlternateProtocolUsage.200Truncated", usage,
+                              ALTERNATE_PROTOCOL_USAGE_MAX);
+  } else if (alternate_protocol_experiment ==
+      ALTERNATE_PROTOCOL_TRUNCATED_1000_SERVERS) {
+    UMA_HISTOGRAM_ENUMERATION("Net.AlternateProtocolUsage.1000Truncated", usage,
+                              ALTERNATE_PROTOCOL_USAGE_MAX);
+  }
 }
 
 void HistogramBrokenAlternateProtocolLocation(

net/http/http_server_properties.h

@@ -19,6 +19,15 @@
 
 namespace net {
 
+enum AlternateProtocolExperiment {
+  // 200 alternate_protocol servers are loaded (persisted 200 MRU servers).
+  ALTERNATE_PROTOCOL_NOT_PART_OF_EXPERIMENT = 0,
+  // 200 alternate_protocol servers are loaded (persisted 1000 MRU servers).
+  ALTERNATE_PROTOCOL_TRUNCATED_200_SERVERS,
+  // 1000 alternate_protocol servers are loaded (persisted 1000 MRU servers).
+  ALTERNATE_PROTOCOL_TRUNCATED_1000_SERVERS,
+};
+
 enum AlternateProtocolUsage {
   // Alternate Protocol was used without racing a normal connection.
   ALTERNATE_PROTOCOL_USAGE_NO_RACE = 0,
@@ -36,8 +45,10 @@ enum AlternateProtocolUsage {
   ALTERNATE_PROTOCOL_USAGE_MAX,
 };
 
-// Log a histogram to reflect |usage|.
-NET_EXPORT void HistogramAlternateProtocolUsage(AlternateProtocolUsage usage);
+// Log a histogram to reflect |usage| and |alternate_protocol_experiment|.
+NET_EXPORT void HistogramAlternateProtocolUsage(
+    AlternateProtocolUsage usage,
+    AlternateProtocolExperiment alternate_protocol_experiment);
 
 enum BrokenAlternateProtocolLocation {
   BROKEN_ALTERNATE_PROTOCOL_LOCATION_HTTP_STREAM_FACTORY_IMPL_JOB = 0,
@@ -159,6 +170,12 @@ class NET_EXPORT HttpServerProperties {
   // Returns all Alternate-Protocol mappings.
   virtual const AlternateProtocolMap& alternate_protocol_map() const = 0;
 
+  virtual void SetAlternateProtocolExperiment(
+      AlternateProtocolExperiment experiment) = 0;
+
+  virtual AlternateProtocolExperiment GetAlternateProtocolExperiment()
+      const = 0;
+
   // Gets a reference to the SettingsMap stored for a host.
   // If no settings are stored, returns an empty SettingsMap.
   virtual const SettingsMap& GetSpdySettings(

net/http/http_server_properties_impl.cc

@@ -29,6 +29,8 @@ const uint64 kBrokenAlternateProtocolDelaySecs = 300;
 HttpServerPropertiesImpl::HttpServerPropertiesImpl()
     : spdy_servers_map_(SpdyServerHostPortMap::NO_AUTO_EVICT),
       alternate_protocol_map_(AlternateProtocolMap::NO_AUTO_EVICT),
+      alternate_protocol_experiment_(
+          ALTERNATE_PROTOCOL_NOT_PART_OF_EXPERIMENT),
       spdy_settings_map_(SpdySettingsMap::NO_AUTO_EVICT),
       pipeline_capability_map_(
           new CachedPipelineCapabilityMap(kDefaultNumHostsToRemember)),
@@ -220,6 +222,19 @@ bool HttpServerPropertiesImpl::HasAlternateProtocol(
   return GetCanonicalHost(server) != canonical_host_to_origin_map_.end();
 }
 
+std::string HttpServerPropertiesImpl::GetCanonicalSuffix(
+    const HostPortPair& server) {
+  // If this host ends with a canonical suffix, then return the canonical
+  // suffix.
+  for (size_t i = 0; i < canoncial_suffixes_.size(); ++i) {
+    std::string canonical_suffix = canoncial_suffixes_[i];
+    if (EndsWith(server.host(), canoncial_suffixes_[i], false)) {
+      return canonical_suffix;
+    }
+  }
+  return std::string();
+}
+
 PortAlternateProtocolPair
 HttpServerPropertiesImpl::GetAlternateProtocol(
     const HostPortPair& server) {
@@ -275,7 +290,8 @@ void HttpServerPropertiesImpl::SetAlternateProtocol(
     // TODO(rch): Consider the case where multiple requests are started
     // before the first completes. In this case, only one of the jobs
     // would reach this code, whereas all of them should should have.
-    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_MAPPING_MISSING);
+    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_MAPPING_MISSING,
+                                    alternate_protocol_experiment_);
   }
 
   alternate_protocol_map_.Put(server, alternate);
@@ -340,6 +356,16 @@ HttpServerPropertiesImpl::alternate_protocol_map() const {
   return alternate_protocol_map_;
 }
 
+void HttpServerPropertiesImpl::SetAlternateProtocolExperiment(
+    AlternateProtocolExperiment experiment) {
+  alternate_protocol_experiment_ = experiment;
+}
+
+AlternateProtocolExperiment
+HttpServerPropertiesImpl::GetAlternateProtocolExperiment() const {
+  return alternate_protocol_experiment_;
+}
+
 const SettingsMap& HttpServerPropertiesImpl::GetSpdySettings(
     const HostPortPair& host_port_pair) {
   SpdySettingsMap::iterator it = spdy_settings_map_.Get(host_port_pair);

net/http/http_server_properties_impl.h

@@ -65,6 +65,10 @@ class NET_EXPORT HttpServerPropertiesImpl
   static void ForceAlternateProtocol(const PortAlternateProtocolPair& pair);
   static void DisableForcedAlternateProtocol();
 
+  // Returns the canonical host suffix for |server|, or std::string() if none
+  // exists.
+  std::string GetCanonicalSuffix(const net::HostPortPair& server);
+
   // Changes the number of host/port pairs we remember pipelining capability
   // for. A larger number means we're more likely to be able to pipeline
   // immediately if a host is known good, but uses more memory. This function
@@ -118,6 +122,12 @@ class NET_EXPORT HttpServerPropertiesImpl
   // Returns all Alternate-Protocol mappings.
   virtual const AlternateProtocolMap& alternate_protocol_map() const OVERRIDE;
 
+  virtual void SetAlternateProtocolExperiment(
+      AlternateProtocolExperiment experiment) OVERRIDE;
+
+  virtual AlternateProtocolExperiment GetAlternateProtocolExperiment()
+      const OVERRIDE;
+
   // Gets a reference to the SettingsMap stored for a host.
   // If no settings are stored, returns an empty SettingsMap.
   virtual const SettingsMap& GetSpdySettings(
@@ -187,6 +197,7 @@ class NET_EXPORT HttpServerPropertiesImpl
   AlternateProtocolMap alternate_protocol_map_;
   BrokenAlternateProtocolList broken_alternate_protocol_list_;
   BrokenAlternateProtocolMap broken_alternate_protocol_map_;
+  AlternateProtocolExperiment alternate_protocol_experiment_;
 
   SpdySettingsMap spdy_settings_map_;
   ServerNetworkStatsMap server_network_stats_map_;

net/http/http_server_properties_impl_unittest.cc

@@ -415,6 +415,10 @@ TEST_F(AlternateProtocolServerPropertiesTest, Canonical) {
       impl_.GetAlternateProtocol(test_host_port_pair);
   EXPECT_EQ(canonical_protocol.port, alternate.port);
   EXPECT_EQ(canonical_protocol.protocol, alternate.protocol);
+
+  // Verify the canonical suffix.
+  EXPECT_EQ(".c.youtube.com", impl_.GetCanonicalSuffix(test_host_port_pair));
+  EXPECT_EQ(".c.youtube.com", impl_.GetCanonicalSuffix(canonical_port_pair));
 }
 
 typedef HttpServerPropertiesImplTest SpdySettingsServerPropertiesTest;

net/http/http_stream_factory_impl.cc

@@ -207,7 +207,9 @@ PortAlternateProtocolPair HttpStreamFactoryImpl::GetAlternateProtocolRequestFor(
   PortAlternateProtocolPair alternate =
       http_server_properties.GetAlternateProtocol(origin);
   if (alternate.protocol == ALTERNATE_PROTOCOL_BROKEN) {
-    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_BROKEN);
+    HistogramAlternateProtocolUsage(
+        ALTERNATE_PROTOCOL_USAGE_BROKEN,
+        http_server_properties.GetAlternateProtocolExperiment());
     return kNoAlternateProtocol;
   }
 

net/http/http_stream_factory_impl_job.cc

@@ -1472,17 +1472,28 @@ bool HttpStreamFactoryImpl::Job::IsOrphaned() const {
 }
 
 void HttpStreamFactoryImpl::Job::ReportJobSuccededForRequest() {
+  net::AlternateProtocolExperiment alternate_protocol_experiment =
+      ALTERNATE_PROTOCOL_NOT_PART_OF_EXPERIMENT;
+  base::WeakPtr<HttpServerProperties> http_server_properties =
+      session_->http_server_properties();
+  if (http_server_properties) {
+    alternate_protocol_experiment =
+        http_server_properties->GetAlternateProtocolExperiment();
+  }
   if (using_existing_quic_session_) {
     // If an existing session was used, then no TCP connection was
     // started.
-    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_NO_RACE);
+    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_NO_RACE,
+                                    alternate_protocol_experiment);
   } else if (original_url_) {
     // This job was the alternate protocol job, and hence won the race.
-    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_WON_RACE);
+    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_WON_RACE,
+                                    alternate_protocol_experiment);
   } else {
     // This job was the normal job, and hence the alternate protocol job lost
     // the race.
-    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_LOST_RACE);
+    HistogramAlternateProtocolUsage(ALTERNATE_PROTOCOL_USAGE_LOST_RACE,
+                                    alternate_protocol_experiment);
   }
 }