Re-add resource speculative prefetching code

This CL fixes the memory leak bug in CL 462423004.
See https://codereview.chromium.org/462423004/ for previous code reviews.

-------------------------
 Original CL description
-------------------------

Revert CL 117933003. Re-add speculative resource prefetching code.
https://codereview.chromium.org/117933003/

The speculative resource prefetching code was experimental code developed by
shishir@. He found that it has little improvement on desktop Chrome (win). We
though this should be beneficial to mobile browsers. After discussing with
tburkard@ and kenjibaheux@, we decided to bring the code back and do more
analysis on mobile devices. Reverting the patchset to re-add the code is the
first step.

The following design doc has discussed all related approaches and action items.
https://docs.google.com/a/google.com/document/d/1ie3hu-zNNXvmTXm3aJAtKUGOh6nZfbNjA0aZE1bzzIg/edit?usp=sharing

BUG=408399, 405690

Review URL: https://codereview.chromium.org/576313002

Cr-Commit-Position: refs/heads/master@{#295626}

chrome/browser/net/resource_prefetch_predictor_observer.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/resource_prefetch_predictor_observer.h"
+
+#include <string>
+
+#include "base/metrics/histogram.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/resource_request_info.h"
+#include "net/url_request/url_request.h"
+#include "url/gurl.h"
+
+using content::BrowserThread;
+using predictors::ResourcePrefetchPredictor;
+
+namespace {
+
+// Enum for measuring statistics pertaining to observed request, responses and
+// redirects.
+enum RequestStats {
+  REQUEST_STATS_TOTAL_RESPONSES = 0,
+  REQUEST_STATS_TOTAL_PROCESSED_RESPONSES = 1,
+  REQUEST_STATS_NO_RESOURCE_REQUEST_INFO = 2,
+  REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO = 3,
+  REQUEST_STATS_MAX = 4,
+};
+
+// Specific to main frame requests.
+enum MainFrameRequestStats {
+  MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS = 0,
+  MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS = 1,
+  MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS = 2,
+  MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS = 3,
+  MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES = 4,
+  MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES = 5,
+  MAIN_FRAME_REQUEST_STATS_MAX = 6,
+};
+
+void ReportRequestStats(RequestStats stat) {
+  UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.RequestStats",
+                            stat,
+                            REQUEST_STATS_MAX);
+}
+
+void ReportMainFrameRequestStats(MainFrameRequestStats stat) {
+  UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.MainFrameRequestStats",
+                            stat,
+                            MAIN_FRAME_REQUEST_STATS_MAX);
+}
+
+bool SummarizeResponse(net::URLRequest* request,
+                       ResourcePrefetchPredictor::URLRequestSummary* summary) {
+  const content::ResourceRequestInfo* info =
+      content::ResourceRequestInfo::ForRequest(request);
+  if (!info) {
+    ReportRequestStats(REQUEST_STATS_NO_RESOURCE_REQUEST_INFO);
+    return false;
+  }
+
+  int render_process_id, render_frame_id;
+  if (!info->GetAssociatedRenderFrame(&render_process_id, &render_frame_id)) {
+    ReportRequestStats(REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO);
+    return false;
+  }
+
+  summary->navigation_id.render_process_id = render_process_id;
+  summary->navigation_id.render_frame_id = render_frame_id;
+  summary->navigation_id.main_frame_url = request->first_party_for_cookies();
+  summary->navigation_id.creation_time = request->creation_time();
+  summary->resource_url = request->original_url();
+  summary->resource_type = info->GetResourceType();
+  request->GetMimeType(&summary->mime_type);
+  summary->was_cached = request->was_cached();
+
+  // Use the mime_type to determine the resource type for subresources since
+  // types such as PREFETCH, SUB_RESOURCE, etc are not useful.
+  if (summary->resource_type != content::RESOURCE_TYPE_MAIN_FRAME) {
+    summary->resource_type =
+        ResourcePrefetchPredictor::GetResourceTypeFromMimeType(
+            summary->mime_type,
+            summary->resource_type);
+  }
+  return true;
+}
+
+}  // namespace
+
+namespace chrome_browser_net {
+
+ResourcePrefetchPredictorObserver::ResourcePrefetchPredictorObserver(
+    ResourcePrefetchPredictor* predictor)
+    : predictor_(predictor->AsWeakPtr()) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+}
+
+ResourcePrefetchPredictorObserver::~ResourcePrefetchPredictorObserver() {
+  CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) ||
+        BrowserThread::CurrentlyOn(BrowserThread::IO));
+}
+
+void ResourcePrefetchPredictorObserver::OnRequestStarted(
+    net::URLRequest* request,
+    content::ResourceType resource_type,
+    int child_id,
+    int frame_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME)
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS);
+
+  if (!ResourcePrefetchPredictor::ShouldRecordRequest(request, resource_type))
+    return;
+
+  ResourcePrefetchPredictor::URLRequestSummary summary;
+  summary.navigation_id.render_process_id = child_id;
+  summary.navigation_id.render_frame_id = frame_id;
+  summary.navigation_id.main_frame_url = request->first_party_for_cookies();
+  summary.resource_url = request->original_url();
+  summary.resource_type = resource_type;
+
+  BrowserThread::PostTask(
+      BrowserThread::UI,
+      FROM_HERE,
+      base::Bind(&ResourcePrefetchPredictor::RecordURLRequest,
+                 predictor_,
+                 summary));
+
+  if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME)
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS);
+}
+
+void ResourcePrefetchPredictorObserver::OnRequestRedirected(
+    const GURL& redirect_url,
+    net::URLRequest* request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  const content::ResourceRequestInfo* request_info =
+      content::ResourceRequestInfo::ForRequest(request);
+  if (request_info &&
+      request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS);
+  }
+
+  if (!ResourcePrefetchPredictor::ShouldRecordRedirect(request))
+    return;
+
+  ResourcePrefetchPredictor::URLRequestSummary summary;
+  if (!SummarizeResponse(request, &summary))
+    return;
+
+  summary.redirect_url = redirect_url;
+
+  BrowserThread::PostTask(
+      BrowserThread::UI,
+      FROM_HERE,
+      base::Bind(&ResourcePrefetchPredictor::RecordURLRedirect,
+                 predictor_,
+                 summary));
+
+  if (request_info &&
+      request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS);
+  }
+}
+
+void ResourcePrefetchPredictorObserver::OnResponseStarted(
+    net::URLRequest* request) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  ReportRequestStats(REQUEST_STATS_TOTAL_RESPONSES);
+
+  const content::ResourceRequestInfo* request_info =
+      content::ResourceRequestInfo::ForRequest(request);
+  if (request_info &&
+      request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES);
+  }
+
+  if (!ResourcePrefetchPredictor::ShouldRecordResponse(request))
+    return;
+  ResourcePrefetchPredictor::URLRequestSummary summary;
+  if (!SummarizeResponse(request, &summary))
+    return;
+
+  BrowserThread::PostTask(
+      BrowserThread::UI,
+      FROM_HERE,
+      base::Bind(&ResourcePrefetchPredictor::RecordURLResponse,
+                 predictor_,
+                 summary));
+
+  ReportRequestStats(REQUEST_STATS_TOTAL_PROCESSED_RESPONSES);
+  if (request_info &&
+      request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+    ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES);
+  }
+}
+
+}  // namespace chrome_browser_net

chrome/browser/net/resource_prefetch_predictor_observer.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_
+#define CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_
+
+#include "base/basictypes.h"
+#include "base/memory/weak_ptr.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "content/public/common/resource_type.h"
+
+namespace net {
+class URLRequest;
+}
+
+class GURL;
+
+namespace chrome_browser_net {
+
+// Observes resource requests in the ResourceDispatcherHostDelegate and notifies
+// the ResourcePrefetchPredictor about the ones it is interested in.
+//  - Has an instance per profile, and is owned by the corresponding
+//    ProfileIOData.
+//  - Needs to be constructed on UI thread. Rest of the functions can only be
+//    called on the IO thread. Can be destroyed on UI or IO thread.
+class ResourcePrefetchPredictorObserver {
+ public:
+  explicit ResourcePrefetchPredictorObserver(
+      predictors::ResourcePrefetchPredictor* predictor);
+  ~ResourcePrefetchPredictorObserver();
+
+  // Parts of the ResourceDispatcherHostDelegate that we want to observe.
+  void OnRequestStarted(net::URLRequest* request,
+                        content::ResourceType resource_type,
+                        int child_id,
+                        int frame_id);
+  void OnRequestRedirected(const GURL& redirect_url, net::URLRequest* request);
+  void OnResponseStarted(net::URLRequest* request);
+
+ private:
+  // Owned by profile.
+  base::WeakPtr<predictors::ResourcePrefetchPredictor> predictor_;
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorObserver);
+};
+
+}  // namespace chrome_browser_net
+
+#endif  // CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_

chrome/browser/predictors/predictor_database.cc

@@ -12,6 +12,8 @@
 #include "base/strings/stringprintf.h"
 #include "chrome/browser/predictors/autocomplete_action_predictor_table.h"
 #include "chrome/browser/predictors/logged_in_predictor_table.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
 #include "chrome/browser/prerender/prerender_field_trial.h"
 #include "chrome/browser/profiles/profile.h"
 #include "content/public/browser/browser_thread.h"
@@ -50,6 +52,7 @@ class PredictorDatabaseInternal
   // Cancels pending DB transactions. Should only be called on the UI thread.
   void SetCancelled();
 
+  bool is_resource_prefetch_predictor_enabled_;
   base::FilePath db_path_;
   scoped_ptr<sql::Connection> db_;
 
@@ -57,6 +60,7 @@ class PredictorDatabaseInternal
   // to using a WeakPtr instead.
   scoped_refptr<AutocompleteActionPredictorTable> autocomplete_table_;
   scoped_refptr<LoggedInPredictorTable> logged_in_table_;
+  scoped_refptr<ResourcePrefetchPredictorTables> resource_prefetch_tables_;
 
   DISALLOW_COPY_AND_ASSIGN(PredictorDatabaseInternal);
 };
@@ -66,8 +70,12 @@ PredictorDatabaseInternal::PredictorDatabaseInternal(Profile* profile)
     : db_path_(profile->GetPath().Append(kPredictorDatabaseName)),
       db_(new sql::Connection()),
       autocomplete_table_(new AutocompleteActionPredictorTable()),
-      logged_in_table_(new LoggedInPredictorTable()) {
+      logged_in_table_(new LoggedInPredictorTable()),
+      resource_prefetch_tables_(new ResourcePrefetchPredictorTables()) {
   db_->set_histogram_tag("Predictor");
+  ResourcePrefetchPredictorConfig config;
+  is_resource_prefetch_predictor_enabled_ =
+      IsSpeculativeResourcePrefetchingEnabled(profile, &config);
 }
 
 PredictorDatabaseInternal::~PredictorDatabaseInternal() {
@@ -89,6 +97,7 @@ void PredictorDatabaseInternal::Initialize() {
 
   autocomplete_table_->Initialize(db_.get());
   logged_in_table_->Initialize(db_.get());
+  resource_prefetch_tables_->Initialize(db_.get());
 
   LogDatabaseStats();
 }
@@ -99,6 +108,7 @@ void PredictorDatabaseInternal::SetCancelled() {
 
   autocomplete_table_->SetCancelled();
   logged_in_table_->SetCancelled();
+  resource_prefetch_tables_->SetCancelled();
 }
 
 void PredictorDatabaseInternal::LogDatabaseStats() {
@@ -113,6 +123,8 @@ void PredictorDatabaseInternal::LogDatabaseStats() {
 
   autocomplete_table_->LogDatabaseStats();
   logged_in_table_->LogDatabaseStats();
+  if (is_resource_prefetch_predictor_enabled_)
+    resource_prefetch_tables_->LogDatabaseStats();
 }
 
 PredictorDatabase::PredictorDatabase(Profile* profile)
@@ -138,6 +150,11 @@ scoped_refptr<LoggedInPredictorTable>
   return db_->logged_in_table_;
 }
 
+scoped_refptr<ResourcePrefetchPredictorTables>
+    PredictorDatabase::resource_prefetch_tables() {
+  return db_->resource_prefetch_tables_;
+}
+
 sql::Connection* PredictorDatabase::GetDatabase() {
   return db_->db_.get();
 }

chrome/browser/predictors/predictor_database.h

@@ -19,6 +19,7 @@ namespace predictors {
 class AutocompleteActionPredictorTable;
 class LoggedInPredictorTable;
 class PredictorDatabaseInternal;
+class ResourcePrefetchPredictorTables;
 
 class PredictorDatabase : public KeyedService {
  public:
@@ -26,6 +27,7 @@ class PredictorDatabase : public KeyedService {
   virtual ~PredictorDatabase();
 
   scoped_refptr<AutocompleteActionPredictorTable> autocomplete_table();
+  scoped_refptr<ResourcePrefetchPredictorTables> resource_prefetch_tables();
   scoped_refptr<LoggedInPredictorTable> logged_in_table();
 
   // Used for testing.

chrome/browser/predictors/resource_prefetch_common.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+
+#include "base/command_line.h"
+#include "base/metrics/field_trial.h"
+#include "base/prefs/pref_service.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/common/pref_names.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/render_view_host.h"
+#include "content/public/browser/web_contents.h"
+
+namespace predictors {
+
+const char kSpeculativePrefetchingTrialName[] =
+    "SpeculativeResourcePrefetching";
+
+bool IsSpeculativeResourcePrefetchingEnabled(
+    Profile* profile,
+    ResourcePrefetchPredictorConfig* config) {
+  DCHECK(config);
+
+  // Off the record - disabled.
+  if (!profile || profile->IsOffTheRecord())
+    return false;
+
+  // If the user has explicitly disabled "predictive actions" - disabled.
+  if (!profile->GetPrefs() ||
+      !profile->GetPrefs()->GetBoolean(prefs::kNetworkPredictionEnabled)) {
+    return false;
+  }
+
+  // The config has the default params already set. The command line with just
+  // enable them with the default params.
+  if (CommandLine::ForCurrentProcess()->HasSwitch(
+          switches::kSpeculativeResourcePrefetching)) {
+    const std::string value =
+        CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+            switches::kSpeculativeResourcePrefetching);
+
+    if (value == switches::kSpeculativeResourcePrefetchingDisabled) {
+      return false;
+    } else if (value == switches::kSpeculativeResourcePrefetchingLearning) {
+      config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+      config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+      return true;
+    } else if (value == switches::kSpeculativeResourcePrefetchingEnabled) {
+      config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+      config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+      config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+      config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+      return true;
+    }
+  }
+
+  std::string trial = base::FieldTrialList::FindFullName(
+      kSpeculativePrefetchingTrialName);
+
+  if (trial == "LearningHost") {
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    return true;
+  } else if (trial == "LearningURL") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    return true;
+  } else if (trial == "Learning") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    return true;
+  } else if (trial == "PrefetchingHost") {
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+    return true;
+  } else if (trial == "PrefetchingURL") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    return true;
+  } else if (trial == "Prefetching") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+    return true;
+  } else if (trial == "PrefetchingLowConfidence") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->min_url_visit_count = 1;
+    config->min_resource_confidence_to_trigger_prefetch = 0.5f;
+    config->min_resource_hits_to_trigger_prefetch = 1;
+    return true;
+  } else if (trial == "PrefetchingHighConfidence") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->min_url_visit_count = 3;
+    config->min_resource_confidence_to_trigger_prefetch = 0.9f;
+    config->min_resource_hits_to_trigger_prefetch = 3;
+    return true;
+  } else if (trial == "PrefetchingMoreResources") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->max_resources_per_entry = 100;
+    return true;
+  } else if (trial == "LearningSmallDB") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+
+    config->max_urls_to_track = 200;
+    config->max_hosts_to_track = 100;
+    return true;
+  } else if (trial == "PrefetchingSmallDB") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->max_urls_to_track = 200;
+    config->max_hosts_to_track = 100;
+    return true;
+  } else if (trial == "PrefetchingSmallDBLowConfidence") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->max_urls_to_track = 200;
+    config->max_hosts_to_track = 100;
+    config->min_url_visit_count = 1;
+    config->min_resource_confidence_to_trigger_prefetch = 0.5f;
+    config->min_resource_hits_to_trigger_prefetch = 1;
+    return true;
+  } else if (trial == "PrefetchingSmallDBHighConfidence") {
+    config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+    config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+    config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+    config->max_urls_to_track = 200;
+    config->max_hosts_to_track = 100;
+    config->min_url_visit_count = 3;
+    config->min_resource_confidence_to_trigger_prefetch = 0.9f;
+    config->min_resource_hits_to_trigger_prefetch = 3;
+    return true;
+  }
+
+  return false;
+}
+
+NavigationID::NavigationID()
+    : render_process_id(-1),
+      render_frame_id(-1) {
+}
+
+NavigationID::NavigationID(const NavigationID& other)
+    : render_process_id(other.render_process_id),
+      render_frame_id(other.render_frame_id),
+      main_frame_url(other.main_frame_url),
+      creation_time(other.creation_time) {
+}
+
+NavigationID::NavigationID(content::WebContents* web_contents)
+    : render_process_id(web_contents->GetRenderProcessHost()->GetID()),
+      render_frame_id(web_contents->GetRenderViewHost()->GetRoutingID()),
+      main_frame_url(web_contents->GetURL()) {
+}
+
+bool NavigationID::is_valid() const {
+  return render_process_id != -1 && render_frame_id != -1 &&
+      !main_frame_url.is_empty();
+}
+
+bool NavigationID::operator<(const NavigationID& rhs) const {
+  DCHECK(is_valid() && rhs.is_valid());
+  if (render_process_id != rhs.render_process_id)
+    return render_process_id < rhs.render_process_id;
+  else if (render_frame_id != rhs.render_frame_id)
+    return render_frame_id < rhs.render_frame_id;
+  else
+    return main_frame_url < rhs.main_frame_url;
+}
+
+bool NavigationID::operator==(const NavigationID& rhs) const {
+  DCHECK(is_valid() && rhs.is_valid());
+  return IsSameRenderer(rhs) && main_frame_url == rhs.main_frame_url;
+}
+
+bool NavigationID::IsSameRenderer(const NavigationID& other) const {
+  DCHECK(is_valid() && other.is_valid());
+  return render_process_id == other.render_process_id &&
+      render_frame_id == other.render_frame_id;
+}
+
+ResourcePrefetchPredictorConfig::ResourcePrefetchPredictorConfig()
+    : mode(0),
+      max_navigation_lifetime_seconds(60),
+      max_urls_to_track(500),
+      max_hosts_to_track(200),
+      min_url_visit_count(2),
+      max_resources_per_entry(50),
+      max_consecutive_misses(3),
+      min_resource_confidence_to_trigger_prefetch(0.8f),
+      min_resource_hits_to_trigger_prefetch(3),
+      max_prefetches_inflight_per_navigation(24),
+      max_prefetches_inflight_per_host_per_navigation(3) {
+}
+
+ResourcePrefetchPredictorConfig::~ResourcePrefetchPredictorConfig() {
+}
+
+bool ResourcePrefetchPredictorConfig::IsLearningEnabled() const {
+  return IsURLLearningEnabled() || IsHostLearningEnabled();
+}
+
+bool ResourcePrefetchPredictorConfig::IsPrefetchingEnabled() const {
+  return IsURLPrefetchingEnabled() || IsHostPrefetchingEnabled();
+}
+
+bool ResourcePrefetchPredictorConfig::IsURLLearningEnabled() const {
+  return (mode & URL_LEARNING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsHostLearningEnabled() const {
+  return (mode & HOST_LEARNING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsURLPrefetchingEnabled() const {
+  return (mode & URL_PREFETCHING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsHostPrefetchingEnabled() const {
+  return (mode & HOST_PRFETCHING) > 0;
+}
+
+}  // namespace predictors

chrome/browser/predictors/resource_prefetch_common.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_
+
+#include "base/time/time.h"
+#include "url/gurl.h"
+
+class Profile;
+
+namespace content {
+class WebContents;
+}
+
+namespace predictors {
+
+struct ResourcePrefetchPredictorConfig;
+
+// Returns true if prefetching is enabled. And will initilize the |config|
+// fields to the appropritate values.
+bool IsSpeculativeResourcePrefetchingEnabled(
+    Profile* profile,
+    ResourcePrefetchPredictorConfig* config);
+
+// Represents the type of key based on which prefetch data is stored.
+enum PrefetchKeyType {
+  PREFETCH_KEY_TYPE_HOST,
+  PREFETCH_KEY_TYPE_URL
+};
+
+// Represents a single navigation for a render frame.
+struct NavigationID {
+  // TODO(shishir): Maybe take process_id, frame_id and url as input in
+  // constructor.
+  NavigationID();
+  NavigationID(const NavigationID& other);
+  explicit NavigationID(content::WebContents* web_contents);
+  bool operator<(const NavigationID& rhs) const;
+  bool operator==(const NavigationID& rhs) const;
+
+  bool IsSameRenderer(const NavigationID& other) const;
+
+  // Returns true iff the render_process_id_, render_frame_id_ and
+  // frame_url_ has been set correctly.
+  bool is_valid() const;
+
+  int render_process_id;
+  int render_frame_id;
+  GURL main_frame_url;
+
+  // NOTE: Even though we store the creation time here, it is not used during
+  // comparison of two NavigationIDs because it cannot always be determined
+  // correctly.
+  base::TimeTicks creation_time;
+};
+
+// Represents the config for the resource prefetch prediction algorithm. It is
+// useful for running experiments.
+struct ResourcePrefetchPredictorConfig {
+  // Initializes the config with default values.
+  ResourcePrefetchPredictorConfig();
+  ~ResourcePrefetchPredictorConfig();
+
+  // The mode the prefetcher is running in. Forms a bit map.
+  enum Mode {
+    URL_LEARNING    = 1 << 0,
+    HOST_LEARNING   = 1 << 1,
+    URL_PREFETCHING = 1 << 2,  // Should also turn on URL_LEARNING.
+    HOST_PRFETCHING = 1 << 3   // Should also turn on HOST_LEARNING.
+  };
+  int mode;
+
+  // Helpers to deal with mode.
+  bool IsLearningEnabled() const;
+  bool IsPrefetchingEnabled() const;
+  bool IsURLLearningEnabled() const;
+  bool IsHostLearningEnabled() const;
+  bool IsURLPrefetchingEnabled() const;
+  bool IsHostPrefetchingEnabled() const;
+
+  // If a navigation hasn't seen a load complete event in this much time, it
+  // is considered abandoned.
+  size_t max_navigation_lifetime_seconds;
+
+  // Size of LRU caches for the URL and host data.
+  size_t max_urls_to_track;
+  size_t max_hosts_to_track;
+
+  // The number of times we should have seen a visit to this URL in history
+  // to start tracking it. This is to ensure we don't bother with oneoff
+  // entries. For hosts we track each one.
+  size_t min_url_visit_count;
+
+  // The maximum number of resources to store per entry.
+  size_t max_resources_per_entry;
+  // The number of consecutive misses after we stop tracking a resource URL.
+  size_t max_consecutive_misses;
+
+  // The minimum confidence (accuracy of hits) required for a resource to be
+  // prefetched.
+  float min_resource_confidence_to_trigger_prefetch;
+  // The minimum number of times we must have a URL on record to prefetch it.
+  size_t min_resource_hits_to_trigger_prefetch;
+
+  // Maximum number of prefetches that can be inflight for a single navigation.
+  size_t max_prefetches_inflight_per_navigation;
+  // Maximum number of prefetches that can be inflight for a host for a single
+  // navigation.
+  size_t max_prefetches_inflight_per_host_per_navigation;
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_

chrome/browser/predictors/resource_prefetch_predictor_factory.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+
+#include "chrome/browser/history/history_service_factory.h"
+#include "chrome/browser/predictors/predictor_database_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/profiles/profile.h"
+#include "components/keyed_service/content/browser_context_dependency_manager.h"
+#include "components/keyed_service/content/browser_context_keyed_service_factory.h"
+
+namespace predictors {
+
+// static
+ResourcePrefetchPredictor* ResourcePrefetchPredictorFactory::GetForProfile(
+    content::BrowserContext* context) {
+  return static_cast<ResourcePrefetchPredictor*>(
+      GetInstance()->GetServiceForBrowserContext(context, true));
+}
+
+// static
+ResourcePrefetchPredictorFactory*
+ResourcePrefetchPredictorFactory::GetInstance() {
+  return Singleton<ResourcePrefetchPredictorFactory>::get();
+}
+
+ResourcePrefetchPredictorFactory::ResourcePrefetchPredictorFactory()
+    : BrowserContextKeyedServiceFactory(
+        "ResourcePrefetchPredictor",
+        BrowserContextDependencyManager::GetInstance()) {
+  DependsOn(HistoryServiceFactory::GetInstance());
+  DependsOn(PredictorDatabaseFactory::GetInstance());
+}
+
+ResourcePrefetchPredictorFactory::~ResourcePrefetchPredictorFactory() {}
+
+KeyedService*
+    ResourcePrefetchPredictorFactory::BuildServiceInstanceFor(
+        content::BrowserContext* context) const {
+  Profile* profile = Profile::FromBrowserContext(context);
+
+  ResourcePrefetchPredictorConfig config;
+  if (!IsSpeculativeResourcePrefetchingEnabled(profile, &config))
+    return NULL;
+
+  return new ResourcePrefetchPredictor(config, profile);
+}
+
+}  // namespace predictors

chrome/browser/predictors/resource_prefetch_predictor_factory.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_
+
+#include "base/basictypes.h"
+#include "base/memory/singleton.h"
+#include "components/keyed_service/content/browser_context_keyed_service_factory.h"
+
+namespace predictors {
+
+class ResourcePrefetchPredictor;
+
+class ResourcePrefetchPredictorFactory
+    : public BrowserContextKeyedServiceFactory {
+ public:
+  static ResourcePrefetchPredictor* GetForProfile(
+      content::BrowserContext* context);
+  static ResourcePrefetchPredictorFactory* GetInstance();
+
+ private:
+  friend struct DefaultSingletonTraits<ResourcePrefetchPredictorFactory>;
+
+  ResourcePrefetchPredictorFactory();
+  virtual ~ResourcePrefetchPredictorFactory();
+
+  // RefcountedBrowserContextKeyedServiceFactory:
+  virtual KeyedService* BuildServiceInstanceFor(
+      content::BrowserContext* context) const OVERRIDE;
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorFactory);
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_

chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h"
+
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+#include "chrome/browser/profiles/profile.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/load_from_memory_cache_details.h"
+
+DEFINE_WEB_CONTENTS_USER_DATA_KEY(
+    predictors::ResourcePrefetchPredictorTabHelper);
+
+using content::BrowserThread;
+
+namespace predictors {
+
+ResourcePrefetchPredictorTabHelper::ResourcePrefetchPredictorTabHelper(
+    content::WebContents* web_contents)
+    : content::WebContentsObserver(web_contents) {
+}
+
+ResourcePrefetchPredictorTabHelper::~ResourcePrefetchPredictorTabHelper() {
+}
+
+void ResourcePrefetchPredictorTabHelper::DocumentOnLoadCompletedInMainFrame() {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+  ResourcePrefetchPredictor* predictor =
+      ResourcePrefetchPredictorFactory::GetForProfile(
+          web_contents()->GetBrowserContext());
+  if (!predictor)
+    return;
+
+  NavigationID navigation_id(web_contents());
+  predictor->RecordMainFrameLoadComplete(navigation_id);
+}
+
+void ResourcePrefetchPredictorTabHelper::DidLoadResourceFromMemoryCache(
+    const content::LoadFromMemoryCacheDetails& details) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+  ResourcePrefetchPredictor* predictor =
+      ResourcePrefetchPredictorFactory::GetForProfile(
+          web_contents()->GetBrowserContext());
+  if (!predictor)
+    return;
+
+  ResourcePrefetchPredictor::URLRequestSummary summary;
+  summary.navigation_id = NavigationID(web_contents());
+  summary.resource_url = details.url;
+  summary.mime_type = details.mime_type;
+  summary.resource_type =
+      ResourcePrefetchPredictor::GetResourceTypeFromMimeType(
+          details.mime_type, details.resource_type);
+  summary.was_cached = true;
+  predictor->RecordURLResponse(summary);
+}
+
+}  // namespace predictors

chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_
+
+#include "content/public/browser/web_contents_observer.h"
+#include "content/public/browser/web_contents_user_data.h"
+
+namespace predictors {
+
+class ResourcePrefetchPredictorTabHelper
+    : public content::WebContentsObserver,
+      public content::WebContentsUserData<ResourcePrefetchPredictorTabHelper> {
+ public:
+  virtual ~ResourcePrefetchPredictorTabHelper();
+
+  // content::WebContentsObserver implementation
+  virtual void DocumentOnLoadCompletedInMainFrame() OVERRIDE;
+  virtual void DidLoadResourceFromMemoryCache(
+      const content::LoadFromMemoryCacheDetails& details) OVERRIDE;
+
+ private:
+  explicit ResourcePrefetchPredictorTabHelper(
+      content::WebContents* web_contents);
+  friend class content::WebContentsUserData<ResourcePrefetchPredictorTabHelper>;
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTabHelper);
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_

chrome/browser/predictors/resource_prefetch_predictor_tables.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_
+
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "base/time/time.h"
+#include "chrome/browser/predictors/predictor_table_base.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "content/public/common/resource_type.h"
+#include "url/gurl.h"
+
+namespace sql {
+class Statement;
+}
+
+namespace predictors {
+
+// Interface for database tables used by the ResourcePrefetchPredictor.
+// All methods except the constructor and destructor need to be called on the DB
+// thread.
+//
+// Currently manages:
+//  - UrlResourceTable - resources per Urls.
+//  - UrlMetadataTable - misc data for Urls (like last visit time).
+//  - HostResourceTable - resources per host.
+//  - HostMetadataTable - misc data for hosts.
+class ResourcePrefetchPredictorTables : public PredictorTableBase {
+ public:
+  // Used in the UrlResourceTable and HostResourceTable to store resources
+  // required for the page or host.
+  struct ResourceRow {
+    ResourceRow();
+    ResourceRow(const ResourceRow& other);
+    ResourceRow(const std::string& main_frame_url,
+                const std::string& resource_url,
+                content::ResourceType resource_type,
+                int number_of_hits,
+                int number_of_misses,
+                int consecutive_misses,
+                double average_position);
+    void UpdateScore();
+    bool operator==(const ResourceRow& rhs) const;
+
+    // Stores the host for host based data, main frame Url for the Url based
+    // data. This field is cleared for efficiency reasons and the code outside
+    // this class should not assume it is set.
+    std::string primary_key;
+
+    GURL resource_url;
+    content::ResourceType resource_type;
+    size_t number_of_hits;
+    size_t number_of_misses;
+    size_t consecutive_misses;
+    double average_position;
+
+    // Not stored.
+    float score;
+  };
+  typedef std::vector<ResourceRow> ResourceRows;
+
+  // Sorts the ResourceRows by score, descending.
+  struct ResourceRowSorter {
+    bool operator()(const ResourceRow& x, const ResourceRow& y) const;
+  };
+
+  // Aggregated data for a Url or Host. Although the data differs slightly, we
+  // store them in the same structure, because most of the fields are common and
+  // it allows us to use the same functions.
+  struct PrefetchData {
+    PrefetchData(PrefetchKeyType key_type, const std::string& primary_key);
+    PrefetchData(const PrefetchData& other);
+    ~PrefetchData();
+    bool operator==(const PrefetchData& rhs) const;
+
+    bool is_host() const { return key_type == PREFETCH_KEY_TYPE_HOST; }
+
+    // Is the data a host as opposed to a Url?
+    PrefetchKeyType key_type;  // Not const to be able to assign.
+    std::string primary_key;  // is_host() ? main frame url : host.
+
+    base::Time last_visit;
+    ResourceRows resources;
+  };
+  // Map from primary key to PrefetchData for the key.
+  typedef std::map<std::string, PrefetchData> PrefetchDataMap;
+
+  // Returns data for all Urls and Hosts.
+  virtual void GetAllData(PrefetchDataMap* url_data_map,
+                          PrefetchDataMap* host_data_map);
+
+  // Updates data for a Url and a host. If either of the |url_data| or
+  // |host_data| has an empty primary key, it will be ignored.
+  // Note that the Urls and primary key in |url_data| and |host_data| should be
+  // less than |kMaxStringLength| in length.
+  virtual void UpdateData(const PrefetchData& url_data,
+                          const PrefetchData& host_data);
+
+  // Delete data for the input |urls| and |hosts|.
+  virtual void DeleteData(const std::vector<std::string>& urls,
+                  const std::vector<std::string>& hosts);
+
+  // Wrapper over DeleteData for convenience.
+  virtual void DeleteSingleDataPoint(const std::string& key,
+                                     PrefetchKeyType key_type);
+
+  // Deletes all data in all the tables.
+  virtual void DeleteAllData();
+
+  // The maximum length of the string that can be stored in the DB.
+  static const size_t kMaxStringLength;
+
+ private:
+  friend class PredictorDatabaseInternal;
+  friend class MockResourcePrefetchPredictorTables;
+
+  ResourcePrefetchPredictorTables();
+  virtual ~ResourcePrefetchPredictorTables();
+
+  // Helper functions below help perform functions on the Url and host table
+  // using the same code.
+  void GetAllDataHelper(PrefetchKeyType key_type,
+                        PrefetchDataMap* data_map,
+                        std::vector<std::string>* to_delete);
+  bool UpdateDataHelper(const PrefetchData& data);
+  void DeleteDataHelper(PrefetchKeyType key_type,
+                        const std::vector<std::string>& keys);
+
+  // Returns true if the strings in the |data| are less than |kMaxStringLength|
+  // in length.
+  bool StringsAreSmallerThanDBLimit(const PrefetchData& data) const;
+
+  // PredictorTableBase methods.
+  virtual void CreateTableIfNonExistent() OVERRIDE;
+  virtual void LogDatabaseStats() OVERRIDE;
+
+  // Helpers to return Statements for cached Statements. The caller must take
+  // ownership of the return Statements.
+  sql::Statement* GetUrlResourceDeleteStatement();
+  sql::Statement* GetUrlResourceUpdateStatement();
+  sql::Statement* GetUrlMetadataDeleteStatement();
+  sql::Statement* GetUrlMetadataUpdateStatement();
+
+  sql::Statement* GetHostResourceDeleteStatement();
+  sql::Statement* GetHostResourceUpdateStatement();
+  sql::Statement* GetHostMetadataDeleteStatement();
+  sql::Statement* GetHostMetadataUpdateStatement();
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTables);
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_

chrome/browser/predictors/resource_prefetcher.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetcher.h"
+
+#include <iterator>
+
+#include "base/stl_util.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/base/io_buffer.h"
+#include "net/base/request_priority.h"
+#include "net/url_request/url_request_context.h"
+
+namespace {
+
+// The size of the buffer used to read the resource.
+static const size_t kResourceBufferSizeBytes = 50000;
+
+}  // namespace
+
+namespace predictors {
+
+ResourcePrefetcher::Request::Request(const GURL& i_resource_url)
+    : resource_url(i_resource_url),
+      prefetch_status(PREFETCH_STATUS_NOT_STARTED),
+      usage_status(USAGE_STATUS_NOT_REQUESTED) {
+}
+
+ResourcePrefetcher::Request::Request(const Request& other)
+    : resource_url(other.resource_url),
+      prefetch_status(other.prefetch_status),
+      usage_status(other.usage_status) {
+}
+
+ResourcePrefetcher::ResourcePrefetcher(
+    Delegate* delegate,
+    const ResourcePrefetchPredictorConfig& config,
+    const NavigationID& navigation_id,
+    PrefetchKeyType key_type,
+    scoped_ptr<RequestVector> requests)
+        : state_(INITIALIZED),
+          delegate_(delegate),
+          config_(config),
+          navigation_id_(navigation_id),
+          key_type_(key_type),
+          request_vector_(requests.Pass()) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+  DCHECK(request_vector_.get());
+
+  std::copy(request_vector_->begin(), request_vector_->end(),
+            std::back_inserter(request_queue_));
+}
+
+ResourcePrefetcher::~ResourcePrefetcher() {
+  // Delete any pending net::URLRequests.
+  STLDeleteContainerPairFirstPointers(inflight_requests_.begin(),
+                                      inflight_requests_.end());
+}
+
+void ResourcePrefetcher::Start() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  CHECK_EQ(state_, INITIALIZED);
+  state_ = RUNNING;
+
+  TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::Stop() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (state_ == FINISHED)
+    return;
+
+  state_ = STOPPED;
+}
+
+void ResourcePrefetcher::TryToLaunchPrefetchRequests() {
+  CHECK(state_ == RUNNING || state_ == STOPPED);
+
+  // Try to launch new requests if the state is RUNNING.
+  if (state_ == RUNNING) {
+    bool request_available = true;
+
+    // Loop through the requests while we are under the
+    // max_prefetches_inflight_per_host_per_navigation limit, looking for a URL
+    // for which the max_prefetches_inflight_per_host_per_navigation limit has
+    // not been reached. Try to launch as many requests as possible.
+    while ((inflight_requests_.size() <
+                config_.max_prefetches_inflight_per_navigation) &&
+           request_available) {
+      std::list<Request*>::iterator request_it = request_queue_.begin();
+      for (; request_it != request_queue_.end(); ++request_it) {
+        const std::string& host = (*request_it)->resource_url.host();
+
+        std::map<std::string, size_t>::iterator host_it =
+            host_inflight_counts_.find(host);
+        if (host_it == host_inflight_counts_.end() ||
+            host_it->second <
+                config_.max_prefetches_inflight_per_host_per_navigation)
+          break;
+      }
+      request_available = request_it != request_queue_.end();
+
+      if (request_available) {
+        SendRequest(*request_it);
+        request_queue_.erase(request_it);
+      }
+    }
+  }
+
+  // If the inflight_requests_ is empty, we cant launch any more. Finish.
+  if (inflight_requests_.empty()) {
+    CHECK(host_inflight_counts_.empty());
+    CHECK(request_queue_.empty() || state_ == STOPPED);
+
+    state_ = FINISHED;
+    delegate_->ResourcePrefetcherFinished(this, request_vector_.release());
+  }
+}
+
+void ResourcePrefetcher::SendRequest(Request* request) {
+  request->prefetch_status = Request::PREFETCH_STATUS_STARTED;
+
+  net::URLRequest* url_request =
+      delegate_->GetURLRequestContext()->CreateRequest(
+          request->resource_url, net::LOW, this, NULL).release();
+
+  inflight_requests_[url_request] = request;
+  host_inflight_counts_[url_request->original_url().host()] += 1;
+
+  url_request->set_method("GET");
+  url_request->set_first_party_for_cookies(navigation_id_.main_frame_url);
+  url_request->SetReferrer(navigation_id_.main_frame_url.spec());
+  StartURLRequest(url_request);
+}
+
+void ResourcePrefetcher::StartURLRequest(net::URLRequest* request) {
+  request->Start();
+}
+
+void ResourcePrefetcher::FinishRequest(net::URLRequest* request,
+                                       Request::PrefetchStatus status) {
+  std::map<net::URLRequest*, Request*>::iterator request_it =
+      inflight_requests_.find(request);
+  CHECK(request_it != inflight_requests_.end());
+
+  const std::string host = request->original_url().host();
+  std::map<std::string, size_t>::iterator host_it = host_inflight_counts_.find(
+      host);
+  CHECK_GT(host_it->second, 0U);
+  host_it->second -= 1;
+  if (host_it->second == 0)
+    host_inflight_counts_.erase(host);
+
+  request_it->second->prefetch_status = status;
+  inflight_requests_.erase(request_it);
+
+  delete request;
+
+  TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::ReadFullResponse(net::URLRequest* request) {
+  bool status = true;
+  while (status) {
+    int bytes_read = 0;
+    scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer(
+        kResourceBufferSizeBytes));
+    status = request->Read(buffer.get(), kResourceBufferSizeBytes, &bytes_read);
+
+    if (status) {
+      status = ShouldContinueReadingRequest(request, bytes_read);
+    } else if (request->status().error()) {
+      FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+      return;
+    }
+  }
+}
+
+bool ResourcePrefetcher::ShouldContinueReadingRequest(net::URLRequest* request,
+                                                      int bytes_read) {
+  if (bytes_read == 0) {  // When bytes_read == 0, no more data.
+    if (request->was_cached())
+      FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE);
+    else
+      FinishRequest(request, Request::PREFETCH_STATUS_FROM_NETWORK);
+    return false;
+  }
+
+  return true;
+}
+
+void ResourcePrefetcher::OnReceivedRedirect(
+    net::URLRequest* request,
+    const net::RedirectInfo& redirect_info,
+    bool* defer_redirect) {
+  FinishRequest(request, Request::PREFETCH_STATUS_REDIRECTED);
+}
+
+void ResourcePrefetcher::OnAuthRequired(net::URLRequest* request,
+                                        net::AuthChallengeInfo* auth_info) {
+  FinishRequest(request, Request::PREFETCH_STATUS_AUTH_REQUIRED);
+}
+
+void ResourcePrefetcher::OnCertificateRequested(
+    net::URLRequest* request,
+    net::SSLCertRequestInfo* cert_request_info) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CERT_REQUIRED);
+}
+
+void ResourcePrefetcher::OnSSLCertificateError(net::URLRequest* request,
+                                               const net::SSLInfo& ssl_info,
+                                               bool fatal) {
+  FinishRequest(request, Request::PREFETCH_STATUS_CERT_ERROR);
+}
+
+void ResourcePrefetcher::OnResponseStarted(net::URLRequest* request) {
+  if (request->status().error()) {
+    FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+    return;
+  }
+
+  // TODO(shishir): Do not read cached entries, or ones that are not cacheable.
+  ReadFullResponse(request);
+}
+
+void ResourcePrefetcher::OnReadCompleted(net::URLRequest* request,
+                                         int bytes_read) {
+  if (request->status().error()) {
+    FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+    return;
+  }
+
+  if (ShouldContinueReadingRequest(request, bytes_read))
+    ReadFullResponse(request);
+}
+
+}  // namespace predictors

chrome/browser/predictors/resource_prefetcher.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_
+
+#include <list>
+#include <map>
+#include <vector>
+
+#include "base/gtest_prod_util.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/scoped_vector.h"
+#include "base/threading/thread_checker.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "net/url_request/redirect_info.h"
+#include "net/url_request/url_request.h"
+#include "url/gurl.h"
+
+namespace net {
+class URLRequestContext;
+}
+
+namespace predictors {
+
+// Responsible for prefetching resources for a single navigation based on the
+// input list of resources.
+//  - Limits the max number of resources in flight for any host and also across
+//    hosts.
+//  - When stopped, will wait for the pending requests to finish.
+//  - Lives entirely on the IO thread.
+class ResourcePrefetcher : public net::URLRequest::Delegate {
+ public:
+  // Denotes the prefetch request for a single subresource.
+  struct Request {
+    explicit Request(const GURL& i_resource_url);
+    Request(const Request& other);
+
+    enum PrefetchStatus {
+      PREFETCH_STATUS_NOT_STARTED,
+      PREFETCH_STATUS_STARTED,
+
+      // Cancellation reasons.
+      PREFETCH_STATUS_REDIRECTED,
+      PREFETCH_STATUS_AUTH_REQUIRED,
+      PREFETCH_STATUS_CERT_REQUIRED,
+      PREFETCH_STATUS_CERT_ERROR,
+      PREFETCH_STATUS_CANCELLED,
+      PREFETCH_STATUS_FAILED,
+
+      // Successful prefetch states.
+      PREFETCH_STATUS_FROM_CACHE,
+      PREFETCH_STATUS_FROM_NETWORK
+    };
+
+    enum UsageStatus {
+      USAGE_STATUS_NOT_REQUESTED,
+      USAGE_STATUS_FROM_CACHE,
+      USAGE_STATUS_FROM_NETWORK,
+      USAGE_STATUS_NAVIGATION_ABANDONED
+    };
+
+    GURL resource_url;
+    PrefetchStatus prefetch_status;
+    UsageStatus usage_status;
+  };
+  typedef ScopedVector<Request> RequestVector;
+
+  // Used to communicate when the prefetching is done. All methods are invoked
+  // on the IO thread.
+  class Delegate {
+   public:
+    virtual ~Delegate() { }
+
+    // Called when the ResourcePrefetcher is finished, i.e. there is nothing
+    // pending in flight. Should take ownership of |requests|.
+    virtual void ResourcePrefetcherFinished(
+        ResourcePrefetcher* prefetcher,
+        RequestVector* requests) = 0;
+
+    virtual net::URLRequestContext* GetURLRequestContext() = 0;
+  };
+
+  // |delegate| has to outlive the ResourcePrefetcher. The ResourcePrefetcher
+  // takes ownership of |requests|.
+  ResourcePrefetcher(Delegate* delegate,
+                     const ResourcePrefetchPredictorConfig& config,
+                     const NavigationID& navigation_id,
+                     PrefetchKeyType key_type,
+                     scoped_ptr<RequestVector> requests);
+  virtual ~ResourcePrefetcher();
+
+  void Start();  // Kicks off the prefetching. Can only be called once.
+  void Stop();   // No additional prefetches will be queued after this.
+
+  const NavigationID& navigation_id() const { return navigation_id_; }
+  PrefetchKeyType key_type() const { return key_type_; }
+
+ private:
+  friend class ResourcePrefetcherTest;
+  friend class TestResourcePrefetcher;
+
+  // Launches new prefetch requests if possible.
+  void TryToLaunchPrefetchRequests();
+
+  // Starts a net::URLRequest for the input |request|.
+  void SendRequest(Request* request);
+
+  // Called by |SendRequest| to start the |request|. This is necessary to stub
+  // out the Start() call to net::URLRequest for unittesting.
+  virtual void StartURLRequest(net::URLRequest* request);
+
+  // Marks the request as finished, with the given status.
+  void FinishRequest(net::URLRequest* request, Request::PrefetchStatus status);
+
+  // Reads the response data from the response - required for the resource to
+  // be cached correctly. Stubbed out during testing.
+  virtual void ReadFullResponse(net::URLRequest* request);
+
+  // Returns true if the request has more data that needs to be read. If it
+  // returns false, the request should not be referenced again.
+  bool ShouldContinueReadingRequest(net::URLRequest* request, int bytes_read);
+
+  // net::URLRequest::Delegate methods.
+  virtual void OnReceivedRedirect(net::URLRequest* request,
+                                  const net::RedirectInfo& redirect_info,
+                                  bool* defer_redirect) OVERRIDE;
+  virtual void OnAuthRequired(net::URLRequest* request,
+                              net::AuthChallengeInfo* auth_info) OVERRIDE;
+  virtual void OnCertificateRequested(
+      net::URLRequest* request,
+      net::SSLCertRequestInfo* cert_request_info) OVERRIDE;
+  virtual void OnSSLCertificateError(net::URLRequest* request,
+                                     const net::SSLInfo& ssl_info,
+                                     bool fatal) OVERRIDE;
+  virtual void OnResponseStarted(net::URLRequest* request) OVERRIDE;
+  virtual void OnReadCompleted(net::URLRequest* request,
+                               int bytes_read) OVERRIDE;
+
+  enum PrefetcherState {
+    INITIALIZED = 0,  // Prefetching hasn't started.
+    RUNNING = 1,      // Prefetching started, allowed to add more requests.
+    STOPPED = 2,      // Prefetching started, not allowed to add more requests.
+    FINISHED = 3      // No more inflight request, new requests not possible.
+  };
+
+  base::ThreadChecker thread_checker_;
+  PrefetcherState state_;
+  Delegate* const delegate_;
+  ResourcePrefetchPredictorConfig const config_;
+  NavigationID navigation_id_;
+  PrefetchKeyType key_type_;
+  scoped_ptr<RequestVector> request_vector_;
+
+  std::map<net::URLRequest*, Request*> inflight_requests_;
+  std::list<Request*> request_queue_;
+  std::map<std::string, size_t> host_inflight_counts_;
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcher);
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_

chrome/browser/predictors/resource_prefetcher_manager.h

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_
+
+#include <map>
+
+#include "base/memory/ref_counted.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "chrome/browser/predictors/resource_prefetcher.h"
+
+namespace net {
+class URLRequestContextGetter;
+}
+
+namespace predictors {
+
+struct NavigationID;
+class ResourcePrefetchPredictor;
+
+// Manages prefetches for multple navigations.
+//  - Created and owned by the resource prefetch predictor.
+//  - Needs to be refcounted as it is de-referenced on two different threads.
+//  - Created on the UI thread, but most functions are called in the IO thread.
+//  - Will only allow one inflight prefresh per main frame URL.
+class ResourcePrefetcherManager
+    :  public ResourcePrefetcher::Delegate,
+       public base::RefCountedThreadSafe<ResourcePrefetcherManager> {
+ public:
+  // The |predictor| should be alive till ShutdownOnIOThread is called.
+  ResourcePrefetcherManager(ResourcePrefetchPredictor* predictor,
+                            const ResourcePrefetchPredictorConfig& config,
+                            net::URLRequestContextGetter* getter);
+
+  // UI thread.
+  void ShutdownOnUIThread();
+
+  // --- IO Thread methods.
+
+  // The prefetchers need to be deleted on the IO thread.
+  void ShutdownOnIOThread();
+
+  // Will create a new ResourcePrefetcher for the main frame url of the input
+  // navigation if there isn't one already for the same URL or host (for host
+  // based).
+  void MaybeAddPrefetch(const NavigationID& navigation_id,
+                        PrefetchKeyType key_type,
+                        scoped_ptr<ResourcePrefetcher::RequestVector> requests);
+
+  // Stops the ResourcePrefetcher for the input navigation, if one was in
+  // progress.
+  void MaybeRemovePrefetch(const NavigationID& navigation_id);
+
+  // ResourcePrefetcher::Delegate methods.
+  virtual void ResourcePrefetcherFinished(
+      ResourcePrefetcher* prefetcher,
+      ResourcePrefetcher::RequestVector* requests) OVERRIDE;
+  virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE;
+
+ private:
+  friend class base::RefCountedThreadSafe<ResourcePrefetcherManager>;
+  friend class MockResourcePrefetcherManager;
+
+  typedef std::map<std::string, ResourcePrefetcher*> PrefetcherMap;
+
+  virtual ~ResourcePrefetcherManager();
+
+  // UI Thread. |predictor_| needs to be called on the UI thread.
+  void ResourcePrefetcherFinishedOnUI(
+      const NavigationID& navigation_id,
+      PrefetchKeyType key_type,
+      scoped_ptr<ResourcePrefetcher::RequestVector> requests);
+
+  ResourcePrefetchPredictor* predictor_;
+  const ResourcePrefetchPredictorConfig config_;
+  net::URLRequestContextGetter* const context_getter_;
+
+  PrefetcherMap prefetcher_map_;  // Owns the ResourcePrefetcher pointers.
+
+  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherManager);
+};
+
+}  // namespace predictors
+
+#endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_

chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc

@@ -67,6 +67,13 @@ IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, NormalPassedFlag) {
 IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, DISABLED_CloudPrintPolicyFlag) {
   CommandLine new_command_line(GetCommandLineForRelaunch());
   new_command_line.AppendSwitch(switches::kCheckCloudPrintConnectorPolicy);
+  // This is important for the test as the way the browser process is launched
+  // here causes the predictor databases to be initialized multiple times. This
+  // is not an issue for production where the process is launched as a service
+  // and a Profile is not created. See http://crbug.com/140466 for more details.
+  new_command_line.AppendSwitchASCII(
+      switches::kSpeculativeResourcePrefetching,
+      switches::kSpeculativeResourcePrefetchingDisabled);
 
   base::ProcessHandle handle;
   bool launched =

chrome/browser/resources/predictors/predictors.js

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 <include src="autocomplete_action_predictor.js"/>
+<include src="resource_prefetch_predictor.js"/>
 
 if (cr.isWindows)
   document.documentElement.setAttribute('os', 'win');