Adding basic prefetch (not serving) functionality to search prefetch

This introduces a few basic checks, but does not robustly handle all
corner cases. This also does not handle serving the response, but rather
just storing it.

Bug: 1138639, 1138640
Change-Id: I9e1698156e717754541a9e18f1316c42326b9949
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2486454
Commit-Queue: Ryan Sturm <ryansturm@chromium.org>
Reviewed-by: Nicolas Ouellet-Payeur <nicolaso@chromium.org>
Reviewed-by: Robert Ogden <robertogden@chromium.org>
Cr-Commit-Position: refs/heads/master@{#819453}

chrome/browser/BUILD.gn

@@ -1268,6 +1268,8 @@ static_library("browser") {
     "predictors/resource_prefetch_predictor_tables.h",
     "prefetch/search_prefetch/field_trial_settings.cc",
     "prefetch/search_prefetch/field_trial_settings.h",
+    "prefetch/search_prefetch/prefetched_response_container.cc",
+    "prefetch/search_prefetch/prefetched_response_container.h",
     "prefetch/search_prefetch/search_prefetch_service.cc",
     "prefetch/search_prefetch/search_prefetch_service.h",
     "prefetch/search_prefetch/search_prefetch_service_factory.cc",

chrome/browser/prefetch/search_prefetch/field_trial_settings.cc

@@ -4,9 +4,16 @@
 
 #include "chrome/browser/prefetch/search_prefetch/field_trial_settings.h"
 
-const base::Feature kSearchPrefetchService{"SearchPrefecthService",
+const base::Feature kSearchPrefetchService{"SearchPrefetchService",
                                            base::FEATURE_DISABLED_BY_DEFAULT};
 
+const base::Feature kSearchPrefetchServicePrefetching{
+    "SearchPrefetchServicePrefetching", base::FEATURE_DISABLED_BY_DEFAULT};
+
 bool SearchPrefetchServiceIsEnabled() {
   return base::FeatureList::IsEnabled(kSearchPrefetchService);
 }
+
+bool SearchPrefetchServicePrefetchingIsEnabled() {
+  return base::FeatureList::IsEnabled(kSearchPrefetchServicePrefetching);
+}

chrome/browser/prefetch/search_prefetch/field_trial_settings.h

@@ -8,7 +8,10 @@
 #include "base/feature_list.h"
 
 extern const base::Feature kSearchPrefetchService;
+extern const base::Feature kSearchPrefetchServicePrefetching;
 
 bool SearchPrefetchServiceIsEnabled();
 
+bool SearchPrefetchServicePrefetchingIsEnabled();
+
 #endif  // CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_FIELD_TRIAL_SETTINGS_H_

chrome/browser/prefetch/search_prefetch/prefetched_response_container.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/prefetch/search_prefetch/prefetched_response_container.h"
+
+PrefetchedResponseContainer::PrefetchedResponseContainer(
+    network::mojom::URLResponseHeadPtr head,
+    std::unique_ptr<std::string> body)
+    : head_(std::move(head)), body_(std::move(body)) {}
+
+PrefetchedResponseContainer::~PrefetchedResponseContainer() = default;
+
+std::unique_ptr<PrefetchedResponseContainer>
+PrefetchedResponseContainer::Clone() const {
+  return std::make_unique<PrefetchedResponseContainer>(
+      head_->Clone(), std::make_unique<std::string>(*body_));
+}
+
+network::mojom::URLResponseHeadPtr PrefetchedResponseContainer::TakeHead() {
+  DCHECK(head_);
+  return std::move(head_);
+}
+
+std::unique_ptr<std::string> PrefetchedResponseContainer::TakeBody() {
+  DCHECK(body_);
+  return std::move(body_);
+}

chrome/browser/prefetch/search_prefetch/prefetched_response_container.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_PREFETCHED_RESPONSE_CONTAINER_H_
+#define CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_PREFETCHED_RESPONSE_CONTAINER_H_
+
+#include <memory>
+#include <string>
+
+#include "services/network/public/mojom/url_response_head.mojom.h"
+#include "url/gurl.h"
+
+// This class encapsulates a whole HTTP response which can be used for caching
+// and later replaying a prefetch request.
+class PrefetchedResponseContainer {
+ public:
+  PrefetchedResponseContainer(network::mojom::URLResponseHeadPtr head,
+                              std::unique_ptr<std::string> body);
+  ~PrefetchedResponseContainer();
+
+  PrefetchedResponseContainer(const PrefetchedResponseContainer&) = delete;
+  PrefetchedResponseContainer& operator=(const PrefetchedResponseContainer&) =
+      delete;
+
+  std::unique_ptr<PrefetchedResponseContainer> Clone() const;
+
+  // Takes ownership of the response head.
+  network::mojom::URLResponseHeadPtr TakeHead();
+
+  // Take ownership of the response body.
+  std::unique_ptr<std::string> TakeBody();
+
+ private:
+  network::mojom::URLResponseHeadPtr head_;
+  std::unique_ptr<std::string> body_;
+};
+
+#endif  // CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_PREFETCHED_RESPONSE_CONTAINER_H_

chrome/browser/prefetch/search_prefetch/search_prefetch_service.cc

@@ -4,7 +4,118 @@
 
 #include "chrome/browser/prefetch/search_prefetch/search_prefetch_service.h"
 
+#include "base/bind.h"
+#include "base/callback.h"
+#include "chrome/browser/prefetch/search_prefetch/field_trial_settings.h"
+#include "chrome/browser/prefetch/search_prefetch/prefetched_response_container.h"
 #include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/search_engines/template_url_service_factory.h"
+#include "components/search_engines/template_url_service.h"
+#include "components/variations/net/variations_http_headers.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/storage_partition.h"
+#include "content/public/common/content_constants.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_status_code.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "services/network/public/cpp/resource_request.h"
+#include "services/network/public/cpp/shared_url_loader_factory.h"
+
+SearchPrefetchService::PrefetchRequest::PrefetchRequest(
+    const GURL& prefetch_url)
+    : prefetch_url_(prefetch_url) {}
+
+SearchPrefetchService::PrefetchRequest::~PrefetchRequest() = default;
+
+void SearchPrefetchService::PrefetchRequest::StartPrefetchRequest(
+    Profile* profile) {
+  net::NetworkTrafficAnnotationTag traffic_annotation =
+      net::DefineNetworkTrafficAnnotation("search_prefetch_service", R"(
+        semantics {
+          sender: "Search Prefetch Service"
+          description:
+            "Prefetches search results page (HTML) based on omnibox hints "
+            "provided by the user's default search engine. This allows the "
+            "prefetched content to be served when the user navigates to the "
+            "omnibox hint."
+          trigger:
+            "User typing in the omnibox and the default search provider "
+            "indicates the provided omnibox hint entry is likely to be "
+            "navigated which would result in loading a search results page for "
+            "that hint."
+          data: "Credentials if user is signed in."
+          destination: OTHER
+          destination_other: "The user's default search engine."
+        }
+        policy {
+          cookies_allowed: YES
+          cookies_store: "user"
+          setting:
+            "Users can control this feature by opting out of 'Preload pages "
+            "for faster browsing and searching'"
+          chrome_policy {
+            DefaultSearchProviderEnabled {
+              policy_options {mode: MANDATORY}
+              DefaultSearchProviderEnabled: false
+            }
+            NetworkPredictionOptions {
+              NetworkPredictionOptions: 2
+            }
+          }
+        })");
+
+  auto resource_request = std::make_unique<network::ResourceRequest>();
+  resource_request->load_flags |= net::LOAD_PREFETCH;
+  resource_request->url = prefetch_url_;
+  resource_request->credentials_mode =
+      network::mojom::CredentialsMode::kInclude;
+  variations::AppendVariationsHeaderUnknownSignedIn(
+      prefetch_url_, variations::InIncognito::kNo, resource_request.get());
+  resource_request->headers.SetHeader(content::kCorsExemptPurposeHeaderName,
+                                      "prefetch");
+  // TODO(ryansturm): Find other headers that may need to be set.
+  // https://crbug.com/1138648
+
+  simple_loader_ = network::SimpleURLLoader::Create(std::move(resource_request),
+                                                    traffic_annotation);
+
+  auto url_loader_factory =
+      content::BrowserContext::GetDefaultStoragePartition(profile)
+          ->GetURLLoaderFactoryForBrowserProcess();
+
+  simple_loader_->DownloadToString(
+      url_loader_factory.get(),
+      base::BindOnce(&SearchPrefetchService::PrefetchRequest::LoadDone,
+                     base::Unretained(this)),
+      1024 * 1024);
+}
+
+void SearchPrefetchService::PrefetchRequest::LoadDone(
+    std::unique_ptr<std::string> response_body) {
+  bool success = simple_loader_->NetError() == net::OK;
+  int response_code = 0;
+
+  // TODO(ryansturm): Handle these errors more robustly by reporting them to the
+  // service. We need to prevent prefetches for x amount of time based on the
+  // error. https://crbug.com/1138641
+  if (!success || response_body->empty()) {
+    current_status_ = SearchPrefetchStatus::kRequestFailed;
+    return;
+  }
+  if (simple_loader_->ResponseInfo() && simple_loader_->ResponseInfo()->headers)
+    response_code = simple_loader_->ResponseInfo()->headers->response_code();
+  if (response_code != net::HTTP_OK) {
+    current_status_ = SearchPrefetchStatus::kRequestFailed;
+    return;
+  }
+  current_status_ = SearchPrefetchStatus::kSuccessfullyCompleted;
+
+  prefetch_response_container_ = std::make_unique<PrefetchedResponseContainer>(
+      simple_loader_->ResponseInfo()->Clone(), std::move(response_body));
+
+  simple_loader_.reset();
+}
 
 SearchPrefetchService::SearchPrefetchService(Profile* profile)
     : profile_(profile) {
@@ -12,3 +123,40 @@ SearchPrefetchService::SearchPrefetchService(Profile* profile)
 }
 
 SearchPrefetchService::~SearchPrefetchService() = default;
+
+bool SearchPrefetchService::MaybePrefetchURL(const GURL& url) {
+  if (!SearchPrefetchServicePrefetchingIsEnabled())
+    return false;
+
+  auto* template_url_service =
+      TemplateURLServiceFactory::GetForProfile(profile_);
+  if (!template_url_service)
+    return false;
+  base::string16 search_terms;
+
+  // Extract the terms directly to make sure this string will match the URL
+  // interception string logic.
+  template_url_service->GetDefaultSearchProvider()->ExtractSearchTermsFromURL(
+      url, template_url_service->search_terms_data(), &search_terms);
+
+  if (search_terms.size() == 0)
+    return false;
+
+  // Don't prefetch the same search terms twice within the expiry duration.
+  if (prefetches_.find(search_terms) != prefetches_.end()) {
+    return false;
+  }
+
+  prefetches_.emplace(search_terms, std::make_unique<PrefetchRequest>(url));
+  prefetches_[search_terms]->StartPrefetchRequest(profile_);
+  return true;
+  // TODO(ryansturm): Expire entries after 60 seconds. https://crbug.com/1138639
+}
+
+base::Optional<SearchPrefetchStatus>
+SearchPrefetchService::GetSearchPrefetchStatusForTesting(
+    base::string16 search_terms) {
+  if (prefetches_.find(search_terms) == prefetches_.end())
+    return base::nullopt;
+  return prefetches_[search_terms]->current_status();
+}

chrome/browser/prefetch/search_prefetch/search_prefetch_service.h

@@ -5,16 +5,82 @@
 #ifndef CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_SEARCH_PREFETCH_SERVICE_H_
 #define CHROME_BROWSER_PREFETCH_SEARCH_PREFETCH_SEARCH_PREFETCH_SERVICE_H_
 
+#include <map>
+
+#include "base/callback.h"
+#include "base/optional.h"
+#include "base/strings/string16.h"
 #include "components/keyed_service/core/keyed_service.h"
+#include "services/network/public/cpp/simple_url_loader.h"
+#include "url/gurl.h"
 
 class Profile;
+class GURL;
+class PrefetchedResponseContainer;
+
+enum class SearchPrefetchStatus {
+  // The request is on the network and may move to any other state.
+  kInFlight = 1,
+  // The request received all the data and is ready to serve.
+  kSuccessfullyCompleted = 2,
+  // The request hit an error and cannot be served.
+  kRequestFailed = 3,
+};
 
 class SearchPrefetchService : public KeyedService {
  public:
   explicit SearchPrefetchService(Profile* profile);
   ~SearchPrefetchService() override;
 
- private:
+  SearchPrefetchService(const SearchPrefetchService&) = delete;
+  SearchPrefetchService& operator=(const SearchPrefetchService&) = delete;
+
+  // Returns whether the prefetch started or not.
+  bool MaybePrefetchURL(const GURL& url);
+
+  // Reports the status of a prefetch for a given search term.
+  base::Optional<SearchPrefetchStatus> GetSearchPrefetchStatusForTesting(
+      base::string16 search_terms);
+
+  // Internal class to represent an ongoing or completed prefetch.
+  class PrefetchRequest {
+   public:
+    // |service| must outlive this class and be able to manage this class's
+    // lifetime.
+    explicit PrefetchRequest(const GURL& prefetch_url);
+    ~PrefetchRequest();
+
+    PrefetchRequest(const PrefetchRequest&) = delete;
+    PrefetchRequest& operator=(const PrefetchRequest&) = delete;
+
+    // Starts the network request to prefetch |prefetch_url_|.
+    void StartPrefetchRequest(Profile* profile);
+
+    SearchPrefetchStatus current_status() const { return current_status_; }
+
+   private:
+    // Called as a callback when the prefetch request is complete. Stores the
+    // response and other metadata in |prefetch_response_container_|.
+    void LoadDone(std::unique_ptr<std::string> response_body);
+
+    SearchPrefetchStatus current_status_ = SearchPrefetchStatus::kInFlight;
+
+    // The URL to prefetch the search terms from.
+    const GURL prefetch_url_;
+
+    // The ongoing prefetch request. Null before and after the fetch.
+    std::unique_ptr<network::SimpleURLLoader> simple_loader_;
+
+    // Once a prefetch is completed successfully, the associated prefetch data
+    // and metadata about the request.
+    std::unique_ptr<PrefetchedResponseContainer> prefetch_response_container_;
+  };
+
+  // Prefetches that are started are stored using search terms as a key. Only
+  // one prefetch should be started for a given search term until the old
+  // prefetch expires.
+  std::map<base::string16, std::unique_ptr<PrefetchRequest>> prefetches_;
+
   Profile* profile_;
 };
 

tools/traffic_annotation/summary/annotations.xml

@@ -302,6 +302,7 @@ Refer to README.md for content description and update process.
  <item id="save_file_manager" added_in_milestone="62" hash_code="56275203" type="0" content_hash_code="56692339" os_list="linux,windows" file_path="content/browser/download/save_file_manager.cc"/>
  <item id="sct_auditing" added_in_milestone="87" hash_code="48603483" type="0" content_hash_code="43567503" os_list="linux,windows" file_path="chrome/browser/ssl/sct_reporting_service.cc"/>
  <item id="sdch_dictionary_fetch" added_in_milestone="62" hash_code="47152935" type="0" deprecated="2017-09-16" content_hash_code="16764294" file_path=""/>
+ <item id="search_prefetch_service" added_in_milestone="88" hash_code="108986091" type="0" content_hash_code="125257414" os_list="linux,windows" file_path="chrome/browser/prefetch/search_prefetch/search_prefetch_service.cc"/>
  <item id="search_suggest_service" added_in_milestone="73" hash_code="57785193" type="0" content_hash_code="132247652" os_list="linux,windows" file_path="chrome/browser/search/search_suggest/search_suggest_loader_impl.cc"/>
  <item id="security_key_socket" added_in_milestone="66" hash_code="31074955" type="0" content_hash_code="13741232" os_list="linux,windows" file_path="remoting/host/security_key/security_key_socket.cc"/>
  <item id="send_message_express" added_in_milestone="85" hash_code="23527666" type="0" deprecated="2020-09-21" content_hash_code="96850228" file_path=""/>

tools/traffic_annotation/summary/grouping.xml

@@ -321,6 +321,7 @@ hidden="true" so that these annotations don't show up in the document.
       <traffic_annotation unique_id="omnibox_documentsuggest"/>
       <traffic_annotation unique_id="omnibox_zerosuggest"/>
       <traffic_annotation unique_id="one_google_bar_service"/>
+      <traffic_annotation unique_id="search_prefetch_service"/>
     </sender>
   </group>
   <group name="Updates, Recovery, and Crash Reports">