Enable fetching of hints from the URLs extracted

Enable fetching of hints from the URLs extracted from the
search result page.
The hints are fetched only on slow connections.

Change-Id: I0c8d0324ff913297c9f2b2a5fdc300771c0c3798
Bug: 1004527
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1839240
Commit-Queue: Tarun Bansal <tbansal@chromium.org>
Reviewed-by: Michael Crouse <mcrouse@chromium.org>
Reviewed-by: Sophie Chang <sophiechang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#703480}

chrome/browser/optimization_guide/optimization_guide_hints_manager.cc

@@ -16,10 +16,13 @@
 #include "base/task_runner_util.h"
 #include "base/time/default_clock.h"
 #include "chrome/browser/browser_process.h"
+#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service.h"
+#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service_factory.h"
 #include "chrome/browser/optimization_guide/optimization_guide_navigation_data.h"
 #include "chrome/browser/optimization_guide/optimization_guide_permissions_util.h"
 #include "chrome/browser/optimization_guide/optimization_guide_web_contents_observer.h"
 #include "chrome/browser/profiles/profile.h"
+#include "components/google/core/common/google_util.h"
 #include "components/optimization_guide/bloom_filter.h"
 #include "components/optimization_guide/hint_cache.h"
 #include "components/optimization_guide/hint_cache_store.h"
@@ -184,6 +187,10 @@ OptimizationGuideHintsManager::OptimizationGuideHintsManager(
       optimization_guide::switches::ShouldPurgeHintCacheStoreOnStartup(),
       base::BindOnce(&OptimizationGuideHintsManager::OnHintCacheInitialized,
                      ui_weak_ptr_factory_.GetWeakPtr()));
+
+  NavigationPredictorKeyedService* navigation_predictor_service =
+      NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
+  navigation_predictor_service->AddObserver(this);
 }
 
 OptimizationGuideHintsManager::~OptimizationGuideHintsManager() {
@@ -192,6 +199,10 @@ OptimizationGuideHintsManager::~OptimizationGuideHintsManager() {
   optimization_guide_service_->RemoveObserver(this);
   g_browser_process->network_quality_tracker()
       ->RemoveEffectiveConnectionTypeObserver(this);
+
+  NavigationPredictorKeyedService* navigation_predictor_service =
+      NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
+  navigation_predictor_service->RemoveObserver(this);
 }
 
 void OptimizationGuideHintsManager::Shutdown() {
@@ -533,6 +544,7 @@ void OptimizationGuideHintsManager::OnPageNavigationHintsFetched(
 }
 
 void OptimizationGuideHintsManager::OnFetchedTopHostsHintsStored() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
   LOCAL_HISTOGRAM_BOOLEAN("OptimizationGuide.FetchedHints.Stored", true);
 
   top_hosts_hints_fetch_timer_.Stop();
@@ -542,11 +554,13 @@ void OptimizationGuideHintsManager::OnFetchedTopHostsHintsStored() {
 }
 
 void OptimizationGuideHintsManager::OnFetchedPageNavigationHintsStored() {
-  for (const auto& url : navigation_urls_last_fetched_real_time_)
-    LoadHintForURL(url, base::DoNothing());
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+  for (const auto& host : navigation_hosts_last_fetched_real_time_)
+    LoadHintForHost(host, base::DoNothing());
 }
 
 base::Time OptimizationGuideHintsManager::GetLastHintsFetchAttemptTime() const {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
   return base::Time::FromDeltaSinceWindowsEpoch(
       base::TimeDelta::FromMicroseconds(pref_service_->GetInt64(
           optimization_guide::prefs::kHintsFetcherLastFetchAttempt)));
@@ -554,6 +568,7 @@ base::Time OptimizationGuideHintsManager::GetLastHintsFetchAttemptTime() const {
 
 void OptimizationGuideHintsManager::SetLastHintsFetchAttemptTime(
     base::Time last_attempt_time) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
   pref_service_->SetInt64(
       optimization_guide::prefs::kHintsFetcherLastFetchAttempt,
       last_attempt_time.ToDeltaSinceWindowsEpoch().InMicroseconds());
@@ -581,20 +596,88 @@ void OptimizationGuideHintsManager::LoadHintForNavigation(
     }
   }
 
-  LoadHintForURL(url, std::move(callback));
+  LoadHintForHost(url.host(), std::move(callback));
 }
 
-void OptimizationGuideHintsManager::LoadHintForURL(const GURL& url,
-                                                   base::OnceClosure callback) {
+void OptimizationGuideHintsManager::LoadHintForHost(
+    const std::string& host,
+    base::OnceClosure callback) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
-  DCHECK(url.has_host());
 
   hint_cache_->LoadHint(
-      url.host(),
+      host,
       base::BindOnce(&OptimizationGuideHintsManager::OnHintLoaded,
                      ui_weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
 }
 
+bool OptimizationGuideHintsManager::IsGoogleURL(const GURL& url) const {
+  return google_util::IsGoogleHostname(url.host(),
+                                       google_util::DISALLOW_SUBDOMAIN);
+}
+
+void OptimizationGuideHintsManager::OnPredictionUpdated(
+    const base::Optional<NavigationPredictorKeyedService::Prediction>&
+        prediction) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  if (!prediction.has_value())
+    return;
+
+  const GURL& source_document_url = prediction->source_document_url();
+
+  // We only extract next predicted navigations from Google URLs.
+  if (!IsGoogleURL(source_document_url))
+    return;
+
+  // Extract the target hosts. Use a flat set to remove duplicates.
+  // |target_hosts_serialized| is the ordered list of non-duplicate hosts.
+  base::flat_set<std::string> target_hosts;
+  std::vector<std::string> target_hosts_serialized;
+  for (const auto& url : prediction->sorted_predicted_urls()) {
+    if (!IsAllowedToFetchNavigationHints(url))
+      continue;
+    if (target_hosts.size() >=
+        optimization_guide::features::
+            MaxHostsForOptimizationGuideServiceHintsFetch()) {
+      break;
+    }
+
+    // Insert the host to |target_hosts|. The host is inserted to
+    // |target_hosts_serialized| only if it was not a duplicate insertion to
+    // |target_hosts|.
+    std::pair<base::flat_set<std::string>::iterator, bool> insert_result =
+        target_hosts.insert(url.host());
+    if (insert_result.second)
+      target_hosts_serialized.push_back(url.host());
+
+    // Ensure that the 2 data structures remain synchronized.
+    DCHECK_EQ(target_hosts.size(), target_hosts_serialized.size());
+  }
+
+  if (target_hosts.empty())
+    return;
+
+  navigation_hosts_last_fetched_real_time_.clear();
+  for (const auto& host : target_hosts)
+    navigation_hosts_last_fetched_real_time_.push_back(host);
+
+  if (!hints_fetcher_) {
+    hints_fetcher_ = std::make_unique<optimization_guide::HintsFetcher>(
+        url_loader_factory_,
+        optimization_guide::features::GetOptimizationGuideServiceURL(),
+        pref_service_);
+  }
+
+  hints_fetcher_->FetchOptimizationGuideServiceHints(
+      target_hosts_serialized,
+      optimization_guide::proto::CONTEXT_PAGE_NAVIGATION,
+      base::BindOnce(&OptimizationGuideHintsManager::OnHintsFetched,
+                     ui_weak_ptr_factory_.GetWeakPtr()));
+
+  for (const auto& host : target_hosts)
+    LoadHintForHost(host, base::DoNothing());
+}
+
 void OptimizationGuideHintsManager::OnHintLoaded(
     base::OnceClosure callback,
     const optimization_guide::proto::Hint* loaded_hint) const {
@@ -841,11 +924,16 @@ void OptimizationGuideHintsManager::OnNavigationStartOrRedirect(
     base::OnceClosure callback) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 
+  if (optimization_guide::switches::
+          DisableFetchingHintsAtNavigationStartForTesting()) {
+    return;
+  }
+
   if (IsAllowedToFetchNavigationHints(navigation_handle->GetURL())) {
     std::vector<std::string> hosts{navigation_handle->GetURL().host()};
-    navigation_urls_last_fetched_real_time_.clear();
-    navigation_urls_last_fetched_real_time_.push_back(
-        navigation_handle->GetURL());
+    navigation_hosts_last_fetched_real_time_.clear();
+    navigation_hosts_last_fetched_real_time_.push_back(
+        navigation_handle->GetURL().host());
 
     if (!hints_fetcher_) {
       hints_fetcher_ = std::make_unique<optimization_guide::HintsFetcher>(

chrome/browser/optimization_guide/optimization_guide_hints_manager.h

@@ -18,6 +18,7 @@
 #include "base/synchronization/lock.h"
 #include "base/time/clock.h"
 #include "base/timer/timer.h"
+#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service.h"
 #include "components/optimization_guide/hints_component_info.h"
 #include "components/optimization_guide/optimization_guide_service_observer.h"
 #include "components/optimization_guide/proto/hints.pb.h"
@@ -60,7 +61,8 @@ class Profile;
 
 class OptimizationGuideHintsManager
     : public optimization_guide::OptimizationGuideServiceObserver,
-      public network::NetworkQualityTracker::EffectiveConnectionTypeObserver {
+      public network::NetworkQualityTracker::EffectiveConnectionTypeObserver,
+      public NavigationPredictorKeyedService::Observer {
  public:
   OptimizationGuideHintsManager(
       optimization_guide::OptimizationGuideService* optimization_guide_service,
@@ -139,6 +141,17 @@ class OptimizationGuideHintsManager
                                    base::OnceClosure callback);
 
  private:
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest, IsGoogleURL);
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
+                           HintsFetched_AtSRP_ECT_SLOW_2G);
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
+                           HintsFetched_AtSRP_ECT_4G);
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
+                           HintsFetched_AtNonSRP_ECT_SLOW_2G);
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
+                           HintsFetched_AtSRP_ECT_SLOW_2G_DuplicatesRemoved);
+  FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
+                           HintsFetched_AtSRP_ECT_SLOW_2G_InsecureHostsRemoved);
   // Processes the hints component.
   //
   // Should always be called on the thread that belongs to
@@ -240,11 +253,20 @@ class OptimizationGuideHintsManager
   void LoadHintForNavigation(content::NavigationHandle* navigation_handle,
                              base::OnceClosure callback);
 
-  // Loads the hint for |url| if available.
+  // Loads the hint for |host| if available.
   // |callback| is run when the request has finished regardless of whether there
-  // was actually a hint for that |url| or not. The callback can be used as a
+  // was actually a hint for that |host| or not. The callback can be used as a
   // signal for tests.
-  void LoadHintForURL(const GURL& url, base::OnceClosure callback);
+  void LoadHintForHost(const std::string& host, base::OnceClosure callback);
+
+  // Returns true if the hostname for |url| matches the host of google web
+  // search results page (www.google.*).
+  bool IsGoogleURL(const GURL& url) const;
+
+  // NavigationPredictorKeyedService::Observer:
+  void OnPredictionUpdated(
+      const base::Optional<NavigationPredictorKeyedService::Prediction>&
+          prediction) override;
 
   // The OptimizationGuideService that this guide is listening to. Not owned.
   optimization_guide::OptimizationGuideService* const
@@ -314,8 +336,8 @@ class OptimizationGuideHintsManager
   // Used in testing to subscribe to an update event in this class.
   base::OnceClosure next_update_closure_;
 
-  // URLs for which hints were last fetched in the real-time.
-  std::vector<GURL> navigation_urls_last_fetched_real_time_;
+  // Hosts for which hints were last fetched in the real-time.
+  std::vector<std::string> navigation_hosts_last_fetched_real_time_;
 
   // Used to get |weak_ptr_| to self on the UI thread.
   base::WeakPtrFactory<OptimizationGuideHintsManager> ui_weak_ptr_factory_{

chrome/browser/optimization_guide/optimization_guide_hints_manager_unittest.cc

@@ -1703,6 +1703,175 @@ TEST_F(OptimizationGuideHintsManagerTest,
   EXPECT_EQ(nullptr, navigation_data->page_hint());
 }
 
+TEST_F(OptimizationGuideHintsManagerTest, IsGoogleURL) {
+  const struct {
+    const char* url;
+    bool expect_is_google_url;
+  } tests[] = {
+      {"https://www.google.com/"
+       "search?q=cats&oq=cq&aqs=foo&ie=UTF-8",
+       true},
+      {"https://www.google.com/", true},
+      {"https://www.google.com/:99", true},
+
+      // Try localized search pages.
+      {"https://www.google.co.in/"
+       "search?q=cats&oq=cq&aqs=foo&ie=UTF-8",
+       true},
+      {"https://www.google.co.in/", true},
+      {"https://www.google.co.in/:99", true},
+
+      // Try Google domain pages that are not web search related.
+      {"https://www.not-google.com/", false},
+      {"https://www.youtube.com/", false},
+      {"https://domain.google.com/", false},
+      {"https://images.google.com/", false},
+  };
+
+  for (const auto& test : tests) {
+    GURL url(test.url);
+    EXPECT_TRUE(url.is_valid());
+    EXPECT_EQ(test.expect_is_google_url, hints_manager()->IsGoogleURL(url));
+  }
+}
+
+TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtSRP_ECT_SLOW_2G) {
+  hints_manager()->RegisterOptimizationTypes(
+      {optimization_guide::proto::DEFER_ALL_SCRIPT});
+  base::test::ScopedFeatureList scoped_list;
+  scoped_list.InitAndEnableFeature(
+      optimization_guide::features::kOptimizationHintsFetching);
+  InitializeWithDefaultConfig("1.0.0.0");
+
+  // Set ECT estimate so hint is activated.
+  hints_manager()->OnEffectiveConnectionTypeChanged(
+      net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
+  std::unique_ptr<content::MockNavigationHandle> navigation_handle =
+      CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
+          url_without_hints());
+  base::HistogramTester histogram_tester;
+  std::vector<GURL> sorted_predicted_urls;
+  sorted_predicted_urls.push_back(GURL("https://foo.com/"));
+  NavigationPredictorKeyedService::Prediction prediction(
+      nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
+
+  hints_manager()->OnPredictionUpdated(prediction);
+  histogram_tester.ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 1, 1);
+}
+
+TEST_F(OptimizationGuideHintsManagerTest,
+       HintsFetched_AtSRP_ECT_SLOW_2G_DuplicatesRemoved) {
+  hints_manager()->RegisterOptimizationTypes(
+      {optimization_guide::proto::DEFER_ALL_SCRIPT});
+  base::test::ScopedFeatureList scoped_list;
+  scoped_list.InitAndEnableFeature(
+      optimization_guide::features::kOptimizationHintsFetching);
+  InitializeWithDefaultConfig("1.0.0.0");
+
+  // Set ECT estimate so hint is activated.
+  hints_manager()->OnEffectiveConnectionTypeChanged(
+      net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
+  std::unique_ptr<content::MockNavigationHandle> navigation_handle =
+      CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
+          url_without_hints());
+  base::HistogramTester histogram_tester;
+  std::vector<GURL> sorted_predicted_urls;
+  sorted_predicted_urls.push_back(GURL("https://foo.com/page1.html"));
+  sorted_predicted_urls.push_back(GURL("https://foo.com/page2.html"));
+  sorted_predicted_urls.push_back(GURL("https://foo.com/page3.html"));
+  sorted_predicted_urls.push_back(GURL("https://bar.com/"));
+
+  NavigationPredictorKeyedService::Prediction prediction(
+      nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
+
+  hints_manager()->OnPredictionUpdated(prediction);
+  // Ensure that we only include 2 hosts in the request. These would be foo.com
+  // and bar.com.
+  histogram_tester.ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 2, 1);
+}
+
+TEST_F(OptimizationGuideHintsManagerTest,
+       HintsFetched_AtSRP_ECT_SLOW_2G_InsecureHostsRemoved) {
+  hints_manager()->RegisterOptimizationTypes(
+      {optimization_guide::proto::DEFER_ALL_SCRIPT});
+  base::test::ScopedFeatureList scoped_list;
+  scoped_list.InitAndEnableFeature(
+      optimization_guide::features::kOptimizationHintsFetching);
+  InitializeWithDefaultConfig("1.0.0.0");
+
+  // Set ECT estimate so hint is activated.
+  hints_manager()->OnEffectiveConnectionTypeChanged(
+      net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
+  std::unique_ptr<content::MockNavigationHandle> navigation_handle =
+      CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
+          url_without_hints());
+  base::HistogramTester histogram_tester;
+  std::vector<GURL> sorted_predicted_urls;
+  sorted_predicted_urls.push_back(GURL("https://foo.com/page1.html"));
+  sorted_predicted_urls.push_back(GURL("http://insecure-bar.com/"));
+
+  NavigationPredictorKeyedService::Prediction prediction(
+      nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
+
+  hints_manager()->OnPredictionUpdated(prediction);
+  // Ensure that we only include 1 secure host in the request. These would be
+  // foo.com.
+  histogram_tester.ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 1, 1);
+}
+
+TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtSRP_ECT_4G) {
+  hints_manager()->RegisterOptimizationTypes(
+      {optimization_guide::proto::DEFER_ALL_SCRIPT});
+  base::test::ScopedFeatureList scoped_list;
+  scoped_list.InitAndEnableFeature(
+      optimization_guide::features::kOptimizationHintsFetching);
+  InitializeWithDefaultConfig("1.0.0.0");
+
+  // Set ECT estimate so hint is activated.
+  hints_manager()->OnEffectiveConnectionTypeChanged(
+      net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_4G);
+  std::unique_ptr<content::MockNavigationHandle> navigation_handle =
+      CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
+          url_without_hints());
+  base::HistogramTester histogram_tester;
+  std::vector<GURL> sorted_predicted_urls;
+  sorted_predicted_urls.push_back(GURL("https://foo.com/"));
+  NavigationPredictorKeyedService::Prediction prediction(
+      nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
+
+  hints_manager()->OnPredictionUpdated(prediction);
+  histogram_tester.ExpectTotalCount(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 0);
+}
+
+TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtNonSRP_ECT_SLOW_2G) {
+  hints_manager()->RegisterOptimizationTypes(
+      {optimization_guide::proto::DEFER_ALL_SCRIPT});
+  base::test::ScopedFeatureList scoped_list;
+  scoped_list.InitAndEnableFeature(
+      optimization_guide::features::kOptimizationHintsFetching);
+  InitializeWithDefaultConfig("1.0.0.0");
+
+  // Set ECT estimate so hint is activated.
+  hints_manager()->OnEffectiveConnectionTypeChanged(
+      net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
+  std::unique_ptr<content::MockNavigationHandle> navigation_handle =
+      CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
+          url_without_hints());
+  base::HistogramTester histogram_tester;
+  std::vector<GURL> sorted_predicted_urls;
+  sorted_predicted_urls.push_back(GURL("https://foo.com/"));
+  NavigationPredictorKeyedService::Prediction prediction(
+      nullptr, GURL("https://www.not-google.com/"), sorted_predicted_urls);
+
+  hints_manager()->OnPredictionUpdated(prediction);
+  histogram_tester.ExpectTotalCount(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 0);
+}
+
 TEST_F(OptimizationGuideHintsManagerTest,
        HintsFetchedAtNavigationTime_ECT_SLOW_2G) {
   hints_manager()->RegisterOptimizationTypes(

chrome/browser/previews/hints_fetcher_browsertest.cc

@@ -10,6 +10,7 @@
 #include "base/command_line.h"
 #include "base/containers/flat_set.h"
 #include "base/run_loop.h"
+#include "base/strings/strcat.h"
 #include "base/task/post_task.h"
 #include "base/task/thread_pool/thread_pool_instance.h"
 #include "base/test/metrics/histogram_tester.h"
@@ -27,6 +28,7 @@
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
 #include "components/data_reduction_proxy/core/common/data_reduction_proxy_features.h"
+#include "components/google/core/common/google_util.h"
 #include "components/optimization_guide/hint_cache_store.h"
 #include "components/optimization_guide/hints_component_info.h"
 #include "components/optimization_guide/hints_component_util.h"
@@ -42,12 +44,18 @@
 #include "components/previews/core/previews_black_list.h"
 #include "components/previews/core/previews_features.h"
 #include "components/previews/core/previews_switches.h"
+#include "components/ukm/test_ukm_recorder.h"
 #include "content/public/browser/browser_task_traits.h"
+#include "content/public/test/browser_test_base.h"
 #include "content/public/test/browser_test_utils.h"
 #include "content/public/test/network_connection_change_simulator.h"
+#include "net/dns/mock_host_resolver.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
+#include "services/metrics/public/cpp/ukm_builders.h"
 #include "services/network/public/cpp/network_quality_tracker.h"
+#include "third_party/blink/public/common/features.h"
 
 namespace {
 
@@ -79,6 +87,10 @@ int RetryForHistogramUntilCountReached(
     total = GetTotalHistogramSamples(histogram_tester, histogram_name);
     if (total >= count)
       return total;
+
+    content::FetchHistogramsFromChildProcesses();
+    SubprocessMetricsProvider::MergeHistogramDeltasForTesting();
+    base::RunLoop().RunUntilIdle();
   }
 }
 
@@ -102,6 +114,24 @@ enum class HintsFetcherRemoteResponseType {
   kMalformed = 2,
 };
 
+constexpr char kGoogleHost[] = "www.google.com";
+
+// Modifies |relative_url|:
+// Scheme of the returned URL matches the scheme of the |server|.
+// Host of the returned URL matches kGoogleHost.
+// Port number of the returned URL matches the port at which |server| is
+// listening.
+// Path of the returned URL is set to |relative_url|.
+GURL GetURLWithGoogleHost(net::EmbeddedTestServer* server,
+                          const std::string& relative_url) {
+  GURL server_base_url = server->base_url();
+  GURL base_url =
+      GURL(base::StrCat({server_base_url.scheme(), "://", kGoogleHost, ":",
+                         server_base_url.port()}));
+  EXPECT_TRUE(base_url.is_valid()) << base_url.possibly_invalid_spec();
+  return base_url.Resolve(relative_url);
+}
+
 }  // namespace
 
 // This test class sets up everything but does not enable any
@@ -121,6 +151,12 @@ class HintsFetcherDisabledBrowserTest
     content::NetworkConnectionChangeSimulator().SetConnectionType(
         network::mojom::ConnectionType::CONNECTION_2G);
 
+    // Ensure that kGoogleHost resolves to the localhost where the embedded test
+    // server is listening.
+    host_resolver()->AddRule("*", "127.0.0.1");
+
+    ukm_recorder_ = std::make_unique<ukm::TestAutoSetUkmRecorder>();
+
     InProcessBrowserTest::SetUpOnMainThread();
   }
 
@@ -128,15 +164,19 @@ class HintsFetcherDisabledBrowserTest
     origin_server_.reset(
         new net::EmbeddedTestServer(net::EmbeddedTestServer::TYPE_HTTPS));
     origin_server_->ServeFilesFromSourceDirectory("chrome/test/data/previews");
-    origin_server_->RegisterRequestHandler(base::BindRepeating(
-        &HintsFetcherDisabledBrowserTest::HandleOriginRequest,
-        base::Unretained(this)));
 
     ASSERT_TRUE(origin_server_->Start());
 
     https_url_ = origin_server_->GetURL("/hint_setup.html");
     ASSERT_TRUE(https_url().SchemeIs(url::kHttpsScheme));
 
+    search_results_page_url_ =
+        GetURLWithGoogleHost(origin_server_.get(), "/search_results_page.html");
+    ASSERT_TRUE(search_results_page_url_.is_valid() &&
+                search_results_page_url_.SchemeIs(url::kHttpsScheme) &&
+                google_util::IsGoogleHostname(search_results_page_url_.host(),
+                                              google_util::DISALLOW_SUBDOMAIN));
+
     hints_server_.reset(
         new net::EmbeddedTestServer(net::EmbeddedTestServer::TYPE_HTTPS));
     hints_server_->ServeFilesFromSourceDirectory("chrome/test/data/previews");
@@ -148,7 +188,9 @@ class HintsFetcherDisabledBrowserTest
 
     if (IsOptimizationGuideKeyedServiceEnabled()) {
       param_feature_list_.InitWithFeatures(
-          {optimization_guide::features::kOptimizationGuideKeyedService}, {});
+          {optimization_guide::features::kOptimizationGuideKeyedService,
+           blink::features::kNavigationPredictor},
+          {});
     } else {
       param_feature_list_.InitWithFeatures(
           {}, {optimization_guide::features::kOptimizationGuideKeyedService});
@@ -158,6 +200,8 @@ class HintsFetcherDisabledBrowserTest
   }
 
   void SetUpCommandLine(base::CommandLine* cmd) override {
+    cmd->AppendSwitch("ignore-certificate-errors");
+
     cmd->AppendSwitch("enable-spdy-proxy-auth");
 
     // Due to race conditions, it's possible that blacklist data is not loaded
@@ -293,6 +337,10 @@ class HintsFetcherDisabledBrowserTest
     return &histogram_tester_;
   }
 
+  const GURL& search_results_page_url() const {
+    return search_results_page_url_;
+  }
+
   void SetExpectedHintsRequestForHosts(
       const base::flat_set<std::string>& hosts) {
     base::AutoLock lock(lock_);
@@ -304,6 +352,37 @@ class HintsFetcherDisabledBrowserTest
     return count_hints_requests_received_;
   }
 
+  void WaitUntilHintsFetcherRequestReceived() {
+    while (true) {
+      {
+        // Acquire the |lock_| inside to avoid starving other consumers of the
+        // lock.
+        base::AutoLock lock(lock_);
+        if (count_hints_requests_received_ > 0)
+          return;
+      }
+      base::RunLoop().RunUntilIdle();
+    }
+  }
+
+  void ResetCountHintsRequestsReceived() {
+    base::AutoLock lock(lock_);
+    count_hints_requests_received_ = 0;
+  }
+
+  // Wait for page layout to happen. This is needed in some tests since the
+  // anchor elements are extracted from the webpage after page layout finishes.
+  void WaitForPageLayout() {
+    const char* entry_name =
+        ukm::builders::NavigationPredictorPageLinkMetrics::kEntryName;
+
+    if (ukm_recorder_->GetEntriesByName(entry_name).empty()) {
+      base::RunLoop run_loop;
+      ukm_recorder_->SetOnAddEntryCallback(entry_name, run_loop.QuitClosure());
+      run_loop.Run();
+    }
+  }
+
  protected:
   base::test::ScopedFeatureList scoped_feature_list_;
   std::unique_ptr<net::EmbeddedTestServer> origin_server_;
@@ -403,6 +482,8 @@ class HintsFetcherDisabledBrowserTest
 
   GURL https_url_;
 
+  GURL search_results_page_url_;
+
   base::HistogramTester histogram_tester_;
 
   optimization_guide::testing::TestHintsComponentCreator
@@ -420,6 +501,8 @@ class HintsFetcherDisabledBrowserTest
   // null, then the verification is not done.
   base::Optional<base::flat_set<std::string>> expect_hints_request_for_hosts_;
 
+  std::unique_ptr<ukm::TestAutoSetUkmRecorder> ukm_recorder_;
+
   DISALLOW_COPY_AND_ASSIGN(HintsFetcherDisabledBrowserTest);
 };
 
@@ -1252,3 +1335,90 @@ IN_PROC_BROWSER_TEST_P(HintsFetcherChangeDefaultBlacklistSizeBrowserTest,
   // the blacklist.
   EXPECT_EQ(engaged_hosts, GetTopHostBlacklistSize());
 }
+
+class HintsFetcherSearchPageBrowserTest : public HintsFetcherBrowserTest {
+  void SetUpCommandLine(base::CommandLine* cmd) override {
+    cmd->AppendSwitch(optimization_guide::switches::
+                          kDisableFetchingHintsAtNavigationStartForTesting);
+    cmd->AppendSwitch("ignore-certificate-errors");
+    HintsFetcherBrowserTest::SetUpCommandLine(cmd);
+  }
+};
+
+// True if testing using the OptimizationGuideKeyedService implementation.
+INSTANTIATE_TEST_SUITE_P(OptimizationGuideKeyedServiceImplementation,
+                         HintsFetcherSearchPageBrowserTest,
+                         testing::Bool());
+
+IN_PROC_BROWSER_TEST_P(
+    HintsFetcherSearchPageBrowserTest,
+    DISABLE_ON_WIN_MAC_CHROMESOS(HintsFetcher_SRP_Slow_Connection)) {
+  g_browser_process->network_quality_tracker()
+      ->ReportEffectiveConnectionTypeForTesting(
+          net::EFFECTIVE_CONNECTION_TYPE_2G);
+
+  const base::HistogramTester* histogram_tester = GetHistogramTester();
+
+  // Whitelist NoScript for https_url()'s' host.
+  SetUpComponentUpdateHints(https_url());
+
+  // Expect that the browser initialization will record at least one sample
+  // in each of the following histograms as One Platform Hints are enabled.
+  EXPECT_GE(RetryForHistogramUntilCountReached(
+                histogram_tester,
+                "OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 1),
+            1);
+
+  EXPECT_GE(RetryForHistogramUntilCountReached(
+                histogram_tester,
+                "OptimizationGuide.HintsFetcher.GetHintsRequest.Status", 1),
+            1);
+
+  histogram_tester->ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.Status", net::HTTP_OK, 1);
+  histogram_tester->ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.NetErrorCode", net::OK,
+      1);
+  histogram_tester->ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.GetHintsRequest.HintCount", 1, 1);
+
+  // Hints fetching on SRP is enabled only when optimization keyed service is
+  // enabled.
+  if (!IsOptimizationGuideKeyedServiceEnabled())
+    return;
+
+  // Populate expected hosts with hosts contained in the html response of
+  // search_results_page_url().
+  base::flat_set<std::string> expected_hosts;
+  expected_hosts.insert(GURL("https://foo.com").host());
+  expected_hosts.insert(GURL("https://example.com").host());
+  expected_hosts.insert(GURL("https://example2.com").host());
+  SetExpectedHintsRequestForHosts(expected_hosts);
+
+  histogram_tester->ExpectTotalCount(
+      optimization_guide::kLoadedHintLocalHistogramString, 0);
+
+  // Navigate to a host not in the seeded site engagement service; it
+  // should be recorded as not covered by the hints fetcher.
+  ResetCountHintsRequestsReceived();
+  ui_test_utils::NavigateToURL(browser(), search_results_page_url());
+  WaitForPageLayout();
+
+  RetryForHistogramUntilCountReached(
+      histogram_tester, "AnchorElementMetrics.Visible.HighestNavigationScore",
+      1);
+
+  RetryForHistogramUntilCountReached(
+      histogram_tester,
+      "OptimizationGuide.HintsFetcher.NavigationHostCoveredByFetch", 1);
+
+  histogram_tester->ExpectUniqueSample(
+      "OptimizationGuide.HintsFetcher.NavigationHostCoveredByFetch", false, 1);
+
+  WaitUntilHintsFetcherRequestReceived();
+  EXPECT_EQ(1u, count_hints_requests_received());
+
+  RetryForHistogramUntilCountReached(
+      histogram_tester, optimization_guide::kLoadedHintLocalHistogramString,
+      IsOptimizationGuideKeyedServiceEnabled() ? 2 : 1);
+}

chrome/test/data/previews/search_results_page.html

+<html>
+  <head>
+  </head>
+  <body>
+    <a id="google" href="https://foo.com/">Google</a>
+    <a id="example" href="https://foo.com/simple_page_with_anchors.html">Same page</a>
+    <a id="example" href="https://foo.com/simple_page_with_anchors.html#">Same page</a>
+    <a id="example" href="https://foo.com/simple_page_with_anchors.html#foobar">Same page</a>
+    <a id="imageSameHref" href="https://foo.com"><img height="1" width="1"></a>
+    <a id="diffHref" href="https://example.com/foo.html">diff</a>
+    <a id="diffHref" href="https://example.com/bar.html">diff</a>
+    <a id="diffHref" href="https://example.com/baz.html">diff</a>
+    <a id="diffHref" href="https://example2.com/foo.html">diff</a>
+  </body>
+</html>
\ No newline at end of file

components/optimization_guide/hints_fetcher.h

@@ -53,7 +53,9 @@ class HintsFetcher {
   // not already in progress. Returns whether a new request was issued.
   // |hints_fetched_callback| is run, passing a GetHintsResponse object, if a
   // fetch was successful or passes nullopt if the fetch fails. Virtualized for
-  // testing.
+  // testing. Hints fetcher may fetch hints for only a subset of the provided
+  // |hosts|. |hosts| should be an ordered list in descending order of
+  // probability that the hints are needed for that host.
   virtual bool FetchOptimizationGuideServiceHints(
       const std::vector<std::string>& hosts,
       optimization_guide::proto::RequestContext request_context,

components/optimization_guide/optimization_guide_switches.cc

@@ -43,6 +43,9 @@ const char kOptimizationGuideServiceAPIKey[] =
 // fresh data.
 const char kPurgeHintCacheStore[] = "purge_hint_cache_store";
 
+const char kDisableFetchingHintsAtNavigationStartForTesting[] =
+    "disable-fetching-hints-at-navigation-start";
+
 bool IsHintComponentProcessingDisabled() {
   return base::CommandLine::ForCurrentProcess()->HasSwitch(kHintsProtoOverride);
 }
@@ -105,5 +108,11 @@ ParseComponentConfigFromCommandLine() {
   return proto_configuration;
 }
 
+bool DisableFetchingHintsAtNavigationStartForTesting() {
+  base::CommandLine* command_line = base::CommandLine::ForCurrentProcess();
+  return command_line->HasSwitch(
+      switches::kDisableFetchingHintsAtNavigationStartForTesting);
+}
+
 }  // namespace switches
 }  // namespace optimization_guide

components/optimization_guide/optimization_guide_switches.h

@@ -24,6 +24,7 @@ extern const char kFetchHintsOverrideTimer[];
 extern const char kOptimizationGuideServiceURL[];
 extern const char kOptimizationGuideServiceAPIKey[];
 extern const char kPurgeHintCacheStore[];
+extern const char kDisableFetchingHintsAtNavigationStartForTesting[];
 
 // Returns whether the hint component should be processed.
 // Available hint components are only processed if a proto override isn't being
@@ -50,6 +51,10 @@ bool ShouldOverrideFetchHintsTimer();
 std::unique_ptr<optimization_guide::proto::Configuration>
 ParseComponentConfigFromCommandLine();
 
+// Returns true if fetching of hints in real-time at the time of navigation
+// start should be disabled. Returns true only in tests.
+bool DisableFetchingHintsAtNavigationStartForTesting();
+
 }  // namespace switches
 }  // namespace optimization_guide