Commit 58510ca8 authored by Tarun Bansal's avatar Tarun Bansal Committed by Commit Bot

Enable fetching of hints from the URLs extracted

Enable fetching of hints from the URLs extracted from the
search result page.
The hints are fetched only on slow connections.

Change-Id: I0c8d0324ff913297c9f2b2a5fdc300771c0c3798
Bug: 1004527
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1839240
Commit-Queue: Tarun Bansal <tbansal@chromium.org>
Reviewed-by: default avatarMichael Crouse <mcrouse@chromium.org>
Reviewed-by: default avatarSophie Chang <sophiechang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#703480}
parent 1d0c7959
......@@ -16,10 +16,13 @@
#include "base/task_runner_util.h"
#include "base/time/default_clock.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service.h"
#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service_factory.h"
#include "chrome/browser/optimization_guide/optimization_guide_navigation_data.h"
#include "chrome/browser/optimization_guide/optimization_guide_permissions_util.h"
#include "chrome/browser/optimization_guide/optimization_guide_web_contents_observer.h"
#include "chrome/browser/profiles/profile.h"
#include "components/google/core/common/google_util.h"
#include "components/optimization_guide/bloom_filter.h"
#include "components/optimization_guide/hint_cache.h"
#include "components/optimization_guide/hint_cache_store.h"
......@@ -184,6 +187,10 @@ OptimizationGuideHintsManager::OptimizationGuideHintsManager(
optimization_guide::switches::ShouldPurgeHintCacheStoreOnStartup(),
base::BindOnce(&OptimizationGuideHintsManager::OnHintCacheInitialized,
ui_weak_ptr_factory_.GetWeakPtr()));
NavigationPredictorKeyedService* navigation_predictor_service =
NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
navigation_predictor_service->AddObserver(this);
}
OptimizationGuideHintsManager::~OptimizationGuideHintsManager() {
......@@ -192,6 +199,10 @@ OptimizationGuideHintsManager::~OptimizationGuideHintsManager() {
optimization_guide_service_->RemoveObserver(this);
g_browser_process->network_quality_tracker()
->RemoveEffectiveConnectionTypeObserver(this);
NavigationPredictorKeyedService* navigation_predictor_service =
NavigationPredictorKeyedServiceFactory::GetForProfile(profile_);
navigation_predictor_service->RemoveObserver(this);
}
void OptimizationGuideHintsManager::Shutdown() {
......@@ -533,6 +544,7 @@ void OptimizationGuideHintsManager::OnPageNavigationHintsFetched(
}
void OptimizationGuideHintsManager::OnFetchedTopHostsHintsStored() {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
LOCAL_HISTOGRAM_BOOLEAN("OptimizationGuide.FetchedHints.Stored", true);
top_hosts_hints_fetch_timer_.Stop();
......@@ -542,11 +554,13 @@ void OptimizationGuideHintsManager::OnFetchedTopHostsHintsStored() {
}
void OptimizationGuideHintsManager::OnFetchedPageNavigationHintsStored() {
for (const auto& url : navigation_urls_last_fetched_real_time_)
LoadHintForURL(url, base::DoNothing());
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
for (const auto& host : navigation_hosts_last_fetched_real_time_)
LoadHintForHost(host, base::DoNothing());
}
base::Time OptimizationGuideHintsManager::GetLastHintsFetchAttemptTime() const {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
return base::Time::FromDeltaSinceWindowsEpoch(
base::TimeDelta::FromMicroseconds(pref_service_->GetInt64(
optimization_guide::prefs::kHintsFetcherLastFetchAttempt)));
......@@ -554,6 +568,7 @@ base::Time OptimizationGuideHintsManager::GetLastHintsFetchAttemptTime() const {
void OptimizationGuideHintsManager::SetLastHintsFetchAttemptTime(
base::Time last_attempt_time) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
pref_service_->SetInt64(
optimization_guide::prefs::kHintsFetcherLastFetchAttempt,
last_attempt_time.ToDeltaSinceWindowsEpoch().InMicroseconds());
......@@ -581,20 +596,88 @@ void OptimizationGuideHintsManager::LoadHintForNavigation(
}
}
LoadHintForURL(url, std::move(callback));
LoadHintForHost(url.host(), std::move(callback));
}
void OptimizationGuideHintsManager::LoadHintForURL(const GURL& url,
void OptimizationGuideHintsManager::LoadHintForHost(
const std::string& host,
base::OnceClosure callback) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
DCHECK(url.has_host());
hint_cache_->LoadHint(
url.host(),
host,
base::BindOnce(&OptimizationGuideHintsManager::OnHintLoaded,
ui_weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
}
bool OptimizationGuideHintsManager::IsGoogleURL(const GURL& url) const {
return google_util::IsGoogleHostname(url.host(),
google_util::DISALLOW_SUBDOMAIN);
}
void OptimizationGuideHintsManager::OnPredictionUpdated(
const base::Optional<NavigationPredictorKeyedService::Prediction>&
prediction) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (!prediction.has_value())
return;
const GURL& source_document_url = prediction->source_document_url();
// We only extract next predicted navigations from Google URLs.
if (!IsGoogleURL(source_document_url))
return;
// Extract the target hosts. Use a flat set to remove duplicates.
// |target_hosts_serialized| is the ordered list of non-duplicate hosts.
base::flat_set<std::string> target_hosts;
std::vector<std::string> target_hosts_serialized;
for (const auto& url : prediction->sorted_predicted_urls()) {
if (!IsAllowedToFetchNavigationHints(url))
continue;
if (target_hosts.size() >=
optimization_guide::features::
MaxHostsForOptimizationGuideServiceHintsFetch()) {
break;
}
// Insert the host to |target_hosts|. The host is inserted to
// |target_hosts_serialized| only if it was not a duplicate insertion to
// |target_hosts|.
std::pair<base::flat_set<std::string>::iterator, bool> insert_result =
target_hosts.insert(url.host());
if (insert_result.second)
target_hosts_serialized.push_back(url.host());
// Ensure that the 2 data structures remain synchronized.
DCHECK_EQ(target_hosts.size(), target_hosts_serialized.size());
}
if (target_hosts.empty())
return;
navigation_hosts_last_fetched_real_time_.clear();
for (const auto& host : target_hosts)
navigation_hosts_last_fetched_real_time_.push_back(host);
if (!hints_fetcher_) {
hints_fetcher_ = std::make_unique<optimization_guide::HintsFetcher>(
url_loader_factory_,
optimization_guide::features::GetOptimizationGuideServiceURL(),
pref_service_);
}
hints_fetcher_->FetchOptimizationGuideServiceHints(
target_hosts_serialized,
optimization_guide::proto::CONTEXT_PAGE_NAVIGATION,
base::BindOnce(&OptimizationGuideHintsManager::OnHintsFetched,
ui_weak_ptr_factory_.GetWeakPtr()));
for (const auto& host : target_hosts)
LoadHintForHost(host, base::DoNothing());
}
void OptimizationGuideHintsManager::OnHintLoaded(
base::OnceClosure callback,
const optimization_guide::proto::Hint* loaded_hint) const {
......@@ -841,11 +924,16 @@ void OptimizationGuideHintsManager::OnNavigationStartOrRedirect(
base::OnceClosure callback) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
if (optimization_guide::switches::
DisableFetchingHintsAtNavigationStartForTesting()) {
return;
}
if (IsAllowedToFetchNavigationHints(navigation_handle->GetURL())) {
std::vector<std::string> hosts{navigation_handle->GetURL().host()};
navigation_urls_last_fetched_real_time_.clear();
navigation_urls_last_fetched_real_time_.push_back(
navigation_handle->GetURL());
navigation_hosts_last_fetched_real_time_.clear();
navigation_hosts_last_fetched_real_time_.push_back(
navigation_handle->GetURL().host());
if (!hints_fetcher_) {
hints_fetcher_ = std::make_unique<optimization_guide::HintsFetcher>(
......
......@@ -18,6 +18,7 @@
#include "base/synchronization/lock.h"
#include "base/time/clock.h"
#include "base/timer/timer.h"
#include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service.h"
#include "components/optimization_guide/hints_component_info.h"
#include "components/optimization_guide/optimization_guide_service_observer.h"
#include "components/optimization_guide/proto/hints.pb.h"
......@@ -60,7 +61,8 @@ class Profile;
class OptimizationGuideHintsManager
: public optimization_guide::OptimizationGuideServiceObserver,
public network::NetworkQualityTracker::EffectiveConnectionTypeObserver {
public network::NetworkQualityTracker::EffectiveConnectionTypeObserver,
public NavigationPredictorKeyedService::Observer {
public:
OptimizationGuideHintsManager(
optimization_guide::OptimizationGuideService* optimization_guide_service,
......@@ -139,6 +141,17 @@ class OptimizationGuideHintsManager
base::OnceClosure callback);
private:
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest, IsGoogleURL);
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_SLOW_2G);
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_4G);
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
HintsFetched_AtNonSRP_ECT_SLOW_2G);
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_SLOW_2G_DuplicatesRemoved);
FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_SLOW_2G_InsecureHostsRemoved);
// Processes the hints component.
//
// Should always be called on the thread that belongs to
......@@ -240,11 +253,20 @@ class OptimizationGuideHintsManager
void LoadHintForNavigation(content::NavigationHandle* navigation_handle,
base::OnceClosure callback);
// Loads the hint for |url| if available.
// Loads the hint for |host| if available.
// |callback| is run when the request has finished regardless of whether there
// was actually a hint for that |url| or not. The callback can be used as a
// was actually a hint for that |host| or not. The callback can be used as a
// signal for tests.
void LoadHintForURL(const GURL& url, base::OnceClosure callback);
void LoadHintForHost(const std::string& host, base::OnceClosure callback);
// Returns true if the hostname for |url| matches the host of google web
// search results page (www.google.*).
bool IsGoogleURL(const GURL& url) const;
// NavigationPredictorKeyedService::Observer:
void OnPredictionUpdated(
const base::Optional<NavigationPredictorKeyedService::Prediction>&
prediction) override;
// The OptimizationGuideService that this guide is listening to. Not owned.
optimization_guide::OptimizationGuideService* const
......@@ -314,8 +336,8 @@ class OptimizationGuideHintsManager
// Used in testing to subscribe to an update event in this class.
base::OnceClosure next_update_closure_;
// URLs for which hints were last fetched in the real-time.
std::vector<GURL> navigation_urls_last_fetched_real_time_;
// Hosts for which hints were last fetched in the real-time.
std::vector<std::string> navigation_hosts_last_fetched_real_time_;
// Used to get |weak_ptr_| to self on the UI thread.
base::WeakPtrFactory<OptimizationGuideHintsManager> ui_weak_ptr_factory_{
......
......@@ -1703,6 +1703,175 @@ TEST_F(OptimizationGuideHintsManagerTest,
EXPECT_EQ(nullptr, navigation_data->page_hint());
}
TEST_F(OptimizationGuideHintsManagerTest, IsGoogleURL) {
const struct {
const char* url;
bool expect_is_google_url;
} tests[] = {
{"https://www.google.com/"
"search?q=cats&oq=cq&aqs=foo&ie=UTF-8",
true},
{"https://www.google.com/", true},
{"https://www.google.com/:99", true},
// Try localized search pages.
{"https://www.google.co.in/"
"search?q=cats&oq=cq&aqs=foo&ie=UTF-8",
true},
{"https://www.google.co.in/", true},
{"https://www.google.co.in/:99", true},
// Try Google domain pages that are not web search related.
{"https://www.not-google.com/", false},
{"https://www.youtube.com/", false},
{"https://domain.google.com/", false},
{"https://images.google.com/", false},
};
for (const auto& test : tests) {
GURL url(test.url);
EXPECT_TRUE(url.is_valid());
EXPECT_EQ(test.expect_is_google_url, hints_manager()->IsGoogleURL(url));
}
}
TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtSRP_ECT_SLOW_2G) {
hints_manager()->RegisterOptimizationTypes(
{optimization_guide::proto::DEFER_ALL_SCRIPT});
base::test::ScopedFeatureList scoped_list;
scoped_list.InitAndEnableFeature(
optimization_guide::features::kOptimizationHintsFetching);
InitializeWithDefaultConfig("1.0.0.0");
// Set ECT estimate so hint is activated.
hints_manager()->OnEffectiveConnectionTypeChanged(
net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
std::unique_ptr<content::MockNavigationHandle> navigation_handle =
CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
url_without_hints());
base::HistogramTester histogram_tester;
std::vector<GURL> sorted_predicted_urls;
sorted_predicted_urls.push_back(GURL("https://foo.com/"));
NavigationPredictorKeyedService::Prediction prediction(
nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
hints_manager()->OnPredictionUpdated(prediction);
histogram_tester.ExpectUniqueSample(
"OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 1, 1);
}
TEST_F(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_SLOW_2G_DuplicatesRemoved) {
hints_manager()->RegisterOptimizationTypes(
{optimization_guide::proto::DEFER_ALL_SCRIPT});
base::test::ScopedFeatureList scoped_list;
scoped_list.InitAndEnableFeature(
optimization_guide::features::kOptimizationHintsFetching);
InitializeWithDefaultConfig("1.0.0.0");
// Set ECT estimate so hint is activated.
hints_manager()->OnEffectiveConnectionTypeChanged(
net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
std::unique_ptr<content::MockNavigationHandle> navigation_handle =
CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
url_without_hints());
base::HistogramTester histogram_tester;
std::vector<GURL> sorted_predicted_urls;
sorted_predicted_urls.push_back(GURL("https://foo.com/page1.html"));
sorted_predicted_urls.push_back(GURL("https://foo.com/page2.html"));
sorted_predicted_urls.push_back(GURL("https://foo.com/page3.html"));
sorted_predicted_urls.push_back(GURL("https://bar.com/"));
NavigationPredictorKeyedService::Prediction prediction(
nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
hints_manager()->OnPredictionUpdated(prediction);
// Ensure that we only include 2 hosts in the request. These would be foo.com
// and bar.com.
histogram_tester.ExpectUniqueSample(
"OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 2, 1);
}
TEST_F(OptimizationGuideHintsManagerTest,
HintsFetched_AtSRP_ECT_SLOW_2G_InsecureHostsRemoved) {
hints_manager()->RegisterOptimizationTypes(
{optimization_guide::proto::DEFER_ALL_SCRIPT});
base::test::ScopedFeatureList scoped_list;
scoped_list.InitAndEnableFeature(
optimization_guide::features::kOptimizationHintsFetching);
InitializeWithDefaultConfig("1.0.0.0");
// Set ECT estimate so hint is activated.
hints_manager()->OnEffectiveConnectionTypeChanged(
net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
std::unique_ptr<content::MockNavigationHandle> navigation_handle =
CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
url_without_hints());
base::HistogramTester histogram_tester;
std::vector<GURL> sorted_predicted_urls;
sorted_predicted_urls.push_back(GURL("https://foo.com/page1.html"));
sorted_predicted_urls.push_back(GURL("http://insecure-bar.com/"));
NavigationPredictorKeyedService::Prediction prediction(
nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
hints_manager()->OnPredictionUpdated(prediction);
// Ensure that we only include 1 secure host in the request. These would be
// foo.com.
histogram_tester.ExpectUniqueSample(
"OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 1, 1);
}
TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtSRP_ECT_4G) {
hints_manager()->RegisterOptimizationTypes(
{optimization_guide::proto::DEFER_ALL_SCRIPT});
base::test::ScopedFeatureList scoped_list;
scoped_list.InitAndEnableFeature(
optimization_guide::features::kOptimizationHintsFetching);
InitializeWithDefaultConfig("1.0.0.0");
// Set ECT estimate so hint is activated.
hints_manager()->OnEffectiveConnectionTypeChanged(
net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_4G);
std::unique_ptr<content::MockNavigationHandle> navigation_handle =
CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
url_without_hints());
base::HistogramTester histogram_tester;
std::vector<GURL> sorted_predicted_urls;
sorted_predicted_urls.push_back(GURL("https://foo.com/"));
NavigationPredictorKeyedService::Prediction prediction(
nullptr, GURL("https://www.google.com/"), sorted_predicted_urls);
hints_manager()->OnPredictionUpdated(prediction);
histogram_tester.ExpectTotalCount(
"OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 0);
}
TEST_F(OptimizationGuideHintsManagerTest, HintsFetched_AtNonSRP_ECT_SLOW_2G) {
hints_manager()->RegisterOptimizationTypes(
{optimization_guide::proto::DEFER_ALL_SCRIPT});
base::test::ScopedFeatureList scoped_list;
scoped_list.InitAndEnableFeature(
optimization_guide::features::kOptimizationHintsFetching);
InitializeWithDefaultConfig("1.0.0.0");
// Set ECT estimate so hint is activated.
hints_manager()->OnEffectiveConnectionTypeChanged(
net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_SLOW_2G);
std::unique_ptr<content::MockNavigationHandle> navigation_handle =
CreateMockNavigationHandleWithOptimizationGuideWebContentsObserver(
url_without_hints());
base::HistogramTester histogram_tester;
std::vector<GURL> sorted_predicted_urls;
sorted_predicted_urls.push_back(GURL("https://foo.com/"));
NavigationPredictorKeyedService::Prediction prediction(
nullptr, GURL("https://www.not-google.com/"), sorted_predicted_urls);
hints_manager()->OnPredictionUpdated(prediction);
histogram_tester.ExpectTotalCount(
"OptimizationGuide.HintsFetcher.GetHintsRequest.HostCount", 0);
}
TEST_F(OptimizationGuideHintsManagerTest,
HintsFetchedAtNavigationTime_ECT_SLOW_2G) {
hints_manager()->RegisterOptimizationTypes(
......
<html>
<head>
</head>
<body>
<a id="google" href="https://foo.com/">Google</a>
<a id="example" href="https://foo.com/simple_page_with_anchors.html">Same page</a>
<a id="example" href="https://foo.com/simple_page_with_anchors.html#">Same page</a>
<a id="example" href="https://foo.com/simple_page_with_anchors.html#foobar">Same page</a>
<a id="imageSameHref" href="https://foo.com"><img height="1" width="1"></a>
<a id="diffHref" href="https://example.com/foo.html">diff</a>
<a id="diffHref" href="https://example.com/bar.html">diff</a>
<a id="diffHref" href="https://example.com/baz.html">diff</a>
<a id="diffHref" href="https://example2.com/foo.html">diff</a>
</body>
</html>
\ No newline at end of file
......@@ -53,7 +53,9 @@ class HintsFetcher {
// not already in progress. Returns whether a new request was issued.
// |hints_fetched_callback| is run, passing a GetHintsResponse object, if a
// fetch was successful or passes nullopt if the fetch fails. Virtualized for
// testing.
// testing. Hints fetcher may fetch hints for only a subset of the provided
// |hosts|. |hosts| should be an ordered list in descending order of
// probability that the hints are needed for that host.
virtual bool FetchOptimizationGuideServiceHints(
const std::vector<std::string>& hosts,
optimization_guide::proto::RequestContext request_context,
......
......@@ -43,6 +43,9 @@ const char kOptimizationGuideServiceAPIKey[] =
// fresh data.
const char kPurgeHintCacheStore[] = "purge_hint_cache_store";
const char kDisableFetchingHintsAtNavigationStartForTesting[] =
"disable-fetching-hints-at-navigation-start";
bool IsHintComponentProcessingDisabled() {
return base::CommandLine::ForCurrentProcess()->HasSwitch(kHintsProtoOverride);
}
......@@ -105,5 +108,11 @@ ParseComponentConfigFromCommandLine() {
return proto_configuration;
}
bool DisableFetchingHintsAtNavigationStartForTesting() {
base::CommandLine* command_line = base::CommandLine::ForCurrentProcess();
return command_line->HasSwitch(
switches::kDisableFetchingHintsAtNavigationStartForTesting);
}
} // namespace switches
} // namespace optimization_guide
......@@ -24,6 +24,7 @@ extern const char kFetchHintsOverrideTimer[];
extern const char kOptimizationGuideServiceURL[];
extern const char kOptimizationGuideServiceAPIKey[];
extern const char kPurgeHintCacheStore[];
extern const char kDisableFetchingHintsAtNavigationStartForTesting[];
// Returns whether the hint component should be processed.
// Available hint components are only processed if a proto override isn't being
......@@ -50,6 +51,10 @@ bool ShouldOverrideFetchHintsTimer();
std::unique_ptr<optimization_guide::proto::Configuration>
ParseComponentConfigFromCommandLine();
// Returns true if fetching of hints in real-time at the time of navigation
// start should be disabled. Returns true only in tests.
bool DisableFetchingHintsAtNavigationStartForTesting();
} // namespace switches
} // namespace optimization_guide
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment