[NTP][RQ] Repeatable Queries Service

Provides repeatable query suggestions to be shown in the NTP MV tiles
when Google is the default search provider. The repeatable queries are
requested from the server for signed-in users and extracted from the
URL database for unauthenticated users.

Fixed: 1106396, 1110077
Change-Id: I749739cb4382d83f21256430428a60c5f1d563bd
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2518773
Commit-Queue: Moe Ahmadi <mahmadi@chromium.org>
Reviewed-by: Tommy Li <tommycli@chromium.org>
Reviewed-by: Colin Blundell <blundell@chromium.org>
Reviewed-by: David Roger <droger@chromium.org>
Reviewed-by: Scott Violet <sky@chromium.org>
Reviewed-by: Ramya Nagarajan <ramyan@chromium.org>
Reviewed-by: Nicolas Ouellet-Payeur <nicolaso@chromium.org>
Cr-Commit-Position: refs/heads/master@{#825036}

chrome/browser/BUILD.gn

@@ -3762,6 +3762,8 @@ static_library("browser") {
       "search/promos/promo_service_factory.cc",
       "search/promos/promo_service_factory.h",
       "search/promos/promo_service_observer.h",
+      "search/repeatable_queries/repeatable_queries_service_factory.cc",
+      "search/repeatable_queries/repeatable_queries_service_factory.h",
       "search/search_engine_base_url_tracker.cc",
       "search/search_engine_base_url_tracker.h",
       "search/search_suggest/search_suggest_data.cc",

chrome/browser/search/repeatable_queries/repeatable_queries_service_factory.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/search/repeatable_queries/repeatable_queries_service_factory.h"
+
+#include "base/feature_list.h"
+#include "chrome/browser/content_settings/cookie_settings_factory.h"
+#include "chrome/browser/history/history_service_factory.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/search_engines/template_url_service_factory.h"
+#include "chrome/browser/signin/identity_manager_factory.h"
+#include "chrome/common/webui_url_constants.h"
+#include "components/keyed_service/content/browser_context_dependency_manager.h"
+#include "components/search/ntp_features.h"
+#include "components/search/repeatable_queries/repeatable_queries_service.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/storage_partition.h"
+
+// static
+RepeatableQueriesService* RepeatableQueriesServiceFactory::GetForProfile(
+    Profile* profile) {
+  return static_cast<RepeatableQueriesService*>(
+      GetInstance()->GetServiceForBrowserContext(profile, true));
+}
+
+// static
+RepeatableQueriesServiceFactory*
+RepeatableQueriesServiceFactory::GetInstance() {
+  return base::Singleton<RepeatableQueriesServiceFactory>::get();
+}
+
+RepeatableQueriesServiceFactory::RepeatableQueriesServiceFactory()
+    : BrowserContextKeyedServiceFactory(
+          "RepeatableQueriesService",
+          BrowserContextDependencyManager::GetInstance()) {
+  DependsOn(CookieSettingsFactory::GetInstance());
+  DependsOn(IdentityManagerFactory::GetInstance());
+}
+
+RepeatableQueriesServiceFactory::~RepeatableQueriesServiceFactory() = default;
+
+KeyedService* RepeatableQueriesServiceFactory::BuildServiceInstanceFor(
+    content::BrowserContext* context) const {
+  if (!base::FeatureList::IsEnabled(ntp_features::kNtpRepeatableQueries)) {
+    return nullptr;
+  }
+
+  Profile* profile = Profile::FromBrowserContext(context);
+  signin::IdentityManager* identity_manager =
+      IdentityManagerFactory::GetForProfile(profile);
+  history::HistoryService* history_service =
+      HistoryServiceFactory::GetForProfile(profile,
+                                           ServiceAccessType::EXPLICIT_ACCESS);
+  TemplateURLService* template_url_service =
+      TemplateURLServiceFactory::GetForProfile(profile);
+  auto url_loader_factory =
+      content::BrowserContext::GetDefaultStoragePartition(context)
+          ->GetURLLoaderFactoryForBrowserProcess();
+  return new RepeatableQueriesService(identity_manager, history_service,
+                                      template_url_service, url_loader_factory,
+                                      GURL(chrome::kChromeUINewTabURL));
+}

chrome/browser/search/repeatable_queries/repeatable_queries_service_factory.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_FACTORY_H_
+#define CHROME_BROWSER_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_FACTORY_H_
+
+#include "base/macros.h"
+#include "base/memory/singleton.h"
+#include "components/keyed_service/content/browser_context_keyed_service_factory.h"
+
+class RepeatableQueriesService;
+class Profile;
+
+class RepeatableQueriesServiceFactory
+    : public BrowserContextKeyedServiceFactory {
+ public:
+  // Returns the RepeatableQueriesService for |profile|.
+  static RepeatableQueriesService* GetForProfile(Profile* profile);
+
+  static RepeatableQueriesServiceFactory* GetInstance();
+
+ private:
+  friend struct base::DefaultSingletonTraits<RepeatableQueriesServiceFactory>;
+
+  RepeatableQueriesServiceFactory();
+  ~RepeatableQueriesServiceFactory() override;
+
+  // Overridden from BrowserContextKeyedServiceFactory:
+  KeyedService* BuildServiceInstanceFor(
+      content::BrowserContext* profile) const override;
+
+  DISALLOW_COPY_AND_ASSIGN(RepeatableQueriesServiceFactory);
+};
+
+#endif  // CHROME_BROWSER_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_FACTORY_H_

components/omnibox/browser/in_memory_url_index.h

@@ -151,6 +151,7 @@ class InMemoryURLIndex : public KeyedService,
   friend class history::HQPPerfTestOnePopularURL;
   friend class InMemoryURLIndexTest;
   friend class InMemoryURLIndexCacheTest;
+  friend class RepeatableQueriesServiceTest;
   FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, ExpireRow);
   FRIEND_TEST_ALL_PREFIXES(LimitedInMemoryURLIndexTest, Initialization);
 

components/search/BUILD.gn

@@ -6,6 +6,9 @@ static_library("search") {
   sources = [
     "ntp_features.cc",
     "ntp_features.h",
+    "repeatable_queries/repeatable_queries_service.cc",
+    "repeatable_queries/repeatable_queries_service.h",
+    "repeatable_queries/repeatable_queries_service_observer.h",
     "search.cc",
     "search.h",
     "search_provider_observer.cc",
@@ -15,7 +18,15 @@ static_library("search") {
   deps = [
     "//base",
     "//components/google/core/common",
+    "//components/history/core/browser",
+    "//components/keyed_service/core",
     "//components/search_engines",
+    "//components/signin/public/identity_manager",
+    "//components/variations/net",
+    "//net",
+    "//net/traffic_annotation",
+    "//services/data_decoder/public/cpp",
+    "//services/network/public/cpp",
     "//url",
   ]
 }
@@ -23,7 +34,10 @@ static_library("search") {
 source_set("unit_tests") {
   testonly = true
 
-  sources = [ "search_unittest.cc" ]
+  sources = [
+    "repeatable_queries/repeatable_queries_service_unittest.cc",
+    "search_unittest.cc",
+  ]
   if (is_android) {
     sources += [ "search_android_unittest.cc" ]
   }
@@ -31,7 +45,17 @@ source_set("unit_tests") {
   deps = [
     ":search",
     "//base",
+    "//base/test:test_support",
+    "//components/bookmarks/browser",
+    "//components/bookmarks/test",
+    "//components/history/core/test",
+    "//components/omnibox/browser:test_support",
+    "//components/search_engines:search_engines",
+    "//components/signin/public/base:test_support",
+    "//components/signin/public/identity_manager:test_support",
     "//components/variations",
+    "//services/data_decoder/public/cpp:test_support",
+    "//testing/gmock",
     "//testing/gtest",
   ]
 }

components/search/DEPS

 include_rules = [
+  "+components/bookmarks/browser",
+  "+components/bookmarks/test",
   "+components/google/core",
+  "+components/history/core",
+  "+components/keyed_service/core",
+  "+components/omnibox/browser",
   "+components/search_engines",
+  "+components/signin/public",
   "+components/variations",
+  "+net",
+  "+services/data_decoder/public",
+  "+services/network/public",
 ]

components/search/repeatable_queries/repeatable_queries_service.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_H_
+#define COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/memory/weak_ptr.h"
+#include "base/observer_list.h"
+#include "components/keyed_service/core/keyed_service.h"
+#include "components/search/repeatable_queries/repeatable_queries_service_observer.h"
+#include "services/data_decoder/public/cpp/data_decoder.h"
+#include "url/gurl.h"
+
+class SearchProviderObserver;
+class TemplateURLService;
+
+namespace history {
+class HistoryService;
+}  // namespace history
+
+namespace network {
+class SimpleURLLoader;
+class SharedURLLoaderFactory;
+}  // namespace network
+
+namespace signin {
+class IdentityManager;
+}  // namespace signin
+
+// Represents a repeatable query suggestion.
+class RepeatableQuery {
+ public:
+  RepeatableQuery() = default;
+  ~RepeatableQuery() = default;
+
+  bool operator==(const RepeatableQuery& other) const {
+    return query == other.query && deletion_url == other.deletion_url;
+  }
+  bool operator!=(const RepeatableQuery& other) const {
+    return !(this == &other);
+  }
+
+  // Repeatable query suggestion.
+  base::string16 query;
+
+  // The endpoint used for deleting the query suggestion on the server.
+  // Populated for server provided queries only.
+  std::string deletion_url;
+};
+
+// Provides repeatable query suggestions to be shown in the NTP Most Visited
+// tiles when Google is the default search provider. The repeatable queries are
+// requested from the server for signed-in users and extracted from the URL
+// database for unauthenticated users.
+class RepeatableQueriesService : public KeyedService {
+ public:
+  RepeatableQueriesService(
+      signin::IdentityManager* identity_manager,
+      history::HistoryService* history_service,
+      TemplateURLService* template_url_service,
+      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
+      const GURL& request_initiator_url);
+  ~RepeatableQueriesService() override;
+  RepeatableQueriesService(const RepeatableQueriesService&) = delete;
+  RepeatableQueriesService& operator=(const RepeatableQueriesService&) = delete;
+
+  // KeyedService:
+  void Shutdown() override;
+
+  // Returns the currently cached repeatable query suggestions, if any.
+  const std::vector<RepeatableQuery>& repeatable_queries() const;
+
+  // If Google is the default search provider, asynchronously requests
+  // repeatable query suggestions from the server for signed-in users and
+  // synchronously extracts them from the URL database for
+  // unauthenticated users. Regardless of success, observers are notified via
+  // RepeatableQueriesServiceObserver::OnRepeatableQueriesUpdated.
+  void Refresh();
+
+  // Add/remove observers.
+  void AddObserver(RepeatableQueriesServiceObserver* observer);
+  void RemoveObserver(RepeatableQueriesServiceObserver* observer);
+
+ protected:
+  class SigninObserver;
+
+  virtual SigninObserver* signin_observer();
+  virtual SearchProviderObserver* search_provider_observer();
+
+  // Called when the default search provider changes.
+  void SearchProviderChanged();
+
+  // Called when the signin status changes.
+  void SigninStatusChanged();
+
+  // Returns the server request URL.
+  GURL GetRequestURL();
+
+ private:
+  // Requests repeatable queries from the server. Called for signed-in users.
+  void GetRepeatableQueriesFromServer();
+  void RepeatableQueriesResponseLoaded(std::unique_ptr<std::string> response);
+  void RepeatableQueriesParsed(data_decoder::DataDecoder::ValueOrError result);
+
+  // Queries the in-memory URLDatabase for the repeatable queries submitted
+  // to the default search provider. Called for unauthenticated users.
+  void GetRepeatableQueriesFromURLDatabase();
+
+  void NotifyObservers();
+
+  history::HistoryService* history_service_;
+
+  TemplateURLService* template_url_service_;
+
+  scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
+
+  const GURL request_initiator_url_;
+
+  std::unique_ptr<SigninObserver> signin_observer_;
+
+  std::unique_ptr<SearchProviderObserver> search_provider_observer_;
+
+  base::ObserverList<RepeatableQueriesServiceObserver, true> observers_;
+
+  std::vector<RepeatableQuery> repeatable_queries_;
+
+  std::unique_ptr<network::SimpleURLLoader> simple_loader_;
+
+  base::WeakPtrFactory<RepeatableQueriesService> weak_ptr_factory_{this};
+};
+
+#endif  // COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_H_

components/search/repeatable_queries/repeatable_queries_service_observer.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_OBSERVER_H_
+#define COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_OBSERVER_H_
+
+// Observer class for the RepeatableQueriesService.
+class RepeatableQueriesServiceObserver : public base::CheckedObserver {
+ public:
+  // Called after a Refresh() call on the service, either directly or as a
+  // result of default search provider or signin status change. Note that this
+  // is called after each Refresh(), even if the network request failed, or if
+  // it didn't result in an actual change to the cached data. Observers can get
+  // the repeatable queries via RepeatableQueriesService::repeatable_queries().
+  virtual void OnRepeatableQueriesUpdated() = 0;
+
+  // Called when the service is shutting down allowing the observers to
+  // unregister themselves and clear references to the service.
+  virtual void OnRepeatableQueriesServiceShuttingDown() {}
+};
+
+#endif  // COMPONENTS_SEARCH_REPEATABLE_QUERIES_REPEATABLE_QUERIES_SERVICE_OBSERVER_H_

components/search/search_provider_observer.cc

@@ -10,21 +10,25 @@ SearchProviderObserver::SearchProviderObserver(TemplateURLService* service,
     : service_(service),
       is_google_(search::DefaultSearchProviderIsGoogle(service_)),
       callback_(std::move(callback)) {
-  DCHECK(service_);
-  service_->AddObserver(this);
+  if (service_) {
+    service_observer_.Add(service_);
+  }
 }
 
-SearchProviderObserver::~SearchProviderObserver() {
-  if (service_)
-    service_->RemoveObserver(this);
+SearchProviderObserver::~SearchProviderObserver() = default;
+
+bool SearchProviderObserver::is_google() {
+  return is_google_;
 }
 
 void SearchProviderObserver::OnTemplateURLServiceChanged() {
+  DCHECK(service_);
   is_google_ = search::DefaultSearchProviderIsGoogle(service_);
   callback_.Run();
 }
 
 void SearchProviderObserver::OnTemplateURLServiceShuttingDown() {
-  service_->RemoveObserver(this);
+  DCHECK(service_);
+  service_observer_.Remove(service_);
   service_ = nullptr;
 }

components/search/search_provider_observer.h

@@ -5,6 +5,7 @@
 #ifndef COMPONENTS_SEARCH_SEARCH_PROVIDER_OBSERVER_H_
 #define COMPONENTS_SEARCH_SEARCH_PROVIDER_OBSERVER_H_
 
+#include "base/scoped_observer.h"
 #include "components/search_engines/template_url_service.h"
 #include "components/search_engines/template_url_service_observer.h"
 
@@ -15,17 +16,18 @@ class SearchProviderObserver : public TemplateURLServiceObserver {
  public:
   explicit SearchProviderObserver(TemplateURLService* service,
                                   base::RepeatingClosure callback);
-
   ~SearchProviderObserver() override;
 
-  bool is_google() { return is_google_; }
-  TemplateURLService* template_url_service() { return service_; }
+  virtual bool is_google();
 
  private:
   // TemplateURLServiceObserver:
   void OnTemplateURLServiceChanged() override;
   void OnTemplateURLServiceShuttingDown() override;
 
+  ScopedObserver<TemplateURLService, TemplateURLServiceObserver>
+      service_observer_{this};
+  // May be nullptr in tests.
   TemplateURLService* service_;
   bool is_google_;
   base::RepeatingClosure callback_;

tools/traffic_annotation/summary/annotations.xml

@@ -276,6 +276,7 @@ Refer to README.md for content description and update process.
  <item id="remoting_telemetry_log_writer" added_in_milestone="86" hash_code="107268760" type="0" content_hash_code="81741595" os_list="linux,windows" file_path="remoting/base/telemetry_log_writer.cc"/>
  <item id="render_view_context_menu" added_in_milestone="62" hash_code="25844439" type="0" content_hash_code="69471170" os_list="linux,windows" file_path="chrome/browser/renderer_context_menu/render_view_context_menu.cc"/>
  <item id="renderer_initiated_download" added_in_milestone="62" hash_code="116443055" type="0" content_hash_code="37846436" os_list="linux,windows" file_path="content/browser/renderer_host/render_frame_host_impl.cc"/>
+ <item id="repeatable_queries_service" added_in_milestone="88" hash_code="5394442" type="0" content_hash_code="115294794" os_list="linux,windows" file_path="components/search/repeatable_queries/repeatable_queries_service.cc"/>
  <item id="reporting" added_in_milestone="62" hash_code="109891200" type="0" content_hash_code="125758928" os_list="linux,windows" file_path="net/reporting/reporting_uploader.cc"/>
  <item id="resource_dispatcher_host" added_in_milestone="62" hash_code="81157007" type="0" deprecated="2019-07-30" content_hash_code="35725167" file_path=""/>
  <item id="resource_prefetch" added_in_milestone="62" hash_code="110815970" type="0" deprecated="2018-02-28" content_hash_code="39251261" file_path=""/>

tools/traffic_annotation/summary/grouping.xml

@@ -392,6 +392,7 @@ hidden="true" so that these annotations don't show up in the document.
       <traffic_annotation unique_id="popular_sites_fetch"/>
       <traffic_annotation unique_id="search_suggest_service"/>
       <traffic_annotation unique_id="remote_suggestions_provider"/>
+      <traffic_annotation unique_id="repeatable_queries_service"/>
       <traffic_annotation unique_id="promo_service"/>
       <traffic_annotation unique_id="task_module_service"/>
     </sender>