Support adding TemplateURL by OSDD in SearchEngineTabHelper.

Enable SearchEngineTabHelper to add new TemplateURL to TemplateURLService by following steps:
1. Receive messages of OSDD(Open Search Description Document) <link> in page from Js;
2. Download, parse OSDD XML file, and create TemplateURL;
3. Add TemplateURL to TemplateURLService.

The OSDD functionaliy on Js side has not been changed yet, but SearchEngineTabHelper
and SearchEngineTabHelperTest is decoupled from it. search_engine.js will
change from positive calling into message mechanism for OSDD in another CL.

Bug: 433824
Change-Id: I113e362ed1fa38cff43cea1e8fe2f591fac469c0
Reviewed-on: https://chromium-review.googlesource.com/c/1309746
Commit-Queue: Yi Su <mrsuyi@chromium.org>
Reviewed-by: Eugene But <eugenebut@chromium.org>
Cr-Commit-Position: refs/heads/master@{#604360}

ios/chrome/browser/search_engines/BUILD.gn

@@ -68,15 +68,21 @@ source_set("unit_tests") {
   testonly = true
   sources = [
     "search_engine_js_unittest.mm",
+    "search_engine_tab_helper_unittest.mm",
   ]
   deps = [
     ":search_engine_js",
+    ":search_engines",
     "//base:base",
     "//base/test:test_support",
+    "//components/search_engines",
+    "//ios/chrome/browser/browser_state",
+    "//ios/chrome/browser/browser_state:test_support",
     "//ios/chrome/browser/web:test_support",
     "//ios/web",
     "//ios/web/public/test",
     "//ios/web/public/test/fakes",
+    "//net:test_support",
     "//testing/gtest",
   ]
 }

ios/chrome/browser/search_engines/search_engine_tab_helper.h

@@ -14,6 +14,23 @@ namespace web {
 class WebState;
 }  // namespace web
 
+// Creates TemplateURLs from attached WebState and adds them to
+// TemplateURLService. To create a TemplateURL, 3 basic elements are needed:
+//   1. A short name (e.g. "Google");
+//   2. A keyword (e.g. "google.com");
+//   3. A searchable URL (e.g. "https://google.com?name=a&q={searchTerms}").
+//
+// Both short name and keyword can be generated from navigation history. For
+// searchable URL, there are 2 methods to create it:
+//   1. If a OSDD(Open Search Description Document) <link> is found in page,
+//      use TemplateURLFetcher to download the XML file, parse it and get the
+//      searchable URL;
+//   2. If a <form> is submitted in page, a searchable URL can be generated
+//      by analysing the <form>'s elements and concatenating "name" and
+//      "value" attributes of them.
+//
+// Both these 2 methods depends on injected JavaScript.
+//
 class SearchEngineTabHelper
     : public web::WebStateObserver,
       public web::WebStateUserData<SearchEngineTabHelper> {
@@ -25,14 +42,24 @@ class SearchEngineTabHelper
 
   explicit SearchEngineTabHelper(web::WebState* web_state);
 
+  // Creates a TemplateURL by downloading and parsing the OSDD
+  void AddTemplateURLByOSDD(const GURL& page_url, const GURL& osdd_url);
+
   // WebStateObserver implementation.
   void WebStateDestroyed(web::WebState* web_state) override;
 
+  // Handles messages from JavaScript. Messages can be:
+  //   1. A OSDD <link> is found;
+  //   2. A searchable URL is generated from <form> submission.
+  bool OnJsMessage(const base::DictionaryValue& message,
+                   const GURL& page_url,
+                   bool has_user_gesture,
+                   bool form_in_main_frame,
+                   web::WebFrame* sender_frame);
+
   // WebState this tab helper is attached to.
   web::WebState* web_state_ = nullptr;
 
-  base::WeakPtrFactory<SearchEngineTabHelper> weak_ptr_factory_;
-
   DISALLOW_COPY_AND_ASSIGN(SearchEngineTabHelper);
 };
 

ios/chrome/browser/search_engines/search_engine_tab_helper.mm

@@ -4,20 +4,163 @@
 
 #import "ios/chrome/browser/search_engines/search_engine_tab_helper.h"
 
+#include "base/strings/utf_string_conversions.h"
+#include "components/search_engines/template_url.h"
+#include "components/search_engines/template_url_fetcher.h"
+#include "ios/chrome/browser/browser_state/chrome_browser_state.h"
+#include "ios/chrome/browser/search_engines/template_url_fetcher_factory.h"
+#include "ios/web/public/favicon_status.h"
+#import "ios/web/public/navigation_item.h"
+#import "ios/web/public/navigation_manager.h"
+#include "ui/base/page_transition_types.h"
+#include "url/gurl.h"
+
 #if !defined(__has_feature) || !__has_feature(objc_arc)
 #error "This file requires ARC support."
 #endif
 
 DEFINE_WEB_STATE_USER_DATA_KEY(SearchEngineTabHelper);
 
+namespace {
+
+const char kCommandPrefix[] = "searchEngine";
+const char kCommandOpenSearch[] = "searchEngine.openSearch";
+const char kOpenSearchPageUrlKey[] = "pageUrl";
+const char kOpenSearchOsddUrlKey[] = "osddUrl";
+
+// Returns true if the |item|'s transition type is FORM_SUBMIT.
+bool IsFormSubmit(const web::NavigationItem* item) {
+  return ui::PageTransitionCoreTypeIs(item->GetTransitionType(),
+                                      ui::PAGE_TRANSITION_FORM_SUBMIT);
+}
+
+// Generates a keyword from |item|. This code is based on:
+// https://cs.chromium.org/chromium/src/chrome/browser/ui/search_engines/search_engine_tab_helper.cc?rcl=4e19b43513aa9590ae89d5c68523bc764f40067f&l=41i
+base::string16 GenerateKeywordFromNavigationItem(
+    const web::NavigationItem* item) {
+  // Don't autogenerate keywords for pages that are the result of form
+  // submissions.
+  if (IsFormSubmit(item))
+    return base::string16();
+
+  // The code from Desktop will try NavigationEntry::GetUserTypedURL() first if
+  // available since that represents what the user typed to get here, and fall
+  // back on the regular URL if not.
+  // TODO(crbug.com/433824): Use GetUserTypedURL() once NavigationItem supports
+  // it.
+  GURL url = item->GetURL();
+  if (!url.is_valid()) {
+    return base::string16();
+  }
+
+  // Don't autogenerate keywords for referrers that
+  // a) are anything other than HTTP/HTTPS or
+  // b) have a path.
+  //
+  // To relax the path constraint, make sure to sanitize the path
+  // elements and update AutocompletePopup to look for keywords using the path.
+  // See http://b/issue?id=863583.
+  if (!url.SchemeIsHTTPOrHTTPS() || url.path().length() > 1) {
+    return base::string16();
+  }
+
+  return TemplateURL::GenerateKeyword(url);
+}
+}
+
 SearchEngineTabHelper::~SearchEngineTabHelper() {}
 
 SearchEngineTabHelper::SearchEngineTabHelper(web::WebState* web_state)
-    : web_state_(web_state), weak_ptr_factory_(this) {
+    : web_state_(web_state) {
   web_state->AddObserver(this);
+  web_state->AddScriptCommandCallback(
+      base::BindRepeating(&SearchEngineTabHelper::OnJsMessage,
+                          base::Unretained(this)),
+      kCommandPrefix);
 }
 
 void SearchEngineTabHelper::WebStateDestroyed(web::WebState* web_state) {
+  web_state->RemoveScriptCommandCallback(kCommandPrefix);
   web_state->RemoveObserver(this);
   web_state_ = nullptr;
 }
+
+bool SearchEngineTabHelper::OnJsMessage(const base::DictionaryValue& message,
+                                        const GURL& page_url,
+                                        bool has_user_gesture,
+                                        bool form_in_main_frame,
+                                        web::WebFrame* sender_frame) {
+  const base::Value* cmd = message.FindKey("command");
+  if (!cmd || !cmd->is_string()) {
+    return false;
+  }
+  std::string cmd_str = cmd->GetString();
+  if (cmd_str == kCommandOpenSearch) {
+    const base::Value* document_url = message.FindKey(kOpenSearchPageUrlKey);
+    if (!document_url || !document_url->is_string())
+      return false;
+    const base::Value* osdd_url = message.FindKey(kOpenSearchOsddUrlKey);
+    if (!osdd_url || !osdd_url->is_string())
+      return false;
+    AddTemplateURLByOSDD(GURL(document_url->GetString()),
+                         GURL(osdd_url->GetString()));
+  }
+  return true;
+}
+
+// Creates a new TemplateURL by OSDD. The TemplateURL will be added to
+// TemplateURLService by TemplateURLFecther. This code is based on:
+// https://cs.chromium.org/chromium/src/chrome/browser/ui/search_engines/search_engine_tab_helper.cc?rcl=50f50d521b18ac53d05c4e159c02bcb609454b8e&l=96
+void SearchEngineTabHelper::AddTemplateURLByOSDD(const GURL& page_url,
+                                                 const GURL& osdd_url) {
+  // Checks to see if we should generate a keyword based on the OSDD, and if
+  // necessary uses TemplateURLFetcher to download the OSDD and create a
+  // keyword.
+
+  // Make sure that the page is the current page and other basic checks.
+  // When |page_url| has file: scheme, this method doesn't work because of
+  // http://b/issue?id=863583. For that reason, this doesn't check and allow
+  // urls referring to osdd urls with same schemes.
+  if (!osdd_url.is_valid() || !osdd_url.SchemeIsHTTPOrHTTPS())
+    return;
+
+  ios::ChromeBrowserState* browser_state =
+      ios::ChromeBrowserState::FromBrowserState(web_state_->GetBrowserState());
+  if ((page_url != web_state_->GetLastCommittedURL()) ||
+      (!ios::TemplateURLFetcherFactory::GetForBrowserState(browser_state)) ||
+      (browser_state->IsOffTheRecord()))
+    return;
+
+  // If the current page is a form submit, find the last page that was not a
+  // form submit and use its url to generate the keyword from.
+  const web::NavigationManager* manager = web_state_->GetNavigationManager();
+  const web::NavigationItem* item = nullptr;
+  for (int index = manager->GetLastCommittedItemIndex(); true; --index) {
+    if (index < 0)
+      return;
+    item = manager->GetItemAtIndex(index);
+    if (!IsFormSubmit(item))
+      break;
+  }
+
+  // Autogenerate a keyword for the autodetected case; in the other cases we'll
+  // generate a keyword later after fetching the OSDD.
+  base::string16 keyword = GenerateKeywordFromNavigationItem(item);
+  if (keyword.empty())
+    return;
+
+  // Download the OpenSearch description document. If this is successful, a
+  // new keyword will be created when done. For the last two args:
+  //   1. Used by newwork::ResourceRequest::render_frame_id. We don't use Blink
+  //   so leave it to be the default value defined here:
+  //      https://cs.chromium.org/chromium/src/services/network/public/cpp/resource_request.h?rcl=39c6fbea496641a6514e34c0ab689871d14e6d52&l=194;
+  //   2. Used by network::ResourceRequest::resource_type. It's a design defect:
+  //      https://cs.chromium.org/chromium/src/services/network/public/cpp/resource_request.h?rcl=39c6fbea496641a6514e34c0ab689871d14e6d52&l=100
+  //      Use the same value as the SearchEngineTabHelper for Desktop.
+  ios::TemplateURLFetcherFactory::GetForBrowserState(browser_state)
+      ->ScheduleDownload(
+          keyword, osdd_url, item->GetFavicon().url,
+          url::Origin::Create(web_state_->GetLastCommittedURL()),
+          browser_state->GetURLLoaderFactory(), MSG_ROUTING_NONE,
+          /* content::ResourceType::RESOURCE_TYPE_SUB_RESOURCE */ 6);
+}

ios/chrome/browser/search_engines/search_engine_tab_helper_unittest.mm

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/chrome/browser/search_engines/search_engine_tab_helper.h"
+
+#include "base/strings/utf_string_conversions.h"
+#import "base/test/ios/wait_util.h"
+#include "components/search_engines/template_url_service.h"
+#include "ios/chrome/browser/browser_state/chrome_browser_state.h"
+#include "ios/chrome/browser/browser_state/test_chrome_browser_state.h"
+#include "ios/chrome/browser/search_engines/template_url_service_factory.h"
+#include "ios/chrome/browser/web/chrome_web_test.h"
+#import "ios/web/public/test/web_test_with_web_state.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "net/test/embedded_test_server/http_request.h"
+#include "net/test/embedded_test_server/http_response.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+using base::test::ios::WaitUntilConditionOrTimeout;
+using base::test::ios::kWaitForJSCompletionTimeout;
+
+namespace {
+const char kOpenSearchXmlFilePath[] =
+    "/ios/testing/data/http_server_files/opensearch.xml";
+
+// A BrowserStateKeyedServiceFactory::TestingFactory that creates a testing
+// TemplateURLService. The created TemplateURLService may contain some default
+// TemplateURLs.
+std::unique_ptr<KeyedService> CreateTestingTemplateURLService(
+    web::BrowserState*) {
+  return std::make_unique<TemplateURLService>(/*initializers=*/nullptr,
+                                              /*count=*/0);
+}
+}
+
+// Test fixture for SearchEngineTabHelper class.
+class SearchEngineTabHelperTest : public ChromeWebTest {
+ protected:
+  SearchEngineTabHelperTest()
+      : ChromeWebTest(web::TestWebThreadBundle::Options::IO_MAINLOOP) {}
+
+  void SetUp() override {
+    WebTestWithWebState::SetUp();
+    SearchEngineTabHelper::CreateForWebState(web_state());
+    server_.ServeFilesFromSourceDirectory(".");
+    ASSERT_TRUE(server_.Start());
+    // Set the TestingFactory of BrowserState of ios::TemplateURLServiceFactory
+    // to provide a testing TemplateURLService. Notice that we should not use
+    // GetBrowserState() because it may be overriden to return incognito
+    // BrowserState but TemplateURLService is always created on regular
+    // BrowserState.
+    ios::TemplateURLServiceFactory::GetInstance()->SetTestingFactory(
+        chrome_browser_state_->GetOriginalChromeBrowserState(),
+        base::BindRepeating(&CreateTestingTemplateURLService));
+    template_url_service()->Load();
+  }
+
+  // Returns the testing TemplateURLService.
+  TemplateURLService* template_url_service() {
+    ios::ChromeBrowserState* browser_state =
+        ios::ChromeBrowserState::FromBrowserState(GetBrowserState());
+    return ios::TemplateURLServiceFactory::GetForBrowserState(browser_state);
+  }
+
+  // Sends a message that a OSDD <link> is found in page, with |page_url| and
+  // |osdd_url| as message content.
+  bool SendMessageOfOpenSearch(const GURL& page_url, const GURL& osdd_url) {
+    id result = ExecuteJavaScript([NSString
+        stringWithFormat:@"__gCrWeb.message.invokeOnHost({'command': "
+                         @"'searchEngine.openSearch', 'pageUrl' : '%s', "
+                         @"'osddUrl': '%s'}); true;",
+                         page_url.spec().c_str(), osdd_url.spec().c_str()]);
+    return [result isEqual:@YES];
+  }
+
+  net::EmbeddedTestServer server_;
+
+  DISALLOW_COPY_AND_ASSIGN(SearchEngineTabHelperTest);
+};
+
+// Tests that SearchEngineTabHelper can add TemplateURL to TemplateURLService
+// when a OSDD <link> is found in web page.
+TEST_F(SearchEngineTabHelperTest, AddTemplateURLByOpenSearch) {
+  GURL page_url("https://chrooome.com");
+  GURL osdd_url = server_.GetURL(kOpenSearchXmlFilePath);
+
+  // Record the original TemplateURLs in TemplateURLService.
+  std::vector<TemplateURL*> old_urls =
+      template_url_service()->GetTemplateURLs();
+
+  // Load an empty page, and send a message of openSearchUrl from Js.
+  LoadHtml(@"<html></html>", page_url);
+  ASSERT_TRUE(SendMessageOfOpenSearch(page_url, osdd_url));
+
+  // Wait for TemplateURL added to TemplateURLService.
+  ASSERT_TRUE(WaitUntilConditionOrTimeout(kWaitForJSCompletionTimeout, ^{
+    base::RunLoop().RunUntilIdle();
+    return template_url_service()->GetTemplateURLs().size() ==
+           old_urls.size() + 1;
+  }));
+
+  // TemplateURLService doesn't guarantee that newly added TemplateURL is
+  // appended, so we should check in a general way that only one TemplateURL is
+  // added and others remain untouched.
+  TemplateURL* new_url = nullptr;
+  for (TemplateURL* url : template_url_service()->GetTemplateURLs()) {
+    if (std::find(old_urls.begin(), old_urls.end(), url) == old_urls.end()) {
+      ASSERT_FALSE(new_url);
+      new_url = url;
+    }
+  }
+  ASSERT_TRUE(new_url);
+  EXPECT_EQ(base::UTF8ToUTF16("chrooome.com"), new_url->data().keyword());
+  EXPECT_EQ(base::UTF8ToUTF16("Chrooome"), new_url->data().short_name());
+  EXPECT_EQ(
+      "https://chrooome.com/index.php?title=chrooome&search={searchTerms}",
+      new_url->data().url());
+}
+
+// Test fixture for SearchEngineTabHelper class in incognito mode.
+class SearchEngineTabHelperIncognitoTest : public SearchEngineTabHelperTest {
+ protected:
+  SearchEngineTabHelperIncognitoTest() {}
+
+  // Overrides WebTest::GetBrowserState.
+  web::BrowserState* GetBrowserState() override {
+    return chrome_browser_state_->GetOffTheRecordChromeBrowserState();
+  }
+
+  DISALLOW_COPY_AND_ASSIGN(SearchEngineTabHelperIncognitoTest);
+};
+
+// Tests that SearchEngineTabHelper doesn't add TemplateURL to
+// TemplateURLService when a OSDD <link> is found in web page under incognito
+// mode.
+TEST_F(SearchEngineTabHelperIncognitoTest,
+       NotAddTemplateURLByOpenSearchUnderIncognito) {
+  GURL page_url("https://chrooome.com");
+  GURL osdd_url = server_.GetURL(kOpenSearchXmlFilePath);
+
+  // Record the original TemplateURLs in TemplateURLService.
+  std::vector<TemplateURL*> old_urls =
+      template_url_service()->GetTemplateURLs();
+
+  // Load an empty page, and send a message of openSearchUrl from Js.
+  LoadHtml(@"<html></html>", page_url);
+  ASSERT_TRUE(SendMessageOfOpenSearch(page_url, osdd_url));
+
+  // No new TemplateURL should be added to TemplateURLService, wait for timeout.
+  ASSERT_FALSE(WaitUntilConditionOrTimeout(kWaitForJSCompletionTimeout, ^{
+    base::RunLoop().RunUntilIdle();
+    return template_url_service()->GetTemplateURLs().size() ==
+           old_urls.size() + 1;
+  }));
+  EXPECT_EQ(old_urls, template_url_service()->GetTemplateURLs());
+}

ios/chrome/browser/web/chrome_web_test.h

@@ -22,8 +22,11 @@ class ChromeWebTest : public web::WebTestWithWebState {
   ~ChromeWebTest() override;
 
  protected:
-  ChromeWebTest();
-  explicit ChromeWebTest(std::unique_ptr<web::WebClient> web_client);
+  ChromeWebTest(web::TestWebThreadBundle::Options =
+                    web::TestWebThreadBundle::Options::DEFAULT);
+  explicit ChromeWebTest(std::unique_ptr<web::WebClient> web_client,
+                         web::TestWebThreadBundle::Options =
+                             web::TestWebThreadBundle::Options::DEFAULT);
   // WebTest implementation.
   void SetUp() override;
   void TearDown() override;

ios/chrome/browser/web/chrome_web_test.mm

@@ -17,12 +17,14 @@
 
 ChromeWebTest::~ChromeWebTest() {}
 
-ChromeWebTest::ChromeWebTest(std::unique_ptr<web::WebClient> web_client)
-    : web::WebTestWithWebState(std::move(web_client)),
+ChromeWebTest::ChromeWebTest(std::unique_ptr<web::WebClient> web_client,
+                             web::TestWebThreadBundle::Options options)
+    : web::WebTestWithWebState(std::move(web_client), options),
       chrome_browser_state_(TestChromeBrowserState::Builder().Build()) {}
 
-ChromeWebTest::ChromeWebTest()
-    : chrome_browser_state_(TestChromeBrowserState::Builder().Build()) {}
+ChromeWebTest::ChromeWebTest(web::TestWebThreadBundle::Options options)
+    : web::WebTestWithWebState(options),
+      chrome_browser_state_(TestChromeBrowserState::Builder().Build()) {}
 
 void ChromeWebTest::SetUp() {
   web::WebTestWithWebState::SetUp();

ios/testing/BUILD.gn

@@ -83,6 +83,7 @@ bundle_data("http_server_bundle_data") {
     "data/http_server_files/multi_field_form.html",
     "data/http_server_files/onload_replacestate_reload.html",
     "data/http_server_files/onload_replacestate_reload.js",
+    "data/http_server_files/opensearch.xml",
     "data/http_server_files/pony.html",
     "data/http_server_files/redirect_refresh.html",
     "data/http_server_files/single_page_wide.pdf",

ios/testing/data/http_server_files/opensearch.xml

+<?xml version="1.0"?>
+<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" xmlns:moz="http://www.mozilla.org/2006/browser/search/">
+    <ShortName>Chrooome</ShortName>
+    <Description>Chrooome</Description>
+    <Image height="16" width="16" type="image/x-icon">https://chrooome.com/favicon.ico</Image>
+    <Url type="text/html" method="get" template="https://chrooome.com/index.php?title=chrooome&amp;search={searchTerms}" />
+    <moz:SearchForm>https://en.cppreference.com/w/Special:Search</moz:SearchForm>
+</OpenSearchDescription>