Create network request service to check if URL resolves

This service uses a HEAD request in order to check if the specified URL
can resolve to an existing page. This will be used in the New Tab Page
in order to check if a new custom link URL (obtained from adding or
editing) can resolve.

Split from https://crrev.com/c/1249567.

Bug: 874194
Change-Id: I46be9a09d6f07988c468e9e025bed4117df61437
Reviewed-on: https://chromium-review.googlesource.com/c/1259664
Commit-Queue: Kristi Park <kristipark@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: Mathieu Perreault <mathp@chromium.org>
Cr-Commit-Position: refs/heads/master@{#597630}

chrome/browser/BUILD.gn

@@ -2925,6 +2925,8 @@ jumbo_split_static_library("browser") {
       "search/one_google_bar/one_google_bar_service_observer.h",
       "search/search_engine_base_url_tracker.cc",
       "search/search_engine_base_url_tracker.h",
+      "search/url_validity_checker_factory.cc",
+      "search/url_validity_checker_factory.h",
       "signin/mutable_profile_oauth2_token_service_delegate.cc",
       "signin/mutable_profile_oauth2_token_service_delegate.h",
       "signin/signin_promo.cc",

chrome/browser/search/url_validity_checker_factory.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/search/url_validity_checker_factory.h"
+
+#include <memory>
+
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/net/system_network_context_manager.h"
+#include "content/public/browser/browser_thread.h"
+#include "services/network/public/cpp/shared_url_loader_factory.h"
+
+// static
+UrlValidityChecker* UrlValidityCheckerFactory::GetInstance() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+  static base::LazyInstance<UrlValidityCheckerFactory>::DestructorAtExit
+      instance = LAZY_INSTANCE_INITIALIZER;
+  return &(instance.Get().url_validity_checker_);
+}
+
+UrlValidityCheckerFactory::UrlValidityCheckerFactory()
+    : url_validity_checker_(g_browser_process->system_network_context_manager()
+                                ->GetSharedURLLoaderFactory()) {}
+
+UrlValidityCheckerFactory::~UrlValidityCheckerFactory() {}

chrome/browser/search/url_validity_checker_factory.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_SEARCH_URL_VALIDITY_CHECKER_FACTORY_H_
+#define CHROME_BROWSER_SEARCH_URL_VALIDITY_CHECKER_FACTORY_H_
+
+#include "base/lazy_instance.h"
+#include "base/macros.h"
+#include "components/search/url_validity_checker_impl.h"
+
+// Singleton that owns a single UrlValidityCheckerImpl instance. Should only be
+// called from the UI thread.
+class UrlValidityCheckerFactory {
+ public:
+  static UrlValidityChecker* GetInstance();
+
+ private:
+  friend struct base::LazyInstanceTraitsBase<UrlValidityCheckerFactory>;
+
+  UrlValidityCheckerFactory();
+  ~UrlValidityCheckerFactory();
+
+  // The only instance that exists.
+  UrlValidityCheckerImpl url_validity_checker_;
+
+  DISALLOW_COPY_AND_ASSIGN(UrlValidityCheckerFactory);
+};
+
+#endif  // CHROME_BROWSER_SEARCH_URL_VALIDITY_CHECKER_FACTORY_H_

components/search/BUILD.gn

@@ -14,6 +14,19 @@ static_library("search") {
     "//components/search_engines",
     "//url",
   ]
+
+  if (!is_ios && !is_android) {
+    sources += [
+      "url_validity_checker.h",
+      "url_validity_checker_impl.cc",
+      "url_validity_checker_impl.h",
+    ]
+
+    deps += [
+      "//net",
+      "//services/network/public/cpp",
+    ]
+  }
 }
 
 source_set("unit_tests") {
@@ -28,4 +41,15 @@ source_set("unit_tests") {
     "//components/variations",
     "//testing/gtest",
   ]
+
+  if (!is_ios && !is_android) {
+    sources += [ "url_validity_checker_impl_unittest.cc" ]
+
+    deps += [
+      "//net",
+      "//net:test_support",
+      "//services/network:test_support",
+      "//testing/gmock",
+    ]
+  }
 }

components/search/DEPS

@@ -2,4 +2,7 @@ include_rules = [
   "+components/google/core",
   "+components/search_engines",
   "+components/variations",
+  "+net",
+  "+services/network/public/cpp",
+  "+services/network/test",
 ]

components/search/url_validity_checker.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_H_
+#define COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_H_
+
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "url/gurl.h"
+
+// A standalone service that validates if the provided URL is able to resolve to
+// a valid page.
+class UrlValidityChecker {
+ public:
+  // The callback invoked when the request completes. Returns true if the
+  // response was |valid| and the request |duration|.
+  using UrlValidityCheckerCallback =
+      base::OnceCallback<void(bool valid, const base::TimeDelta& duration)>;
+
+  virtual ~UrlValidityChecker() = default;
+
+  // Creates a HEAD request to check if |url| resolves to an existing page.
+  // Returns true if the URL resolves and the request duration. Redirects (3xx)
+  // and 2xx response codes are considered as resolving.
+  virtual void DoesUrlResolve(
+      const GURL& url,
+      net::NetworkTrafficAnnotationTag traffic_annotation,
+      UrlValidityCheckerCallback callback) = 0;
+};
+
+#endif  // COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_H_

components/search/url_validity_checker_impl.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/search/url_validity_checker_impl.h"
+
+#include "base/bind.h"
+#include "base/time/time.h"
+#include "base/timer/timer.h"
+#include "net/http/http_request_headers.h"
+#include "net/http/http_status_code.h"
+
+// Stores the pending request and associated metadata. Deleted once the request
+// finishes.
+struct UrlValidityCheckerImpl::PendingRequest {
+  PendingRequest() = default;
+
+  GURL url;
+  base::TimeTicks time_created;
+  UrlValidityCheckerCallback callback;
+  std::unique_ptr<network::SimpleURLLoader> loader;
+
+  DISALLOW_COPY_AND_ASSIGN(PendingRequest);
+};
+
+UrlValidityCheckerImpl::UrlValidityCheckerImpl(
+    scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory)
+    : url_loader_factory_(url_loader_factory) {}
+
+UrlValidityCheckerImpl::~UrlValidityCheckerImpl() = default;
+
+void UrlValidityCheckerImpl::DoesUrlResolve(
+    const GURL& url,
+    net::NetworkTrafficAnnotationTag traffic_annotation,
+    UrlValidityCheckerCallback callback) {
+  auto resource_request = std::make_unique<network::ResourceRequest>();
+  resource_request->url = url;
+  resource_request->method = "HEAD";
+  resource_request->allow_credentials = false;
+
+  auto request_iter = pending_requests_.emplace(pending_requests_.begin());
+  request_iter->url = url;
+  request_iter->time_created = NowTicks();
+  request_iter->callback = std::move(callback);
+  request_iter->loader = network::SimpleURLLoader::Create(
+      std::move(resource_request), traffic_annotation);
+  // Don't follow redirects to prevent leaking URL data to HTTP sites.
+  request_iter->loader->SetOnRedirectCallback(
+      base::BindRepeating(&UrlValidityCheckerImpl::OnSimpleLoaderRedirect,
+                          weak_ptr_factory_.GetWeakPtr(), request_iter));
+  request_iter->loader->DownloadToString(
+      url_loader_factory_.get(),
+      base::BindOnce(&UrlValidityCheckerImpl::OnSimpleLoaderComplete,
+                     weak_ptr_factory_.GetWeakPtr(), request_iter),
+      /*max_body_size=*/1);
+}
+
+void UrlValidityCheckerImpl::OnSimpleLoaderRedirect(
+    std::list<PendingRequest>::iterator request_iter,
+    const net::RedirectInfo& redirect_info,
+    const network::ResourceResponseHead& response_head,
+    std::vector<std::string>* to_be_removed_headers) {
+  // Assume the URL is valid if a redirect is returned.
+  OnSimpleLoaderHandler(request_iter, true);
+}
+
+void UrlValidityCheckerImpl::OnSimpleLoaderComplete(
+    std::list<PendingRequest>::iterator request_iter,
+    std::unique_ptr<std::string> response_body) {
+  // |response_body| is null for non-2xx responses.
+  OnSimpleLoaderHandler(request_iter, response_body.get() != nullptr);
+}
+
+void UrlValidityCheckerImpl::OnSimpleLoaderHandler(
+    std::list<PendingRequest>::iterator request_iter,
+    bool valid) {
+  base::TimeDelta elapsed_time = NowTicks() - request_iter->time_created;
+  std::move(request_iter->callback).Run(valid, elapsed_time);
+  pending_requests_.erase(request_iter);
+}
+
+base::TimeTicks UrlValidityCheckerImpl::NowTicks() const {
+  if (!time_ticks_for_testing_.is_null())
+    return time_ticks_for_testing_;
+  return base::TimeTicks::Now();
+}

components/search/url_validity_checker_impl.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_IMPL_H_
+#define COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_IMPL_H_
+
+#include <list>
+#include <vector>
+
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "build/build_config.h"
+#include "components/search/url_validity_checker.h"
+#include "services/network/public/cpp/shared_url_loader_factory.h"
+#include "services/network/public/cpp/simple_url_loader.h"
+
+namespace net {
+struct RedirectInfo;
+}  // namespace net
+
+namespace network {
+struct ResourceResponseHead;
+class SharedURLLoaderFactory;
+}  // namespace network
+
+class UrlValidityCheckerImpl : public UrlValidityChecker {
+ public:
+  explicit UrlValidityCheckerImpl(
+      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory);
+  ~UrlValidityCheckerImpl() override;
+
+  void DoesUrlResolve(const GURL& url,
+                      net::NetworkTrafficAnnotationTag traffic_annotation,
+                      UrlValidityCheckerCallback callback) override;
+
+  // Used for testing.
+  void SetTimeTicksForTesting(const base::TimeTicks& time_ticks) {
+    time_ticks_for_testing_ = time_ticks;
+  }
+
+ private:
+  struct PendingRequest;
+
+  void OnSimpleLoaderRedirect(
+      std::list<PendingRequest>::iterator request_iter,
+      const net::RedirectInfo& redirect_info,
+      const network::ResourceResponseHead& response_head,
+      std::vector<std::string>* to_be_removed_headers);
+  void OnSimpleLoaderComplete(std::list<PendingRequest>::iterator request_iter,
+                              std::unique_ptr<std::string> response_body);
+  // Called when the request from |DoesUrlResolve| finishes. Invokes the
+  // associated callback with the request status and duration.
+  void OnSimpleLoaderHandler(std::list<PendingRequest>::iterator request_iter,
+                             bool valid);
+
+  // Returns base::TimeTicks::Now() or the test TimeTicks if not null.
+  base::TimeTicks NowTicks() const;
+
+  // Stores any ongoing network requests. Once a request is completed, it is
+  // deleted from the list.
+  std::list<PendingRequest> pending_requests_;
+  scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
+
+  // Test time ticks used for testing.
+  base::TimeTicks time_ticks_for_testing_;
+
+  base::WeakPtrFactory<UrlValidityCheckerImpl> weak_ptr_factory_{this};
+
+  DISALLOW_COPY_AND_ASSIGN(UrlValidityCheckerImpl);
+};
+
+#endif  // COMPONENTS_SEARCH_URL_VALIDITY_CHECKER_IMPL_H_

components/search/url_validity_checker_impl_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/search/url_validity_checker_impl.h"
+
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/run_loop.h"
+#include "base/test/bind_test_util.h"
+#include "base/test/mock_callback.h"
+#include "base/test/scoped_task_environment.h"
+#include "base/test/simple_test_tick_clock.h"
+#include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
+#include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
+#include "services/network/test/test_url_loader_factory.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "url/gurl.h"
+
+using ::testing::_;
+
+class UrlValidityCheckerImplTest : public testing::Test {
+ protected:
+  UrlValidityCheckerImplTest()
+      : test_shared_loader_factory_(
+            base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
+                &test_url_loader_factory_)),
+        url_checker_(test_shared_loader_factory_) {
+    // Start |clock_| at non-zero.
+    clock_.Advance(base::TimeDelta::FromSeconds(1));
+  }
+
+  ~UrlValidityCheckerImplTest() override {}
+
+  void SetUp() override {
+    test_shared_loader_factory_ =
+        base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
+            url_loader_factory());
+    url_checker()->SetTimeTicksForTesting(clock_.NowTicks());
+  }
+
+  UrlValidityCheckerImpl* url_checker() { return &url_checker_; }
+
+  network::TestURLLoaderFactory* url_loader_factory() {
+    return &test_url_loader_factory_;
+  }
+
+  void AdvanceClock(const base::TimeDelta& delta) {
+    clock_.Advance(delta);
+    url_checker()->SetTimeTicksForTesting(clock_.NowTicks());
+  }
+
+  base::test::ScopedTaskEnvironment scoped_task_environment_;
+
+ private:
+  network::TestURLLoaderFactory test_url_loader_factory_;
+  scoped_refptr<network::SharedURLLoaderFactory> test_shared_loader_factory_;
+
+  base::SimpleTestTickClock clock_;
+  UrlValidityCheckerImpl url_checker_;
+
+  DISALLOW_COPY_AND_ASSIGN(UrlValidityCheckerImplTest);
+};
+
+TEST_F(UrlValidityCheckerImplTest, DoesUrlResolve_OnSuccess) {
+  const GURL kUrl("https://www.foo.com");
+  const int kTimeAdvance = 10;
+  base::TimeDelta expected_duration =
+      base::TimeDelta::FromSeconds(kTimeAdvance);
+
+  network::ResourceResponseHead response;
+  response.headers = new net::HttpResponseHeaders(
+      "HTTP/1.1 200 OK\nContent-type: text/html\n\n");
+  url_loader_factory()->SetInterceptor(
+      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
+        AdvanceClock(expected_duration);
+        url_loader_factory()->AddResponse(
+            request.url, response, std::string(),
+            network::URLLoaderCompletionStatus(net::OK));
+      }));
+  base::MockCallback<UrlValidityChecker::UrlValidityCheckerCallback>
+      callback_ok;
+  EXPECT_CALL(callback_ok, Run(true, expected_duration));
+
+  url_checker()->DoesUrlResolve(kUrl, TRAFFIC_ANNOTATION_FOR_TESTS,
+                                callback_ok.Get());
+  scoped_task_environment_.RunUntilIdle();
+
+  response.headers =
+      new net::HttpResponseHeaders("HTTP/1.1 204 No Content\r\n\r\n");
+  base::MockCallback<UrlValidityChecker::UrlValidityCheckerCallback>
+      callback_no_content;
+  EXPECT_CALL(callback_no_content, Run(true, expected_duration));
+
+  url_checker()->DoesUrlResolve(kUrl, TRAFFIC_ANNOTATION_FOR_TESTS,
+                                callback_no_content.Get());
+  scoped_task_environment_.RunUntilIdle();
+}
+
+TEST_F(UrlValidityCheckerImplTest, DoesUrlResolve_OnFailure) {
+  const GURL kUrl("https://www.foo.com");
+  const int kTimeAdvance = 20;
+  base::TimeDelta expected_duration =
+      base::TimeDelta::FromSeconds(kTimeAdvance);
+
+  url_loader_factory()->SetInterceptor(
+      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
+        AdvanceClock(expected_duration);
+        url_loader_factory()->AddResponse(
+            request.url, network::ResourceResponseHead(), std::string(),
+            network::URLLoaderCompletionStatus(net::ERR_FAILED));
+      }));
+  base::MockCallback<UrlValidityChecker::UrlValidityCheckerCallback> callback;
+  EXPECT_CALL(callback, Run(false, expected_duration));
+
+  url_checker()->DoesUrlResolve(kUrl, TRAFFIC_ANNOTATION_FOR_TESTS,
+                                callback.Get());
+  scoped_task_environment_.RunUntilIdle();
+}
+
+TEST_F(UrlValidityCheckerImplTest, DoesUrlResolve_OnRedirect) {
+  const GURL kUrl("https://www.foo.com");
+  const GURL kRedirectUrl("https://www.foo2.com");
+  const int kTimeAdvance = 30;
+  base::TimeDelta expected_duration =
+      base::TimeDelta::FromSeconds(kTimeAdvance);
+
+  net::RedirectInfo redirect_info;
+  redirect_info.status_code = 301;
+  redirect_info.new_url = kRedirectUrl;
+  network::TestURLLoaderFactory::Redirects redirects{
+      {redirect_info, network::ResourceResponseHead()}};
+  url_loader_factory()->SetInterceptor(
+      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
+        AdvanceClock(expected_duration);
+        url_loader_factory()->AddResponse(
+            request.url, network::ResourceResponseHead(), std::string(),
+            network::URLLoaderCompletionStatus(), redirects);
+      }));
+  base::MockCallback<UrlValidityChecker::UrlValidityCheckerCallback> callback;
+  EXPECT_CALL(callback, Run(true, expected_duration));
+
+  url_checker()->DoesUrlResolve(kUrl, TRAFFIC_ANNOTATION_FOR_TESTS,
+                                callback.Get());
+  scoped_task_environment_.RunUntilIdle();
+}