Support network service in headless

- extract all URLRequestContextGettter and NetworkContext creation logic into HeadlessRequestContextManager;
- reconcile network service and non-network service code paths as much as possible;
- fix a couple of issues with handling of redirect URLs in DevToolsURLLoaderInterceptor that
    were different from URLRequestJob-based interception and caused headless tests to fail;
- plumb a call to NetworkServiceTestHelper::RegisterNetworkBinders() in headless utility
    process when running tests to let network service tests inject the magic client certificates;
- update traffic annotations as they just moved;

Bug: 838291,792676

Cq-Include-Trybots: luci.chromium.try:linux_mojo
Change-Id: Ia29143a85b0c0293fae13079ec9f88ff5ba924f9
Reviewed-on: https://chromium-review.googlesource.com/c/1263566Reviewed-by: Georges Khalil <georgesak@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: Dmitry Gozman <dgozman@chromium.org>
Commit-Queue: Andrey Kosyakov <caseq@chromium.org>
Cr-Commit-Position: refs/heads/master@{#598893}

content/browser/devtools/devtools_url_loader_interceptor.cc

@@ -207,7 +207,7 @@ class InterceptionJob : public network::mojom::URLLoaderClient,
   Response InnerContinueRequest(std::unique_ptr<Modifications> modifications);
   Response ProcessAuthResponse(AuthChallengeResponse* auth_challenge_response);
   Response ProcessResponseOverride(const std::string& response);
-  Response ProcessRedirectByClient(const std::string& location);
+  Response ProcessRedirectByClient(const GURL& redirect_url);
   void SendResponse(const base::StringPiece& body);
   void ApplyModificationsToRequest(
       std::unique_ptr<Modifications> modifications);
@@ -821,7 +821,8 @@ Response InterceptionJob::InnerContinueRequest(
       auto* headers = response_metadata_->head.headers.get();
       headers->RemoveHeader("location");
       headers->AddHeader("location: " + location);
-      return ProcessRedirectByClient(location);
+      return ProcessRedirectByClient(
+          create_loader_params_->request.url.Resolve(location));
     }
     client_->OnReceiveRedirect(*response_metadata_->redirect_info,
                                response_metadata_->head);
@@ -941,7 +942,9 @@ Response InterceptionJob::ProcessResponseOverride(const std::string& response) {
   head->headers = new net::HttpResponseHeaders(std::move(raw_headers));
   head->headers->GetMimeTypeAndCharset(&head->mime_type, &head->charset);
   if (head->mime_type.empty()) {
-    net::SniffMimeType(response.data() + header_size, body_size,
+    size_t bytes_to_sniff =
+        std::min(body_size, static_cast<size_t>(net::kMaxBytesToSniff));
+    net::SniffMimeType(response.data() + header_size, bytes_to_sniff,
                        create_loader_params_->request.url, "",
                        net::ForceSniffFileUrlsForHtml::kDisabled,
                        &head->mime_type);
@@ -953,9 +956,12 @@ Response InterceptionJob::ProcessResponseOverride(const std::string& response) {
   head->request_start = start_ticks_;
   head->response_start = base::TimeTicks::Now();
 
-  std::string location_url;
-  if (head->headers->IsRedirect(&location_url))
-    return ProcessRedirectByClient(location_url);
+  std::string location;
+  if (head->headers->IsRedirect(&location)) {
+    GURL redirect_url = create_loader_params_->request.url.Resolve(location);
+    if (redirect_url.is_valid())
+      return ProcessRedirectByClient(redirect_url);
+  }
 
   response_metadata_->transfer_size = body_size;
 
@@ -968,9 +974,7 @@ Response InterceptionJob::ProcessResponseOverride(const std::string& response) {
   return Response::OK();
 }
 
-Response InterceptionJob::ProcessRedirectByClient(const std::string& location) {
-  GURL redirect_url = create_loader_params_->request.url.Resolve(location);
-
+Response InterceptionJob::ProcessRedirectByClient(const GURL& redirect_url) {
   if (!redirect_url.is_valid())
     return Response::Error("Invalid redirect URL in overriden headers");
 
@@ -989,7 +993,7 @@ Response InterceptionJob::ProcessRedirectByClient(const std::string& location) {
           first_party_url_policy, request.referrer_policy,
           request.referrer.spec(), &headers, headers.response_code(),
           redirect_url, false /* insecure_scheme_was_upgraded */,
-          false /* copy_fragment */));
+          true /* copy_fragment */));
 
   client_->OnReceiveRedirect(*response_metadata_->redirect_info,
                              response_metadata_->head);

headless/BUILD.gn

@@ -308,12 +308,12 @@ jumbo_component("headless") {
     "lib/browser/headless_platform_event_source.h",
     "lib/browser/headless_quota_permission_context.cc",
     "lib/browser/headless_quota_permission_context.h",
+    "lib/browser/headless_request_context_manager.cc",
+    "lib/browser/headless_request_context_manager.h",
     "lib/browser/headless_resource_dispatcher_host_delegate.cc",
     "lib/browser/headless_resource_dispatcher_host_delegate.h",
     "lib/browser/headless_shell_application_mac.h",
     "lib/browser/headless_shell_application_mac.mm",
-    "lib/browser/headless_url_request_context_getter.cc",
-    "lib/browser/headless_url_request_context_getter.h",
     "lib/browser/headless_window_tree_host.h",
     "lib/browser/protocol/browser_handler.cc",
     "lib/browser/protocol/browser_handler.h",

headless/lib/browser/DEPS

@@ -5,6 +5,7 @@ include_rules = [
   "+components/security_state",
   "+components/viz",
   "+printing",
+  "+services/network",
   "+storage/browser/quota",
   "+storage/common/quota",
   "+third_party/skia/include",

headless/lib/browser/headless_browser_context_impl.cc

@@ -14,72 +14,32 @@
 #include "base/task/post_task.h"
 #include "content/public/browser/browser_task_traits.h"
 #include "content/public/browser/browser_thread.h"
-#include "content/public/browser/resource_context.h"
 #include "content/public/browser/storage_partition.h"
 #include "headless/grit/headless_lib_resources.h"
 #include "headless/lib/browser/headless_browser_context_options.h"
 #include "headless/lib/browser/headless_browser_impl.h"
 #include "headless/lib/browser/headless_browser_main_parts.h"
 #include "headless/lib/browser/headless_permission_manager.h"
-#include "headless/lib/browser/headless_url_request_context_getter.h"
 #include "headless/lib/browser/headless_web_contents_impl.h"
 #include "net/url_request/url_request_context.h"
 #include "ui/base/resource/resource_bundle.h"
 
 namespace headless {
 
-// Contains net::URLRequestContextGetter required for resource loading.
-// Must be destructed on the IO thread as per content::ResourceContext
-// requirements.
-class HeadlessResourceContext : public content::ResourceContext {
- public:
-  HeadlessResourceContext();
-  ~HeadlessResourceContext() override;
-
-  // ResourceContext implementation:
-  net::URLRequestContext* GetRequestContext() override;
-
-  // Configure the URL request context getter to be used for resource fetching.
-  // Must be called before any of the other methods of this class are used. Must
-  // be called on the browser UI thread.
-  void set_url_request_context_getter(
-      scoped_refptr<net::URLRequestContextGetter> url_request_context_getter) {
-    DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
-    url_request_context_getter_ = std::move(url_request_context_getter);
-  }
-
-  net::URLRequestContextGetter* url_request_context_getter() {
-    return url_request_context_getter_.get();
-  }
-
- private:
-  scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_;
-
-  DISALLOW_COPY_AND_ASSIGN(HeadlessResourceContext);
-};
-
-HeadlessResourceContext::HeadlessResourceContext() {
-  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
-}
-
-HeadlessResourceContext::~HeadlessResourceContext() {
-  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO));
-}
-
-net::URLRequestContext* HeadlessResourceContext::GetRequestContext() {
-  CHECK(url_request_context_getter_);
-  return url_request_context_getter_->GetURLRequestContext();
-}
-
 HeadlessBrowserContextImpl::HeadlessBrowserContextImpl(
     HeadlessBrowserImpl* browser,
     std::unique_ptr<HeadlessBrowserContextOptions> context_options)
     : browser_(browser),
       context_options_(std::move(context_options)),
-      resource_context_(std::make_unique<HeadlessResourceContext>()),
       permission_controller_delegate_(
           std::make_unique<HeadlessPermissionManager>(this)) {
   InitWhileIOAllowed();
+  base::FilePath user_data_path =
+      IsOffTheRecord() || context_options_->user_data_dir().empty()
+          ? base::FilePath()
+          : path_;
+  request_context_manager_ = std::make_unique<HeadlessRequestContextManager>(
+      context_options_.get(), user_data_path);
 }
 
 HeadlessBrowserContextImpl::~HeadlessBrowserContextImpl() {
@@ -89,20 +49,12 @@ HeadlessBrowserContextImpl::~HeadlessBrowserContextImpl() {
   // Destroy all web contents before shutting down storage partitions.
   web_contents_map_.clear();
 
-  if (resource_context_) {
+  if (request_context_manager_) {
     content::BrowserThread::DeleteSoon(content::BrowserThread::IO, FROM_HERE,
-                                       resource_context_.release());
+                                       request_context_manager_.release());
   }
 
   ShutdownStoragePartitions();
-
-  if (url_request_getter_) {
-    base::PostTaskWithTraits(
-        FROM_HERE, {content::BrowserThread::IO},
-        base::BindOnce(
-            &HeadlessURLRequestContextGetter::NotifyContextShuttingDown,
-            url_request_getter_));
-  }
 }
 
 // static
@@ -221,7 +173,7 @@ bool HeadlessBrowserContextImpl::IsOffTheRecord() const {
 }
 
 content::ResourceContext* HeadlessBrowserContextImpl::GetResourceContext() {
-  return resource_context_.get();
+  return request_context_manager_->GetResourceContext();
 }
 
 content::DownloadManagerDelegate*
@@ -273,19 +225,8 @@ HeadlessBrowserContextImpl::GetBrowsingDataRemoverDelegate() {
 net::URLRequestContextGetter* HeadlessBrowserContextImpl::CreateRequestContext(
     content::ProtocolHandlerMap* protocol_handlers,
     content::URLRequestInterceptorScopedVector request_interceptors) {
-  base::FilePath user_data_path =
-      IsOffTheRecord() || context_options_->user_data_dir().empty()
-          ? base::FilePath()
-          : path_;
-
-  url_request_getter_ = base::MakeRefCounted<HeadlessURLRequestContextGetter>(
-      base::CreateSingleThreadTaskRunnerWithTraits(
-          {content::BrowserThread::IO}),
-      protocol_handlers, context_options_->TakeProtocolHandlers(),
-      std::move(request_interceptors), context_options_.get(),
-      std::move(user_data_path));
-  resource_context_->set_url_request_context_getter(url_request_getter_);
-  return url_request_getter_.get();
+  return request_context_manager_->CreateRequestContext(
+      protocol_handlers, std::move(request_interceptors));
 }
 
 net::URLRequestContextGetter*
@@ -299,7 +240,7 @@ HeadlessBrowserContextImpl::CreateRequestContextForStoragePartition(
 
 net::URLRequestContextGetter*
 HeadlessBrowserContextImpl::CreateMediaRequestContext() {
-  return resource_context_->url_request_context_getter();
+  return request_context_manager_->url_request_context_getter();
 }
 
 net::URLRequestContextGetter*
@@ -363,6 +304,14 @@ const std::string& HeadlessBrowserContextImpl::Id() const {
   return UniqueId();
 }
 
+::network::mojom::NetworkContextPtr
+HeadlessBrowserContextImpl::CreateNetworkContext(
+    bool in_memory,
+    const base::FilePath& relative_partition_path) {
+  return request_context_manager_->CreateNetworkContext(
+      in_memory, relative_partition_path);
+}
+
 HeadlessBrowserContext::Builder::Builder(HeadlessBrowserImpl* browser)
     : browser_(browser),
       options_(new HeadlessBrowserContextOptions(browser->options())) {}

headless/lib/browser/headless_browser_context_impl.h

@@ -16,14 +16,13 @@
 #include "content/public/browser/global_routing_id.h"
 #include "content/public/browser/resource_context.h"
 #include "headless/lib/browser/headless_browser_context_options.h"
-#include "headless/lib/browser/headless_url_request_context_getter.h"
+#include "headless/lib/browser/headless_request_context_manager.h"
 #include "headless/public/headless_browser.h"
 #include "headless/public/headless_browser_context.h"
 #include "headless/public/headless_export.h"
 
 namespace headless {
 class HeadlessBrowserImpl;
-class HeadlessResourceContext;
 class HeadlessWebContentsImpl;
 
 class HEADLESS_EXPORT HeadlessBrowserContextImpl final
@@ -109,6 +108,10 @@ class HEADLESS_EXPORT HeadlessBrowserContextImpl final
   const base::UnguessableToken* GetDevToolsFrameTokenForFrameTreeNodeId(
       int frame_tree_node_id) const;
 
+  ::network::mojom::NetworkContextPtr CreateNetworkContext(
+      bool in_memory,
+      const base::FilePath& relative_partition_path);
+
  private:
   HeadlessBrowserContextImpl(
       HeadlessBrowserImpl* browser,
@@ -120,8 +123,6 @@ class HEADLESS_EXPORT HeadlessBrowserContextImpl final
 
   HeadlessBrowserImpl* browser_;  // Not owned.
   std::unique_ptr<HeadlessBrowserContextOptions> context_options_;
-  std::unique_ptr<HeadlessResourceContext> resource_context_;
-  scoped_refptr<HeadlessURLRequestContextGetter> url_request_getter_;
   base::FilePath path_;
 
   std::unordered_map<std::string, std::unique_ptr<HeadlessWebContents>>
@@ -140,6 +141,8 @@ class HEADLESS_EXPORT HeadlessBrowserContextImpl final
   std::unique_ptr<content::PermissionControllerDelegate>
       permission_controller_delegate_;
 
+  std::unique_ptr<HeadlessRequestContextManager> request_context_manager_;
+
   DISALLOW_COPY_AND_ASSIGN(HeadlessBrowserContextImpl);
 };
 

headless/lib/browser/headless_browser_impl.cc

@@ -93,21 +93,12 @@ void HeadlessBrowserImpl::Shutdown() {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 
   weak_ptr_factory_.InvalidateWeakPtrs();
-
   browser_contexts_.clear();
-#if defined(USE_NSS_CERTS)
-  if (system_url_request_getter_) {
-    base::PostTaskWithTraits(
-        FROM_HERE, {content::BrowserThread::IO},
-        base::BindOnce(
-            [](scoped_refptr<HeadlessURLRequestContextGetter> getter) {
-              net::SetURLRequestContextForNSSHttpIO(nullptr);
-              getter->NotifyContextShuttingDown();
-            },
-            std::move(system_url_request_getter_)));
-    DCHECK(!system_url_request_getter_);  // Posted task grabs ownership.
+  if (system_request_context_manager_) {
+    content::BrowserThread::DeleteSoon(
+        content::BrowserThread::IO, FROM_HERE,
+        system_request_context_manager_.release());
   }
-#endif
   browser_main_parts_->QuitMainMessageLoop();
 }
 
@@ -179,27 +170,11 @@ void HeadlessBrowserImpl::SetDefaultBrowserContext(
 
   default_browser_context_ = browser_context;
 
-#if defined(USE_NSS_CERTS)
-  if (!system_url_request_getter_ && browser_context) {
-    ProtocolHandlerMap empty_protocol_handlers;
-    system_url_request_getter_ =
-        base::MakeRefCounted<HeadlessURLRequestContextGetter>(
-            base::CreateSingleThreadTaskRunnerWithTraits(
-                {content::BrowserThread::IO}),
-            &empty_protocol_handlers, ProtocolHandlerMap(),
-            content::URLRequestInterceptorScopedVector(),
-            HeadlessBrowserContextImpl::From(browser_context)->options(),
-            base::FilePath());
-    base::PostTaskWithTraits(
-        FROM_HERE, {content::BrowserThread::IO},
-        base::BindOnce(
-            [](HeadlessURLRequestContextGetter* getter) {
-              net::SetURLRequestContextForNSSHttpIO(
-                  getter->GetURLRequestContext());
-            },
-            base::Unretained(system_url_request_getter_.get())));
+  if (default_browser_context_ && !system_request_context_manager_) {
+    system_request_context_manager_ =
+        HeadlessRequestContextManager::CreateSystemContext(
+            HeadlessBrowserContextImpl::From(browser_context)->options());
   }
-#endif  // defined(USE_NSS_CERTS)
 }
 
 HeadlessBrowserContext* HeadlessBrowserImpl::GetDefaultBrowserContext() {

headless/lib/browser/headless_browser_impl.h

@@ -30,7 +30,7 @@ namespace headless {
 
 class HeadlessBrowserContextImpl;
 class HeadlessBrowserMainParts;
-class HeadlessURLRequestContextGetter;
+class HeadlessRequestContextManager;
 class HeadlessWebContentsImpl;
 
 extern const base::FilePath::CharType kDefaultProfileName[];
@@ -104,11 +104,8 @@ class HEADLESS_EXPORT HeadlessBrowserImpl : public HeadlessBrowser,
   HeadlessBrowserContext* default_browser_context_;  // Not owned.
 
   scoped_refptr<content::DevToolsAgentHost> agent_host_;
-
-#if defined(USE_NSS_CERTS)
-  scoped_refptr<HeadlessURLRequestContextGetter> system_url_request_getter_;
-#endif
-
+  std::unique_ptr<HeadlessRequestContextManager>
+      system_request_context_manager_;
   base::WeakPtrFactory<HeadlessBrowserImpl> weak_ptr_factory_;
 
  private:

headless/lib/browser/headless_content_browser_client.cc

@@ -338,4 +338,13 @@ bool HeadlessContentBrowserClient::ShouldEnableStrictSiteIsolation() {
   return false;
 }
 
+::network::mojom::NetworkContextPtr
+HeadlessContentBrowserClient::CreateNetworkContext(
+    content::BrowserContext* context,
+    bool in_memory,
+    const base::FilePath& relative_partition_path) {
+  return HeadlessBrowserContextImpl::From(context)->CreateNetworkContext(
+      in_memory, relative_partition_path);
+}
+
 }  // namespace headless

headless/lib/browser/headless_content_browser_client.h

@@ -61,6 +61,11 @@ class HeadlessContentBrowserClient : public content::ContentBrowserClient {
                                        const GURL& effective_site_url) override;
   bool ShouldEnableStrictSiteIsolation() override;
 
+  ::network::mojom::NetworkContextPtr CreateNetworkContext(
+      content::BrowserContext* context,
+      bool in_memory,
+      const base::FilePath& relative_partition_path) override;
+
  private:
   std::unique_ptr<base::Value> GetBrowserServiceManifestOverlay();
   std::unique_ptr<base::Value> GetRendererServiceManifestOverlay();

headless/lib/browser/headless_url_request_context_getter.h → headless/lib/browser/headless_request_context_manager.h

-// Copyright 2015 The Chromium Authors. All rights reserved.
+// Copyright 2018 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#ifndef HEADLESS_LIB_BROWSER_HEADLESS_URL_REQUEST_CONTEXT_GETTER_H_
-#define HEADLESS_LIB_BROWSER_HEADLESS_URL_REQUEST_CONTEXT_GETTER_H_
+#ifndef HEADLESS_LIB_BROWSER_HEADLESS_REQUEST_CONTEXT_MANAGER_H_
+#define HEADLESS_LIB_BROWSER_HEADLESS_REQUEST_CONTEXT_MANAGER_H_
 
-#include <memory>
-#include <string>
-
-#include "base/compiler_specific.h"
 #include "base/files/file_path.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
-#include "base/single_thread_task_runner.h"
 #include "content/public/browser/browser_context.h"
-#include "headless/public/headless_browser.h"
-#include "net/http/http_auth_preferences.h"
-#include "net/proxy_resolution/proxy_config.h"
-#include "net/proxy_resolution/proxy_config_service.h"
-#include "net/url_request/url_request_context_getter.h"
-#include "net/url_request/url_request_job_factory.h"
+#include "services/network/public/mojom/network_context.mojom.h"
 
-namespace net {
-class ProxyConfigService;
+#include <string>
+
+namespace content {
+class ResourceContext;
 }
 
+namespace net {
+class URLRequestContextGetter;
+}  // namespace net
+
 namespace headless {
+
 class HeadlessBrowserContextOptions;
+class HeadlessProxyConfigMonitor;
+class HeadlessResourceContext;
+class HeadlessURLRequestContextGetter;
 
-class HeadlessURLRequestContextGetter : public net::URLRequestContextGetter {
+class HeadlessRequestContextManager {
  public:
-  HeadlessURLRequestContextGetter(
-      scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
-      content::ProtocolHandlerMap* protocol_handlers,
-      content::ProtocolHandlerMap context_protocol_handlers,
-      content::URLRequestInterceptorScopedVector request_interceptors,
+  static std::unique_ptr<HeadlessRequestContextManager> CreateSystemContext(
+      const HeadlessBrowserContextOptions* options);
+
+  explicit HeadlessRequestContextManager(
       const HeadlessBrowserContextOptions* options,
       base::FilePath user_data_path);
+  ~HeadlessRequestContextManager();
 
-  // net::URLRequestContextGetter implementation:
-  net::URLRequestContext* GetURLRequestContext() override;
-  scoped_refptr<base::SingleThreadTaskRunner> GetNetworkTaskRunner()
-      const override;
+  net::URLRequestContextGetter* CreateRequestContext(
+      content::ProtocolHandlerMap* protocol_handlers,
+      content::URLRequestInterceptorScopedVector request_interceptors);
 
-  void NotifyContextShuttingDown();
+  ::network::mojom::NetworkContextPtr CreateNetworkContext(
+      bool in_memory,
+      const base::FilePath& relative_partition_path);
 
- protected:
-  ~HeadlessURLRequestContextGetter() override;
+  content::ResourceContext* GetResourceContext();
+  net::URLRequestContextGetter* url_request_context_getter();
 
  private:
+  void Initialize();
+  void InitializeOnIO();
+
+  ::network::mojom::NetworkContextParamsPtr CreateNetworkContextParams();
+
+  const bool network_service_enabled_;
+
   scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
 
-  // The |options| object given to the constructor is not guaranteed to outlive
-  // this class, so we make copies of the parts we need to access on the IO
-  // thread.
+  base::FilePath user_data_path_;
   std::string accept_language_;
   std::string user_agent_;
-  std::string host_resolver_rules_;
-  const net::ProxyConfig* proxy_config_;  // Not owned.
+  std::unique_ptr<net::ProxyConfig> proxy_config_;
+  std::unique_ptr<HeadlessProxyConfigMonitor> proxy_config_monitor_;
+  bool is_system_context_;
+
+  ::network::mojom::NetworkContextPtr network_context_;
+  ::network::mojom::NetworkContextRequest network_context_request_;
 
-  std::unique_ptr<net::ProxyConfigService> proxy_config_service_;
-  std::unique_ptr<net::URLRequestContext> url_request_context_;
   content::ProtocolHandlerMap protocol_handlers_;
   content::URLRequestInterceptorScopedVector request_interceptors_;
-  net::HttpAuthPreferences prefs_;
-  base::FilePath user_data_path_;
-  bool shut_down_ = false;
 
-  DISALLOW_COPY_AND_ASSIGN(HeadlessURLRequestContextGetter);
+  std::unique_ptr<::network::mojom::NetworkContext> network_context_owner_;
+  scoped_refptr<HeadlessURLRequestContextGetter> url_request_context_getter_;
+  std::unique_ptr<HeadlessResourceContext> resource_context_;
+
+  DISALLOW_COPY_AND_ASSIGN(HeadlessRequestContextManager);
 };
 
 }  // namespace headless
 
-#endif  // HEADLESS_LIB_BROWSER_HEADLESS_URL_REQUEST_CONTEXT_GETTER_H_
+#endif  // HEADLESS_LIB_BROWSER_HEADLESS_REQUEST_CONTEXT_MANAGER_H_

headless/lib/browser/headless_url_request_context_getter.cc

-// Copyright 2016 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "headless/lib/browser/headless_url_request_context_getter.h"
-
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "base/task/post_task.h"
-#include "build/build_config.h"
-#include "components/cookie_config/cookie_store_util.h"
-#include "content/public/browser/browser_task_traits.h"
-#include "content/public/browser/browser_thread.h"
-#include "content/public/browser/cookie_store_factory.h"
-#include "content/public/browser/devtools_network_transaction_factory.h"
-#include "headless/app/headless_shell_switches.h"
-#include "headless/lib/browser/headless_browser_context_impl.h"
-#include "headless/lib/browser/headless_browser_context_options.h"
-#include "net/base/network_delegate_impl.h"
-#include "net/cert_net/nss_ocsp.h"
-#include "net/cookies/cookie_store.h"
-#include "net/dns/mapped_host_resolver.h"
-#include "net/http/http_auth_handler_factory.h"
-#include "net/http/http_auth_scheme.h"
-#include "net/http/http_transaction_factory.h"
-#include "net/http/http_util.h"
-#include "net/proxy_resolution/proxy_resolution_service.h"
-#include "net/ssl/channel_id_service.h"
-#include "net/ssl/default_channel_id_store.h"
-#include "net/traffic_annotation/network_traffic_annotation.h"
-#include "net/url_request/url_request_context.h"
-#include "net/url_request/url_request_context_builder.h"
-#include "services/network/public/cpp/network_switches.h"
-
-#if defined(OS_LINUX) && !defined(OS_CHROMEOS)
-#include "base/command_line.h"
-#include "components/os_crypt/key_storage_config_linux.h"
-#include "components/os_crypt/os_crypt.h"
-#endif
-
-namespace headless {
-
-namespace {
-
-class DelegateImpl : public net::NetworkDelegateImpl {
- public:
-  DelegateImpl() = default;
-  ~DelegateImpl() override = default;
-
- private:
-  // net::NetworkDelegateImpl implementation.
-  bool OnCanAccessFile(const net::URLRequest& request,
-                       const base::FilePath& original_path,
-                       const base::FilePath& absolute_path) const override {
-    return true;
-  }
-
-  DISALLOW_COPY_AND_ASSIGN(DelegateImpl);
-};
-
-}  // namespace
-
-HeadlessURLRequestContextGetter::HeadlessURLRequestContextGetter(
-    scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
-    content::ProtocolHandlerMap* protocol_handlers,
-    content::ProtocolHandlerMap context_protocol_handlers,
-    content::URLRequestInterceptorScopedVector request_interceptors,
-    const HeadlessBrowserContextOptions* options,
-    base::FilePath user_data_path)
-    : io_task_runner_(std::move(io_task_runner)),
-      accept_language_(options->accept_language()),
-      user_agent_(options->user_agent()),
-      proxy_config_(options->proxy_config()),
-      request_interceptors_(std::move(request_interceptors)),
-      user_data_path_(std::move(user_data_path)) {
-  // Must first be created on the UI thread.
-  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
-
-  std::swap(protocol_handlers_, *protocol_handlers);
-  for (auto& pair : context_protocol_handlers) {
-    protocol_handlers_[pair.first] = std::move(pair.second);
-  }
-
-  base::CommandLine* command_line = base::CommandLine::ForCurrentProcess();
-  prefs_.SetServerWhitelist(
-      command_line->GetSwitchValueASCII(switches::kAuthServerWhitelist));
-
-  host_resolver_rules_ = command_line->GetSwitchValueASCII(
-      ::network::switches::kHostResolverRules);
-
-  // We must create the proxy config service on the UI loop on Linux because it
-  // must synchronously run on the glib message loop. This will be passed to
-  // the URLRequestContextStorage on the IO thread in GetURLRequestContext().
-  if (!proxy_config_) {
-    proxy_config_service_ =
-        net::ProxyResolutionService::CreateSystemProxyConfigService(
-            io_task_runner_);
-  }
-}
-
-HeadlessURLRequestContextGetter::~HeadlessURLRequestContextGetter() = default;
-
-net::URLRequestContext*
-HeadlessURLRequestContextGetter::GetURLRequestContext() {
-  DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
-
-  if (shut_down_)
-    return nullptr;
-
-  if (url_request_context_)
-    return url_request_context_.get();
-
-  net::URLRequestContextBuilder builder;
-
-  // Don't store cookies in incognito mode or if no user-data-dir was
-  // specified
-  // TODO: Enable this always once saving/restoring sessions is implemented
-  // (https://crbug.com/617931)
-  if (!user_data_path_.empty()) {
-#if defined(OS_LINUX) && !defined(OS_CHROMEOS)
-    std::unique_ptr<os_crypt::Config> config(new os_crypt::Config());
-    base::CommandLine* command_line = base::CommandLine::ForCurrentProcess();
-    config->store = command_line->GetSwitchValueASCII(switches::kPasswordStore);
-    config->product_name = "HeadlessChrome";
-    // OSCrypt may target keyring, which requires calls from the main
-    // thread.
-    config->main_thread_runner = base::CreateSingleThreadTaskRunnerWithTraits(
-        {content::BrowserThread::UI});
-    config->should_use_preference = false;
-    config->user_data_path = user_data_path_;
-    OSCrypt::SetConfig(std::move(config));
-#endif
-
-    content::CookieStoreConfig cookie_config(
-        user_data_path_.Append(FILE_PATH_LITERAL("Cookies")), false, true,
-        NULL);
-    cookie_config.crypto_delegate = cookie_config::GetCookieCryptoDelegate();
-    // TODO(crbug.com/801910): Hook up logging by passing in a non-null netlog.
-    std::unique_ptr<net::CookieStore> cookie_store =
-        CreateCookieStore(cookie_config, nullptr /* netlog*/);
-    std::unique_ptr<net::ChannelIDService> channel_id_service =
-        std::make_unique<net::ChannelIDService>(
-            new net::DefaultChannelIDStore(nullptr));
-
-    cookie_store->SetChannelIDServiceID(channel_id_service->GetUniqueID());
-    builder.SetCookieAndChannelIdStores(std::move(cookie_store),
-                                        std::move(channel_id_service));
-  }
-
-  builder.set_accept_language(
-      net::HttpUtil::GenerateAcceptLanguageHeader(accept_language_));
-  builder.set_user_agent(user_agent_);
-  // TODO(skyostil): Make these configurable.
-  builder.set_data_enabled(true);
-  builder.set_file_enabled(true);
-  if (proxy_config_) {
-    net::NetworkTrafficAnnotationTag traffic_annotation =
-        net::DefineNetworkTrafficAnnotation("proxy_config_headless", R"(
-      semantics {
-        sender: "Proxy Config"
-        description:
-          "Creates a proxy based on configuration received from headless "
-          "command prompt."
-        trigger:
-          "User starts headless with proxy config."
-        data:
-          "Proxy configurations."
-        destination: OTHER
-        destination_other:
-          "The proxy server specified in the configuration."
-      }
-      policy {
-        cookies_allowed: NO
-        setting:
-          "This config is only used for headless mode and provided by user."
-        policy_exception_justification:
-          "This config is only used for headless mode and provided by user."
-      })");
-
-    builder.set_proxy_resolution_service(
-        net::ProxyResolutionService::CreateFixed(net::ProxyConfigWithAnnotation(
-            *proxy_config_, traffic_annotation)));
-  } else {
-    builder.set_proxy_config_service(std::move(proxy_config_service_));
-  }
-
-  builder.set_network_delegate(std::make_unique<DelegateImpl>());
-  std::unique_ptr<net::HostResolver> host_resolver(
-      net::HostResolver::CreateDefaultResolver(nullptr));
-
-  if (!host_resolver_rules_.empty()) {
-    std::unique_ptr<net::MappedHostResolver> mapped_host_resolver(
-        new net::MappedHostResolver(std::move(host_resolver)));
-    mapped_host_resolver->SetRulesFromString(host_resolver_rules_);
-    host_resolver = std::move(mapped_host_resolver);
-  }
-
-  std::unique_ptr<net::HttpAuthHandlerRegistryFactory> factory =
-      net::HttpAuthHandlerRegistryFactory::CreateDefault(host_resolver.get());
-  factory->SetHttpAuthPreferences(net::kNegotiateAuthScheme, &prefs_);
-  builder.SetHttpAuthHandlerFactory(std::move(factory));
-  builder.set_host_resolver(std::move(host_resolver));
-
-  // Extra headers are required for network emulation and are removed in
-  // DevToolsNetworkTransaction. If a protocol handler is set for http or
-  // https, then it is likely that the HttpTransactionFactoryCallback will
-  // not be called and DevToolsNetworkTransaction would not remove the header.
-  bool has_http_handler = false;
-  for (auto& pair : protocol_handlers_) {
-    builder.SetProtocolHandler(pair.first, std::move(pair.second));
-    if (pair.first == url::kHttpScheme || pair.first == url::kHttpsScheme)
-      has_http_handler = true;
-  }
-  protocol_handlers_.clear();
-  builder.SetInterceptors(std::move(request_interceptors_));
-
-  if (!has_http_handler) {
-    builder.SetCreateHttpTransactionFactoryCallback(
-        base::BindOnce(&content::CreateDevToolsNetworkTransactionFactory));
-  }
-
-  url_request_context_ = builder.Build();
-
-  return url_request_context_.get();
-}
-
-scoped_refptr<base::SingleThreadTaskRunner>
-HeadlessURLRequestContextGetter::GetNetworkTaskRunner() const {
-  return io_task_runner_;
-}
-
-void HeadlessURLRequestContextGetter::NotifyContextShuttingDown() {
-  DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
-  shut_down_ = true;
-  net::URLRequestContextGetter::NotifyContextShuttingDown();
-  url_request_context_ = nullptr;  // deletes it
-}
-
-}  // namespace headless

headless/lib/utility/headless_content_utility_client.cc

@@ -4,6 +4,7 @@
 
 #include "headless/lib/utility/headless_content_utility_client.h"
 
+#include "base/lazy_instance.h"
 #include "printing/buildflags/buildflags.h"
 
 #if BUILDFLAG(ENABLE_PRINTING)
@@ -13,6 +14,18 @@
 
 namespace headless {
 
+namespace {
+base::LazyInstance<
+    HeadlessContentUtilityClient::NetworkBinderCreationCallback>::Leaky
+    g_network_binder_creation_callback = LAZY_INSTANCE_INITIALIZER;
+};
+
+// static
+void HeadlessContentUtilityClient::SetNetworkBinderCreationCallbackForTests(
+    NetworkBinderCreationCallback callback) {
+  g_network_binder_creation_callback.Get() = std::move(callback);
+}
+
 HeadlessContentUtilityClient::HeadlessContentUtilityClient(
     const std::string& user_agent)
     : user_agent_(user_agent) {}
@@ -29,4 +42,10 @@ void HeadlessContentUtilityClient::RegisterServices(
 #endif
 }
 
+void HeadlessContentUtilityClient::RegisterNetworkBinders(
+    service_manager::BinderRegistry* registry) {
+  if (g_network_binder_creation_callback.Get())
+    g_network_binder_creation_callback.Get().Run(registry);
+}
+
 }  // namespace headless

headless/lib/utility/headless_content_utility_client.h

@@ -7,17 +7,28 @@
 
 #include <string>
 
+#include "base/callback.h"
 #include "content/public/utility/content_utility_client.h"
+#include "headless/public/headless_export.h"
 
 namespace headless {
 
-class HeadlessContentUtilityClient : public content::ContentUtilityClient {
+class HEADLESS_EXPORT HeadlessContentUtilityClient
+    : public content::ContentUtilityClient {
  public:
+  using NetworkBinderCreationCallback =
+      base::RepeatingCallback<void(service_manager::BinderRegistry*)>;
+
+  static void SetNetworkBinderCreationCallbackForTests(
+      NetworkBinderCreationCallback callback);
+
   explicit HeadlessContentUtilityClient(const std::string& user_agent);
   ~HeadlessContentUtilityClient() override;
 
   // content::ContentUtilityClient:
   void RegisterServices(StaticServiceMap* services) override;
+  void RegisterNetworkBinders(
+      service_manager::BinderRegistry* registry) override;
 
  private:
   const std::string user_agent_;

headless/test/data/protocol/helpers/http-interceptor.js

@@ -39,7 +39,8 @@
           + (event.params.request.urlFragment || '');
       this.requestedUrls_.push(url);
 
-      if (this.responses_.has(url)) {
+      var response = this.responses_.get(url);
+      if (response) {
         if (!this.disabledRequestedUrlsLogging) {
           this.testRunner_.log(`requested url: ${url}`);
         }
@@ -47,12 +48,12 @@
         this.testRunner_.log(`requested url: ${url} is not known`);
         this.logResponses();
       }
-      const body = this.responses_.get(url).body || '';
-      const headers = this.responses_.get(url).headers || [];
-      const response = headers.join('\r\n') + '\r\n\r\n' + body;
+      const body = (response && response.body) || '';
+      const headers = (response && response.headers) || [];
+      const headers_with_body = headers.join('\r\n') + '\r\n\r\n' + body;
       this.dp_.Network.continueInterceptedRequest({
         interceptionId: event.params.interceptionId,
-        rawResponse: btoa(response)
+        rawResponse: btoa(headers_with_body)
       });
     });
 

headless/test/headless_test_launcher.cc

@@ -8,10 +8,13 @@
 #include "base/command_line.h"
 #include "base/macros.h"
 #include "base/test/launcher/test_launcher.h"
+#include "content/public/common/content_switches.h"
 #include "content/public/test/content_test_suite_base.h"
+#include "content/public/test/network_service_test_helper.h"
 #include "content/public/test/test_launcher.h"
 #include "headless/lib/browser/headless_browser_impl.h"
 #include "headless/lib/headless_content_main_delegate.h"
+#include "headless/lib/utility/headless_content_utility_client.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace headless {
@@ -72,6 +75,25 @@ int main(int argc, char** argv) {
   if (parallel_jobs > 1U) {
     parallel_jobs /= 2U;
   }
+
+  // Setup a working test environment for the network service in case it's used.
+  // Only create this object in the utility process, so that its members don't
+  // interfere with other test objects in the browser process.
+  std::unique_ptr<content::NetworkServiceTestHelper>
+      network_service_test_helper;
+  if (base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+          switches::kProcessType) == switches::kUtilityProcess) {
+    network_service_test_helper =
+        std::make_unique<content::NetworkServiceTestHelper>();
+    headless::HeadlessContentUtilityClient::
+        SetNetworkBinderCreationCallbackForTests(base::BindRepeating(
+            [](content::NetworkServiceTestHelper* helper,
+               service_manager::BinderRegistry* registry) {
+              helper->RegisterNetworkBinders(registry);
+            },
+            network_service_test_helper.get()));
+  }
+
   headless::HeadlessTestLauncherDelegate launcher_delegate;
   return LaunchTests(&launcher_delegate, parallel_jobs, argc, argv);
 }

tools/traffic_annotation/summary/annotations.xml

@@ -198,7 +198,7 @@ Refer to README.md for content description and update process.
  <item id="profile_avatar" hash_code="51164680" type="0" content_hash_code="113550845" os_list="linux,windows" file_path="chrome/browser/profiles/profile_avatar_downloader.cc"/>
  <item id="profile_resetter_upload" hash_code="105330607" type="0" content_hash_code="129329171" os_list="linux,windows" file_path="chrome/browser/profile_resetter/reset_report_uploader.cc"/>
  <item id="proxy_config_direct" hash_code="119015679" type="0" content_hash_code="119931568" os_list="linux,windows" file_path="net/proxy_resolution/proxy_config_with_annotation.cc"/>
- <item id="proxy_config_headless" hash_code="133221587" type="0" content_hash_code="77459277" os_list="linux,windows" file_path="headless/lib/browser/headless_url_request_context_getter.cc"/>
+ <item id="proxy_config_headless" hash_code="133221587" type="0" content_hash_code="77459277" os_list="linux,windows" file_path="headless/lib/browser/headless_request_context_manager.cc"/>
  <item id="proxy_config_settings" hash_code="136468456" type="0" content_hash_code="19527377" os_list="linux,windows" file_path="components/proxy_config/pref_proxy_config_tracker_impl.cc"/>
  <item id="proxy_config_system" hash_code="11258689" type="0" content_hash_code="77057929" os_list="linux,windows" file_path="net/proxy_resolution/proxy_resolution_service.cc"/>
  <item id="proxy_script_fetcher" hash_code="37531401" type="0" deprecated="2018-03-16" content_hash_code="31866133" file_path=""/>
@@ -233,6 +233,7 @@ Refer to README.md for content description and update process.
  <item id="sdch_dictionary_fetch" hash_code="47152935" type="0" deprecated="2017-09-16" content_hash_code="16764294" file_path=""/>
  <item id="security_key_socket" hash_code="31074955" type="0" content_hash_code="11296409" os_list="linux,windows" file_path="remoting/host/security_key/security_key_socket.cc"/>
  <item id="service_worker_navigation_preload" hash_code="129872904" type="0" content_hash_code="79473248" os_list="linux,windows" file_path="content/browser/service_worker/service_worker_fetch_dispatcher.cc"/>
+ <item id="service_worker_update_checker" hash_code="130931413" type="0" content_hash_code="46608086" os_list="linux,mac,windows" file_path="content/browser/service_worker/service_worker_single_script_update_checker.cc"/>
  <item id="service_worker_write_to_cache_job" hash_code="117963307" type="0" content_hash_code="18065724" os_list="linux,windows" file_path="content/browser/service_worker/service_worker_write_to_cache_job.cc"/>
  <item id="services_http_server_error_response" hash_code="59302801" type="0" content_hash_code="127774041" os_list="linux,windows" file_path="services/network/public/cpp/server/http_server.cc"/>
  <item id="shared_worker_script_load" hash_code="107347227" type="0" content_hash_code="64293499" os_list="linux,mac,windows" file_path="content/browser/shared_worker/shared_worker_script_fetcher.cc"/>
@@ -284,5 +285,4 @@ Refer to README.md for content description and update process.
  <item id="webstore_installer" hash_code="18764319" type="0" content_hash_code="11030110" os_list="linux,windows" file_path="chrome/browser/extensions/webstore_installer.cc"/>
  <item id="webui_content_scripts_download" hash_code="100545943" type="0" content_hash_code="119898059" os_list="linux,windows" file_path="extensions/browser/guest_view/web_view/web_ui/web_ui_url_fetcher.cc"/>
  <item id="xmpp_signal_strategy" hash_code="88906454" type="0" content_hash_code="88958321" os_list="linux,windows" file_path="remoting/signaling/xmpp_signal_strategy.cc"/>
- <item id="service_worker_update_checker" hash_code="130931413" type="0" content_hash_code="46608086" os_list="linux,mac,windows" file_path="content/browser/service_worker/service_worker_single_script_update_checker.cc"/>
 </annotations>