Automation: allow overriding network proxy per browser context.

Bug: 1090797
Change-Id: I53bb57df832253aab581f448ea8c773e0107b579
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2226298Reviewed-by: Nicolas Ouellet-Payeur <nicolaso@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Commit-Queue: Pavel Feldman <pfeldman@chromium.org>
Auto-Submit: Pavel Feldman <pfeldman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#774679}

content/browser/devtools/devtools_instrumentation.cc

@@ -9,6 +9,7 @@
 #include "content/browser/devtools/browser_devtools_agent_host.h"
 #include "content/browser/devtools/devtools_url_loader_interceptor.h"
 #include "content/browser/devtools/protocol/audits_handler.h"
+#include "content/browser/devtools/protocol/browser_handler.h"
 #include "content/browser/devtools/protocol/emulation_handler.h"
 #include "content/browser/devtools/protocol/fetch_handler.h"
 #include "content/browser/devtools/protocol/log_handler.h"
@@ -23,14 +24,17 @@
 #include "content/browser/web_contents/web_contents_impl.h"
 #include "content/browser/web_package/signed_exchange_envelope.h"
 #include "content/common/navigation_params.mojom.h"
+#include "content/public/browser/browser_context.h"
 #include "mojo/public/cpp/bindings/pending_remote.h"
 #include "mojo/public/cpp/bindings/self_owned_receiver.h"
 #include "net/base/load_flags.h"
 #include "net/cookies/canonical_cookie.h"
 #include "net/cookies/cookie_inclusion_status.h"
 #include "net/http/http_request_headers.h"
+#include "net/proxy_resolution/proxy_config.h"
 #include "net/quic/quic_transport_error.h"
 #include "net/ssl/ssl_info.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
 #include "services/network/public/mojom/network_context.mojom.h"
 #include "services/network/public/mojom/url_loader_factory.mojom.h"
 #include "third_party/blink/public/mojom/devtools/inspector_issue.mojom.h"
@@ -827,6 +831,18 @@ void OnQuicTransportHandshakeFailed(
   DispatchToAgents(ftn, &protocol::LogHandler::EntryAdded, entry.get());
 }
 
+void ApplyNetworkContextParamsOverrides(
+    BrowserContext* browser_context,
+    network::mojom::NetworkContextParams* context_params) {
+  for (auto* agent_host : BrowserDevToolsAgentHost::Instances()) {
+    for (auto* target_handler :
+         protocol::TargetHandler::ForAgentHost(agent_host)) {
+      target_handler->ApplyNetworkContextParamsOverrides(browser_context,
+                                                         context_params);
+    }
+  }
+}
+
 }  // namespace devtools_instrumentation
 
 }  // namespace content

content/browser/devtools/devtools_instrumentation.h

@@ -43,13 +43,14 @@ class DownloadItem;
 }  // namespace download
 
 namespace content {
-class SignedExchangeEnvelope;
+class BrowserContext;
 class FrameTreeNode;
 class NavigationHandle;
 class NavigationRequest;
 class NavigationThrottle;
 class RenderFrameHostImpl;
 class RenderProcessHost;
+class SignedExchangeEnvelope;
 class WebContents;
 
 struct SignedExchangeError;
@@ -202,6 +203,10 @@ void OnQuicTransportHandshakeFailed(
     const GURL& url,
     const base::Optional<net::QuicTransportError>& error);
 
+void ApplyNetworkContextParamsOverrides(
+    BrowserContext* browser_context,
+    network::mojom::NetworkContextParams* network_context_params);
+
 }  // namespace devtools_instrumentation
 
 }  // namespace content

content/browser/devtools/protocol/target_handler.cc

@@ -30,6 +30,35 @@ namespace protocol {
 
 namespace {
 
+constexpr net::NetworkTrafficAnnotationTag
+    kSettingsProxyConfigTrafficAnnotation =
+        net::DefineNetworkTrafficAnnotation("devtools_proxy_config", R"(
+      semantics {
+        sender: "Proxy Configuration over Developer Tools"
+        description:
+          "Used to fetch HTTP/HTTPS/SOCKS5/PAC proxy configuration when "
+          "proxy is configured by DevTools. It is equivalent to the one "
+          "configured via the --proxy-server command line flag. "
+          "When proxy implies automatic configuration, it can send network "
+          "requests in the scope of this annotation."
+        trigger:
+          "Whenever a network request is made when the system proxy settings "
+          "are used, and they indicate to use a proxy server."
+        data:
+          "Proxy configuration."
+        destination: OTHER
+        destination_other: "The proxy server specified in the configuration."
+      }
+      policy {
+        cookies_allowed: NO
+        setting:
+          "This request cannot be disabled in settings. However it will never "
+          "be made if user does not run with '--remote-debugging-*' switches "
+          "and does not explicitly send this data over Chrome remote debugging."
+        policy_exception_justification:
+          "Not implemented, only used in DevTools and is behind a switch."
+      })");
+
 static const char kNotAllowedError[] = "Not allowed";
 static const char kMethod[] = "method";
 static const char kResumeMethod[] = "Runtime.runIfWaitingForDebugger";
@@ -560,6 +589,7 @@ Response TargetHandler::Disable() {
     }
     dispose_on_detach_context_ids_.clear();
   }
+  contexts_with_overridden_proxy_.clear();
   return Response::Success();
 }
 
@@ -950,6 +980,8 @@ void TargetHandler::DevToolsAgentHostCrashed(DevToolsAgentHost* host,
 
 protocol::Response TargetHandler::CreateBrowserContext(
     Maybe<bool> dispose_on_detach,
+    Maybe<std::string> proxy_server,
+    Maybe<std::string> proxy_bypass_list,
     std::string* out_context_id) {
   if (access_mode_ != AccessMode::kBrowser)
     return Response::ServerError(kNotAllowedError);
@@ -961,6 +993,18 @@ protocol::Response TargetHandler::CreateBrowserContext(
   BrowserContext* context = delegate->CreateBrowserContext();
   if (!context)
     return Response::ServerError("Failed to create browser context.");
+
+  if (proxy_server.isJust()) {
+    net::ProxyConfig proxy_config;
+    proxy_config.proxy_rules().ParseFromString(proxy_server.fromJust());
+    if (proxy_bypass_list.isJust()) {
+      proxy_config.proxy_rules().bypass_rules.ParseFromString(
+          proxy_bypass_list.fromJust());
+    }
+    contexts_with_overridden_proxy_[context->UniqueId()] =
+        std::move(proxy_config);
+  }
+
   *out_context_id = context->UniqueId();
   if (dispose_on_detach.fromMaybe(false))
     dispose_on_detach_context_ids_.insert(*out_context_id);
@@ -1024,5 +1068,15 @@ void TargetHandler::DisposeBrowserContext(
           std::move(callback)));
 }
 
+void TargetHandler::ApplyNetworkContextParamsOverrides(
+    BrowserContext* browser_context,
+    network::mojom::NetworkContextParams* context_params) {
+  auto it = contexts_with_overridden_proxy_.find(browser_context->UniqueId());
+  if (it == contexts_with_overridden_proxy_.end())
+    return;
+  context_params->initial_proxy_config = net::ProxyConfigWithAnnotation(
+      it->second, kSettingsProxyConfigTrafficAnnotation);
+}
+
 }  // namespace protocol
 }  // namespace content

content/browser/devtools/protocol/target_handler.h

@@ -8,12 +8,15 @@
 #include <map>
 #include <set>
 
+#include "base/containers/flat_map.h"
 #include "base/containers/flat_set.h"
 #include "base/memory/weak_ptr.h"
 #include "content/browser/devtools/protocol/devtools_domain_handler.h"
 #include "content/browser/devtools/protocol/target.h"
 #include "content/browser/devtools/protocol/target_auto_attacher.h"
 #include "content/public/browser/devtools_agent_host_observer.h"
+#include "net/proxy_resolution/proxy_config.h"
+#include "services/network/public/mojom/network_context.mojom.h"
 
 namespace content {
 
@@ -86,6 +89,8 @@ class TargetHandler : public DevToolsDomainHandler,
   Response ExposeDevToolsProtocol(const std::string& target_id,
                                   Maybe<std::string> binding_name) override;
   Response CreateBrowserContext(Maybe<bool> dispose_on_detach,
+                                Maybe<std::string> proxy_server,
+                                Maybe<std::string> proxy_bypass_list,
                                 std::string* out_context_id) override;
   void DisposeBrowserContext(
       const std::string& context_id,
@@ -104,6 +109,10 @@ class TargetHandler : public DevToolsDomainHandler,
       std::unique_ptr<protocol::Array<Target::TargetInfo>>* target_infos)
       override;
 
+  void ApplyNetworkContextParamsOverrides(
+      BrowserContext* browser_context,
+      network::mojom::NetworkContextParams* network_context_params);
+
  private:
   class Session;
   class Throttle;
@@ -143,6 +152,7 @@ class TargetHandler : public DevToolsDomainHandler,
   std::map<DevToolsAgentHost*, Session*> auto_attached_sessions_;
   std::set<DevToolsAgentHost*> reported_hosts_;
   base::flat_set<std::string> dispose_on_detach_context_ids_;
+  base::flat_map<std::string, net::ProxyConfig> contexts_with_overridden_proxy_;
   AccessMode access_mode_;
   std::string owner_target_id_;
   DevToolsSession* root_session_;

content/browser/storage_partition_impl.cc

@@ -2371,6 +2371,8 @@ void StoragePartitionImpl::InitNetworkContext() {
   GetContentClient()->browser()->ConfigureNetworkContextParams(
       browser_context_, is_in_memory_, relative_partition_path_,
       context_params.get(), cert_verifier_creation_params.get());
+  devtools_instrumentation::ApplyNetworkContextParamsOverrides(
+      browser_context_, context_params.get());
   DCHECK(!context_params->cert_verifier_params)
       << "|cert_verifier_params| should not be set in the NetworkContextParams,"
          "as they will be replaced with either the newly configured "

headless/test/data/protocol/sanity/browser-set-initial-proxy-config-expected.txt

+Tests that headless session can configure proxy.
+
+No proxy page title: Page with title
+Bogus proxy page title: 
+Good proxy page title: Page with title
\ No newline at end of file

headless/test/data/protocol/sanity/browser-set-initial-proxy-config.js

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+(async function(testRunner) {
+  testRunner.log('Tests that headless session can configure proxy.\n');
+  const { result: { sessionId } } =
+  await testRunner.browserP().Target.attachToBrowserTarget({});
+  const { protocol: bProtocol } = new TestRunner.Session(testRunner, sessionId);
+
+  async function dumpWithProxyServer(proxyServer) {
+    const { result: { browserContextId } } =
+        await bProtocol.Target.createBrowserContext({ proxyServer });
+    const { result: { targetId }} =
+        await bProtocol.Target.createTarget({
+      browserContextId: browserContextId,
+      url: 'about:blank'
+    });
+
+    const { result: { sessionId } } =
+        await bProtocol.Target.attachToTarget({ targetId, flatten: true });
+    const { protocol: pProtocol } =
+        new TestRunner.Session(testRunner, sessionId);
+    await pProtocol.Page.enable({});
+    await pProtocol.Page.navigate({
+      url: testRunner._testBaseURL + 'resources/title.html'
+    });
+    await pProtocol.Page.onceLoadEventFired();
+    const { result: { result: { value } } } =
+        await pProtocol.Runtime.evaluate({ expression: 'document.title' });
+    return value;
+  }
+
+  testRunner.log(`No proxy page title: ${await dumpWithProxyServer()}`);
+  testRunner.log(`Bogus proxy page title: ${await dumpWithProxyServer(
+      'bogus')}`);
+  testRunner.log(`Good proxy page title: ${await dumpWithProxyServer(
+      new URL(testRunner._targetBaseURL).host)}`);
+
+  testRunner.completeTest();
+})

headless/test/data/protocol/sanity/resources/title.html

+<html>
+	<title>Page with title</title>
+</html>
\ No newline at end of file

headless/test/headless_protocol_browsertest.cc

@@ -438,5 +438,7 @@ HEADLESS_PROTOCOL_COMPOSITOR_TEST(RendererCanvas, "sanity/renderer-canvas.js")
 
 HEADLESS_PROTOCOL_COMPOSITOR_TEST(RendererOpacityAnimation,
                                   "sanity/renderer-opacity-animation.js")
+HEADLESS_PROTOCOL_COMPOSITOR_TEST(BrowserSetInitialProxyConfig,
+                                  "sanity/browser-set-initial-proxy-config.js")
 
 }  // namespace headless

third_party/blink/public/devtools_protocol/browser_protocol.pdl

@@ -7075,6 +7075,11 @@ domain Target
     parameters
       # If specified, disposes this context when debugging session disconnects.
       optional boolean disposeOnDetach
+      # Proxy server, similar to the one passed to --proxy-server
+      optional string proxyServer
+      # Proxy bypass list, similar to the one passed to --proxy-bypass-list
+      optional string proxyBypassList
+
     returns
       # The id of the context created.
       Browser.BrowserContextID browserContextId

third_party/blink/web_tests/http/tests/inspector-protocol/resources/inspector-protocol-test.js

@@ -329,11 +329,14 @@ TestRunner.Session = class {
     return session;
   }
 
-  sendCommand(method, params) {
+  async sendCommand(method, params) {
     var requestId = ++this._requestId;
     if (this._testRunner._dumpInspectorProtocolMessages)
       this._testRunner.log(`frontend => backend: ${JSON.stringify({method, params, sessionId: this._sessionId})}`);
-    return DevToolsAPI._sendCommand(this._sessionId, method, params);
+    const result = await DevToolsAPI._sendCommand(this._sessionId, method, params);
+    if (this._testRunner._dumpInspectorProtocolMessages)
+      this._testRunner.log(`backend => frontend: ${JSON.stringify(result)}`);
+    return result;
   }
 
   async evaluate(code, ...args) {

tools/traffic_annotation/summary/annotations.xml

@@ -78,6 +78,7 @@ Refer to README.md for content description and update process.
  <item id="devtools_http_handler" hash_code="49160454" type="0" content_hash_code="88414393" os_list="linux,windows" file_path="content/browser/devtools/devtools_http_handler.cc"/>
  <item id="devtools_interceptor" hash_code="98123737" type="0" deprecated="2019-07-31" content_hash_code="64591843" file_path=""/>
  <item id="devtools_network_resource" hash_code="129652775" type="0" content_hash_code="32810159" os_list="linux,windows" file_path="chrome/browser/devtools/devtools_ui_bindings.cc"/>
+ <item id="devtools_proxy_config" hash_code="79904729" type="0" content_hash_code="31996982" os_list="linux,windows" file_path="content/browser/devtools/protocol/target_handler.cc"/>
  <item id="dial_get_app_info" hash_code="15952025" type="0" deprecated="2018-02-27" content_hash_code="90542080" file_path=""/>
  <item id="dial_get_device_description" hash_code="50422598" type="0" deprecated="2018-02-27" content_hash_code="129827780" file_path=""/>
  <item id="dial_url_fetcher" hash_code="41424546" type="0" content_hash_code="129828432" os_list="linux,windows" file_path="chrome/browser/media/router/discovery/dial/dial_url_fetcher.cc"/>