Network service changes for custom proxy config

This CL handles network service changes needed to support DRP proxy
config. This allows setting a custom proxy config that will be used if
the request would otherwise be sent direct. Additionally, headers can be
set at the config or request level for requests sent using the proxy.
These headers can either be added pre or post cache, which allows
caching of the response to be affected by the Vary: tag for some headers.

Bug: 721403
Cq-Include-Trybots: luci.chromium.try:linux_mojo
Change-Id: Ie0faf02d1499e75bf27d023a3dc73efbeab6e787
Reviewed-on: https://chromium-review.googlesource.com/c/1260262Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Eric Roman <eroman@chromium.org>
Reviewed-by: Tarun Bansal <tbansal@chromium.org>
Commit-Queue: Clark DuVall <cduvall@chromium.org>
Cr-Commit-Position: refs/heads/master@{#597642}

services/network/BUILD.gn

@@ -67,6 +67,8 @@ jumbo_component("network_service") {
     "network_service.h",
     "network_service_network_delegate.cc",
     "network_service_network_delegate.h",
+    "network_service_proxy_delegate.cc",
+    "network_service_proxy_delegate.h",
     "network_usage_accumulator.cc",
     "network_usage_accumulator.h",
     "p2p/socket.cc",
@@ -245,6 +247,7 @@ source_set("tests") {
     "network_change_manager_unittest.cc",
     "network_context_unittest.cc",
     "network_quality_estimator_manager_unittest.cc",
+    "network_service_proxy_delegate_unittest.cc",
     "network_service_unittest.cc",
     "network_usage_accumulator_unittest.cc",
     "p2p/socket_tcp_server_unittest.cc",

services/network/network_context.cc

@@ -74,6 +74,7 @@
 #include "services/network/mojo_net_log.h"
 #include "services/network/network_service.h"
 #include "services/network/network_service_network_delegate.h"
+#include "services/network/network_service_proxy_delegate.h"
 #include "services/network/p2p/socket_manager.h"
 #include "services/network/proxy_config_service_mojo.h"
 #include "services/network/proxy_lookup_request.h"
@@ -1480,6 +1481,12 @@ URLRequestContextOwner NetworkContext::MakeURLRequestContext() {
       std::make_unique<NetworkServiceNetworkDelegate>(this);
   builder.set_network_delegate(std::move(network_delegate));
 
+  if (params_->custom_proxy_config_client_request) {
+    proxy_delegate_ = std::make_unique<NetworkServiceProxyDelegate>(
+        std::move(params_->custom_proxy_config_client_request));
+    builder.set_shared_proxy_delegate(proxy_delegate_.get());
+  }
+
   // |network_service_| may be nullptr in tests.
   auto result = ApplyContextParamsToBuilder(&builder);
 

services/network/network_context.h

@@ -61,6 +61,7 @@ class CookieManager;
 class ExpectCTReporter;
 class HostResolver;
 class NetworkService;
+class NetworkServiceProxyDelegate;
 class P2PSocketManager;
 class ProxyLookupRequest;
 class ResourceScheduler;
@@ -279,6 +280,10 @@ class COMPONENT_EXPORT(NETWORK_SERVICE) NetworkContext
     return proxy_lookup_requests_.size();
   }
 
+  NetworkServiceProxyDelegate* proxy_delegate() const {
+    return proxy_delegate_.get();
+  }
+
  private:
   class ContextNetworkDelegate;
 
@@ -394,6 +399,8 @@ class COMPONENT_EXPORT(NETWORK_SERVICE) NetworkContext
   std::set<std::unique_ptr<HostResolver>, base::UniquePtrComparator>
       host_resolvers_;
 
+  std::unique_ptr<NetworkServiceProxyDelegate> proxy_delegate_;
+
   // Used for Signed Exchange certificate verification.
   int next_cert_verify_id_ = 0;
   struct PendingCertVerify {

services/network/network_service_network_delegate.cc

@@ -7,6 +7,7 @@
 #include "services/network/cookie_manager.h"
 #include "services/network/network_context.h"
 #include "services/network/network_service.h"
+#include "services/network/network_service_proxy_delegate.h"
 #include "services/network/public/cpp/features.h"
 #include "services/network/url_loader.h"
 
@@ -24,6 +25,28 @@ NetworkServiceNetworkDelegate::NetworkServiceNetworkDelegate(
 
 NetworkServiceNetworkDelegate::~NetworkServiceNetworkDelegate() = default;
 
+int NetworkServiceNetworkDelegate::OnBeforeStartTransaction(
+    net::URLRequest* request,
+    net::CompletionOnceCallback callback,
+    net::HttpRequestHeaders* headers) {
+  if (network_context_->proxy_delegate()) {
+    network_context_->proxy_delegate()->OnBeforeStartTransaction(request,
+                                                                 headers);
+  }
+  return net::OK;
+}
+
+void NetworkServiceNetworkDelegate::OnBeforeSendHeaders(
+    net::URLRequest* request,
+    const net::ProxyInfo& proxy_info,
+    const net::ProxyRetryInfoMap& proxy_retry_info,
+    net::HttpRequestHeaders* headers) {
+  if (network_context_->proxy_delegate()) {
+    network_context_->proxy_delegate()->OnBeforeSendHeaders(request, proxy_info,
+                                                            headers);
+  }
+}
+
 int NetworkServiceNetworkDelegate::OnHeadersReceived(
     net::URLRequest* request,
     net::CompletionOnceCallback callback,

services/network/network_service_network_delegate.h

@@ -22,6 +22,13 @@ class COMPONENT_EXPORT(NETWORK_SERVICE) NetworkServiceNetworkDelegate
 
  private:
   // net::NetworkDelegateImpl implementation.
+  void OnBeforeSendHeaders(net::URLRequest* request,
+                           const net::ProxyInfo& proxy_info,
+                           const net::ProxyRetryInfoMap& proxy_retry_info,
+                           net::HttpRequestHeaders* headers) override;
+  int OnBeforeStartTransaction(net::URLRequest* request,
+                               net::CompletionOnceCallback callback,
+                               net::HttpRequestHeaders* headers) override;
   int OnHeadersReceived(
       net::URLRequest* request,
       net::CompletionOnceCallback callback,

services/network/network_service_proxy_delegate.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "services/network/network_service_proxy_delegate.h"
+#include "net/base/url_util.h"
+#include "net/http/http_request_headers.h"
+#include "net/http/http_util.h"
+#include "net/proxy_resolution/proxy_info.h"
+#include "services/network/url_loader.h"
+
+namespace network {
+namespace {
+
+void GetAlternativeProxy(const GURL& url,
+                         const net::ProxyRetryInfoMap& proxy_retry_info,
+                         net::ProxyInfo* result) {
+  net::ProxyServer resolved_proxy_server = result->proxy_server();
+  DCHECK(resolved_proxy_server.is_valid());
+
+  // Right now, HTTPS proxies are assumed to support quic. If this needs to
+  // change, add a setting in CustomProxyConfig to control this behavior.
+  if (!resolved_proxy_server.is_https())
+    return;
+
+  net::ProxyInfo alternative_proxy_info;
+  alternative_proxy_info.UseProxyServer(net::ProxyServer(
+      net::ProxyServer::SCHEME_QUIC, resolved_proxy_server.host_port_pair()));
+  alternative_proxy_info.DeprioritizeBadProxies(proxy_retry_info);
+
+  if (alternative_proxy_info.is_empty())
+    return;
+
+  result->SetAlternativeProxy(alternative_proxy_info.proxy_server());
+}
+
+bool ApplyProxyConfigToProxyInfo(const net::ProxyConfig::ProxyRules& rules,
+                                 const net::ProxyRetryInfoMap& proxy_retry_info,
+                                 const GURL& url,
+                                 net::ProxyInfo* proxy_info) {
+  DCHECK(proxy_info);
+  if (rules.empty())
+    return false;
+
+  rules.Apply(url, proxy_info);
+  proxy_info->DeprioritizeBadProxies(proxy_retry_info);
+  return !proxy_info->proxy_server().is_direct();
+}
+
+// Checks if |target_proxy| is in |proxy_list|.
+bool CheckProxyList(const net::ProxyList& proxy_list,
+                    const net::ProxyServer& target_proxy) {
+  for (const auto& proxy : proxy_list.GetAll()) {
+    if (proxy.host_port_pair().Equals(target_proxy.host_port_pair()))
+      return true;
+  }
+  return false;
+}
+
+}  // namespace
+
+NetworkServiceProxyDelegate::NetworkServiceProxyDelegate(
+    mojom::CustomProxyConfigClientRequest config_client_request)
+    : binding_(this, std::move(config_client_request)) {}
+
+void NetworkServiceProxyDelegate::OnBeforeStartTransaction(
+    net::URLRequest* request,
+    net::HttpRequestHeaders* headers) {
+  if (!MayProxyURL(request->url()))
+    return;
+
+  headers->MergeFrom(proxy_config_->pre_cache_headers);
+
+  auto* url_loader = URLLoader::ForRequest(*request);
+  if (url_loader) {
+    headers->MergeFrom(url_loader->custom_proxy_pre_cache_headers());
+  }
+}
+
+void NetworkServiceProxyDelegate::OnBeforeSendHeaders(
+    net::URLRequest* request,
+    const net::ProxyInfo& proxy_info,
+    net::HttpRequestHeaders* headers) {
+  auto* url_loader = URLLoader::ForRequest(*request);
+  if (IsInProxyConfig(proxy_info.proxy_server())) {
+    headers->MergeFrom(proxy_config_->post_cache_headers);
+
+    if (url_loader) {
+      headers->MergeFrom(url_loader->custom_proxy_post_cache_headers());
+    }
+    // TODO(crbug.com/721403): This check may be incorrect if a new proxy config
+    // is set between OnBeforeStartTransaction and here.
+  } else if (MayProxyURL(request->url())) {
+    for (const auto& kv : proxy_config_->pre_cache_headers.GetHeaderVector()) {
+      headers->RemoveHeader(kv.key);
+    }
+
+    if (url_loader) {
+      for (const auto& kv :
+           url_loader->custom_proxy_pre_cache_headers().GetHeaderVector()) {
+        headers->RemoveHeader(kv.key);
+      }
+    }
+  }
+}
+
+NetworkServiceProxyDelegate::~NetworkServiceProxyDelegate() {}
+
+void NetworkServiceProxyDelegate::OnResolveProxy(
+    const GURL& url,
+    const std::string& method,
+    const net::ProxyRetryInfoMap& proxy_retry_info,
+    net::ProxyInfo* result) {
+  if (!EligibleForProxy(*result, url, method))
+    return;
+
+  net::ProxyInfo proxy_info;
+  if (ApplyProxyConfigToProxyInfo(proxy_config_->rules, proxy_retry_info, url,
+                                  &proxy_info)) {
+    DCHECK(!proxy_info.is_empty() && !proxy_info.is_direct());
+    result->OverrideProxyList(proxy_info.proxy_list());
+    GetAlternativeProxy(url, proxy_retry_info, result);
+  }
+}
+
+void NetworkServiceProxyDelegate::OnFallback(const net::ProxyServer& bad_proxy,
+                                             int net_error) {}
+
+void NetworkServiceProxyDelegate::OnCustomProxyConfigUpdated(
+    mojom::CustomProxyConfigPtr proxy_config) {
+  DCHECK(proxy_config->rules.empty() ||
+         !proxy_config->rules.proxies_for_http.IsEmpty());
+  proxy_config_ = std::move(proxy_config);
+}
+
+bool NetworkServiceProxyDelegate::IsInProxyConfig(
+    const net::ProxyServer& proxy_server) const {
+  if (!proxy_server.is_valid() || proxy_server.is_direct())
+    return false;
+
+  return CheckProxyList(proxy_config_->rules.proxies_for_http, proxy_server);
+}
+
+bool NetworkServiceProxyDelegate::MayProxyURL(const GURL& url) const {
+  return url.SchemeIs(url::kHttpScheme) && !proxy_config_->rules.empty() &&
+         !net::IsLocalhost(url);
+}
+
+bool NetworkServiceProxyDelegate::EligibleForProxy(
+    const net::ProxyInfo& proxy_info,
+    const GURL& url,
+    const std::string& method) const {
+  return proxy_info.is_direct() && proxy_info.proxy_list().size() == 1 &&
+         MayProxyURL(url) && net::HttpUtil::IsMethodIdempotent(method);
+}
+
+}  // namespace network

services/network/network_service_proxy_delegate.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SERVICES_NETWORK_NETWORK_SERVICE_PROXY_DELEGATE_H_
+#define SERVICES_NETWORK_NETWORK_SERVICE_PROXY_DELEGATE_H_
+
+#include "base/component_export.h"
+#include "base/macros.h"
+#include "mojo/public/cpp/bindings/binding.h"
+#include "net/base/proxy_delegate.h"
+#include "services/network/public/mojom/network_context.mojom.h"
+
+namespace net {
+class HttpRequestHeaders;
+class URLRequest;
+}  // namespace net
+
+namespace network {
+
+// NetworkServiceProxyDelegate is used to support the custom proxy
+// configuration, which can be set in
+// NetworkContextParams.custom_proxy_config_client_request.
+class COMPONENT_EXPORT(NETWORK_SERVICE) NetworkServiceProxyDelegate
+    : public net::ProxyDelegate,
+      public mojom::CustomProxyConfigClient {
+ public:
+  explicit NetworkServiceProxyDelegate(
+      mojom::CustomProxyConfigClientRequest config_client_request);
+  ~NetworkServiceProxyDelegate() override;
+
+  // These methods are forwarded from the NetworkDelegate.
+  void OnBeforeStartTransaction(net::URLRequest* request,
+                                net::HttpRequestHeaders* headers);
+  void OnBeforeSendHeaders(net::URLRequest* request,
+                           const net::ProxyInfo& proxy_info,
+                           net::HttpRequestHeaders* headers);
+
+  // net::ProxyDelegate implementation:
+  void OnResolveProxy(const GURL& url,
+                      const std::string& method,
+                      const net::ProxyRetryInfoMap& proxy_retry_info,
+                      net::ProxyInfo* result) override;
+  void OnFallback(const net::ProxyServer& bad_proxy, int net_error) override;
+
+ private:
+  // Checks whether |proxy_server| is present in the current proxy config.
+  bool IsInProxyConfig(const net::ProxyServer& proxy_server) const;
+
+  // Whether the current config may proxy |url|.
+  bool MayProxyURL(const GURL& url) const;
+
+  // Whether the |url| with current |proxy_info| is eligible to be proxied.
+  bool EligibleForProxy(const net::ProxyInfo& proxy_info,
+                        const GURL& url,
+                        const std::string& method) const;
+
+  // mojom::CustomProxyConfigClient implementation:
+  void OnCustomProxyConfigUpdated(
+      mojom::CustomProxyConfigPtr proxy_config) override;
+
+  mojom::CustomProxyConfigPtr proxy_config_;
+  mojo::Binding<mojom::CustomProxyConfigClient> binding_;
+
+  DISALLOW_COPY_AND_ASSIGN(NetworkServiceProxyDelegate);
+};
+
+}  // namespace network
+
+#endif  // SERVICES_NETWORK_NETWORK_SERVICE_PROXY_DELEGATE_H_

services/network/public/cpp/network_ipc_param_traits.h

@@ -169,6 +169,8 @@ IPC_STRUCT_TRAITS_BEGIN(network::ResourceRequest)
   IPC_STRUCT_TRAITS_MEMBER(upgrade_if_insecure)
   IPC_STRUCT_TRAITS_MEMBER(is_revalidating)
   IPC_STRUCT_TRAITS_MEMBER(throttling_profile_id)
+  IPC_STRUCT_TRAITS_MEMBER(custom_proxy_pre_cache_headers)
+  IPC_STRUCT_TRAITS_MEMBER(custom_proxy_post_cache_headers)
 IPC_STRUCT_TRAITS_END()
 
 IPC_STRUCT_TRAITS_BEGIN(network::ResourceResponseInfo)

services/network/public/cpp/proxy_config.typemap

@@ -17,7 +17,7 @@ type_mappings = [
   "network.mojom.ProxyBypassRules=net::ProxyBypassRules",
   "network.mojom.ProxyList=net::ProxyList",
   "network.mojom.ProxyRulesType=net::ProxyConfig::ProxyRules::Type",
-  "network.mojom.ProxyRule=net::ProxyConfig::ProxyRule",
+  "network.mojom.ProxyRules=net::ProxyConfig::ProxyRules",
   "network.mojom.ProxyConfig=net::ProxyConfig",
   "network.mojom.ProxyConfigWithAnnotation=net::ProxyConfigWithAnnotation",
 ]

services/network/public/cpp/resource_request.h

@@ -224,6 +224,12 @@ struct COMPONENT_EXPORT(NETWORK_CPP_BASE) ResourceRequest {
 
   // The profile ID of network conditions to throttle the network request.
   base::Optional<base::UnguessableToken> throttling_profile_id;
+
+  // Headers that will be added pre and post cache if the network context uses
+  // the custom proxy for this request. The custom proxy is used for requests
+  // that match the custom proxy config, and would otherwise be made direct.
+  net::HttpRequestHeaders custom_proxy_pre_cache_headers;
+  net::HttpRequestHeaders custom_proxy_post_cache_headers;
 };
 
 }  // namespace network

services/network/public/mojom/network_context.mojom

@@ -30,10 +30,45 @@ import "services/network/public/mojom/udp_socket.mojom";
 import "services/network/public/mojom/url_loader.mojom";
 import "services/network/public/mojom/url_loader_factory.mojom";
 import "services/network/public/mojom/websocket.mojom";
+import "services/network/public/mojom/http_request_headers.mojom";
 import "services/proxy_resolver/public/mojom/proxy_resolver.mojom";
 import "url/mojom/origin.mojom";
 import "url/mojom/url.mojom";
 
+// Config for setting a custom proxy config that will be used if a request
+// matches the proxy rules and would otherwise be direct. This config allows
+// headers to be set on requests to the proxies from the config before and/or
+// after the caching layer. Currently only supports proxying http requests.
+struct CustomProxyConfig {
+  // The custom proxy rules to use. Right now this is limited to proxies for
+  // http requests.
+  ProxyRules rules;
+
+  // The custom proxy can set these headers in this config which will be added
+  // to all requests using the proxy. This allows setting headers that may be
+  // privacy/security sensitive which we don't want to send to the renderer.
+  // Headers that require per-request logic can be added through the
+  // |custom_proxy_pre_cache_headers| and |custom_proxy_post_cache_headers|
+  // fields in ResourceRequest.
+  //
+  // Headers that will be set before the cache for http requests. If the request
+  // does not use a custom proxy, these headers will be removed before sending
+  // to the network. If a request already has one of these headers set, it may
+  // be overwritten if a custom proxy is used, or removed if a custom proxy is
+  // not used.
+  HttpRequestHeaders pre_cache_headers;
+
+  // Headers that will be set after the cache for requests that are issued
+  // through a custom proxy. Headers here will overwrite matching headers on the
+  // request if a custom proxy is used.
+  HttpRequestHeaders post_cache_headers;
+};
+
+// Client to update the custom proxy config.
+interface CustomProxyConfigClient {
+  OnCustomProxyConfigUpdated(CustomProxyConfig proxy_config);
+};
+
 // Parameters for constructing a network context.
 struct NetworkContextParams {
   // Name used by memory tools to identify the context.
@@ -141,6 +176,11 @@ struct NetworkContextParams {
   ProxyConfigWithAnnotation? initial_proxy_config;
   ProxyConfigClient&? proxy_config_client_request;
 
+  // If |custom_proxy_config_client_request| is set, this context will listen
+  // for updates to the custom proxy config, and use it if applicable for
+  // requests which would otherwise be made direct.
+  CustomProxyConfigClient&? custom_proxy_config_client_request;
+
   // If |proxy_config_client_request| is non-null, this is called during
   // periods of network activity, and can be used as a signal for polling-based
   // logic to determine the proxy config.

services/network/url_loader.cc

@@ -321,6 +321,8 @@ URLLoader::URLLoader(
       keepalive_statistics_recorder_(std::move(keepalive_statistics_recorder)),
       network_usage_accumulator_(std::move(network_usage_accumulator)),
       first_auth_attempt_(true),
+      custom_proxy_pre_cache_headers_(request.custom_proxy_pre_cache_headers),
+      custom_proxy_post_cache_headers_(request.custom_proxy_post_cache_headers),
       weak_ptr_factory_(this) {
   DCHECK(delete_callback_);
   if (!base::FeatureList::IsEnabled(features::kNetworkService)) {

services/network/url_loader.h

@@ -103,6 +103,14 @@ class COMPONENT_EXPORT(NETWORK_SERVICE) URLLoader
   uint32_t GetRenderFrameId() const;
   uint32_t GetProcessId() const;
 
+  const net::HttpRequestHeaders& custom_proxy_pre_cache_headers() const {
+    return custom_proxy_pre_cache_headers_;
+  }
+
+  const net::HttpRequestHeaders& custom_proxy_post_cache_headers() const {
+    return custom_proxy_post_cache_headers_;
+  }
+
   // Gets the URLLoader associated with this request.
   static URLLoader* ForRequest(const net::URLRequest& request);
 
@@ -255,6 +263,9 @@ class COMPONENT_EXPORT(NETWORK_SERVICE) URLLoader
 
   std::unique_ptr<ScopedThrottlingToken> throttling_token_;
 
+  net::HttpRequestHeaders custom_proxy_pre_cache_headers_;
+  net::HttpRequestHeaders custom_proxy_post_cache_headers_;
+
   base::WeakPtrFactory<URLLoader> weak_ptr_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(URLLoader);