Send raw headers and flagged cookies to DevTools via NetworkServiceClient

Bug: 856777, 849483, 868407
Change-Id: I694eeab0d4041b4e521b9d2c47ded5a8d7df6ba3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1643748Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: Maks Orlovich <morlovich@chromium.org>
Reviewed-by: Joey Arhar <jarhar@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Aaron Tagliaboschi <aarontag@chromium.org>
Cr-Commit-Position: refs/heads/master@{#680083}

content/browser/devtools/devtools_instrumentation.cc

@@ -433,7 +433,7 @@ void OnRequestWillBeSentExtraInfo(
     int routing_id,
     const std::string& devtools_request_id,
     const net::CookieStatusList& request_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& request_headers) {
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& request_headers) {
   FrameTreeNode* ftn = GetFtnForNetworkRequest(process_id, routing_id);
   if (!ftn)
     return;
@@ -447,7 +447,7 @@ void OnResponseReceivedExtraInfo(
     int routing_id,
     const std::string& devtools_request_id,
     const net::CookieAndLineStatusList& response_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& response_headers,
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& response_headers,
     const base::Optional<std::string>& response_headers_text) {
   FrameTreeNode* ftn = GetFtnForNetworkRequest(process_id, routing_id);
   if (!ftn)

content/browser/devtools/devtools_instrumentation.h

@@ -14,6 +14,7 @@
 #include "base/optional.h"
 #include "content/common/navigation_params.mojom.h"
 #include "content/public/browser/certificate_request_result_type.h"
+#include "services/network/public/mojom/network_service.mojom.h"
 #include "services/network/public/mojom/url_loader_factory.mojom.h"
 #include "third_party/blink/public/mojom/choosers/file_chooser.mojom.h"
 
@@ -118,13 +119,13 @@ void OnRequestWillBeSentExtraInfo(
     int routing_id,
     const std::string& devtools_request_id,
     const net::CookieStatusList& request_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& request_headers);
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& request_headers);
 void OnResponseReceivedExtraInfo(
     int process_id,
     int routing_id,
     const std::string& devtools_request_id,
     const net::CookieAndLineStatusList& response_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& response_headers,
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& response_headers,
     const base::Optional<std::string>& response_headers_text);
 
 std::vector<std::unique_ptr<NavigationThrottle>> CreateNavigationThrottles(

content/browser/devtools/protocol/network_handler.cc

@@ -695,6 +695,19 @@ std::unique_ptr<Object> GetHeaders(const base::StringPairs& pairs) {
   return Object::fromValue(headers_dict.get(), nullptr);
 }
 
+std::unique_ptr<Object> GetRawHeaders(
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& headers) {
+  std::unique_ptr<DictionaryValue> headers_dict(DictionaryValue::create());
+  for (const auto& header : headers) {
+    std::string value;
+    bool merge_with_another = headers_dict->getString(header->key, &value);
+    headers_dict->setString(header->key, merge_with_another
+                                             ? value + '\n' + header->value
+                                             : header->value);
+  }
+  return Object::fromValue(headers_dict.get(), nullptr);
+}
+
 String GetProtocol(const GURL& url, const network::ResourceResponseInfo& info) {
   std::string protocol = info.alpn_negotiated_protocol;
   if (protocol.empty() || protocol == "unknown") {
@@ -2454,26 +2467,26 @@ void NetworkHandler::SetNetworkConditions(
 void NetworkHandler::OnRequestWillBeSentExtraInfo(
     const std::string& devtools_request_id,
     const net::CookieStatusList& request_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& request_headers) {
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& request_headers) {
   if (!enabled_)
     return;
 
   frontend_->RequestWillBeSentExtraInfo(
       devtools_request_id, BuildProtocolBlockedCookies(request_cookie_list),
-      GetHeaders(request_headers));
+      GetRawHeaders(request_headers));
 }
 
 void NetworkHandler::OnResponseReceivedExtraInfo(
     const std::string& devtools_request_id,
     const net::CookieAndLineStatusList& response_cookie_list,
-    const std::vector<std::pair<std::string, std::string>>& response_headers,
+    const std::vector<network::mojom::HttpRawHeaderPairPtr>& response_headers,
     const base::Optional<std::string>& response_headers_text) {
   if (!enabled_)
     return;
 
   frontend_->ResponseReceivedExtraInfo(
       devtools_request_id, BuildProtocolBlockedSetCookies(response_cookie_list),
-      GetHeaders(response_headers),
+      GetRawHeaders(response_headers),
       response_headers_text.has_value() ? response_headers_text.value()
                                         : Maybe<String>());
 }

content/browser/devtools/protocol/network_handler.h

@@ -191,11 +191,11 @@ class NetworkHandler : public DevToolsDomainHandler,
   void OnRequestWillBeSentExtraInfo(
       const std::string& devtools_request_id,
       const net::CookieStatusList& request_cookie_list,
-      const std::vector<std::pair<std::string, std::string>>& request_headers);
+      const std::vector<network::mojom::HttpRawHeaderPairPtr>& request_headers);
   void OnResponseReceivedExtraInfo(
       const std::string& devtools_request_id,
       const net::CookieAndLineStatusList& response_cookie_list,
-      const std::vector<std::pair<std::string, std::string>>& response_headers,
+      const std::vector<network::mojom::HttpRawHeaderPairPtr>& response_headers,
       const base::Optional<std::string>& response_headers_text);
 
   bool enabled() const { return enabled_; }

content/browser/network_service_client.cc

@@ -13,6 +13,7 @@
 #include "base/threading/sequence_bound.h"
 #include "base/unguessable_token.h"
 #include "content/browser/browsing_data/clear_site_data_handler.h"
+#include "content/browser/devtools/devtools_instrumentation.h"
 #include "content/browser/devtools/devtools_url_loader_interceptor.h"
 #include "content/browser/frame_host/frame_tree_node.h"
 #include "content/browser/loader/resource_dispatcher_host_impl.h"
@@ -702,4 +703,26 @@ void NetworkServiceClient::OnGenerateHttpNegotiateAuthToken(
 }
 #endif
 
+void NetworkServiceClient::OnRawRequest(
+    int32_t process_id,
+    int32_t routing_id,
+    const std::string& devtools_request_id,
+    const net::CookieStatusList& cookies_with_status,
+    std::vector<network::mojom::HttpRawHeaderPairPtr> headers) {
+  devtools_instrumentation::OnRequestWillBeSentExtraInfo(
+      process_id, routing_id, devtools_request_id, cookies_with_status,
+      headers);
+}
+
+void NetworkServiceClient::OnRawResponse(
+    int32_t process_id,
+    int32_t routing_id,
+    const std::string& devtools_request_id,
+    const net::CookieAndLineStatusList& cookies_with_status,
+    std::vector<network::mojom::HttpRawHeaderPairPtr> headers,
+    const base::Optional<std::string>& raw_response_headers) {
+  devtools_instrumentation::OnResponseReceivedExtraInfo(
+      process_id, routing_id, devtools_request_id, cookies_with_status, headers,
+      raw_response_headers);
+}
 }  // namespace content

content/browser/network_service_client.h

@@ -86,6 +86,19 @@ class CONTENT_EXPORT NetworkServiceClient
       const std::string& spn,
       OnGenerateHttpNegotiateAuthTokenCallback callback) override;
 #endif
+  void OnRawRequest(
+      int32_t process_id,
+      int32_t routing_id,
+      const std::string& devtools_request_id,
+      const net::CookieStatusList& cookies_with_status,
+      std::vector<network::mojom::HttpRawHeaderPairPtr> headers) override;
+  void OnRawResponse(
+      int32_t process_id,
+      int32_t routing_id,
+      const std::string& devtools_request_id,
+      const net::CookieAndLineStatusList& cookies_with_status,
+      std::vector<network::mojom::HttpRawHeaderPairPtr> headers,
+      const base::Optional<std::string>& raw_response_headers) override;
   // net::CertDatabase::Observer implementation:
   void OnCertDBChanged() override;
 

net/test/embedded_test_server/default_handlers.cc

@@ -206,6 +206,22 @@ std::unique_ptr<HttpResponse> HandleSetCookie(const HttpRequest& request) {
   return http_response;
 }
 
+// /set-invalid-cookie
+// Sets invalid response cookies "\x01" (chosen via fuzzer to not be a parsable
+// cookie).
+std::unique_ptr<HttpResponse> HandleSetInvalidCookie(
+    const HttpRequest& request) {
+  auto http_response = std::make_unique<BasicHttpResponse>();
+  http_response->set_content_type("text/html");
+  std::string content;
+  GURL request_url = request.GetURL();
+
+  http_response->AddCustomHeader("Set-Cookie", "\x01");
+
+  http_response->set_content("TEST");
+  return http_response;
+}
+
 // /set-many-cookies?N
 // Sets N cookies in the response.
 std::unique_ptr<HttpResponse> HandleSetManyCookies(const HttpRequest& request) {
@@ -365,6 +381,9 @@ std::unique_ptr<HttpResponse> HandleAuthBasic(const HttpRequest& request) {
                                    "Basic realm=\"" + realm + "\"");
     if (query.find("set-cookie-if-challenged") != query.end())
       http_response->AddCustomHeader("Set-Cookie", "got_challenged=true");
+    if (query.find("set-secure-cookie-if-challenged") != query.end())
+      http_response->AddCustomHeader("Set-Cookie",
+                                     "got_challenged=true;Secure");
     http_response->set_content(base::StringPrintf(
         "<html><head><title>Denied: %s</title></head>"
         "<body>auth=%s<p>b64str=%s<p>username: %s<p>userpass: %s<p>"
@@ -542,6 +561,27 @@ std::unique_ptr<HttpResponse> HandleServerRedirectWithCookie(
   return http_response;
 }
 
+// /server-redirect-with-secure-cookie?URL
+// Returns a server redirect to URL, and sets the cookie
+// server-redirect=true;Secure.
+std::unique_ptr<HttpResponse> HandleServerRedirectWithSecureCookie(
+    HttpStatusCode redirect_code,
+    const HttpRequest& request) {
+  GURL request_url = request.GetURL();
+  std::string dest = UnescapeBinaryURLComponent(request_url.query_piece());
+  RequestQuery query = ParseQuery(request_url);
+
+  auto http_response = std::make_unique<BasicHttpResponse>();
+  http_response->set_code(redirect_code);
+  http_response->AddCustomHeader("Location", dest);
+  http_response->AddCustomHeader("Set-Cookie", "server-redirect=true;Secure");
+  http_response->set_content_type("text/html");
+  http_response->set_content(base::StringPrintf(
+      "<html><head></head><body>Redirecting to %s</body></html>",
+      dest.c_str()));
+  return http_response;
+}
+
 // /cross-site?URL
 // Returns a cross-site redirect to URL.
 std::unique_ptr<HttpResponse> HandleCrossSiteRedirect(
@@ -752,6 +792,8 @@ void RegisterDefaultHandlers(EmbeddedTestServer* server) {
   server->RegisterDefaultHandler(PREFIXED_HANDLER("/echo-raw", &HandleEchoRaw));
   server->RegisterDefaultHandler(
       PREFIXED_HANDLER("/set-cookie", &HandleSetCookie));
+  server->RegisterDefaultHandler(
+      PREFIXED_HANDLER("/set-invalid-cookie", &HandleSetInvalidCookie));
   server->RegisterDefaultHandler(
       PREFIXED_HANDLER("/set-many-cookies", &HandleSetManyCookies));
   server->RegisterDefaultHandler(
@@ -783,6 +825,9 @@ void RegisterDefaultHandlers(EmbeddedTestServer* server) {
   server->RegisterDefaultHandler(SERVER_REDIRECT_HANDLER(
       "/server-redirect-with-cookie", &HandleServerRedirectWithCookie,
       HTTP_MOVED_PERMANENTLY));
+  server->RegisterDefaultHandler(SERVER_REDIRECT_HANDLER(
+      "/server-redirect-with-secure-cookie",
+      &HandleServerRedirectWithSecureCookie, HTTP_MOVED_PERMANENTLY));
 
   server->RegisterDefaultHandler(
       base::BindRepeating(&HandleCrossSiteRedirect, server));

services/network/public/mojom/BUILD.gn

@@ -85,6 +85,7 @@ mojom("mojom") {
     "digitally_signed.mojom",
     "fetch_api.mojom",
     "host_resolver.mojom",
+    "http_raw_headers.mojom",
     "http_request_headers.mojom",
     "mdns_responder.mojom",
     "net_log.mojom",

services/network/public/mojom/http_raw_headers.mojom

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module network.mojom;
+
+struct HttpRawHeaderPair {
+  string key;
+  string value;
+};
\ No newline at end of file

services/network/public/mojom/network_service.mojom

@@ -13,6 +13,7 @@ import "mojo/public/mojom/base/unguessable_token.mojom";
 import "mojo/public/mojom/base/values.mojom";
 import "services/network/public/mojom/cookie_manager.mojom";
 import "services/network/public/mojom/host_resolver.mojom";
+import "services/network/public/mojom/http_raw_headers.mojom";
 import "services/network/public/mojom/net_log.mojom";
 import "services/network/public/mojom/network_change_manager.mojom";
 import "services/network/public/mojom/network_context.mojom";
@@ -171,6 +172,27 @@ interface NetworkServiceClient {
                                    string auth_negotiate_android_account_type,
                                    string spn) =>
                                    (int32 result, string auth_token);
+
+  // Called to send raw header information and information about excluded
+  // cookies. Only called when |devtool_request_id| is available to the
+  // URLLoader.
+  OnRawRequest(
+    int32 process_id,
+    int32 routing_id,
+    string devtool_request_id,
+    array<CookieWithStatus> cookies_with_status,
+    array<HttpRawHeaderPair> headers);
+
+  // Called to send information about the cookies blocked from storage from a
+  // received response. Only called when |devtool_request_id| is available to
+  // the URLLoader.
+  OnRawResponse(
+    int32 process_id,
+    int32 routing_id,
+    string devtool_request_id,
+    array<CookieAndLineWithStatus> cookies_with_status,
+    array<HttpRawHeaderPair> headers,
+    string? raw_response_headers);
 };
 
 // Values for configuring HTTP authentication that can only be set once.

services/network/test/test_network_service_client.cc

@@ -112,4 +112,19 @@ void TestNetworkServiceClient::OnGenerateHttpNegotiateAuthToken(
 }
 #endif
 
+void TestNetworkServiceClient::OnRawRequest(
+    int32_t process_id,
+    int32_t routing_id,
+    const std::string& devtools_request_id,
+    const net::CookieStatusList& cookies_with_status,
+    std::vector<network::mojom::HttpRawHeaderPairPtr> headers) {}
+
+void TestNetworkServiceClient::OnRawResponse(
+    int32_t process_id,
+    int32_t routing_id,
+    const std::string& devtools_request_id,
+    const net::CookieAndLineStatusList& cookies_with_status,
+    std::vector<network::mojom::HttpRawHeaderPairPtr> headers,
+    const base::Optional<std::string>& raw_response_headers) {}
+
 }  // namespace network

services/network/test/test_network_service_client.h

@@ -76,6 +76,19 @@ class TestNetworkServiceClient : public network::mojom::NetworkServiceClient {
       const std::string& spn,
       OnGenerateHttpNegotiateAuthTokenCallback callback) override;
 #endif
+  void OnRawRequest(
+      int32_t process_id,
+      int32_t routing_id,
+      const std::string& devtools_request_id,
+      const net::CookieStatusList& cookies_with_status,
+      std::vector<network::mojom::HttpRawHeaderPairPtr> headers) override;
+  void OnRawResponse(
+      int32_t process_id,
+      int32_t routing_id,
+      const std::string& devtools_request_id,
+      const net::CookieAndLineStatusList& cookies_with_status,
+      std::vector<network::mojom::HttpRawHeaderPairPtr> headers,
+      const base::Optional<std::string>& raw_response_headers) override;
 
  private:
   bool upload_files_invalid_ = false;

services/network/url_loader.cc

@@ -18,6 +18,7 @@
 #include "base/memory/weak_ptr.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/optional.h"
+#include "base/strings/strcat.h"
 #include "base/task/post_task.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
@@ -1407,6 +1408,23 @@ void URLLoader::SetRawResponseHeaders(
 
 void URLLoader::SetRawRequestHeadersAndNotify(
     net::HttpRawRequestHeaders headers) {
+  if (network_service_client_ && devtools_request_id()) {
+    std::vector<network::mojom::HttpRawHeaderPairPtr> header_array;
+    header_array.reserve(headers.headers().size());
+
+    for (const auto& header : headers.headers()) {
+      network::mojom::HttpRawHeaderPairPtr pair =
+          network::mojom::HttpRawHeaderPair::New();
+      pair->key = header.first;
+      pair->value = header.second;
+      header_array.push_back(std::move(pair));
+    }
+
+    network_service_client_->OnRawRequest(
+        GetProcessId(), GetRenderFrameId(), devtools_request_id().value(),
+        url_request_->maybe_sent_cookies(), std::move(header_array));
+  }
+
   if (network_context_client_) {
     net::CookieStatusList reported_cookies;
     for (const auto& cookie_and_status : url_request_->maybe_sent_cookies()) {
@@ -1424,8 +1442,6 @@ void URLLoader::SetRawRequestHeadersAndNotify(
     }
   }
 
-  // TODO(crbug.com/856777): Add OnRawRequest once implemented
-
   if (want_raw_headers_)
     raw_request_headers_.Assign(std::move(headers));
 }
@@ -1588,6 +1604,39 @@ URLLoader::BlockResponseForCorbResult URLLoader::BlockResponseForCorb() {
 }
 
 void URLLoader::ReportFlaggedResponseCookies() {
+  if (network_service_client_ && devtools_request_id() &&
+      url_request_->response_headers()) {
+    std::vector<network::mojom::HttpRawHeaderPairPtr> header_array;
+
+    size_t iterator = 0;
+    std::string name, value;
+    while (url_request_->response_headers()->EnumerateHeaderLines(
+        &iterator, &name, &value)) {
+      network::mojom::HttpRawHeaderPairPtr pair =
+          network::mojom::HttpRawHeaderPair::New();
+      pair->key = name;
+      pair->value = value;
+      header_array.push_back(std::move(pair));
+    }
+
+    // Only send the "raw" header text when the headers were actually send in
+    // text form (i.e. not QUIC or SPDY)
+    base::Optional<std::string> raw_response_headers;
+
+    const net::HttpResponseInfo& response_info = url_request_->response_info();
+
+    if (!response_info.DidUseQuic() && !response_info.was_fetched_via_spdy) {
+      raw_response_headers =
+          base::make_optional(net::HttpUtil::ConvertHeadersBackToHTTPResponse(
+              url_request_->response_headers()->raw_headers()));
+    }
+
+    network_service_client_->OnRawResponse(
+        GetProcessId(), GetRenderFrameId(), devtools_request_id().value(),
+        url_request_->maybe_stored_cookies(), std::move(header_array),
+        raw_response_headers);
+  }
+
   if (network_context_client_) {
     net::CookieStatusList reported_cookies;
     for (const auto& cookie_line_and_status :
@@ -1606,9 +1655,6 @@ void URLLoader::ReportFlaggedResponseCookies() {
           reported_cookies);
     }
   }
-
-  // TODO(crbug.com/856777): add OnRawResponse once implemented.
-  // (might want to change method name at that point).
 }
 
 }  // namespace network