Add mime sniffing to the network service.

This doesn't handle all the cases (e.g. plugins, downloads) but these don't work
yet with the network service. However it does fix 900 new wpt tests that 
started failing after r487304 since these generated *worker.html files weren't
being served with a mime type which was causing their load to fail.

BUG=746144,729849

Change-Id: I232c9af1dca32f1c641736c8ccd61978096559ef
Reviewed-on: https://chromium-review.googlesource.com/577507Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: John Abd-El-Malek <jam@chromium.org>
Cr-Commit-Position: refs/heads/master@{#488445}

content/browser/loader/mime_sniffing_resource_handler.cc

@@ -22,6 +22,7 @@
 #include "content/browser/loader/resource_dispatcher_host_impl.h"
 #include "content/browser/loader/resource_request_info_impl.h"
 #include "content/browser/loader/stream_resource_handler.h"
+#include "content/common/loader_util.h"
 #include "content/public/browser/content_browser_client.h"
 #include "content/public/browser/download_item.h"
 #include "content/public/browser/download_save_info.h"
@@ -194,7 +195,9 @@ void MimeSniffingResourceHandler::OnResponseStarted(
   // the response, and so must be skipped for 304 responses.
   if (!(response_->head.headers.get() &&
         response_->head.headers->response_code() == 304)) {
-    if (ShouldSniffContent()) {
+    // MIME sniffing should be disabled for a request initiated by fetch().
+    if (request_context_type_ != REQUEST_CONTEXT_TYPE_FETCH &&
+        ShouldSniffContent(request(), response_.get())) {
       controller->Resume();
       return;
     }
@@ -425,34 +428,6 @@ void MimeSniffingResourceHandler::ReplayReadCompleted() {
                                  base::MakeUnique<Controller>(this));
 }
 
-bool MimeSniffingResourceHandler::ShouldSniffContent() {
-  if (request_context_type_ == REQUEST_CONTEXT_TYPE_FETCH) {
-    // MIME sniffing should be disabled for a request initiated by fetch().
-    return false;
-  }
-
-  const std::string& mime_type = response_->head.mime_type;
-
-  std::string content_type_options;
-  request()->GetResponseHeaderByName("x-content-type-options",
-                                     &content_type_options);
-
-  bool sniffing_blocked =
-      base::LowerCaseEqualsASCII(content_type_options, "nosniff");
-  bool we_would_like_to_sniff =
-      net::ShouldSniffMimeType(request()->url(), mime_type);
-
-  if (!sniffing_blocked && we_would_like_to_sniff) {
-    // We're going to look at the data before deciding what the content type
-    // is.  That means we need to delay sending the ResponseStarted message
-    // over the IPC channel.
-    VLOG(1) << "To buffer: " << request()->url().spec();
-    return true;
-  }
-
-  return false;
-}
-
 bool MimeSniffingResourceHandler::MaybeStartInterception() {
   if (!CanBeIntercepted())
     return true;

content/browser/loader/mime_sniffing_resource_handler.h

@@ -130,10 +130,6 @@ class CONTENT_EXPORT MimeSniffingResourceHandler
 
   // --------------------------------------------------------------------------
 
-  // Whether the response body should be sniffed in order to determine the MIME
-  // type of the response.
-  bool ShouldSniffContent();
-
   // Checks whether this request should be intercepted as a stream or a
   // download. If this is the case, sets up the new ResourceHandler that will be
   // used for interception.

content/browser/loader/navigation_url_loader_network_service.cc

@@ -137,9 +137,8 @@ class NavigationURLLoaderNetworkService::URLLoaderRequestController
           webui_factory_ptr_.get(),
           GetContentClient()->browser()->CreateURLLoaderThrottles(
               web_contents_getter_),
-          0 /* routing_id? */, 0 /* request_id? */,
-          mojom::kURLLoadOptionSendSSLInfo, *resource_request_, this,
-          kTrafficAnnotation);
+          0 /* routing_id? */, 0 /* request_id? */, mojom::kURLLoadOptionNone,
+          *resource_request_, this, kTrafficAnnotation);
       return;
     }
 
@@ -220,8 +219,8 @@ class NavigationURLLoaderNetworkService::URLLoaderRequestController
         GetContentClient()->browser()->CreateURLLoaderThrottles(
             web_contents_getter_),
         0 /* routing_id? */, 0 /* request_id? */,
-        mojom::kURLLoadOptionSendSSLInfo, *resource_request_, this,
-        kTrafficAnnotation);
+        mojom::kURLLoadOptionSendSSLInfo | mojom::kURLLoadOptionSniffMimeType,
+        *resource_request_, this, kTrafficAnnotation);
   }
 
   void FollowRedirect() {

content/child/resource_dispatcher.cc

@@ -35,6 +35,7 @@
 #include "content/public/child/fixed_received_data.h"
 #include "content/public/child/request_peer.h"
 #include "content/public/child/resource_dispatcher_delegate.h"
+#include "content/public/common/content_features.h"
 #include "content/public/common/resource_request.h"
 #include "content/public/common/resource_request_completion_status.h"
 #include "content/public/common/resource_response.h"
@@ -691,11 +692,21 @@ int ResourceDispatcher::StartAsync(
         loading_task_runner ? loading_task_runner : thread_task_runner_;
     std::unique_ptr<URLLoaderClientImpl> client(
         new URLLoaderClientImpl(request_id, this, task_runner));
+
+    uint32_t options = mojom::kURLLoadOptionNone;
+    // TODO(jam): use this flag for ResourceDispatcherHost code path once
+    // MojoLoading is the only IPC code path.
+    if (base::FeatureList::IsEnabled(features::kNetworkService) &&
+        request->fetch_request_context_type != REQUEST_CONTEXT_TYPE_FETCH) {
+      // MIME sniffing should be disabled for a request initiated by fetch().
+      options |= mojom::kURLLoadOptionSniffMimeType;
+    }
+
     std::unique_ptr<ThrottlingURLLoader> url_loader =
         ThrottlingURLLoader::CreateLoaderAndStart(
             url_loader_factory, std::move(throttles), routing_id, request_id,
-            mojom::kURLLoadOptionNone, *request, client.get(),
-            traffic_annotation, std::move(task_runner));
+            options, *request, client.get(), traffic_annotation,
+            std::move(task_runner));
     pending_requests_[request_id]->url_loader = std::move(url_loader);
     pending_requests_[request_id]->url_loader_client = std::move(client);
   } else {

content/common/BUILD.gn

@@ -207,6 +207,8 @@ source_set("common") {
     "inter_process_time_ticks_converter.h",
     "layer_tree_settings_factory.cc",
     "layer_tree_settings_factory.h",
+    "loader_util.cc",
+    "loader_util.h",
     "mac/app_nap_activity.h",
     "mac/app_nap_activity.mm",
     "mac/attributed_string_coder.h",

content/common/loader_util.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/loader_util.h"
+
+#include <string>
+
+#include "content/public/common/resource_response.h"
+#include "net/base/mime_sniffer.h"
+#include "net/url_request/url_request.h"
+
+namespace content {
+
+bool ShouldSniffContent(net::URLRequest* url_request,
+                        ResourceResponse* response) {
+  const std::string& mime_type = response->head.mime_type;
+
+  std::string content_type_options;
+  url_request->GetResponseHeaderByName("x-content-type-options",
+                                       &content_type_options);
+
+  bool sniffing_blocked =
+      base::LowerCaseEqualsASCII(content_type_options, "nosniff");
+  bool we_would_like_to_sniff =
+      net::ShouldSniffMimeType(url_request->url(), mime_type);
+
+  if (!sniffing_blocked && we_would_like_to_sniff) {
+    // We're going to look at the data before deciding what the content type
+    // is.  That means we need to delay sending the response started IPC.
+    VLOG(1) << "To buffer: " << url_request->url().spec();
+    return true;
+  }
+
+  return false;
+}
+
+}  // namespace content

content/common/loader_util.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_COMMON_LOADER_UTIL_H_
+#define CONTENT_COMMON_LOADER_UTIL_H_
+
+namespace net {
+class URLRequest;
+}
+
+namespace content {
+struct ResourceResponse;
+
+// Helper utilities shared between network service and ResourceDispatcherHost
+// code paths.
+
+// Whether the response body should be sniffed in order to determine the MIME
+// type of the response.
+bool ShouldSniffContent(net::URLRequest* url_request,
+                        ResourceResponse* response);
+
+}  // namespace content
+
+#endif  // CONTENT_COMMON_LOADER_UTIL_H_

content/network/DEPS

@@ -3,6 +3,7 @@ include_rules = [
   "+components/network_session_configurator",
   "-content",
   "+content/common/content_export.h",
+  "+content/common/loader_util.h",
   "+content/common/net_adapters.h",
   "+content/network",
   "+content/public/common/content_client.h",

content/network/url_loader_impl.cc

@@ -7,6 +7,7 @@
 #include "base/task_scheduler/post_task.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
+#include "content/common/loader_util.h"
 #include "content/common/net_adapters.h"
 #include "content/network/network_context.h"
 #include "content/public/common/referrer.h"
@@ -14,6 +15,7 @@
 #include "content/public/common/url_loader_factory.mojom.h"
 #include "net/base/elements_upload_data_stream.h"
 #include "net/base/load_flags.h"
+#include "net/base/mime_sniffer.h"
 #include "net/base/upload_bytes_element_reader.h"
 #include "net/base/upload_file_element_reader.h"
 #include "net/url_request/url_request_context.h"
@@ -251,32 +253,14 @@ void URLLoaderImpl::OnResponseStarted(net::URLRequest* url_request,
     NotifyCompleted(net_error);
     return;
   }
-  // TODO: Add support for optional MIME sniffing.
 
-  scoped_refptr<ResourceResponse> response = new ResourceResponse();
-  PopulateResourceResponse(url_request_.get(), response.get());
-  response->head.encoded_data_length = url_request_->raw_header_size();
-
-  base::Optional<net::SSLInfo> ssl_info;
-  if (options_ & mojom::kURLLoadOptionSendSSLInfo)
-    ssl_info = url_request_->ssl_info();
-  mojom::DownloadedTempFilePtr downloaded_file_ptr;
-  url_loader_client_->OnReceiveResponse(response->head, ssl_info,
-                                        std::move(downloaded_file_ptr));
-
-  net::IOBufferWithSize* metadata = url_request->response_info().metadata.get();
-  if (metadata) {
-    const uint8_t* data = reinterpret_cast<const uint8_t*>(metadata->data());
-
-    url_loader_client_->OnReceiveCachedMetadata(
-        std::vector<uint8_t>(data, data + metadata->size()));
-  }
+  response_ = new ResourceResponse();
+  PopulateResourceResponse(url_request_.get(), response_.get());
+  response_->head.encoded_data_length = url_request_->raw_header_size();
 
   mojo::DataPipe data_pipe(kDefaultAllocationSize);
-
   response_body_stream_ = std::move(data_pipe.producer_handle);
-  url_loader_client_->OnStartLoadingResponseBody(
-      std::move(data_pipe.consumer_handle));
+  consumer_handle_ = std::move(data_pipe.consumer_handle);
   peer_closed_handle_watcher_.Watch(
       response_body_stream_.get(), MOJO_HANDLE_SIGNAL_PEER_CLOSED,
       base::Bind(&URLLoaderImpl::OnResponseBodyStreamClosed,
@@ -288,6 +272,10 @@ void URLLoaderImpl::OnResponseStarted(net::URLRequest* url_request,
       base::Bind(&URLLoaderImpl::OnResponseBodyStreamReady,
                  base::Unretained(this)));
 
+  if (!(options_ & mojom::kURLLoadOptionSniffMimeType) ||
+      !ShouldSniffContent(url_request_.get(), response_.get()))
+    SendResponseToClient();
+
   // Start reading...
   ReadMore();
 }
@@ -332,6 +320,24 @@ void URLLoaderImpl::ReadMore() {
 
 void URLLoaderImpl::DidRead(uint32_t num_bytes, bool completed_synchronously) {
   DCHECK(url_request_->status().is_success());
+  if (consumer_handle_.is_valid()) {
+    const std::string& type_hint = response_->head.mime_type;
+    std::string new_type;
+    bool made_final_decision =
+        net::SniffMimeType(pending_write_->buffer(), num_bytes,
+                           url_request_->url(), type_hint, &new_type);
+    // SniffMimeType() returns false if there is not enough data to determine
+    // the mime type. However, even if it returns false, it returns a new type
+    // that is probably better than the current one.
+    response_->head.mime_type.assign(new_type);
+
+    if (!made_final_decision) {
+      // TODO: handle case where the initial read didn't have enough bytes.
+      // http://crbug.com/746144
+      LOG(ERROR) << "URLLoaderImpl couldn't make final sniffing decision.";
+    }
+    SendResponseToClient();
+  }
   response_body_stream_ = pending_write_->Complete(num_bytes);
   pending_write_ = nullptr;
   if (completed_synchronously) {
@@ -362,6 +368,9 @@ base::WeakPtr<URLLoaderImpl> URLLoaderImpl::GetWeakPtrForTests() {
 }
 
 void URLLoaderImpl::NotifyCompleted(int error_code) {
+  if (consumer_handle_.is_valid())
+    SendResponseToClient();
+
   ResourceRequestCompletionStatus request_complete_data;
   request_complete_data.error_code = error_code;
   request_complete_data.exists_in_cache =
@@ -399,4 +408,25 @@ void URLLoaderImpl::DeleteIfNeeded() {
     delete this;
 }
 
+void URLLoaderImpl::SendResponseToClient() {
+  base::Optional<net::SSLInfo> ssl_info;
+  if (options_ & mojom::kURLLoadOptionSendSSLInfo)
+    ssl_info = url_request_->ssl_info();
+  mojom::DownloadedTempFilePtr downloaded_file_ptr;
+  url_loader_client_->OnReceiveResponse(response_->head, ssl_info,
+                                        std::move(downloaded_file_ptr));
+
+  net::IOBufferWithSize* metadata =
+      url_request_->response_info().metadata.get();
+  if (metadata) {
+    const uint8_t* data = reinterpret_cast<const uint8_t*>(metadata->data());
+
+    url_loader_client_->OnReceiveCachedMetadata(
+        std::vector<uint8_t>(data, data + metadata->size()));
+  }
+
+  url_loader_client_->OnStartLoadingResponseBody(std::move(consumer_handle_));
+  response_ = nullptr;
+}
+
 }  // namespace content

content/network/url_loader_impl.h

@@ -13,6 +13,7 @@
 #include "content/common/content_export.h"
 #include "content/public/common/url_loader.mojom.h"
 #include "mojo/public/cpp/bindings/associated_binding.h"
+#include "mojo/public/cpp/system/data_pipe.h"
 #include "mojo/public/cpp/system/simple_watcher.h"
 #include "net/traffic_annotation/network_traffic_annotation.h"
 #include "net/url_request/url_request.h"
@@ -21,6 +22,7 @@ namespace content {
 
 class NetworkContext;
 class NetToMojoPendingBuffer;
+struct ResourceResponse;
 
 class CONTENT_EXPORT URLLoaderImpl : public mojom::URLLoader,
                                      public net::URLRequest::Delegate {
@@ -59,6 +61,7 @@ class CONTENT_EXPORT URLLoaderImpl : public mojom::URLLoader,
   void OnResponseBodyStreamClosed(MojoResult result);
   void OnResponseBodyStreamReady(MojoResult result);
   void DeleteIfNeeded();
+  void SendResponseToClient();
 
   NetworkContext* context_;
   int32_t options_;
@@ -72,6 +75,11 @@ class CONTENT_EXPORT URLLoaderImpl : public mojom::URLLoader,
   mojo::SimpleWatcher writable_handle_watcher_;
   mojo::SimpleWatcher peer_closed_handle_watcher_;
 
+  // Used when deferring sending the data to the client until mime sniffing is
+  // finished.
+  scoped_refptr<ResourceResponse> response_;
+  mojo::ScopedDataPipeConsumerHandle consumer_handle_;
+
   base::WeakPtrFactory<URLLoaderImpl> weak_ptr_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(URLLoaderImpl);

content/network/url_loader_unittest.cc

@@ -113,6 +113,15 @@ class URLLoaderImplTest : public testing::Test {
     CHECK_EQ(data, file_contents);
   }
 
+  std::string LoadAndReturnMimeType(bool sniff, const std::string& path) {
+    TestURLLoaderClient client;
+    GURL url = test_server()->GetURL(path);
+    int32_t options =
+        sniff ? mojom::kURLLoadOptionSniffMimeType : mojom::kURLLoadOptionNone;
+    Load(url, &client, options);
+    return client.response_head().mime_type;
+  }
+
   net::EmbeddedTestServer* test_server() { return &test_server_; }
   NetworkContext* context() { return context_.get(); }
   void DestroyContext() { context_.reset(); }
@@ -191,4 +200,39 @@ TEST_F(URLLoaderImplTest, DestroyContextWithLiveRequest) {
   EXPECT_EQ(0u, client.download_data_length());
 }
 
+TEST_F(URLLoaderImplTest, DoNotSniffUnlessSpecified) {
+  std::string mime_type =
+      LoadAndReturnMimeType(false, "/content-sniffer-test0.html");
+  ASSERT_TRUE(mime_type.empty());
+}
+
+TEST_F(URLLoaderImplTest, SniffMimeType) {
+  std::string mime_type =
+      LoadAndReturnMimeType(true, "/content-sniffer-test0.html");
+  ASSERT_EQ(std::string("text/html"), mime_type);
+}
+
+TEST_F(URLLoaderImplTest, RespectNoSniff) {
+  std::string mime_type = LoadAndReturnMimeType(true, "/nosniff-test.html");
+  ASSERT_TRUE(mime_type.empty());
+}
+
+TEST_F(URLLoaderImplTest, DoNotSniffHTMLFromTextPlain) {
+  std::string mime_type =
+      LoadAndReturnMimeType(true, "/content-sniffer-test1.html");
+  ASSERT_EQ(std::string("text/plain"), mime_type);
+}
+
+TEST_F(URLLoaderImplTest, DoNotSniffHTMLFromImageGIF) {
+  std::string mime_type =
+      LoadAndReturnMimeType(true, "/content-sniffer-test2.html");
+  ASSERT_EQ(std::string("image/gif"), mime_type);
+}
+
+TEST_F(URLLoaderImplTest, CantSniffEmptyHtml) {
+  std::string mime_type =
+      LoadAndReturnMimeType(true, "/content-sniffer-test4.html");
+  ASSERT_TRUE(mime_type.empty());
+}
+
 }  // namespace content

content/public/common/url_loader_factory.mojom

@@ -13,6 +13,8 @@ struct URLSyncLoadResult;
 const uint32 kURLLoadOptionNone = 0;
 // Sends the net::SSLInfo struct in OnReceiveResponse.
 const uint32 kURLLoadOptionSendSSLInfo = 1;
+// Enables mime sniffing. NOTE: this is only used with the network service.
+const uint32 kURLLoadOptionSniffMimeType = 2;
 
 interface URLLoaderFactory {
   // Creats a URLLoader and starts loading with the given |request|. |client|'s

content/test/data/content-sniffer-test4.html.mock-http-headers

+HTTP/1.1 200 OK