Declarative Net Request: Collapse blocked <img> elements.

With Declarative Net Request, there is no easy way for extensions to collapse
the DOM elements corresponding to the resources they blocked. This CL implements
the ability for the blocked <img> elements to be automatically collapsed in the
DOM. In subsequent CLs, similar function will be implemented for other DOM
elements.

To do this, following changes are made:
 - A new field called kCollapsedByClient is added to
   blink::ResourceRequestBlockedReason. Whenever an extension ruleset wants to
   collapse the DOM element corresponding to a network request, the
   extended_error_code is set to kCollapsedByClient, which is then used as a
   signal by the renderer to perform the collapsing.
 - Changes to blink::ResourceRequestBlockedReason also necessiate changes to the
   Devtools protocol. As a good side-effect, resources collapsed by extensions
   would be marked as "collapsed-by-client" by Devtools.
 - Changes are made to allow content embedders to set a
   blink::ResourceRequestBlockedReason for a request.

BUG=848842

Cq-Include-Trybots: luci.chromium.try:linux_mojo
Change-Id: I0bfd1a60bee79ef651741b7655fade6be4a7d419
Reviewed-on: https://chromium-review.googlesource.com/1088189
Commit-Queue: Matt Menke <mmenke@chromium.org>
Reviewed-by: Istiaque Ahmed <lazyboy@chromium.org>
Reviewed-by: Nasko Oskov <nasko@chromium.org>
Reviewed-by: Nate Chapin <japhet@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: Dmitry Gozman <dgozman@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#568332}

chrome/browser/extensions/api/declarative_net_request/declarative_net_request_browsertest.cc

@@ -1549,6 +1549,37 @@ IN_PROC_BROWSER_TEST_P(DeclarativeNetRequestBrowserTest,
       embedded_test_server()->GetURL("/pages_with_script/script.js")));
 }
 
+// Ensures that any <img> elements blocked by the API are collapsed.
+IN_PROC_BROWSER_TEST_P(DeclarativeNetRequestBrowserTest, ImageCollapsed) {
+  // Loads a page with an image and returns whether the image was collapsed.
+  auto is_image_collapsed = [this]() {
+    ui_test_utils::NavigateToURL(
+        browser(), embedded_test_server()->GetURL("google.com", "/image.html"));
+    EXPECT_EQ(content::PAGE_TYPE_NORMAL, GetPageType());
+    bool is_image_collapsed = false;
+    const std::string script =
+        "domAutomationController.send(!!window.imageCollapsed);";
+    EXPECT_TRUE(content::ExecuteScriptAndExtractBool(web_contents(), script,
+                                                     &is_image_collapsed));
+    return is_image_collapsed;
+  };
+
+  // Initially the image shouldn't be collapsed.
+  EXPECT_FALSE(is_image_collapsed());
+
+  // Load an extension which blocks all images.
+  std::vector<TestRule> rules;
+  TestRule rule = CreateGenericRule();
+  rule.condition->url_filter = std::string("*");
+  rule.condition->resource_types = std::vector<std::string>({"image"});
+  rules.push_back(rule);
+  ASSERT_NO_FATAL_FAILURE(LoadExtensionWithRules(rules));
+
+  // Now the image request should be blocked and the corresponding DOM element
+  // should be collapsed.
+  EXPECT_TRUE(is_image_collapsed());
+}
+
 // Test fixture to verify that host permissions for the request url and the
 // request initiator are properly checked. Loads an example.com url with four
 // sub-frames named frame_[1..4] from hosts frame_[1..4].com. The initiator for

chrome/browser/net/chrome_extensions_network_delegate.cc

@@ -195,9 +195,18 @@ int ChromeExtensionsNetworkDelegateImpl::OnBeforeURLRequest(
       &active_requests_[request];
   *web_request_info = std::make_unique<extensions::WebRequestInfo>(request);
 
-  return ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
+  bool should_collapse_initiator = false;
+  int result = ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
       profile_, extension_info_map_.get(), web_request_info->get(),
-      std::move(callback), new_url);
+      std::move(callback), new_url, &should_collapse_initiator);
+  if (should_collapse_initiator) {
+    auto* info = ResourceRequestInfo::ForRequest(request);
+    DCHECK(info);
+    info->SetResourceRequestBlockedReason(
+        blink::ResourceRequestBlockedReason::kCollapsedByClient);
+  }
+
+  return result;
 }
 
 int ChromeExtensionsNetworkDelegateImpl::OnBeforeStartTransaction(

chrome/test/data/extensions/declarative_net_request/image.html

+<!DOCTYPE html>
+<html>
+<head>
+<script type="text/javascript">
+imageCollapsed = false;
+window.addEventListener('load', () => {
+  var image = document.getElementById('image');
+  var display = window.getComputedStyle(image).display;
+  imageCollapsed = display == 'none';
+});
+</script>
+</head>
+
+<body>
+  <img src="subresources/image.png" id="image" >
+</body>
+
+</html>

content/browser/devtools/devtools_url_interceptor_request_job.cc

@@ -1112,7 +1112,7 @@ void DevToolsURLInterceptorRequestJob::ProcessInterceptionResponse(
       // far, to minimize risk of breaking other usages.
       ResourceRequestInfoImpl* resource_request_info =
           ResourceRequestInfoImpl::ForRequest(request());
-      resource_request_info->set_resource_request_blocked_reason(
+      resource_request_info->SetResourceRequestBlockedReason(
           blink::ResourceRequestBlockedReason::kInspector);
     }
     NotifyStartError(net::URLRequestStatus(net::URLRequestStatus::FAILED,

content/browser/devtools/protocol/network_handler.cc

@@ -1526,6 +1526,8 @@ String blockedReason(blink::ResourceRequestBlockedReason reason) {
       return protocol::Network::BlockedReasonEnum::ContentType;
     case blink::ResourceRequestBlockedReason::kOther:
       return protocol::Network::BlockedReasonEnum::Other;
+    case blink::ResourceRequestBlockedReason::kCollapsedByClient:
+      return protocol::Network::BlockedReasonEnum::CollapsedByClient;
   }
   NOTREACHED();
   return protocol::Network::BlockedReasonEnum::Other;

content/browser/loader/resource_request_info_impl.cc

@@ -35,6 +35,11 @@ const void* const kResourceRequestInfoImplKey = &kResourceRequestInfoImplKey;
 // ----------------------------------------------------------------------------
 // ResourceRequestInfo
 
+// static
+ResourceRequestInfo* ResourceRequestInfo::ForRequest(net::URLRequest* request) {
+  return ResourceRequestInfoImpl::ForRequest(request);
+}
+
 // static
 const ResourceRequestInfo* ResourceRequestInfo::ForRequest(
     const net::URLRequest* request) {
@@ -317,6 +322,11 @@ ResourceRequestInfo::DevToolsStatus ResourceRequestInfoImpl::GetDevToolsStatus()
   return devtools_status_;
 }
 
+void ResourceRequestInfoImpl::SetResourceRequestBlockedReason(
+    blink::ResourceRequestBlockedReason reason) {
+  resource_request_blocked_reason_ = reason;
+}
+
 base::Optional<blink::ResourceRequestBlockedReason>
 ResourceRequestInfoImpl::GetResourceRequestBlockedReason() const {
   return resource_request_blocked_reason_;

content/browser/loader/resource_request_info_impl.h

@@ -98,6 +98,8 @@ class ResourceRequestInfoImpl : public ResourceRequestInfo,
   PreviewsState GetPreviewsState() const override;
   NavigationUIData* GetNavigationUIData() const override;
   DevToolsStatus GetDevToolsStatus() const override;
+  void SetResourceRequestBlockedReason(
+      blink::ResourceRequestBlockedReason reason) override;
   base::Optional<blink::ResourceRequestBlockedReason>
   GetResourceRequestBlockedReason() const override;
   base::StringPiece GetCustomCancelReason() const override;
@@ -188,11 +190,6 @@ class ResourceRequestInfoImpl : public ResourceRequestInfo,
     devtools_status_ = devtools_status;
   }
 
-  void set_resource_request_blocked_reason(
-      base::Optional<blink::ResourceRequestBlockedReason> reason) {
-    resource_request_blocked_reason_ = reason;
-  }
-
   void SetBlobHandles(BlobHandles blob_handles);
 
   bool should_report_corb_blocking() const {

content/public/browser/resource_request_info.h

@@ -29,6 +29,8 @@ class WebContents;
 class ResourceRequestInfo {
  public:
   // Returns the ResourceRequestInfo associated with the given URLRequest.
+  CONTENT_EXPORT static ResourceRequestInfo* ForRequest(
+      net::URLRequest* request);
   CONTENT_EXPORT static const ResourceRequestInfo* ForRequest(
       const net::URLRequest* request);
 
@@ -187,8 +189,13 @@ class ResourceRequestInfo {
   // If and why this request was canceled by DevTools. TODO(johannes): Remove.
   virtual DevToolsStatus GetDevToolsStatus() const = 0;
 
-  // For net::ERR_BLOCKED_BY_CLIENT and net::ERR_BLOCKED_BY_RESPONSE
-  // errors, this will return the reason, otherwise base::nullopt.
+  // Used to annotate requests blocked using net::ERR_BLOCKED_BY_CLIENT and
+  // net::ERR_BLOCKED_BY_RESPONSE errors, with a ResourceRequestBlockedReason.
+  virtual void SetResourceRequestBlockedReason(
+      blink::ResourceRequestBlockedReason) = 0;
+
+  // Returns the ResourceRequestBlockedReason for this request, else
+  // base::nullopt.
   virtual base::Optional<blink::ResourceRequestBlockedReason>
   GetResourceRequestBlockedReason() const = 0;
 

extensions/browser/api/declarative_net_request/ruleset_manager.cc

@@ -230,6 +230,12 @@ bool IsRequestPageAllowed(const WebRequestInfo& request,
          allowed_pages.MatchesURL(*request.frame_data->pending_main_frame_url);
 }
 
+bool ShouldCollapseResourceType(flat_rule::ElementType type) {
+  // TODO(crbug.com/848842): Add support for other element types like
+  // SUBDOCUMENT, OBJECT.
+  return type == flat_rule::ElementType_IMAGE;
+}
+
 }  // namespace
 
 RulesetManager::RulesetManager(const InfoMap* info_map) : info_map_(info_map) {
@@ -344,7 +350,8 @@ RulesetManager::Action RulesetManager::EvaluateRequest(
 
       if (ruleset_data->matcher->ShouldBlockRequest(
               url, first_party_origin, element_type, is_third_party)) {
-        return Action::BLOCK;
+        return ShouldCollapseResourceType(element_type) ? Action::COLLAPSE
+                                                        : Action::BLOCK;
       }
     }
   }

extensions/browser/api/declarative_net_request/ruleset_manager.h

@@ -31,7 +31,11 @@ class RulesetManager {
  public:
   enum class Action {
     NONE,
+    // Block the network request.
     BLOCK,
+    // Block the network request and collapse the corresponding DOM element.
+    COLLAPSE,
+    // Redirect the network request.
     REDIRECT,
   };
 

extensions/browser/api/web_request/web_request_api.cc

@@ -797,7 +797,10 @@ int ExtensionWebRequestEventRouter::OnBeforeRequest(
     const InfoMap* extension_info_map,
     WebRequestInfo* request,
     net::CompletionOnceCallback callback,
-    GURL* new_url) {
+    GURL* new_url,
+    bool* should_collapse_initiator) {
+  DCHECK(should_collapse_initiator);
+
   if (ShouldHideEvent(browser_context, extension_info_map, *request))
     return net::OK;
 
@@ -820,10 +823,17 @@ int ExtensionWebRequestEventRouter::OnBeforeRequest(
 
     Action action = extension_info_map->GetRulesetManager()->EvaluateRequest(
         *request, is_incognito_context, new_url);
-    if (action == Action::BLOCK)
-      return net::ERR_BLOCKED_BY_CLIENT;
-    if (action == Action::REDIRECT)
-      return net::OK;
+    switch (action) {
+      case Action::NONE:
+        break;
+      case Action::BLOCK:
+        return net::ERR_BLOCKED_BY_CLIENT;
+      case Action::COLLAPSE:
+        *should_collapse_initiator = true;
+        return net::ERR_BLOCKED_BY_CLIENT;
+      case Action::REDIRECT:
+        return net::OK;
+    }
   }
 
   // Whether to initialized |blocked_requests_|.

extensions/browser/api/web_request/web_request_api.h

@@ -327,12 +327,16 @@ class ExtensionWebRequestEventRouter {
 
   // Dispatches the OnBeforeRequest event to any extensions whose filters match
   // the given request. Returns net::ERR_IO_PENDING if an extension is
-  // intercepting the request, OK otherwise.
+  // intercepting the request and OK if the request should proceed normally.
+  // net::ERR_BLOCKED_BY_CLIENT is returned if the request should be blocked. In
+  // this case, |should_collapse_initiator| might be set to true indicating
+  // whether the DOM element which initiated the request should be blocked.
   int OnBeforeRequest(void* browser_context,
                       const extensions::InfoMap* extension_info_map,
                       WebRequestInfo* request,
                       net::CompletionOnceCallback callback,
-                      GURL* new_url);
+                      GURL* new_url,
+                      bool* should_collapse_initiator);
 
   // Dispatches the onBeforeSendHeaders event. This is fired for HTTP(s)
   // requests only, and allows modification of the outgoing request headers.

extensions/browser/api/web_request/web_request_proxying_url_loader_factory.cc

@@ -11,6 +11,7 @@
 #include "content/public/browser/global_request_id.h"
 #include "extensions/browser/extension_navigation_ui_data.h"
 #include "net/http/http_util.h"
+#include "third_party/blink/public/platform/resource_request_blocked_reason.h"
 
 namespace extensions {
 
@@ -75,13 +76,19 @@ void WebRequestProxyingURLLoaderFactory::InProgressRequest::Restart() {
       base::BindRepeating(&InProgressRequest::ContinueToBeforeSendHeaders,
                           weak_factory_.GetWeakPtr());
   redirect_url_ = GURL();
+  bool should_collapse_initiator = false;
   int result = ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
       factory_->browser_context_, factory_->info_map_, &info_.value(),
-      continuation, &redirect_url_);
+      continuation, &redirect_url_, &should_collapse_initiator);
   if (result == net::ERR_BLOCKED_BY_CLIENT) {
     // The request was cancelled synchronously. Dispatch an error notification
     // and terminate the request.
-    OnRequestError(network::URLLoaderCompletionStatus(result));
+    network::URLLoaderCompletionStatus status(result);
+    if (should_collapse_initiator) {
+      status.extended_error_code = static_cast<int>(
+          blink::ResourceRequestBlockedReason::kCollapsedByClient);
+    }
+    OnRequestError(status);
     return;
   }
 

extensions/browser/api/web_request/web_request_proxying_websocket.cc

@@ -79,9 +79,14 @@ void WebRequestProxyingWebSocket::AddChannelRequest(
 
   // TODO(yhirano): Consider having throttling here (probably with aligned with
   // WebRequestProxyingURLLoaderFactory).
+  bool should_collapse_initiator = false;
   int result = ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
-      browser_context_, info_map_, &info_.value(), continuation,
-      &redirect_url_);
+      browser_context_, info_map_, &info_.value(), continuation, &redirect_url_,
+      &should_collapse_initiator);
+
+  // It doesn't make sense to collapse WebSocket requests since they won't be
+  // associated with a DOM element.
+  DCHECK(!should_collapse_initiator);
 
   if (result == net::ERR_BLOCKED_BY_CLIENT) {
     OnError(result);

extensions/shell/browser/shell_network_delegate.cc

@@ -56,9 +56,18 @@ int ShellNetworkDelegate::OnBeforeURLRequest(
     net::CompletionOnceCallback callback,
     GURL* new_url) {
   WebRequestInfo* web_request_info = GetWebRequestInfo(request);
-  return ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
+  bool should_collapse_initiator = false;
+  int result = ExtensionWebRequestEventRouter::GetInstance()->OnBeforeRequest(
       browser_context_, extension_info_map_.get(), web_request_info,
-      std::move(callback), new_url);
+      std::move(callback), new_url, &should_collapse_initiator);
+  if (should_collapse_initiator) {
+    auto* info = content::ResourceRequestInfo::ForRequest(request);
+    DCHECK(info);
+    info->SetResourceRequestBlockedReason(
+        blink::ResourceRequestBlockedReason::kCollapsedByClient);
+  }
+
+  return result;
 }
 
 int ShellNetworkDelegate::OnBeforeStartTransaction(

third_party/blink/public/platform/resource_request_blocked_reason.h

@@ -16,6 +16,7 @@ enum class ResourceRequestBlockedReason {
   kInspector,
   kSubresourceFilter,
   kContentType,
+  kCollapsedByClient,
 };
 }  // namespace blink
 

third_party/blink/renderer/core/inspector/browser_protocol.pdl

@@ -3477,6 +3477,7 @@ domain Network
       inspector
       subresource-filter
       content-type
+      collapsed-by-client
 
   # HTTP response data.
   type Response extends object

third_party/blink/renderer/core/inspector/inspector_network_agent.cc

@@ -362,6 +362,8 @@ String BuildBlockedReason(ResourceRequestBlockedReason reason) {
       return protocol::Network::BlockedReasonEnum::ContentType;
     case ResourceRequestBlockedReason::kOther:
       return protocol::Network::BlockedReasonEnum::Other;
+    case ResourceRequestBlockedReason::kCollapsedByClient:
+      return protocol::Network::BlockedReasonEnum::CollapsedByClient;
   }
   NOTREACHED();
   return protocol::Network::BlockedReasonEnum::Other;

third_party/blink/renderer/devtools/front_end/network/NetworkDataGridNode.js

@@ -864,6 +864,9 @@ Network.NetworkRequestNode = class extends Network.NetworkNode {
         case Protocol.Network.BlockedReason.ContentType:
           reason = Common.UIString('content-type');
           break;
+        case Protocol.Network.BlockedReason.CollapsedByClient:
+          reason = Common.UIString('extension');
+          break;
       }
       this._setTextAndTitle(cell, Common.UIString('(blocked:%s)', reason));
     } else if (this._request.finished) {

third_party/blink/renderer/platform/loader/fetch/resource_error.cc

@@ -55,7 +55,7 @@ ResourceError ResourceError::CancelledDueToAccessCheckError(
     ResourceRequestBlockedReason blocked_reason) {
   ResourceError error = CancelledError(url);
   error.is_access_check_ = true;
-  error.should_collapse_initiator_ =
+  error.blocked_by_subresource_filter_ =
       blocked_reason == ResourceRequestBlockedReason::kSubresourceFilter;
   return error;
 }
@@ -171,6 +171,12 @@ bool ResourceError::WasBlockedByResponse() const {
   return error_code_ == net::ERR_BLOCKED_BY_RESPONSE;
 }
 
+bool ResourceError::ShouldCollapseInitiator() const {
+  return blocked_by_subresource_filter_ ||
+         GetResourceRequestBlockedReason() ==
+             ResourceRequestBlockedReason::kCollapsedByClient;
+}
+
 base::Optional<ResourceRequestBlockedReason>
 ResourceError::GetResourceRequestBlockedReason() const {
   if (error_code_ != net::ERR_BLOCKED_BY_CLIENT &&
@@ -207,6 +213,9 @@ String DescriptionForBlockedByClientOrResponse(int error, int extended_error) {
     case ResourceRequestBlockedReason::kContentType:
       detail = "ContentType";
       break;
+    case ResourceRequestBlockedReason::kCollapsedByClient:
+      detail = "Collapsed";
+      break;
   }
   return WebString::FromASCII(net::ErrorToString(error) + "." + detail);
 }

third_party/blink/renderer/platform/loader/fetch/resource_error.h

@@ -80,7 +80,7 @@ class PLATFORM_EXPORT ResourceError final {
   bool IsTimeout() const;
   bool IsCacheMiss() const;
   bool WasBlockedByResponse() const;
-  bool ShouldCollapseInitiator() const { return should_collapse_initiator_; }
+  bool ShouldCollapseInitiator() const;
   base::Optional<ResourceRequestBlockedReason> GetResourceRequestBlockedReason()
       const;
 
@@ -104,7 +104,7 @@ class PLATFORM_EXPORT ResourceError final {
   String localized_description_;
   bool is_access_check_ = false;
   bool has_copy_in_cache_ = false;
-  bool should_collapse_initiator_ = false;
+  bool blocked_by_subresource_filter_ = false;
   base::Optional<network::CORSErrorStatus> cors_error_status_;
 };