[devtools] Exponential back-off/retry for 'loadNetworkResource'

This CL adds a simple exponential back-off & retry strategy to the
inspector's loadNetworkResource method. A retry is only attempted
if the load failed with net::ERR_INSUFFICIENT_RESOURCES.

Retrying ensures that the kMaxOutstandingRequestsPerProcess limit in
the network component does not cause load failures to the front-end.
Before this CL, bursty loads from the DevTools front-end, such as they
occur during fetching of source maps for large webpages, could result
in load failures of source maps, when in reality the problem was that
too many source maps where loaded at once.

The initial back-off delay was chosen to be 250ms, as the intention is
to give the network stack some time to finish outstanding requests,
which are expected to take some time if there are so many of them that
the request limit triggers. The back-off delay is 250ms*1.3^n and
retrying is aborted when the delay exceeds 10s (14 tries in total).

Bug: chromium:1013124
Change-Id: I2759cfa035fc3952d960d43958536d5e9df613b1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1852212
Commit-Queue: Sigurd Schneider <sigurds@chromium.org>
Reviewed-by: Benedikt Meurer <bmeurer@chromium.org>
Reviewed-by: Yang Guo <yangguo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#705537}

chrome/browser/devtools/devtools_ui_bindings.cc

@@ -381,26 +381,84 @@ GURL SanitizeFrontendURL(const GURL& url,
   return result;
 }
 
+constexpr base::TimeDelta kInitialBackoffDelay =
+    base::TimeDelta::FromMilliseconds(250);
+constexpr base::TimeDelta kMaxBackoffDelay = base::TimeDelta::FromSeconds(10);
+
 }  // namespace
 
 class DevToolsUIBindings::NetworkResourceLoader
     : public network::SimpleURLLoaderStreamConsumer {
  public:
-  NetworkResourceLoader(int stream_id,
+  class URLLoaderFactoryHolder {
+   public:
+    network::mojom::URLLoaderFactory* get() {
+      return ptr_.get() ? ptr_.get() : refptr_.get();
+    }
+    void operator=(std::unique_ptr<network::mojom::URLLoaderFactory>&& ptr) {
+      ptr_ = std::move(ptr);
+    }
+    void operator=(scoped_refptr<network::SharedURLLoaderFactory>&& refptr) {
+      refptr_ = std::move(refptr);
+    }
+
+   private:
+    std::unique_ptr<network::mojom::URLLoaderFactory> ptr_;
+    scoped_refptr<network::SharedURLLoaderFactory> refptr_;
+  };
+
+  static void Create(int stream_id,
+                     DevToolsUIBindings* bindings,
+                     const network::ResourceRequest& resource_request,
+                     const net::NetworkTrafficAnnotationTag& traffic_annotation,
+                     URLLoaderFactoryHolder url_loader_factory,
+                     const DevToolsUIBindings::DispatchCallback& callback,
+                     base::TimeDelta retry_delay = base::TimeDelta()) {
+    auto resource_loader =
+        std::make_unique<DevToolsUIBindings::NetworkResourceLoader>(
+            stream_id, bindings, resource_request, traffic_annotation,
+            std::move(url_loader_factory), callback, retry_delay);
+    bindings->loaders_.insert(std::move(resource_loader));
+  }
+
+  NetworkResourceLoader(
+      int stream_id,
       DevToolsUIBindings* bindings,
-                        std::unique_ptr<network::SimpleURLLoader> loader,
-                        network::mojom::URLLoaderFactory* url_loader_factory,
-                        const DispatchCallback& callback)
+      const network::ResourceRequest& resource_request,
+      const net::NetworkTrafficAnnotationTag& traffic_annotation,
+      URLLoaderFactoryHolder url_loader_factory,
+      const DispatchCallback& callback,
+      base::TimeDelta delay)
       : stream_id_(stream_id),
         bindings_(bindings),
-        loader_(std::move(loader)),
-        callback_(callback) {
+        resource_request_(resource_request),
+        traffic_annotation_(traffic_annotation),
+        loader_(network::SimpleURLLoader::Create(
+            std::make_unique<network::ResourceRequest>(resource_request),
+            traffic_annotation)),
+        url_loader_factory_(std::move(url_loader_factory)),
+        callback_(callback),
+        retry_delay_(delay) {
     loader_->SetOnResponseStartedCallback(base::BindOnce(
         &NetworkResourceLoader::OnResponseStarted, base::Unretained(this)));
-    loader_->DownloadAsStream(url_loader_factory, this);
+    timer_.Start(FROM_HERE, delay,
+                 base::BindRepeating(&NetworkResourceLoader::DownloadAsStream,
+                                     base::Unretained(this)));
   }
 
  private:
+  void DownloadAsStream() {
+    loader_->DownloadAsStream(url_loader_factory_.get(), this);
+  }
+
+  base::TimeDelta GetNextExponentialBackoffDelay(const base::TimeDelta& delta) {
+    if (delta.is_zero()) {
+      return kInitialBackoffDelay;
+    } else {
+      return delta * 1.3;
+    }
+  }
+
   void OnResponseStarted(const GURL& final_url,
                          const network::mojom::URLResponseHead& response_head) {
     response_headers_ = response_head.headers;
@@ -427,9 +485,21 @@ class DevToolsUIBindings::NetworkResourceLoader
   }
 
   void OnComplete(bool success) override {
+    if (!success && loader_->NetError() == net::ERR_INSUFFICIENT_RESOURCES &&
+        retry_delay_ < kMaxBackoffDelay) {
+      const base::TimeDelta delay =
+          GetNextExponentialBackoffDelay(retry_delay_);
+      LOG(WARNING) << "DevToolsUIBindings::NetworkResourceLoader id = "
+                   << stream_id_
+                   << " failed with insufficient resources, retrying in "
+                   << delay << "." << std::endl;
+      NetworkResourceLoader::Create(
+          stream_id_, bindings_, resource_request_, traffic_annotation_,
+          std::move(url_loader_factory_), callback_, delay);
+    } else {
       auto response = BuildObjectForResponse(response_headers_.get(), success);
       callback_.Run(response.get());
-
+    }
     bindings_->loaders_.erase(bindings_->loaders_.find(this));
   }
 
@@ -437,9 +507,14 @@ class DevToolsUIBindings::NetworkResourceLoader
 
   const int stream_id_;
   DevToolsUIBindings* const bindings_;
+  const network::ResourceRequest resource_request_;
+  const net::NetworkTrafficAnnotationTag traffic_annotation_;
   std::unique_ptr<network::SimpleURLLoader> loader_;
+  URLLoaderFactoryHolder url_loader_factory_;
   DispatchCallback callback_;
   scoped_refptr<net::HttpResponseHeaders> response_headers_;
+  base::OneShotTimer timer_;
+  base::TimeDelta retry_delay_;
 
   DISALLOW_COPY_AND_ASSIGN(NetworkResourceLoader);
 };
@@ -747,22 +822,18 @@ void DevToolsUIBindings::LoadNetworkResource(const DispatchCallback& callback,
           }
         })");
 
-  auto resource_request = std::make_unique<network::ResourceRequest>();
-  resource_request->url = gurl;
+  network::ResourceRequest resource_request;
+  resource_request.url = gurl;
   // TODO(caseq): this preserves behavior of URLFetcher-based implementation.
   // We really need to pass proper first party origin from the front-end.
-  resource_request->site_for_cookies = gurl;
-  resource_request->headers.AddHeadersFromString(headers);
+  resource_request.site_for_cookies = gurl;
+  resource_request.headers.AddHeadersFromString(headers);
 
-  std::unique_ptr<network::mojom::URLLoaderFactory> file_url_loader_factory;
-  scoped_refptr<network::SharedURLLoaderFactory> network_url_loader_factory;
-  std::unique_ptr<network::mojom::URLLoaderFactory> webui_url_loader_factory;
-  network::mojom::URLLoaderFactory* url_loader_factory;
+  NetworkResourceLoader::URLLoaderFactoryHolder url_loader_factory;
   if (gurl.SchemeIsFile()) {
-    file_url_loader_factory = content::CreateFileURLLoaderFactory(
+    url_loader_factory = content::CreateFileURLLoaderFactory(
         base::FilePath() /* profile_path */,
         nullptr /* shared_cors_origin_access_list */);
-    url_loader_factory = file_url_loader_factory.get();
   } else if (content::HasWebUIScheme(gurl)) {
     content::WebContents* target_tab;
 #ifndef NDEBUG
@@ -778,10 +849,9 @@ void DevToolsUIBindings::LoadNetworkResource(const DispatchCallback& callback,
     if (allow_web_ui_scheme) {
       std::vector<std::string> allowed_webui_hosts;
       content::RenderFrameHost* frame_host = web_contents()->GetMainFrame();
-      webui_url_loader_factory = content::CreateWebUIURLLoader(
+      url_loader_factory = content::CreateWebUIURLLoader(
           frame_host, target_tab->GetURL().scheme(),
           std::move(allowed_webui_hosts));
-      url_loader_factory = webui_url_loader_factory.get();
     } else {
       base::DictionaryValue response;
       response.SetInteger("statusCode", 403);
@@ -791,17 +861,12 @@ void DevToolsUIBindings::LoadNetworkResource(const DispatchCallback& callback,
   } else {
     auto* partition = content::BrowserContext::GetStoragePartitionForSite(
         web_contents_->GetBrowserContext(), gurl);
-    network_url_loader_factory =
-        partition->GetURLLoaderFactoryForBrowserProcess();
-    url_loader_factory = network_url_loader_factory.get();
+    url_loader_factory = partition->GetURLLoaderFactoryForBrowserProcess();
   }
 
-  auto simple_url_loader = network::SimpleURLLoader::Create(
-      std::move(resource_request), traffic_annotation);
-  auto resource_loader = std::make_unique<NetworkResourceLoader>(
-      stream_id, this, std::move(simple_url_loader), url_loader_factory,
-      callback);
-  loaders_.insert(std::move(resource_loader));
+  NetworkResourceLoader::Create(stream_id, this, resource_request,
+                                traffic_annotation,
+                                std::move(url_loader_factory), callback);
 }
 
 void DevToolsUIBindings::OpenInNewTab(const std::string& url) {