Adds the new URLRequest::OnBeforeNetworkStart hook to the

ResourceThrottle so that the ResourceScheduler can eventually throttle
closer to the start of network usage.

This is the second step in the design doc:
https://docs.google.com/document/d/1TSI3jwozVB_nueWJxzi8uKbgDfsGZRZqw8tvtD-P1Iw/edit?usp=sharing

The third step is to create the new ResourceScheduler.

BUG=328741

Review URL: https://codereview.chromium.org/129173002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@243994 0039d316-1c4b-4281-b951-d872f2087c98

content/browser/download/download_resource_handler.cc

@@ -222,6 +222,12 @@ bool DownloadResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool DownloadResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                   const GURL& url,
+                                                   bool* defer) {
+  return true;
+}
+
 // Create a new buffer, which will be handed to the download thread for file
 // writing and deletion.
 bool DownloadResourceHandler::OnWillRead(int request_id,

content/browser/download/download_resource_handler.h

@@ -64,6 +64,11 @@ class CONTENT_EXPORT DownloadResourceHandler
                            const GURL& url,
                            bool* defer) OVERRIDE;
 
+  // Pass-through implementation.
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
+
   // Create a new buffer, which will be handed to the download thread for file
   // writing and deletion.
   virtual bool OnWillRead(int request_id,

content/browser/download/save_file_resource_handler.cc

@@ -74,6 +74,12 @@ bool SaveFileResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool SaveFileResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                   const GURL& url,
+                                                   bool* defer) {
+  return true;
+}
+
 bool SaveFileResourceHandler::OnWillRead(int request_id,
                                          scoped_refptr<net::IOBuffer>* buf,
                                          int* buf_size,

content/browser/download/save_file_resource_handler.h

@@ -45,6 +45,11 @@ class SaveFileResourceHandler : public ResourceHandler {
                            const GURL& url,
                            bool* defer) OVERRIDE;
 
+  // Pass-through implementation.
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
+
   // Creates a new buffer, which will be handed to the download thread for file
   // writing and deletion.
   virtual bool OnWillRead(int request_id,

content/browser/loader/async_resource_handler.cc

@@ -220,6 +220,12 @@ bool AsyncResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool AsyncResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                const GURL& url,
+                                                bool* defer) {
+  return true;
+}
+
 bool AsyncResourceHandler::OnWillRead(int request_id,
                                       scoped_refptr<net::IOBuffer>* buf,
                                       int* buf_size,

content/browser/loader/async_resource_handler.h

@@ -49,6 +49,9 @@ class AsyncResourceHandler : public ResourceHandler,
   virtual bool OnWillStart(int request_id,
                            const GURL& url,
                            bool* defer) OVERRIDE;
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,
                           int* buf_size,

content/browser/loader/certificate_resource_handler.cc

@@ -56,6 +56,12 @@ bool CertificateResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool CertificateResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                      const GURL& url,
+                                                      bool* defer) {
+  return true;
+}
+
 bool CertificateResourceHandler::OnWillRead(int request_id,
                                            scoped_refptr<net::IOBuffer>* buf,
                                            int* buf_size,

content/browser/loader/certificate_resource_handler.h

@@ -54,6 +54,11 @@ class CertificateResourceHandler : public ResourceHandler {
                            const GURL& url,
                            bool* defer) OVERRIDE;
 
+  // Pass-through implementation.
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
+
   // Create a new buffer to store received data.
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,

content/browser/loader/detachable_resource_handler.cc

@@ -133,6 +133,20 @@ bool DetachableResourceHandler::OnWillStart(int request_id, const GURL& url,
   return ret;
 }
 
+bool DetachableResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                     const GURL& url,
+                                                     bool* defer) {
+  DCHECK(!is_deferred_);
+
+  if (!next_handler_)
+    return true;
+
+  bool ret =
+      next_handler_->OnBeforeNetworkStart(request_id, url, &is_deferred_);
+  *defer = is_deferred_;
+  return ret;
+}
+
 bool DetachableResourceHandler::OnWillRead(int request_id,
                                            scoped_refptr<net::IOBuffer>* buf,
                                            int* buf_size,

content/browser/loader/detachable_resource_handler.h

@@ -58,6 +58,9 @@ class DetachableResourceHandler : public ResourceHandler,
                                  bool* defer) OVERRIDE;
   virtual bool OnWillStart(int request_id, const GURL& url,
                            bool* defer) OVERRIDE;
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,
                           int* buf_size,

content/browser/loader/layered_resource_handler.cc

@@ -56,6 +56,13 @@ bool LayeredResourceHandler::OnWillStart(int request_id, const GURL& url,
   return next_handler_->OnWillStart(request_id, url, defer);
 }
 
+bool LayeredResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                  const GURL& url,
+                                                  bool* defer) {
+  DCHECK(next_handler_.get());
+  return next_handler_->OnBeforeNetworkStart(request_id, url, defer);
+}
+
 bool LayeredResourceHandler::OnWillRead(int request_id,
                                         scoped_refptr<net::IOBuffer>* buf,
                                         int* buf_size,

content/browser/loader/layered_resource_handler.h

@@ -35,6 +35,9 @@ class CONTENT_EXPORT LayeredResourceHandler : public ResourceHandler {
                                  bool* defer) OVERRIDE;
   virtual bool OnWillStart(int request_id, const GURL& url,
                            bool* defer) OVERRIDE;
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,
                           int* buf_size,

content/browser/loader/resource_dispatcher_host_unittest.cc

@@ -210,6 +210,33 @@ class ForwardingFilter : public ResourceMessageFilter {
   DISALLOW_COPY_AND_ASSIGN(ForwardingFilter);
 };
 
+// This class is a variation on URLRequestTestJob that will call
+// URLRequest::OnBeforeNetworkStart before starting.
+class URLRequestTestDelayedNetworkJob : public net::URLRequestTestJob {
+ public:
+  URLRequestTestDelayedNetworkJob(net::URLRequest* request,
+                                  net::NetworkDelegate* network_delegate)
+      : net::URLRequestTestJob(request, network_delegate) {}
+
+  // Only start if not deferred for network start.
+  virtual void Start() OVERRIDE {
+    bool defer = false;
+    NotifyBeforeNetworkStart(&defer);
+    if (defer)
+      return;
+    net::URLRequestTestJob::Start();
+  }
+
+  virtual void ResumeNetworkStart() OVERRIDE {
+    net::URLRequestTestJob::StartAsync();
+  }
+
+ private:
+  virtual ~URLRequestTestDelayedNetworkJob() {}
+
+  DISALLOW_COPY_AND_ASSIGN(URLRequestTestDelayedNetworkJob);
+};
+
 // This class is a variation on URLRequestTestJob in that it does
 // not complete start upon entry, only when specifically told to.
 class URLRequestTestDelayedStartJob : public net::URLRequestTestJob {
@@ -394,7 +421,8 @@ enum GenericResourceThrottleFlags {
   NONE                      = 0,
   DEFER_STARTING_REQUEST    = 1 << 0,
   DEFER_PROCESSING_RESPONSE = 1 << 1,
-  CANCEL_BEFORE_START       = 1 << 2
+  CANCEL_BEFORE_START       = 1 << 2,
+  DEFER_NETWORK_START       = 1 << 3
 };
 
 // Throttle that tracks the current throttle blocking a request.  Only one
@@ -441,6 +469,15 @@ class GenericResourceThrottle : public ResourceThrottle {
     }
   }
 
+  virtual void OnBeforeNetworkStart(bool* defer) OVERRIDE {
+    ASSERT_EQ(NULL, active_throttle_);
+
+    if (flags_ & DEFER_NETWORK_START) {
+      active_throttle_ = this;
+      *defer = true;
+    }
+  }
+
   virtual const char* GetNameForLogging() const OVERRIDE {
     return "GenericResourceThrottle";
   }
@@ -573,6 +610,7 @@ class ResourceDispatcherHostTest : public testing::Test,
     EnsureTestSchemeIsAllowed();
     delay_start_ = false;
     delay_complete_ = false;
+    network_start_notification_ = false;
     url_request_jobs_created_count_ = 0;
   }
 
@@ -670,6 +708,8 @@ class ResourceDispatcherHostTest : public testing::Test,
       } else if (delay_complete_) {
         return new URLRequestTestDelayedCompletionJob(request,
                                                       network_delegate);
+      } else if (network_start_notification_) {
+        return new URLRequestTestDelayedNetworkJob(request, network_delegate);
       } else if (scheme == "big-job") {
         return new URLRequestBigJob(request, network_delegate);
       } else {
@@ -703,6 +743,10 @@ class ResourceDispatcherHostTest : public testing::Test,
     delay_complete_ = delay_job_complete;
   }
 
+  void SetNetworkStartNotificationJobGeneration(bool notification) {
+    network_start_notification_ = notification;
+  }
+
   void GenerateDataReceivedACK(const IPC::Message& msg) {
     EXPECT_EQ(ResourceMsg_DataReceived::ID, msg.type());
 
@@ -736,12 +780,14 @@ class ResourceDispatcherHostTest : public testing::Test,
   static ResourceDispatcherHostTest* test_fixture_;
   static bool delay_start_;
   static bool delay_complete_;
+  static bool network_start_notification_;
   static int url_request_jobs_created_count_;
 };
 // Static.
 ResourceDispatcherHostTest* ResourceDispatcherHostTest::test_fixture_ = NULL;
 bool ResourceDispatcherHostTest::delay_start_ = false;
 bool ResourceDispatcherHostTest::delay_complete_ = false;
+bool ResourceDispatcherHostTest::network_start_notification_ = false;
 int ResourceDispatcherHostTest::url_request_jobs_created_count_ = 0;
 
 void ResourceDispatcherHostTest::MakeTestRequest(int render_view_id,
@@ -1228,6 +1274,33 @@ TEST_F(ResourceDispatcherHostTest, PausedStartError) {
   EXPECT_EQ(0, host_.pending_requests());
 }
 
+// Test the OnBeforeNetworkStart throttle.
+TEST_F(ResourceDispatcherHostTest, ThrottleNetworkStart) {
+  // Arrange to have requests deferred before processing response headers.
+  TestResourceDispatcherHostDelegate delegate;
+  delegate.set_flags(DEFER_NETWORK_START);
+  host_.SetDelegate(&delegate);
+
+  SetNetworkStartNotificationJobGeneration(true);
+  MakeTestRequest(0, 1, net::URLRequestTestJob::test_url_2());
+
+  // Should have deferred for network start.
+  GenericResourceThrottle* first_throttle =
+      GenericResourceThrottle::active_throttle();
+  ASSERT_TRUE(first_throttle);
+  EXPECT_EQ(0, network_delegate()->completed_requests());
+  EXPECT_EQ(1, host_.pending_requests());
+
+  first_throttle->Resume();
+
+  // Flush all the pending requests.
+  while (net::URLRequestTestJob::ProcessOnePendingMessage()) {}
+  base::MessageLoop::current()->RunUntilIdle();
+
+  EXPECT_EQ(1, network_delegate()->completed_requests());
+  EXPECT_EQ(0, host_.pending_requests());
+}
+
 TEST_F(ResourceDispatcherHostTest, ThrottleAndResumeTwice) {
   // Arrange to have requests deferred before starting.
   TestResourceDispatcherHostDelegate delegate;

content/browser/loader/resource_handler.h

@@ -73,6 +73,15 @@ class CONTENT_EXPORT ResourceHandler
   // until someone calls ResourceDispatcherHost::StartDeferredRequest().
   virtual bool OnWillStart(int request_id, const GURL& url, bool* defer) = 0;
 
+  // Called before the net::URLRequest for |request_id| (whose url is |url|}
+  // uses the network for the first time to load the resource. If the handler
+  // returns false, then the request is cancelled. Otherwise if the return value
+  // is true, the ResourceHandler can delay the request from starting by setting
+  // |*defer = true|. Call controller()->Resume() to continue if deferred.
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) = 0;
+
   // Data will be read for the response.  Upon success, this method places the
   // size and address of the buffer where the data is to be written in its
   // out-params.  This call will be followed by either OnReadCompleted or

content/browser/loader/resource_loader.cc

@@ -289,6 +289,20 @@ void ResourceLoader::OnSSLCertificateError(net::URLRequest* request,
       fatal);
 }
 
+void ResourceLoader::OnBeforeNetworkStart(net::URLRequest* unused,
+                                          bool* defer) {
+  DCHECK_EQ(request_.get(), unused);
+
+  // Give the handler a chance to delay the URLRequest from using the network.
+  if (!handler_->OnBeforeNetworkStart(
+           GetRequestInfo()->GetRequestID(), request_->url(), defer)) {
+    Cancel();
+    return;
+  } else if (*defer) {
+    deferred_stage_ = DEFERRED_NETWORK_START;
+  }
+}
+
 void ResourceLoader::OnResponseStarted(net::URLRequest* unused) {
   DCHECK_EQ(request_.get(), unused);
 
@@ -391,6 +405,9 @@ void ResourceLoader::Resume() {
     case DEFERRED_START:
       StartRequestInternal();
       break;
+    case DEFERRED_NETWORK_START:
+      request_->ResumeNetworkStart();
+      break;
     case DEFERRED_REDIRECT:
       request_->FollowDeferredRedirect();
       break;

content/browser/loader/resource_loader.h

@@ -66,6 +66,8 @@ class CONTENT_EXPORT ResourceLoader : public net::URLRequest::Delegate,
   virtual void OnSSLCertificateError(net::URLRequest* request,
                                      const net::SSLInfo& info,
                                      bool fatal) OVERRIDE;
+  virtual void OnBeforeNetworkStart(net::URLRequest* request,
+                                    bool* defer) OVERRIDE;
   virtual void OnResponseStarted(net::URLRequest* request) OVERRIDE;
   virtual void OnReadCompleted(net::URLRequest* request,
                                int bytes_read) OVERRIDE;
@@ -117,6 +119,7 @@ class CONTENT_EXPORT ResourceLoader : public net::URLRequest::Delegate,
   enum DeferredStage {
     DEFERRED_NONE,
     DEFERRED_START,
+    DEFERRED_NETWORK_START,
     DEFERRED_REDIRECT,
     DEFERRED_READ,
     DEFERRED_FINISH

content/browser/loader/resource_loader_unittest.cc

@@ -94,6 +94,12 @@ class ResourceHandlerStub : public ResourceHandler {
     return true;
   }
 
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE {
+    return true;
+  }
+
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,
                           int* buf_size,

content/browser/loader/stream_resource_handler.cc

@@ -57,6 +57,12 @@ bool StreamResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool StreamResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                 const GURL& url,
+                                                 bool* defer) {
+  return true;
+}
+
 bool StreamResourceHandler::OnWillRead(int request_id,
                                        scoped_refptr<net::IOBuffer>* buf,
                                        int* buf_size,

content/browser/loader/stream_resource_handler.h

@@ -49,6 +49,10 @@ class StreamResourceHandler : public StreamWriteObserver,
                            const GURL& url,
                            bool* defer) OVERRIDE;
 
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
+
   // Create a new buffer to store received data.
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,

content/browser/loader/sync_resource_handler.cc

@@ -101,6 +101,12 @@ bool SyncResourceHandler::OnWillStart(int request_id,
   return true;
 }
 
+bool SyncResourceHandler::OnBeforeNetworkStart(int request_id,
+                                               const GURL& url,
+                                               bool* defer) {
+  return true;
+}
+
 bool SyncResourceHandler::OnWillRead(int request_id,
                                      scoped_refptr<net::IOBuffer>* buf,
                                      int* buf_size,

content/browser/loader/sync_resource_handler.h

@@ -46,6 +46,9 @@ class SyncResourceHandler : public ResourceHandler {
   virtual bool OnWillStart(int request_id,
                            const GURL& url,
                            bool* defer) OVERRIDE;
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                    const GURL& url,
+                                    bool* defer) OVERRIDE;
   virtual bool OnWillRead(int request_id,
                           scoped_refptr<net::IOBuffer>* buf,
                           int* buf_size,

content/browser/loader/throttling_resource_handler.cc

@@ -84,6 +84,31 @@ bool ThrottlingResourceHandler::OnWillStart(int request_id,
   return next_handler_->OnWillStart(request_id, url, defer);
 }
 
+bool ThrottlingResourceHandler::OnBeforeNetworkStart(int request_id,
+                                                     const GURL& url,
+                                                     bool* defer) {
+  DCHECK(!cancelled_by_resource_throttle_);
+
+  *defer = false;
+  while (next_index_ < throttles_.size()) {
+    int index = next_index_;
+    throttles_[index]->OnBeforeNetworkStart(defer);
+    next_index_++;
+    if (cancelled_by_resource_throttle_)
+      return false;
+    if (*defer) {
+      OnRequestDefered(index);
+      deferred_stage_ = DEFERRED_NETWORK_START;
+      deferred_url_ = url;
+      return true;  // Do not cancel.
+    }
+  }
+
+  next_index_ = 0;  // Reset for next time.
+
+  return next_handler_->OnBeforeNetworkStart(request_id, url, defer);
+}
+
 bool ThrottlingResourceHandler::OnResponseStarted(int request_id,
                                                   ResourceResponse* response,
                                                   bool* defer) {
@@ -137,6 +162,9 @@ void ThrottlingResourceHandler::Resume() {
     case DEFERRED_START:
       ResumeStart();
       break;
+    case DEFERRED_NETWORK_START:
+      ResumeNetworkStart();
+      break;
     case DEFERRED_REDIRECT:
       ResumeRedirect();
       break;
@@ -160,6 +188,20 @@ void ThrottlingResourceHandler::ResumeStart() {
   }
 }
 
+void ThrottlingResourceHandler::ResumeNetworkStart() {
+  DCHECK(!cancelled_by_resource_throttle_);
+
+  GURL url = deferred_url_;
+  deferred_url_ = GURL();
+
+  bool defer = false;
+  if (!OnBeforeNetworkStart(GetRequestID(), url, &defer)) {
+    controller()->Cancel();
+  } else if (!defer) {
+    controller()->Resume();
+  }
+}
+
 void ThrottlingResourceHandler::ResumeRedirect() {
   DCHECK(!cancelled_by_resource_throttle_);
 

content/browser/loader/throttling_resource_handler.h

@@ -39,6 +39,9 @@ class ThrottlingResourceHandler : public LayeredResourceHandler,
                                  bool* defer) OVERRIDE;
   virtual bool OnWillStart(int request_id, const GURL& url,
                            bool* defer) OVERRIDE;
+  virtual bool OnBeforeNetworkStart(int request_id,
+                                  const GURL& url,
+                                  bool* defer) OVERRIDE;
 
   // ResourceController implementation:
   virtual void Cancel() OVERRIDE;
@@ -48,6 +51,7 @@ class ThrottlingResourceHandler : public LayeredResourceHandler,
 
  private:
   void ResumeStart();
+  void ResumeNetworkStart();
   void ResumeRedirect();
   void ResumeResponse();
 
@@ -58,6 +62,7 @@ class ThrottlingResourceHandler : public LayeredResourceHandler,
   enum DeferredStage {
     DEFERRED_NONE,
     DEFERRED_START,
+    DEFERRED_NETWORK_START,
     DEFERRED_REDIRECT,
     DEFERRED_RESPONSE
   };

content/public/browser/resource_throttle.h

@@ -25,6 +25,7 @@ class ResourceThrottle {
   virtual void WillStartRequest(bool* defer) {}
   virtual void WillRedirectRequest(const GURL& new_url, bool* defer) {}
   virtual void WillProcessResponse(bool* defer) {}
+  virtual void OnBeforeNetworkStart(bool* defer) {}
 
   // Returns the name of the throttle, as a UTF-8 C-string, for logging
   // purposes.  NULL is not allowed.  Caller does *not* take ownership of the