Disable QUIC for 5 miuntes, subject to exponential backoff, when

sessions timeout with open streams.

Review-Url: https://codereview.chromium.org/2322423002
Cr-Commit-Position: refs/heads/master@{#418682}

net/quic/chromium/quic_stream_factory.cc

@@ -80,6 +80,9 @@ const int32_t kQuicStreamMaxRecvWindowSize = 6 * 1024 * 1024;    // 6 MB
 // Set the maximum number of undecryptable packets the connection will store.
 const int32_t kMaxUndecryptablePackets = 100;
 
+// How long QUIC will be disabled for because of timeouts with open streams.
+const int kDisableQuicTimeoutSecs = 5 * 60;
+
 std::unique_ptr<base::Value> NetLogQuicConnectionMigrationTriggerCallback(
     std::string trigger,
     NetLogCaptureMode capture_mode) {
@@ -769,6 +772,8 @@ QuicStreamFactory::QuicStreamFactory(
       prefer_aes_(prefer_aes),
       disable_quic_on_timeout_with_open_streams_(
           disable_quic_on_timeout_with_open_streams),
+      consecutive_disabled_count_(0),
+      need_to_evaluate_consecutive_disabled_count_(false),
       socket_receive_buffer_size_(socket_receive_buffer_size),
       delay_tcp_race_(delay_tcp_race),
       ping_timeout_(QuicTime::Delta::FromSeconds(kPingTimeoutSecs)),
@@ -1203,8 +1208,18 @@ void QuicStreamFactory::OnTimeoutWithOpenStreams() {
   if (ping_timeout_ > reduced_ping_timeout_) {
     ping_timeout_ = reduced_ping_timeout_;
   }
-  if (disable_quic_on_timeout_with_open_streams_)
+  if (disable_quic_on_timeout_with_open_streams_) {
+    if (status_ == OPEN) {
+      task_runner_->PostDelayedTask(
+          FROM_HERE, base::Bind(&QuicStreamFactory::OpenFactory,
+                                weak_factory_.GetWeakPtr()),
+          base::TimeDelta::FromSeconds(kDisableQuicTimeoutSecs *
+                                       (1 << consecutive_disabled_count_)));
+      consecutive_disabled_count_++;
+      need_to_evaluate_consecutive_disabled_count_ = true;
+    }
     status_ = CLOSED;
+  }
 }
 
 void QuicStreamFactory::CancelRequest(QuicStreamRequest* request) {
@@ -1584,6 +1599,15 @@ int QuicStreamFactory::CreateSession(
     base::TimeTicks dns_resolution_end_time,
     const BoundNetLog& net_log,
     QuicChromiumClientSession** session) {
+  if (need_to_evaluate_consecutive_disabled_count_) {
+    task_runner_->PostDelayedTask(
+        FROM_HERE,
+        base::Bind(&QuicStreamFactory::MaybeClearConsecutiveDisabledCount,
+                   weak_factory_.GetWeakPtr()),
+        base::TimeDelta::FromSeconds(kDisableQuicTimeoutSecs));
+
+    need_to_evaluate_consecutive_disabled_count_ = false;
+  }
   TRACE_EVENT0("net", "QuicStreamFactory::CreateSession");
   IPEndPoint addr = *address_list.begin();
   bool enable_port_selection = enable_port_selection_;
@@ -1876,4 +1900,13 @@ void QuicStreamFactory::ProcessGoingAwaySession(
       alternative_service);
 }
 
+void QuicStreamFactory::OpenFactory() {
+  status_ = OPEN;
+}
+
+void QuicStreamFactory::MaybeClearConsecutiveDisabledCount() {
+  if (status_ == OPEN)
+    consecutive_disabled_count_ = 0;
+}
+
 }  // namespace net

net/quic/chromium/quic_stream_factory.h

@@ -483,6 +483,12 @@ class NET_EXPORT_PRIVATE QuicStreamFactory
       bool close_session_on_error,
       const BoundNetLog& bound_net_log);
 
+  // Called to re-enable QUIC when QUIC has been disabled.
+  void OpenFactory();
+  // If QUIC has been working well after having been recently
+  // disabled, clear the |consecutive_disabled_count_|.
+  void MaybeClearConsecutiveDisabledCount();
+
   bool require_confirmation_;
   NetLog* net_log_;
   HostResolver* host_resolver_;
@@ -569,6 +575,10 @@ class NET_EXPORT_PRIVATE QuicStreamFactory
   // streams.
   bool disable_quic_on_timeout_with_open_streams_;
 
+  // Number of times in a row that QUIC has been disabled.
+  int consecutive_disabled_count_;
+  bool need_to_evaluate_consecutive_disabled_count_;
+
   // Size of the UDP receive buffer.
   int socket_receive_buffer_size_;
 

net/quic/chromium/quic_stream_factory_test.cc

@@ -4112,6 +4112,196 @@ TEST_P(QuicStreamFactoryTest, DisableQuicWhenTimeoutsWithOpenStreams) {
   EXPECT_TRUE(socket_data.AllWriteDataConsumed());
 }
 
+TEST_P(QuicStreamFactoryTest,
+       DisableQuicWhenTimeoutsWithOpenStreamsExponentialBackoff) {
+  disable_disk_cache_ = true;
+  disable_quic_on_timeout_with_open_streams_ = true;
+  Initialize();
+  ProofVerifyDetailsChromium verify_details = DefaultProofVerifyDetails();
+  crypto_client_stream_factory_.AddProofVerifyDetails(&verify_details);
+  crypto_client_stream_factory_.AddProofVerifyDetails(&verify_details);
+  QuicStreamFactoryPeer::SetTaskRunner(factory_.get(), runner_.get());
+
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  MockQuicData socket_data;
+  socket_data.AddRead(SYNCHRONOUS, ERR_IO_PENDING);
+  socket_data.AddSocketDataToFactory(&socket_factory_);
+
+  MockQuicData socket_data2;
+  socket_data2.AddRead(SYNCHRONOUS, ERR_IO_PENDING);
+  socket_data2.AddSocketDataToFactory(&socket_factory_);
+
+  crypto_client_stream_factory_.set_handshake_mode(
+      MockCryptoClientStream::CONFIRM_HANDSHAKE);
+  host_resolver_.set_synchronous_mode(true);
+  host_resolver_.rules()->AddIPLiteralRule(host_port_pair_.host(),
+                                           "192.168.0.1", "");
+
+  // Test first timeouts with open streams will disable QUIC.
+  QuicStreamRequest request(factory_.get());
+  EXPECT_EQ(OK, request.Request(host_port_pair_, privacy_mode_,
+                                /*cert_verify_flags=*/0, url_, "GET", net_log_,
+                                callback_.callback()));
+
+  QuicChromiumClientSession* session = GetActiveSession(host_port_pair_);
+
+  std::unique_ptr<QuicHttpStream> stream = request.CreateStream();
+  EXPECT_TRUE(stream.get());
+  HttpRequestInfo request_info;
+  EXPECT_EQ(OK, stream->InitializeStream(&request_info, DEFAULT_PRIORITY,
+                                         net_log_, CompletionCallback()));
+
+  DVLOG(1)
+      << "Created 1st session and initialized a stream. Now trigger timeout."
+      << "Will disable QUIC.";
+  session->connection()->CloseConnection(QUIC_NETWORK_IDLE_TIMEOUT, "test",
+                                         ConnectionCloseBehavior::SILENT_CLOSE);
+  EXPECT_TRUE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_EQ(1u, runner_->GetPostedTasks().size());
+  ASSERT_EQ(clock_->NowInTicks() + base::TimeDelta::FromMinutes(5),
+            runner_->GetPostedTasks()[0].GetTimeToRun());
+  runner_->RunNextTask();
+
+  // Need to spin the loop now to ensure that
+  // QuicStreamFactory::OnSessionClosed() runs.
+  base::RunLoop run_loop;
+  run_loop.RunUntilIdle();
+
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_TRUE(runner_->GetPostedTasks().empty());
+
+  // Create a new session which will cause a task to be posted to
+  // clear the exponential backoff.
+  QuicStreamRequest request2(factory_.get());
+  EXPECT_EQ(OK, request2.Request(host_port_pair_, privacy_mode_,
+                                 /*cert_verify_flags=*/0, url_, "GET", net_log_,
+                                 callback_.callback()));
+  QuicChromiumClientSession* session2 = GetActiveSession(host_port_pair_);
+  std::unique_ptr<QuicHttpStream> stream2 = request2.CreateStream();
+  EXPECT_TRUE(stream2.get());
+  HttpRequestInfo request_info2;
+  EXPECT_EQ(OK, stream2->InitializeStream(&request_info2, DEFAULT_PRIORITY,
+                                          net_log_, CompletionCallback()));
+
+  // Check that the clear task has been posted.
+  ASSERT_EQ(1u, runner_->GetPostedTasks().size());
+  ASSERT_EQ(clock_->NowInTicks() + base::TimeDelta::FromMinutes(5),
+            runner_->GetPostedTasks()[0].GetTimeToRun());
+
+  session2->connection()->CloseConnection(
+      QUIC_NETWORK_IDLE_TIMEOUT, "test", ConnectionCloseBehavior::SILENT_CLOSE);
+  EXPECT_TRUE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_EQ(2u, runner_->GetPostedTasks().size());
+  ASSERT_EQ(clock_->NowInTicks() + base::TimeDelta::FromMinutes(10),
+            runner_->GetPostedTasks()[1].GetTimeToRun());
+  runner_->RunNextTask();
+
+  EXPECT_TRUE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  EXPECT_TRUE(socket_data.AllReadDataConsumed());
+  EXPECT_TRUE(socket_data.AllWriteDataConsumed());
+}
+
+TEST_P(QuicStreamFactoryTest,
+       DisableQuicWhenTimeoutsWithOpenStreamsExponentialBackoffReset) {
+  disable_disk_cache_ = true;
+  disable_quic_on_timeout_with_open_streams_ = true;
+  Initialize();
+  ProofVerifyDetailsChromium verify_details = DefaultProofVerifyDetails();
+  crypto_client_stream_factory_.AddProofVerifyDetails(&verify_details);
+  crypto_client_stream_factory_.AddProofVerifyDetails(&verify_details);
+  QuicStreamFactoryPeer::SetTaskRunner(factory_.get(), runner_.get());
+
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  MockQuicData socket_data;
+  socket_data.AddRead(SYNCHRONOUS, ERR_IO_PENDING);
+  socket_data.AddSocketDataToFactory(&socket_factory_);
+
+  MockQuicData socket_data2;
+  socket_data2.AddRead(SYNCHRONOUS, ERR_IO_PENDING);
+  socket_data2.AddSocketDataToFactory(&socket_factory_);
+
+  crypto_client_stream_factory_.set_handshake_mode(
+      MockCryptoClientStream::CONFIRM_HANDSHAKE);
+  host_resolver_.set_synchronous_mode(true);
+  host_resolver_.rules()->AddIPLiteralRule(host_port_pair_.host(),
+                                           "192.168.0.1", "");
+
+  // Test first timeouts with open streams will disable QUIC.
+  QuicStreamRequest request(factory_.get());
+  EXPECT_EQ(OK, request.Request(host_port_pair_, privacy_mode_,
+                                /*cert_verify_flags=*/0, url_, "GET", net_log_,
+                                callback_.callback()));
+
+  QuicChromiumClientSession* session = GetActiveSession(host_port_pair_);
+
+  std::unique_ptr<QuicHttpStream> stream = request.CreateStream();
+  EXPECT_TRUE(stream.get());
+  HttpRequestInfo request_info;
+  EXPECT_EQ(OK, stream->InitializeStream(&request_info, DEFAULT_PRIORITY,
+                                         net_log_, CompletionCallback()));
+
+  DVLOG(1)
+      << "Created 1st session and initialized a stream. Now trigger timeout."
+      << "Will disable QUIC.";
+  session->connection()->CloseConnection(QUIC_NETWORK_IDLE_TIMEOUT, "test",
+                                         ConnectionCloseBehavior::SILENT_CLOSE);
+  EXPECT_TRUE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_EQ(1u, runner_->GetPostedTasks().size());
+  ASSERT_EQ(clock_->NowInTicks() + base::TimeDelta::FromMinutes(5),
+            runner_->GetPostedTasks()[0].GetTimeToRun());
+  runner_->RunNextTask();
+
+  // Need to spin the loop now to ensure that
+  // QuicStreamFactory::OnSessionClosed() runs.
+  base::RunLoop run_loop;
+  run_loop.RunUntilIdle();
+
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_TRUE(runner_->GetPostedTasks().empty());
+
+  // Create a new session which will cause a task to be posted to
+  // clear the exponential backoff.
+  QuicStreamRequest request2(factory_.get());
+  EXPECT_EQ(OK, request2.Request(host_port_pair_, privacy_mode_,
+                                 /*cert_verify_flags=*/0, url_, "GET", net_log_,
+                                 callback_.callback()));
+  QuicChromiumClientSession* session2 = GetActiveSession(host_port_pair_);
+  std::unique_ptr<QuicHttpStream> stream2 = request2.CreateStream();
+  EXPECT_TRUE(stream2.get());
+  HttpRequestInfo request_info2;
+  EXPECT_EQ(OK, stream2->InitializeStream(&request_info2, DEFAULT_PRIORITY,
+                                          net_log_, CompletionCallback()));
+
+  // Run the clear task and verify that the next disabling is
+  // back to the default timeout.
+  runner_->RunNextTask();
+
+  // QUIC should still be enabled.
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  session2->connection()->CloseConnection(
+      QUIC_NETWORK_IDLE_TIMEOUT, "test", ConnectionCloseBehavior::SILENT_CLOSE);
+  EXPECT_TRUE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  ASSERT_EQ(1u, runner_->GetPostedTasks().size());
+  ASSERT_EQ(clock_->NowInTicks() + base::TimeDelta::FromMinutes(5),
+            runner_->GetPostedTasks()[0].GetTimeToRun());
+  runner_->RunNextTask();
+
+  EXPECT_FALSE(QuicStreamFactoryPeer::IsQuicDisabled(factory_.get()));
+
+  EXPECT_TRUE(socket_data.AllReadDataConsumed());
+  EXPECT_TRUE(socket_data.AllWriteDataConsumed());
+}
+
 TEST_P(QuicStreamFactoryTest, EnableDelayTcpRace) {
   Initialize();
   ProofVerifyDetailsChromium verify_details = DefaultProofVerifyDetails();