Experiment with redirecting all BrowserThreads (but UI/IO) to TaskScheduler

This is part of the migration phase for TaskScheduler, design doc:
https://docs.google.com/document/d/1S2AAeoo1xa_vsLbDYBsDHCqhrkfiMgoIPlyRi6kxa5k/edit

Also generalizes BrowserThreadImpl away from the underlying thread's MessageLoop
and id in favor of TaskRunners and BrowserThreadState.

Cool side-effect: BrowserThreadImpl::StartWithOptions() no longer blocks on the
underlying thread starting (it used to via GetThreadId()), it now takes a reference
to its TaskRunner and lets the thread itself start on its own schedule :).
This wasn't previously intended to block (BrowserThreadImpl::StartAndWaitForTesting() is
supposed to be used for that) but had unintentionally regressed in http://crrev.com/408295
because GetThreadId() implicitly waits (http://crbug.com/672977).

Another nice improvement is that ~BrowserThreadImpl() now cleans the BrowserThreadGlobals
associated with it, this has the side-effect to force proper destruction order of
TestBrowserThread in tests which brought forth a few precursor cleanups
(https://crbug.com/653916#c24) which will from now on be enforced by this CL.

When redirection is disabled, the logic should be the exact same as before.
 - Threads are brought up.
 - Tasks are posted to their MessageLoop (albeit through their TaskRunner now).
 - On shutdown, threads are joined one by one.

When redirection is enabled, we try to mimic this logic.
 - Redirection TaskRunners are only live (|accepting_tasks|) for the same period that the
   matching thread would be.
 - On shutdown, we block on each TaskRunner's queue being flushed one-by-one
   in the same order as in the no-redirection case (this almost identical to
   real threads join % one slight difference documented in detail in
   BrowserThreadImpl::StopRedirectionOfThreadID()).

BUG=653916, 672977
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:linux_chromium_tsan_rel_ng

TEST=
 A) "out\Release\chrome.exe"
    Runs/shuts down the exact same as before.

 B) "out\Release\chrome.exe --force-fieldtrials=BrowserScheduler/Enabled/ --force-fieldtrial-params=BrowserScheduler.Enabled:Background/3;3;1;0;10000/BackgroundFileIO/3;3;1;0;10000/Foreground/3;3;1;0;10000/ForegroundFileIO/3;3;1;0;10000/RedirectSequencedWorkerPools/true/RedirectNonUINonIOBrowserThreads/true"
    Runs/shuts down smoothly and chrome://tracing confirms that redirected BrowserThreads are running on TaskSchedulerWorkers.

Review-Url: https://codereview.chromium.org/2464233002
Cr-Commit-Position: refs/heads/master@{#439139}

chrome/browser/chrome_content_browser_client.cc

@@ -3372,7 +3372,13 @@ void ChromeContentBrowserClient::
       MaybePerformBrowserTaskSchedulerRedirection();
 }
 
-//static
+bool ChromeContentBrowserClient::
+    RedirectNonUINonIOBrowserThreadsToTaskScheduler() {
+  return variations::GetVariationParamValue(
+             "BrowserScheduler", "RedirectNonUINonIOBrowserThreads") == "true";
+}
+
+// static
 void ChromeContentBrowserClient::SetDefaultQuotaSettingsForTesting(
     const storage::QuotaSettings* settings) {
   g_default_quota_settings = settings;

chrome/browser/chrome_content_browser_client.h

@@ -324,8 +324,8 @@ class ChromeContentBrowserClient : public content::ContentBrowserClient {
       std::vector<base::SchedulerWorkerPoolParams>* params_vector,
       base::TaskScheduler::WorkerPoolIndexForTraitsCallback*
           index_to_traits_callback) override;
-
   void PerformExperimentalTaskSchedulerRedirections() override;
+  bool RedirectNonUINonIOBrowserThreadsToTaskScheduler() override;
 
  private:
   friend class DisableWebRtcEncryptionFlagTest;

content/browser/browser_main_loop.cc

@@ -30,8 +30,10 @@
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
+#include "base/synchronization/waitable_event.h"
 #include "base/system_monitor/system_monitor.h"
 #include "base/task_scheduler/initialization_util.h"
+#include "base/task_scheduler/post_task.h"
 #include "base/task_scheduler/scheduler_worker_pool_params.h"
 #include "base/task_scheduler/task_scheduler.h"
 #include "base/task_scheduler/task_traits.h"
@@ -309,35 +311,57 @@ MSVC_PUSH_DISABLE_WARNING(4748)
 NOINLINE void ResetThread_DB(std::unique_ptr<BrowserProcessSubThread> thread) {
   volatile int inhibit_comdat = __LINE__;
   ALLOW_UNUSED_LOCAL(inhibit_comdat);
+  if (thread) {
     thread.reset();
+  } else {
+    BrowserThreadImpl::StopRedirectionOfThreadID(BrowserThread::DB);
+  }
 }
 
 NOINLINE void ResetThread_FILE(
     std::unique_ptr<BrowserProcessSubThread> thread) {
   volatile int inhibit_comdat = __LINE__;
   ALLOW_UNUSED_LOCAL(inhibit_comdat);
+  if (thread) {
     thread.reset();
+  } else {
+    BrowserThreadImpl::StopRedirectionOfThreadID(BrowserThread::FILE);
+  }
 }
 
 NOINLINE void ResetThread_FILE_USER_BLOCKING(
     std::unique_ptr<BrowserProcessSubThread> thread) {
   volatile int inhibit_comdat = __LINE__;
   ALLOW_UNUSED_LOCAL(inhibit_comdat);
+  if (thread) {
     thread.reset();
+  } else {
+    BrowserThreadImpl::StopRedirectionOfThreadID(
+        BrowserThread::FILE_USER_BLOCKING);
+  }
 }
 
 NOINLINE void ResetThread_PROCESS_LAUNCHER(
     std::unique_ptr<BrowserProcessSubThread> thread) {
   volatile int inhibit_comdat = __LINE__;
   ALLOW_UNUSED_LOCAL(inhibit_comdat);
+  if (thread) {
     thread.reset();
+  } else {
+    BrowserThreadImpl::StopRedirectionOfThreadID(
+        BrowserThread::PROCESS_LAUNCHER);
+  }
 }
 
 NOINLINE void ResetThread_CACHE(
     std::unique_ptr<BrowserProcessSubThread> thread) {
   volatile int inhibit_comdat = __LINE__;
   ALLOW_UNUSED_LOCAL(inhibit_comdat);
+  if (thread) {
     thread.reset();
+  } else {
+    BrowserThreadImpl::StopRedirectionOfThreadID(BrowserThread::CACHE);
+  }
 }
 
 NOINLINE void ResetThread_IO(std::unique_ptr<BrowserProcessSubThread> thread) {
@@ -963,62 +987,118 @@ int BrowserMainLoop::CreateThreads() {
   base::Thread::Options ui_message_loop_options;
   ui_message_loop_options.message_loop_type = base::MessageLoop::TYPE_UI;
 
-  // Start threads in the order they occur in the BrowserThread::ID
-  // enumeration, except for BrowserThread::UI which is the main
-  // thread.
+  const bool redirect_nonUInonIO_browser_threads =
+      GetContentClient()
+          ->browser()
+          ->RedirectNonUINonIOBrowserThreadsToTaskScheduler();
+
+  // Start threads in the order they occur in the BrowserThread::ID enumeration,
+  // except for BrowserThread::UI which is the main thread.
   //
   // Must be size_t so we can increment it.
   for (size_t thread_id = BrowserThread::UI + 1;
        thread_id < BrowserThread::ID_COUNT;
        ++thread_id) {
-    std::unique_ptr<BrowserProcessSubThread>* thread_to_start = NULL;
+    // If this thread ID is backed by a real thread, |thread_to_start| will be
+    // set to the appropriate BrowserProcessSubThread*. And |options| can be
+    // updated away from its default.
+    std::unique_ptr<BrowserProcessSubThread>* thread_to_start = nullptr;
     base::Thread::Options options;
 
+    // Otherwise this thread ID will be backed by a SingleThreadTaskRunner using
+    // |non_ui_non_io_task_runner_traits| (which can be augmented below).
+    // TODO(gab): Existing non-UI/non-IO BrowserThreads allow waiting so the
+    // initial redirection will as well but they probably don't need to.
+    base::TaskTraits non_ui_non_io_task_runner_traits =
+        base::TaskTraits().WithFileIO().WithWait();
+
     switch (thread_id) {
       case BrowserThread::DB:
         TRACE_EVENT_BEGIN1("startup",
             "BrowserMainLoop::CreateThreads:start",
             "Thread", "BrowserThread::DB");
+        if (redirect_nonUInonIO_browser_threads) {
+          non_ui_non_io_task_runner_traits
+              .WithPriority(base::TaskPriority::USER_VISIBLE)
+              .WithShutdownBehavior(base::TaskShutdownBehavior::BLOCK_SHUTDOWN);
+        } else {
           thread_to_start = &db_thread_;
           options.timer_slack = base::TIMER_SLACK_MAXIMUM;
+        }
         break;
       case BrowserThread::FILE_USER_BLOCKING:
         TRACE_EVENT_BEGIN1("startup",
             "BrowserMainLoop::CreateThreads:start",
             "Thread", "BrowserThread::FILE_USER_BLOCKING");
+        if (redirect_nonUInonIO_browser_threads) {
+          non_ui_non_io_task_runner_traits
+              .WithPriority(base::TaskPriority::USER_BLOCKING)
+              .WithShutdownBehavior(base::TaskShutdownBehavior::BLOCK_SHUTDOWN);
+        } else {
           thread_to_start = &file_user_blocking_thread_;
+        }
         break;
       case BrowserThread::FILE:
         TRACE_EVENT_BEGIN1("startup",
             "BrowserMainLoop::CreateThreads:start",
             "Thread", "BrowserThread::FILE");
-        thread_to_start = &file_thread_;
+
 #if defined(OS_WIN)
-        // On Windows, the FILE thread needs to be have a UI message loop
-        // which pumps messages in such a way that Google Update can
-        // communicate back to us.
+        // On Windows, the FILE thread needs to have a UI message loop which
+        // pumps messages in such a way that Google Update can communicate back
+        // to us.
+        // TODO(robliao): Need to support COM in TaskScheduler before
+        // redirecting the FILE thread on Windows. http://crbug.com/662122
+        thread_to_start = &file_thread_;
         options = ui_message_loop_options;
 #else
+        if (redirect_nonUInonIO_browser_threads) {
+          non_ui_non_io_task_runner_traits
+              .WithPriority(base::TaskPriority::BACKGROUND)
+              .WithShutdownBehavior(base::TaskShutdownBehavior::BLOCK_SHUTDOWN);
+        } else {
+          thread_to_start = &file_thread_;
           options = io_message_loop_options;
-#endif
           options.timer_slack = base::TIMER_SLACK_MAXIMUM;
+        }
+#endif
         break;
       case BrowserThread::PROCESS_LAUNCHER:
         TRACE_EVENT_BEGIN1("startup",
             "BrowserMainLoop::CreateThreads:start",
             "Thread", "BrowserThread::PROCESS_LAUNCHER");
+        if (redirect_nonUInonIO_browser_threads) {
+          non_ui_non_io_task_runner_traits
+              .WithPriority(base::TaskPriority::USER_BLOCKING)
+              .WithShutdownBehavior(base::TaskShutdownBehavior::BLOCK_SHUTDOWN);
+        } else {
           thread_to_start = &process_launcher_thread_;
           options.timer_slack = base::TIMER_SLACK_MAXIMUM;
+        }
         break;
       case BrowserThread::CACHE:
         TRACE_EVENT_BEGIN1("startup",
             "BrowserMainLoop::CreateThreads:start",
             "Thread", "BrowserThread::CACHE");
-        thread_to_start = &cache_thread_;
 #if defined(OS_WIN)
+        // TaskScheduler doesn't support async I/O on Windows as CACHE thread is
+        // the only user and this use case is going away in
+        // https://codereview.chromium.org/2216583003/.
+        // TODO(gavinp): Remove this ifdef (and thus enable redirection of the
+        // CACHE thread on Windows) once that CL lands.
+        thread_to_start = &cache_thread_;
         options = io_message_loop_options;
-#endif  // defined(OS_WIN)
         options.timer_slack = base::TIMER_SLACK_MAXIMUM;
+#else  // OS_WIN
+        if (redirect_nonUInonIO_browser_threads) {
+          non_ui_non_io_task_runner_traits
+              .WithPriority(base::TaskPriority::USER_BLOCKING)
+              .WithShutdownBehavior(base::TaskShutdownBehavior::BLOCK_SHUTDOWN);
+        } else {
+          thread_to_start = &cache_thread_;
+          options.timer_slack = base::TIMER_SLACK_MAXIMUM;
+        }
+#endif  // OS_WIN
         break;
       case BrowserThread::IO:
         TRACE_EVENT_BEGIN1("startup",
@@ -1032,9 +1112,8 @@ int BrowserMainLoop::CreateThreads() {
         options.priority = base::ThreadPriority::DISPLAY;
 #endif
         break;
-      case BrowserThread::UI:
-      case BrowserThread::ID_COUNT:
-      default:
+      case BrowserThread::UI:        // Falls through.
+      case BrowserThread::ID_COUNT:  // Falls through.
         NOTREACHED();
         break;
     }
@@ -1043,11 +1122,15 @@ int BrowserMainLoop::CreateThreads() {
 
     if (thread_to_start) {
       (*thread_to_start).reset(new BrowserProcessSubThread(id));
-      if (!(*thread_to_start)->StartWithOptions(options)) {
+      if (!(*thread_to_start)->StartWithOptions(options))
         LOG(FATAL) << "Failed to start the browser thread: id == " << id;
-      }
     } else {
-      NOTREACHED();
+      scoped_refptr<base::SingleThreadTaskRunner> redirection_task_runner =
+          base::CreateSingleThreadTaskRunnerWithTraits(
+              non_ui_non_io_task_runner_traits);
+      DCHECK(redirection_task_runner);
+      BrowserThreadImpl::RedirectThreadIDToTaskRunner(
+          id, std::move(redirection_task_runner));
     }
 
     TRACE_EVENT_END0("startup", "BrowserMainLoop::CreateThreads:start");
@@ -1203,8 +1286,8 @@ void BrowserMainLoop::ShutdownThreadsAndCleanUp() {
       }
       case BrowserThread::FILE: {
         TRACE_EVENT0("shutdown", "BrowserMainLoop::Subsystem:FileThread");
-        // Clean up state that lives on or uses the file_thread_ before
-        // it goes away.
+        // Clean up state that lives on or uses the FILE thread before it goes
+        // away.
         save_file_manager_->Shutdown();
         ResetThread_FILE(std::move(file_thread_));
         break;
@@ -1232,7 +1315,6 @@ void BrowserMainLoop::ShutdownThreadsAndCleanUp() {
       }
       case BrowserThread::UI:
       case BrowserThread::ID_COUNT:
-      default:
         NOTREACHED();
         break;
     }

content/browser/browser_main_loop.h

@@ -26,10 +26,10 @@ namespace base {
 class CommandLine;
 class FilePath;
 class HighResolutionTimerManager;
+class MemoryPressureMonitor;
 class MessageLoop;
 class PowerMonitor;
 class SystemMonitor;
-class MemoryPressureMonitor;
 namespace trace_event {
 class TraceEventSystemStatsMonitor;
 }  // namespace trace_event
@@ -273,6 +273,9 @@ class CONTENT_EXPORT BrowserMainLoop {
 #endif
 
   // Members initialized in |CreateThreads()| ----------------------------------
+  // Note: some |*_thread_| members below may never be initialized when
+  // redirection to TaskScheduler is enabled. (ref.
+  // ContentBrowserClient::RedirectNonUINonIOBrowserThreadsToTaskScheduler()).
   std::unique_ptr<BrowserProcessSubThread> db_thread_;
   std::unique_ptr<BrowserProcessSubThread> file_user_blocking_thread_;
   std::unique_ptr<BrowserProcessSubThread> file_thread_;

content/browser/browser_thread_impl.h

@@ -5,7 +5,8 @@
 #ifndef CONTENT_BROWSER_BROWSER_THREAD_IMPL_H_
 #define CONTENT_BROWSER_BROWSER_THREAD_IMPL_H_
 
-#include "base/threading/platform_thread.h"
+#include "base/memory/ref_counted.h"
+#include "base/single_thread_task_runner.h"
 #include "base/threading/thread.h"
 #include "content/common/content_export.h"
 #include "content/public/browser/browser_thread.h"
@@ -32,8 +33,26 @@ class CONTENT_EXPORT BrowserThreadImpl : public BrowserThread,
   bool StartWithOptions(const Options& options);
   bool StartAndWaitForTesting();
 
+  // Redirects tasks posted to |identifier| to |task_runner|.
+  static void RedirectThreadIDToTaskRunner(
+      BrowserThread::ID identifier,
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner);
+
+  // Makes this |identifier| no longer accept tasks and synchronously flushes
+  // any tasks previously posted to it.
+  // Can only be called after a matching RedirectThreadIDToTaskRunner call.
+  static void StopRedirectionOfThreadID(BrowserThread::ID identifier);
+
   static void ShutdownThreadPool();
 
+  // Resets globals for |identifier|. Used in tests to clear global state that
+  // would otherwise leak to the next test. Globals are not otherwise fully
+  // cleaned up in ~BrowserThreadImpl() as there are subtle differences between
+  // UNINITIALIZED and SHUTDOWN state (e.g. globals.task_runners are kept around
+  // on shutdown). Must be called after ~BrowserThreadImpl() for the given
+  // |identifier|.
+  static void ResetGlobalsForTesting(BrowserThread::ID identifier);
+
  protected:
   void Init() override;
   void Run(base::RunLoop* run_loop) override;

content/public/browser/content_browser_client.cc

@@ -441,4 +441,8 @@ ContentBrowserClient::GetMemoryCoordinatorDelegate() {
   return nullptr;
 }
 
+bool ContentBrowserClient::RedirectNonUINonIOBrowserThreadsToTaskScheduler() {
+  return false;
+}
+
 }  // namespace content

content/public/browser/content_browser_client.h

@@ -814,6 +814,10 @@ class CONTENT_EXPORT ContentBrowserClient {
 
   // Performs any necessary PostTask API redirection to the task scheduler.
   virtual void PerformExperimentalTaskSchedulerRedirections() {}
+
+  // If this returns true, all BrowserThreads (but UI/IO) that support it on
+  // this platform will experimentally be redirected to TaskScheduler.
+  virtual bool RedirectNonUINonIOBrowserThreadsToTaskScheduler();
 };
 
 }  // namespace content

content/public/test/test_browser_thread.cc

@@ -40,15 +40,34 @@ class TestBrowserThreadImpl : public BrowserThreadImpl {
 };
 
 TestBrowserThread::TestBrowserThread(BrowserThread::ID identifier)
-    : impl_(new TestBrowserThreadImpl(identifier)) {
-}
+    : impl_(new TestBrowserThreadImpl(identifier)), identifier_(identifier) {}
 
 TestBrowserThread::TestBrowserThread(BrowserThread::ID identifier,
                                      base::MessageLoop* message_loop)
-    : impl_(new TestBrowserThreadImpl(identifier, message_loop)) {}
+    : impl_(new TestBrowserThreadImpl(identifier, message_loop)),
+      identifier_(identifier) {}
 
 TestBrowserThread::~TestBrowserThread() {
-  Stop();
+  // The upcoming BrowserThreadImpl::ResetGlobalsForTesting() call requires that
+  // |impl_| have triggered the shutdown phase for its BrowserThread::ID. This
+  // either happens when the thread is stopped (if real) or destroyed (when fake
+  // -- i.e. using an externally provided MessageLoop).
+  impl_.reset();
+
+  // Resets BrowserThreadImpl's globals so that |impl_| is no longer bound to
+  // |identifier_|. This is fine since the underlying MessageLoop has already
+  // been flushed and deleted in Stop(). In the case of an externally provided
+  // MessageLoop however, this means that TaskRunners obtained through
+  // |BrowserThreadImpl::GetTaskRunnerForThread(identifier_)| will no longer
+  // recognize their BrowserThreadImpl for RunsTasksOnCurrentThread(). This
+  // happens most often when such verifications are made from
+  // MessageLoop::DestructionObservers. Callers that care to work around that
+  // should instead use this shutdown sequence:
+  //   1) TestBrowserThread::Stop()
+  //   2) ~MessageLoop()
+  //   3) ~TestBrowserThread()
+  // (~TestBrowserThreadBundle() does this).
+  BrowserThreadImpl::ResetGlobalsForTesting(identifier_);
 }
 
 bool TestBrowserThread::Start() {

content/public/test/test_browser_thread.h

@@ -52,6 +52,8 @@ class TestBrowserThread {
  private:
   std::unique_ptr<TestBrowserThreadImpl> impl_;
 
+  const BrowserThread::ID identifier_;
+
   DISALLOW_COPY_AND_ASSIGN(TestBrowserThread);
 };
 

content/public/test/test_browser_thread_bundle.cc

@@ -4,6 +4,7 @@
 
 #include "content/public/test/test_browser_thread_bundle.h"
 
+#include "base/logging.h"
 #include "base/message_loop/message_loop.h"
 #include "base/run_loop.h"
 #include "content/browser/browser_thread_impl.h"
@@ -27,29 +28,35 @@ TestBrowserThreadBundle::~TestBrowserThreadBundle() {
   BrowserThreadImpl::FlushThreadPoolHelperForTesting();
 
   // To ensure a clean teardown, each thread's message loop must be flushed
-  // just before the thread is destroyed. But destroying a fake thread does not
+  // just before the thread is destroyed. But stopping a fake thread does not
   // automatically flush the message loop, so we have to do it manually.
   // See http://crbug.com/247525 for discussion.
   base::RunLoop().RunUntilIdle();
-  io_thread_.reset();
+  io_thread_->Stop();
   base::RunLoop().RunUntilIdle();
-  cache_thread_.reset();
+  cache_thread_->Stop();
   base::RunLoop().RunUntilIdle();
-  process_launcher_thread_.reset();
+  process_launcher_thread_->Stop();
   base::RunLoop().RunUntilIdle();
-  file_user_blocking_thread_.reset();
+  file_user_blocking_thread_->Stop();
   base::RunLoop().RunUntilIdle();
-  file_thread_.reset();
+  file_thread_->Stop();
   base::RunLoop().RunUntilIdle();
-  db_thread_.reset();
+  db_thread_->Stop();
   base::RunLoop().RunUntilIdle();
   // This is the point at which we normally shut down the thread pool. So flush
   // it again in case any shutdown tasks have been posted to the pool from the
   // threads above.
   BrowserThreadImpl::FlushThreadPoolHelperForTesting();
   base::RunLoop().RunUntilIdle();
-  ui_thread_.reset();
+  ui_thread_->Stop();
   base::RunLoop().RunUntilIdle();
+
+  // |message_loop_| needs to explicitly go away before fake threads in order
+  // for DestructionObservers hooked to |message_loop_| to be able to invoke
+  // BrowserThread::CurrentlyOn() -- ref. ~TestBrowserThread().
+  CHECK(message_loop_->IsIdleForTesting());
+  message_loop_.reset();
 }
 
 void TestBrowserThreadBundle::Init() {