Add a hang monitor for the audio thread.

We've seen a bunch of reports in the field that the audio thread is
getting hung for significant (perhaps indefinite) amounts of time;
to help debug these reports, add a hang monitor which will crash the
browser process and ensure we get a thread dump of what's going on.

The hang timer is set extremely conservatively right now, at 1 minute,
which is insanely long for audio tasks. I'll monitor crash reports and
adjust it as necessary to find culprits.

The hang monitor uses the browser I/O thread for hang checks, which is
similar to what other hung task monitors seem to do. To do this I had
to move AudioManager and MidiManager creation to after the browser
threads have been spun up. Which shouldn't be an issue...

BUG=422522, 469827
TEST=hang time exceeded ==> crash.

Review URL: https://codereview.chromium.org/1036703002

Cr-Commit-Position: refs/heads/master@{#322276}

content/browser/browser_main_loop.cc

@@ -515,15 +515,6 @@ void BrowserMainLoop::MainMessageLoopStart() {
     TRACE_EVENT0("startup", "BrowserMainLoop::Subsystem:MediaFeatures");
     media::InitializeCPUSpecificMediaFeatures();
   }
-  {
-    TRACE_EVENT0("startup", "BrowserMainLoop::Subsystem:AudioMan");
-    audio_manager_.reset(media::AudioManager::Create(
-        MediaInternals::GetInstance()));
-  }
-  {
-    TRACE_EVENT0("startup", "BrowserMainLoop::Subsystem:MidiManager");
-    midi_manager_.reset(media::MidiManager::Create());
-  }
   {
     TRACE_EVENT0("startup",
                  "BrowserMainLoop::Subsystem:ContentWebUIController");
@@ -1085,6 +1076,17 @@ int BrowserMainLoop::BrowserThreadsStarted() {
   BrowserGpuChannelHostFactory::Initialize(established_gpu_channel);
 #endif
 
+  {
+    TRACE_EVENT0("startup", "BrowserThreadsStarted::Subsystem:AudioMan");
+    audio_manager_.reset(media::AudioManager::CreateWithHangTimer(
+        MediaInternals::GetInstance(), io_thread_->task_runner()));
+  }
+
+  {
+    TRACE_EVENT0("startup", "BrowserThreadsStarted::Subsystem:MidiManager");
+    midi_manager_.reset(media::MidiManager::Create());
+  }
+
 #if defined(OS_LINUX) && defined(USE_UDEV)
   device_monitor_linux_.reset(new DeviceMonitorLinux());
 #elif defined(OS_MACOSX)

media/audio/audio_manager.cc

@@ -9,12 +9,66 @@
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/message_loop/message_loop.h"
+#include "build/build_config.h"
 #include "media/audio/fake_audio_log_factory.h"
 
 namespace media {
 namespace {
 AudioManager* g_last_created = NULL;
-static base::LazyInstance<FakeAudioLogFactory>::Leaky g_fake_log_factory =
+
+// Helper class for managing global AudioManager data and hang timers. If the
+// audio thread is unresponsive for more than a minute we want to crash the
+// process so we can catch offenders quickly in the field.
+class AudioManagerHelper {
+ public:
+  AudioManagerHelper() : max_hung_task_time_(base::TimeDelta::FromMinutes(1)) {}
+  ~AudioManagerHelper() {}
+
+  void StartHangTimer(
+      const scoped_refptr<base::SingleThreadTaskRunner>& monitor_task_runner) {
+    CHECK(!monitor_task_runner_);
+    monitor_task_runner_ = monitor_task_runner;
+    UpdateLastAudioThreadTimeTick();
+    CrashOnAudioThreadHang();
+  }
+
+  // Runs on |monitor_task_runner| typically, but may be started on any thread.
+  void CrashOnAudioThreadHang() {
+    base::AutoLock lock(hang_lock_);
+    CHECK(base::TimeTicks::Now() - last_audio_thread_timer_tick_ <=
+          max_hung_task_time_);
+    monitor_task_runner_->PostDelayedTask(
+        FROM_HERE, base::Bind(&AudioManagerHelper::CrashOnAudioThreadHang,
+                              base::Unretained(this)),
+        max_hung_task_time_);
+  }
+
+  // Runs on the audio thread typically, but may be started on any thread.
+  void UpdateLastAudioThreadTimeTick() {
+    base::AutoLock lock(hang_lock_);
+    last_audio_thread_timer_tick_ = base::TimeTicks::Now();
+    g_last_created->GetTaskRunner()->PostDelayedTask(
+        FROM_HERE,
+        base::Bind(&AudioManagerHelper::UpdateLastAudioThreadTimeTick,
+                   base::Unretained(this)),
+        max_hung_task_time_ / 2);
+  }
+
+  AudioLogFactory* fake_log_factory() { return &fake_log_factory_; }
+
+ private:
+  FakeAudioLogFactory fake_log_factory_;
+
+  const base::TimeDelta max_hung_task_time_;
+  scoped_refptr<base::SingleThreadTaskRunner> monitor_task_runner_;
+
+  base::Lock hang_lock_;
+  base::TimeTicks last_audio_thread_timer_tick_;
+
+  DISALLOW_COPY_AND_ASSIGN(AudioManagerHelper);
+};
+
+static base::LazyInstance<AudioManagerHelper>::Leaky g_helper =
     LAZY_INSTANCE_INITIALIZER;
 }
 
@@ -35,9 +89,22 @@ AudioManager* AudioManager::Create(AudioLogFactory* audio_log_factory) {
   return g_last_created;
 }
 
+// static
+AudioManager* AudioManager::CreateWithHangTimer(
+    AudioLogFactory* audio_log_factory,
+    const scoped_refptr<base::SingleThreadTaskRunner>& monitor_task_runner) {
+  AudioManager* manager = Create(audio_log_factory);
+  // On OSX the audio thread is the UI thread, for which a hang monitor is not
+  // necessary.
+#if !defined(OS_MACOSX)
+  g_helper.Pointer()->StartHangTimer(monitor_task_runner);
+#endif
+  return manager;
+}
+
 // static
 AudioManager* AudioManager::CreateForTesting() {
-  return Create(g_fake_log_factory.Pointer());
+  return Create(g_helper.Pointer()->fake_log_factory());
 }
 
 // static

media/audio/audio_manager.h

@@ -34,6 +34,13 @@ class MEDIA_EXPORT AudioManager {
   // such |audio_log_factory| must outlive the AudioManager.
   static AudioManager* Create(AudioLogFactory* audio_log_factory);
 
+  // Similar to Create() except also schedules a monitor on the given task
+  // runner to ensure the audio thread is not stuck for more than 60 seconds; if
+  // a hang is detected, the process will be crashed.
+  static AudioManager* CreateWithHangTimer(
+      AudioLogFactory* audio_log_factory,
+      const scoped_refptr<base::SingleThreadTaskRunner>& monitor_task_runner);
+
   // Similar to Create() except uses a FakeAudioLogFactory for testing.
   static AudioManager* CreateForTesting();