Attempt to fix audio wedges by restarting all streams on OSX.

Introduces two new methods to AudioOutputDispatcher:
CloseStreamsForWedgeFix() and RestartStreamsForWedgeFix().

Respectively, each method closes or restarts all active
streams owned by a given dispatcher.  The process is
completely transparent to upstream clients.

A new method on AudioManager, FixWedgedAudio() calls
CloseStreamsForWedgeFix() for all dispatchers and then
calls RestartStreamsForWedgeFix() afterward.

FixWedgedAudio() is called by each AudioOutputController
when a wedge is detected.  Multiple in flight wedge checks
are serialized by the audio thread.  The hope is that wedges
will be fixed before the next WedgeCheck() fires.

While the methods are available on all platforms, FixWedgedAudio()
is only wired up on OSX.

BUG=160920
TEST=unittest. fake wedge and observe stream recreation.
R=scherkus@chromium.org

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=238325

Review URL: https://codereview.chromium.org/61203008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@238501 0039d316-1c4b-4281-b951-d872f2087c98

media/audio/audio_manager.h

@@ -175,6 +175,11 @@ class MEDIA_EXPORT AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) = 0;
 
+  // Called when a component has detected a OS level audio wedge.  Shuts down
+  // all active audio streams and then restarts them transparently.  See
+  // http://crbug.com/160920
+  virtual void FixWedgedAudio() = 0;
+
  protected:
   AudioManager();
 

media/audio/audio_manager_base.cc

@@ -418,4 +418,32 @@ int AudioManagerBase::GetUserBufferSize() {
   return 0;
 }
 
+void AudioManagerBase::FixWedgedAudio() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+#if defined(OS_MACOSX)
+  // Through trial and error, we've found that one way to restore audio after a
+  // hang is to close all outstanding audio streams.  Once all streams have been
+  // closed, new streams appear to work correctly.
+  //
+  // In Chrome terms, this means we need to ask all AudioOutputDispatchers to
+  // close all Open()'d streams.  Once all streams across all dispatchers have
+  // been closed, we ask for all previously Start()'d streams to be recreated
+  // using the same AudioSourceCallback they had before.
+  //
+  // Since this operation takes place on the audio thread we can be sure that no
+  // other state-changing stream operations will take place while the fix is in
+  // progress.
+  //
+  // See http://crbug.com/160920 for additional details.
+  for (AudioOutputDispatchers::iterator it = output_dispatchers_.begin();
+       it != output_dispatchers_.end(); ++it) {
+    (*it)->dispatcher->CloseStreamsForWedgeFix();
+  }
+  for (AudioOutputDispatchers::iterator it = output_dispatchers_.begin();
+       it != output_dispatchers_.end(); ++it) {
+    (*it)->dispatcher->RestartStreamsForWedgeFix();
+  }
+#endif
+}
+
 }  // namespace media

media/audio/audio_manager_base.h

@@ -115,6 +115,8 @@ class MEDIA_EXPORT AudioManagerBase : public AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) OVERRIDE;
 
+  virtual void FixWedgedAudio() OVERRIDE;
+
  protected:
   AudioManagerBase();
 

media/audio/audio_output_controller.cc

@@ -478,8 +478,15 @@ void AudioOutputController::WedgeCheck() {
 
   // If we should be playing and we haven't, that's a wedge.
   if (state_ == kPlaying) {
-    UMA_HISTOGRAM_BOOLEAN("Media.AudioOutputControllerPlaybackStartupSuccess",
-                          base::AtomicRefCountIsOne(&on_more_io_data_called_));
+    const bool playback_success =
+        base::AtomicRefCountIsOne(&on_more_io_data_called_);
+
+    UMA_HISTOGRAM_BOOLEAN(
+        "Media.AudioOutputControllerPlaybackStartupSuccess", playback_success);
+
+    // Let the AudioManager try and fix it.
+    if (!playback_success)
+      audio_manager_->FixWedgedAudio();
   }
 }
 

media/audio/audio_output_dispatcher.h

@@ -66,6 +66,13 @@ class MEDIA_EXPORT AudioOutputDispatcher
   // Called on the audio thread when the AudioManager is shutting down.
   virtual void Shutdown() = 0;
 
+  // Called by the AudioManager to restart streams when a wedge is detected.  A
+  // wedge means the OS failed to request any audio after StartStream().  When a
+  // wedge is detected all streams across all dispatchers must be closed.  After
+  // all streams are closed, streams are restarted.  See http://crbug.com/160920
+  virtual void CloseStreamsForWedgeFix() = 0;
+  virtual void RestartStreamsForWedgeFix() = 0;
+
   // Accessor to the input device id used by unified IO.
   const std::string& input_device_id() const { return input_device_id_; }
 

media/audio/audio_output_dispatcher_impl.cc

@@ -59,6 +59,8 @@ bool AudioOutputDispatcherImpl::StartStream(
     AudioOutputStream::AudioSourceCallback* callback,
     AudioOutputProxy* stream_proxy) {
   DCHECK(message_loop_->BelongsToCurrentThread());
+  DCHECK(proxy_to_physical_map_.find(stream_proxy) ==
+         proxy_to_physical_map_.end());
 
   if (idle_streams_.empty() && !CreateAndOpenStream())
     return false;
@@ -186,4 +188,21 @@ void AudioOutputDispatcherImpl::ClosePendingStreams() {
   }
 }
 
+void AudioOutputDispatcherImpl::CloseStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  while (!pausing_streams_.empty()) {
+    idle_streams_.push_back(pausing_streams_.back());
+    pausing_streams_.pop_back();
+  }
+  ClosePendingStreams();
+}
+
+void AudioOutputDispatcherImpl::RestartStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  // Should only be called when the dispatcher is used with fake streams which
+  // don't need to be shutdown or restarted.
+  CHECK_EQ(params_.format(), AudioParameters::AUDIO_FAKE);
+}
+
 }  // namespace media

media/audio/audio_output_dispatcher_impl.h

@@ -59,6 +59,9 @@ class MEDIA_EXPORT AudioOutputDispatcherImpl : public AudioOutputDispatcher {
 
   virtual void Shutdown() OVERRIDE;
 
+  virtual void CloseStreamsForWedgeFix() OVERRIDE;
+  virtual void RestartStreamsForWedgeFix() OVERRIDE;
+
  private:
   typedef std::map<AudioOutputProxy*, AudioOutputStream*> AudioStreamMap;
   friend class base::RefCountedThreadSafe<AudioOutputDispatcherImpl>;

media/audio/audio_output_proxy_unittest.cc

@@ -748,4 +748,64 @@ TEST_F(AudioOutputResamplerTest, LowLatencyOpenEventuallyFails) {
   EXPECT_FALSE(stream3.start_called());
 }
 
+// Ensures the methods used to fix audio output wedges are working correctly.
+TEST_F(AudioOutputResamplerTest, WedgeFix) {
+  MockAudioOutputStream stream1(&manager_, params_);
+  MockAudioOutputStream stream2(&manager_, params_);
+  MockAudioOutputStream stream3(&manager_, params_);
+
+  // Setup the mock such that all three streams are successfully created.
+  EXPECT_CALL(manager(), MakeAudioOutputStream(_, _, _))
+      .WillOnce(Return(&stream1))
+      .WillOnce(Return(&stream2))
+      .WillOnce(Return(&stream3));
+
+  // Stream1 should be able to successfully open and start.
+  EXPECT_CALL(stream1, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream1, Close());
+  EXPECT_CALL(stream1, SetVolume(_));
+  EXPECT_CALL(stream2, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream2, Close());
+
+  // Open and start the first proxy and stream.
+  AudioOutputProxy* proxy1 = new AudioOutputProxy(resampler_.get());
+  EXPECT_TRUE(proxy1->Open());
+  proxy1->Start(&callback_);
+  OnStart();
+
+  // Open, but do not start the second proxy.
+  AudioOutputProxy* proxy2 = new AudioOutputProxy(resampler_.get());
+  EXPECT_TRUE(proxy2->Open());
+
+  // Wait for stream to timeout and shutdown.
+  WaitForCloseTimer(kTestCloseDelayMs);
+
+  resampler_->CloseStreamsForWedgeFix();
+
+  // Stream3 should take Stream1's place after RestartStreamsForWedgeFix().
+  EXPECT_CALL(stream3, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream3, Close());
+  EXPECT_CALL(stream3, SetVolume(_));
+
+  resampler_->RestartStreamsForWedgeFix();
+  OnStart();
+
+  // Perform the required Stop()/Close() shutdown dance for each proxy.
+  proxy2->Close();
+  proxy1->Stop();
+  proxy1->Close();
+
+  // Wait for all of the messages to fly and then verify stream behavior.
+  WaitForCloseTimer(kTestCloseDelayMs);
+  EXPECT_TRUE(stream1.stop_called());
+  EXPECT_TRUE(stream1.start_called());
+  EXPECT_FALSE(stream2.stop_called());
+  EXPECT_FALSE(stream2.start_called());
+  EXPECT_TRUE(stream3.stop_called());
+  EXPECT_TRUE(stream3.start_called());
+}
+
 }  // namespace media

media/audio/audio_output_resampler.cc

@@ -291,6 +291,30 @@ void AudioOutputResampler::Shutdown() {
   DCHECK(callbacks_.empty());
 }
 
+void AudioOutputResampler::CloseStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  // Stop and close all active streams.  Once all streams across all dispatchers
+  // have been closed the AudioManager will call RestartStreamsForWedgeFix().
+  for (CallbackMap::iterator it = callbacks_.begin(); it != callbacks_.end();
+       ++it) {
+    dispatcher_->StopStream(it->first);
+    dispatcher_->CloseStream(it->first);
+  }
+
+  // Close all idle streams as well.
+  dispatcher_->CloseStreamsForWedgeFix();
+}
+
+void AudioOutputResampler::RestartStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  for (CallbackMap::iterator it = callbacks_.begin(); it != callbacks_.end();
+       ++it) {
+    dispatcher_->OpenStream();
+    dispatcher_->StartStream(it->second, it->first);
+  }
+}
+
 OnMoreDataConverter::OnMoreDataConverter(const AudioParameters& input_params,
                                          const AudioParameters& output_params)
     : io_ratio_(static_cast<double>(input_params.GetBytesPerSecond()) /

media/audio/audio_output_resampler.h

@@ -53,6 +53,8 @@ class MEDIA_EXPORT AudioOutputResampler : public AudioOutputDispatcher {
                                double volume) OVERRIDE;
   virtual void CloseStream(AudioOutputProxy* stream_proxy) OVERRIDE;
   virtual void Shutdown() OVERRIDE;
+  virtual void CloseStreamsForWedgeFix() OVERRIDE;
+  virtual void RestartStreamsForWedgeFix() OVERRIDE;
 
  private:
   friend class base::RefCountedThreadSafe<AudioOutputResampler>;

media/audio/mock_audio_manager.cc

@@ -103,4 +103,6 @@ std::string MockAudioManager::GetAssociatedOutputDeviceID(
   return std::string();
 }
 
+void MockAudioManager::FixWedgedAudio() {}
+
 }  // namespace media.

media/audio/mock_audio_manager.h

@@ -67,6 +67,8 @@ class MockAudioManager : public media::AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) OVERRIDE;
 
+  virtual void FixWedgedAudio() OVERRIDE;
+
  protected:
   virtual ~MockAudioManager();