Determine the audible state in MediaSourcePlayer

We set the audible state every time after the pipeline
has successfully decoded the frame: to false if prerolling
and true otherwise, taking the volume into account.

Also, we set the state to false if the flow is going to be
 interrupted because of a coded error, abort or Pause.

As well, we set audible state to false when the starvation
happens and there is no data in the MediaDecoderJob queue,
we believe this indicates the demuxer freeze.

BUG=414810

Review URL: https://codereview.chromium.org/1008093002

Cr-Commit-Position: refs/heads/master@{#322102}

media/base/android/audio_decoder_job.h

@@ -34,6 +34,7 @@ class AudioDecoderJob : public MediaDecoderJob {
 
   // Sets the volume of the audio output.
   void SetVolume(double volume);
+  double volume() const { return volume_; }
 
   // Sets the base timestamp for |audio_timestamp_helper_|.
   void SetBaseTimestamp(base::TimeDelta base_timestamp);

media/base/android/media_decoder_job.h

@@ -108,6 +108,9 @@ class MediaDecoderJob {
 
   bool prerolling() const { return prerolling_; }
 
+  // Returns true if this object has data to decode.
+  bool HasData() const;
+
  protected:
   // Creates a new MediaDecoderJob instance.
   // |decoder_task_runner| - Thread on which the decoder task will run.
@@ -162,9 +165,6 @@ class MediaDecoderJob {
   // Queues an access unit into |media_codec_bridge_|'s input buffer.
   MediaCodecStatus QueueInputBuffer(const AccessUnit& unit);
 
-  // Returns true if this object has data to decode.
-  bool HasData() const;
-
   // Initiates a request for more data.
   // |done_cb| is called when more data is available in |received_data_|.
   void RequestData(const base::Closure& done_cb);

media/base/android/media_source_player.cc

@@ -185,6 +185,8 @@ void MediaSourcePlayer::Release() {
   playing_ = false;
 
   decoder_starvation_callback_.Cancel();
+
+  SetAudible(false);
   DetachListener();
 }
 
@@ -229,6 +231,7 @@ void MediaSourcePlayer::OnDemuxerConfigsAvailable(
     const DemuxerConfigs& configs) {
   DVLOG(1) << __FUNCTION__;
   DCHECK(!HasAudio() && !HasVideo());
+
   duration_ = configs.duration;
 
   audio_decoder_job_->SetDemuxerConfigs(configs);
@@ -390,6 +393,7 @@ void MediaSourcePlayer::ProcessPendingEvents() {
 
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling PREFETCH_REQUEST_EVENT.";
+
     int count = (AudioFinished() ? 0 : 1) + (VideoFinished() ? 0 : 1);
 
     // It is possible that all streams have finished decode, yet starvation
@@ -484,12 +488,18 @@ void MediaSourcePlayer::MediaDecoderCallback(
     return;
   }
 
-  if (status == MEDIA_CODEC_OUTPUT_END_OF_STREAM)
+  if (status == MEDIA_CODEC_OUTPUT_END_OF_STREAM) {
+    if (is_audio)
+      SetAudible(false);
     return;
+  }
 
   if (!playing_) {
     if (is_clock_manager)
       interpolator_.StopInterpolating();
+
+    if (is_audio)
+      SetAudible(false);
     return;
   }
 
@@ -498,6 +508,9 @@ void MediaSourcePlayer::MediaDecoderCallback(
       DVLOG(2) << __FUNCTION__ << ": Key was added during decoding.";
       ResumePlaybackAfterKeyAdded();
     } else {
+      if (is_audio)
+        SetAudible(false);
+
       is_waiting_for_key_ = true;
       manager()->OnWaitingForDecryptionKey(player_id());
     }
@@ -517,8 +530,11 @@ void MediaSourcePlayer::MediaDecoderCallback(
   // If the status is MEDIA_CODEC_ABORT, stop decoding new data. The player is
   // in the middle of a seek or stop event and needs to wait for the IPCs to
   // come.
-  if (status == MEDIA_CODEC_ABORT)
+  if (status == MEDIA_CODEC_ABORT) {
+    if (is_audio)
+      SetAudible(false);
     return;
+  }
 
   if (prerolling_ && IsPrerollFinished(is_audio)) {
     if (IsPrerollFinished(!is_audio)) {
@@ -528,6 +544,13 @@ void MediaSourcePlayer::MediaDecoderCallback(
     return;
   }
 
+  // We successfully decoded a frame and going to the next one.
+  // Set the audible state.
+  if (is_audio) {
+    bool is_audible = !prerolling_ && audio_decoder_job_->volume() > 0;
+    SetAudible(is_audible);
+  }
+
   if (is_clock_manager) {
     // If we have a valid timestamp, start the starvation callback. Otherwise,
     // reset the |start_time_ticks_| so that the next frame will not suffer
@@ -639,6 +662,14 @@ bool MediaSourcePlayer::VideoFinished() {
 
 void MediaSourcePlayer::OnDecoderStarved() {
   DVLOG(1) << __FUNCTION__;
+
+  if (HasAudio()) {
+    // If the starvation timer fired but there are no encoded frames
+    // in the queue we believe the demuxer (i.e. renderer process) froze.
+    if (!audio_decoder_job_->HasData())
+      SetAudible(false);
+  }
+
   SetPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
   ProcessPendingEvents();
 }

media/base/android/media_source_player_unittest.cc

@@ -46,7 +46,9 @@ class MockMediaPlayerManager : public MediaPlayerManager {
         playback_completed_(false),
         num_resources_requested_(0),
         num_metadata_changes_(0),
-        timestamp_updated_(false) {}
+        timestamp_updated_(false),
+        is_audible_(false),
+        is_delay_expired_(false) {}
   ~MockMediaPlayerManager() override {}
 
   // MediaPlayerManager implementation.
@@ -75,12 +77,15 @@ class MockMediaPlayerManager : public MediaPlayerManager {
                       const base::TimeDelta& current_time) override {}
   void OnError(int player_id, int error) override {}
   void OnVideoSizeChanged(int player_id, int width, int height) override {}
-  void OnAudibleStateChanged(int player_id, bool is_audible_now) override {}
   void OnWaitingForDecryptionKey(int player_id) override {}
   MediaPlayerAndroid* GetFullscreenPlayer() override { return NULL; }
   MediaPlayerAndroid* GetPlayer(int player_id) override { return NULL; }
   void RequestFullScreen(int player_id) override {}
 
+  void OnAudibleStateChanged(int player_id, bool is_audible_now) override {
+    is_audible_ = is_audible_now;
+  }
+
   bool playback_completed() const {
     return playback_completed_;
   }
@@ -105,6 +110,18 @@ class MockMediaPlayerManager : public MediaPlayerManager {
     timestamp_updated_ = false;
   }
 
+  bool is_audible() const {
+    return is_audible_;
+  }
+
+  bool is_delay_expired() const {
+    return is_delay_expired_;
+  }
+
+  void SetDelayExpired(bool value) {
+    is_delay_expired_ = value;
+  }
+
  private:
   base::MessageLoop* message_loop_;
   bool playback_completed_;
@@ -114,6 +131,10 @@ class MockMediaPlayerManager : public MediaPlayerManager {
   int num_metadata_changes_;
   // Playback timestamp was updated.
   bool timestamp_updated_;
+  // Audible state of the pipeline
+  bool is_audible_;
+  // Helper flag to ensure delay for WaitForDelay().
+  bool is_delay_expired_;
 
   DISALLOW_COPY_AND_ASSIGN(MockMediaPlayerManager);
 };
@@ -171,6 +192,7 @@ class MediaSourcePlayerTest : public testing::Test {
                 GURL()),
         decoder_callback_hook_executed_(false),
         surface_texture_a_is_next_(true) {}
+
   ~MediaSourcePlayerTest() override {}
 
  protected:
@@ -534,6 +556,37 @@ class MediaSourcePlayerTest : public testing::Test {
     EXPECT_LE(target_timestamp, player_.GetCurrentTime());
   }
 
+  void PlayAudioForTimeInterval(const base::TimeDelta& start_timestamp,
+                                const base::TimeDelta& target_timestamp ) {
+
+    DemuxerData data = CreateReadFromDemuxerAckForAudio(1);
+    int current_timestamp = start_timestamp.InMilliseconds();
+    int stop_timestamp = target_timestamp.InMilliseconds();
+    while (current_timestamp < stop_timestamp) {
+      data.access_units[0].timestamp =
+          base::TimeDelta::FromMilliseconds(current_timestamp);
+      player_.OnDemuxerDataAvailable(data);
+      current_timestamp += 30;
+      WaitForAudioDecodeDone();
+    }
+  }
+
+  void WaitForDelay(const base::TimeDelta& delay) {
+    // Let the message_loop_ process events.
+    // We post delayed task and RunUnitilIdle() until it signals.
+
+    manager_.SetDelayExpired(false);
+    message_loop_.PostDelayedTask(
+        FROM_HERE,
+        base::Bind(&MockMediaPlayerManager::SetDelayExpired,
+                   base::Unretained(&manager_),
+                   true),
+        delay);
+
+    while (!manager_.is_delay_expired())
+      message_loop_.RunUntilIdle();
+  }
+
   DemuxerData CreateReadFromDemuxerAckWithConfigChanged(
       bool is_audio,
       int config_unit_index,
@@ -837,6 +890,7 @@ class MediaSourcePlayerTest : public testing::Test {
     return GetMediaDecoderJob(is_audio)->drain_decoder_;
   }
 
+ protected:
   base::MessageLoop message_loop_;
   MockMediaPlayerManager manager_;
   MockDemuxerAndroid* demuxer_;  // Owned by |player_|.
@@ -857,6 +911,8 @@ class MediaSourcePlayerTest : public testing::Test {
   bool surface_texture_a_is_next_;
   int next_texture_id_;
 
+  bool verify_not_audible_is_called_;
+
   DISALLOW_COPY_AND_ASSIGN(MediaSourcePlayerTest);
 };
 
@@ -888,6 +944,107 @@ TEST_F(MediaSourcePlayerTest, StartAudioDecoderWithInvalidConfig) {
   EXPECT_FALSE(GetMediaCodecBridge(true));
 }
 
+// timav
+TEST_F(MediaSourcePlayerTest, AudioDecoderSetsAudibleState) {
+  SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();
+
+  // No data arrived yet
+  EXPECT_FALSE(manager_.is_audible());
+
+  // Initialize decoder
+  StartAudioDecoderJob();
+  player_.SetVolume(1.0);
+
+  // Process frames until prerolling is done.
+  SeekPlayerWithAbort(true, base::TimeDelta::FromMilliseconds(100));
+  EXPECT_TRUE(IsPrerolling(true));
+  PrerollDecoderToTime(
+      true, base::TimeDelta(), base::TimeDelta::FromMilliseconds(100), false);
+  EXPECT_TRUE(IsPrerolling(false));
+
+  // Send more packets
+  PlayAudioForTimeInterval(base::TimeDelta::FromMilliseconds(150),
+                           base::TimeDelta::FromMilliseconds(220));
+
+  // The player should trigger audible status
+  EXPECT_TRUE(manager_.is_audible());
+
+  // The player release should report a non-audible state.
+  ReleasePlayer();
+  EXPECT_FALSE(manager_.is_audible());
+}
+
+TEST_F(MediaSourcePlayerTest, AudioDecoderRemovesAudibleStateWhenPaused) {
+  SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();
+
+  // No data arrived yet
+  EXPECT_FALSE(manager_.is_audible());
+
+  // Initialize decoder
+  StartAudioDecoderJob();
+  player_.SetVolume(1.0);
+
+  // Process frames until prerolling is done.
+  SeekPlayerWithAbort(true, base::TimeDelta::FromMilliseconds(100));
+  EXPECT_TRUE(IsPrerolling(true));
+  PrerollDecoderToTime(
+      true, base::TimeDelta(), base::TimeDelta::FromMilliseconds(100), false);
+  EXPECT_TRUE(IsPrerolling(false));
+
+  // Send more packets
+  PlayAudioForTimeInterval(base::TimeDelta::FromMilliseconds(150),
+                           base::TimeDelta::FromMilliseconds(220));
+
+  // The player should trigger audible status
+  EXPECT_TRUE(manager_.is_audible());
+
+  // Pause the player
+  player_.Pause(true);
+
+  // Send more packets
+  PlayAudioForTimeInterval(base::TimeDelta::FromMilliseconds(240),
+                           base::TimeDelta::FromMilliseconds(280));
+
+  // The player should trigger audible status again
+  EXPECT_FALSE(manager_.is_audible());
+
+  player_.Release();
+}
+
+TEST_F(MediaSourcePlayerTest, AudioDecoderRemovesAudibleStateWhenIdle) {
+  SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();
+
+  // No data arrived yet
+  EXPECT_FALSE(manager_.is_audible());
+
+  // Initialize decoder
+  StartAudioDecoderJob();
+  player_.SetVolume(1.0);
+
+  // Process frames until prerolling is done.
+  SeekPlayerWithAbort(true, base::TimeDelta::FromMilliseconds(100));
+  EXPECT_TRUE(IsPrerolling(true));
+  PrerollDecoderToTime(
+      true, base::TimeDelta(), base::TimeDelta::FromMilliseconds(100), false);
+  EXPECT_TRUE(IsPrerolling(false));
+
+  // Send more packets
+  PlayAudioForTimeInterval(base::TimeDelta::FromMilliseconds(150),
+                           base::TimeDelta::FromMilliseconds(220));
+
+  // The player should trigger audible status
+  EXPECT_TRUE(manager_.is_audible());
+
+  // Simulate the freeze on demuxer: wait for 300 ms
+  WaitForDelay(base::TimeDelta::FromMilliseconds(300));
+
+  // By this time the player should have reported
+  // that there is no audio.
+  EXPECT_FALSE(manager_.is_audible());
+
+  ReleasePlayer();
+}
+
 TEST_F(MediaSourcePlayerTest, StartVideoCodecWithValidSurface) {
   SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();