Make media::AudioClock track frames written to compute time.

Instead of using AudioRendererAlgorithm's timestamp as the ending
timestamp and counting "backwards", count "forwards" from a starting
timestamp to compute the current media time. Doing so produces more
accurate time calculations during periods where the playback rate
is changing.

last_endpoint_timestamp() is replaced by a new method that computes
the amount of contiguous media data buffered by audio hardware. Using
this value gives a more accurate maximum time value to use when doing
linear interpolation.

BUG=367343,370634

Review URL: https://codereview.chromium.org/436053002

Cr-Commit-Position: refs/heads/master@{#288445}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@288445 0039d316-1c4b-4281-b951-d872f2087c98

media/filters/audio_clock.cc

@@ -4,144 +4,137 @@
 
 #include "media/filters/audio_clock.h"
 
+#include <algorithm>
+
 #include "base/logging.h"
 #include "media/base/buffers.h"
 
 namespace media {
 
-AudioClock::AudioClock(int sample_rate)
-    : sample_rate_(sample_rate), last_endpoint_timestamp_(kNoTimestamp()) {
+AudioClock::AudioClock(base::TimeDelta start_timestamp, int sample_rate)
+    : start_timestamp_(start_timestamp),
+      sample_rate_(sample_rate),
+      microseconds_per_frame_(
+          static_cast<double>(base::Time::kMicrosecondsPerSecond) /
+          sample_rate),
+      total_buffered_frames_(0),
+      current_media_timestamp_(start_timestamp),
+      audio_data_buffered_(0) {
 }
 
 AudioClock::~AudioClock() {
 }
 
-void AudioClock::WroteAudio(int frames,
+void AudioClock::WroteAudio(int frames_written,
+                            int frames_requested,
                             int delay_frames,
-                            float playback_rate,
-                            base::TimeDelta timestamp) {
-  CHECK_GT(playback_rate, 0);
-  CHECK(timestamp != kNoTimestamp());
-  DCHECK_GE(frames, 0);
+                            float playback_rate) {
+  DCHECK_GE(frames_written, 0);
+  DCHECK_LE(frames_written, frames_requested);
   DCHECK_GE(delay_frames, 0);
+  DCHECK_GE(playback_rate, 0);
+
+  // First write: initialize buffer with silence.
+  if (start_timestamp_ == current_media_timestamp_ && buffered_.empty())
+    PushBufferedAudioData(delay_frames, 0.0f);
+
+  // Move frames from |buffered_| into the computed timestamp based on
+  // |delay_frames|.
+  //
+  // The ordering of compute -> push -> pop eliminates unnecessary memory
+  // reallocations in cases where |buffered_| gets emptied.
+  int64_t frames_played =
+      std::max(INT64_C(0), total_buffered_frames_ - delay_frames);
+  current_media_timestamp_ += ComputeBufferedMediaTime(frames_played);
+  PushBufferedAudioData(frames_written, playback_rate);
+  PushBufferedAudioData(frames_requested - frames_written, 0.0f);
+  PopBufferedAudioData(frames_played);
+
+  // Update cached values.
+  double scaled_frames = 0;
+  double scaled_frames_at_same_rate = 0;
+  bool found_silence = false;
+  audio_data_buffered_ = false;
+  for (size_t i = 0; i < buffered_.size(); ++i) {
+    if (buffered_[i].playback_rate == 0) {
+      found_silence = true;
+      continue;
+    }
 
-  if (last_endpoint_timestamp_ == kNoTimestamp())
-    PushBufferedAudio(delay_frames, 0, kNoTimestamp());
-
-  TrimBufferedAudioToMatchDelay(delay_frames);
-  PushBufferedAudio(frames, playback_rate, timestamp);
+    audio_data_buffered_ = true;
 
-  last_endpoint_timestamp_ = timestamp;
-}
+    // Any buffered silence breaks our contiguous stretch of audio data.
+    if (found_silence)
+      break;
 
-void AudioClock::WroteSilence(int frames, int delay_frames) {
-  DCHECK_GE(frames, 0);
-  DCHECK_GE(delay_frames, 0);
+    scaled_frames += (buffered_[i].frames * buffered_[i].playback_rate);
 
-  if (last_endpoint_timestamp_ == kNoTimestamp())
-    PushBufferedAudio(delay_frames, 0, kNoTimestamp());
+    if (i == 0)
+      scaled_frames_at_same_rate = scaled_frames;
+  }
 
-  TrimBufferedAudioToMatchDelay(delay_frames);
-  PushBufferedAudio(frames, 0, kNoTimestamp());
+  contiguous_audio_data_buffered_ = base::TimeDelta::FromMicroseconds(
+      scaled_frames * microseconds_per_frame_);
+  contiguous_audio_data_buffered_at_same_rate_ =
+      base::TimeDelta::FromMicroseconds(scaled_frames_at_same_rate *
+                                        microseconds_per_frame_);
 }
 
-base::TimeDelta AudioClock::CurrentMediaTimestamp(
+base::TimeDelta AudioClock::CurrentMediaTimestampSinceWriting(
     base::TimeDelta time_since_writing) const {
-  int frames_to_skip =
-      static_cast<int>(time_since_writing.InSecondsF() * sample_rate_);
-  int silence_frames = 0;
-  for (size_t i = 0; i < buffered_audio_.size(); ++i) {
-    int frames = buffered_audio_[i].frames;
-    if (frames_to_skip > 0) {
-      if (frames <= frames_to_skip) {
-        frames_to_skip -= frames;
-        continue;
-      }
-      frames -= frames_to_skip;
-      frames_to_skip = 0;
-    }
-
-    // Account for silence ahead of the buffer closest to being played.
-    if (buffered_audio_[i].playback_rate == 0) {
-      silence_frames += frames;
-      continue;
-    }
-
-    // Multiply by playback rate as frames represent time-scaled audio.
-    return buffered_audio_[i].endpoint_timestamp -
-           base::TimeDelta::FromMicroseconds(
-               ((frames * buffered_audio_[i].playback_rate) + silence_frames) /
-               sample_rate_ * base::Time::kMicrosecondsPerSecond);
-  }
+  int64_t frames_played_since_writing = std::min(
+      total_buffered_frames_,
+      static_cast<int64_t>(time_since_writing.InSecondsF() * sample_rate_));
+  return current_media_timestamp_ +
+         ComputeBufferedMediaTime(frames_played_since_writing);
+}
 
-  // Either:
-  //   1) AudioClock is uninitialziated and we'll return kNoTimestamp()
-  //   2) All previously buffered audio has been replaced by silence,
-  //      meaning media time is now at the last endpoint
-  return last_endpoint_timestamp_;
+AudioClock::AudioData::AudioData(int64_t frames, float playback_rate)
+    : frames(frames), playback_rate(playback_rate) {
 }
 
-void AudioClock::TrimBufferedAudioToMatchDelay(int delay_frames) {
-  if (buffered_audio_.empty())
+void AudioClock::PushBufferedAudioData(int64_t frames, float playback_rate) {
+  if (frames == 0)
     return;
 
-  size_t i = buffered_audio_.size() - 1;
-  while (true) {
-    if (buffered_audio_[i].frames <= delay_frames) {
-      // Reached the end before accounting for all of |delay_frames|. This
-      // means we haven't written enough audio data yet to account for hardware
-      // delay. In this case, do nothing.
-      if (i == 0)
-        return;
-
-      // Keep accounting for |delay_frames|.
-      delay_frames -= buffered_audio_[i].frames;
-      --i;
-      continue;
-    }
+  total_buffered_frames_ += frames;
 
-    // All of |delay_frames| has been accounted for: adjust amount of frames
-    // left in current buffer. All preceeding elements with index < |i| should
-    // be considered played out and hence discarded.
-    buffered_audio_[i].frames = delay_frames;
-    break;
+  // Avoid creating extra elements where possible.
+  if (!buffered_.empty() && buffered_.back().playback_rate == playback_rate) {
+    buffered_.back().frames += frames;
+    return;
   }
 
-  // At this point |i| points at what will be the new head of |buffered_audio_|
-  // however if it contains no audio it should be removed as well.
-  if (buffered_audio_[i].frames == 0)
-    ++i;
-
-  buffered_audio_.erase(buffered_audio_.begin(), buffered_audio_.begin() + i);
+  buffered_.push_back(AudioData(frames, playback_rate));
 }
 
-void AudioClock::PushBufferedAudio(int frames,
-                                   float playback_rate,
-                                   base::TimeDelta endpoint_timestamp) {
-  if (playback_rate == 0)
-    DCHECK(endpoint_timestamp == kNoTimestamp());
+void AudioClock::PopBufferedAudioData(int64_t frames) {
+  DCHECK_LE(frames, total_buffered_frames_);
 
-  if (frames == 0)
-    return;
+  total_buffered_frames_ -= frames;
 
-  // Avoid creating extra elements where possible.
-  if (!buffered_audio_.empty() &&
-      buffered_audio_.back().playback_rate == playback_rate) {
-    buffered_audio_.back().frames += frames;
-    buffered_audio_.back().endpoint_timestamp = endpoint_timestamp;
-    return;
-  }
+  while (frames > 0) {
+    int64_t frames_to_pop = std::min(buffered_.front().frames, frames);
+    buffered_.front().frames -= frames_to_pop;
+    if (buffered_.front().frames == 0)
+      buffered_.pop_front();
 
-  buffered_audio_.push_back(
-      BufferedAudio(frames, playback_rate, endpoint_timestamp));
+    frames -= frames_to_pop;
+  }
 }
 
-AudioClock::BufferedAudio::BufferedAudio(int frames,
-                                         float playback_rate,
-                                         base::TimeDelta endpoint_timestamp)
-    : frames(frames),
-      playback_rate(playback_rate),
-      endpoint_timestamp(endpoint_timestamp) {
+base::TimeDelta AudioClock::ComputeBufferedMediaTime(int64_t frames) const {
+  DCHECK_LE(frames, total_buffered_frames_);
+
+  double scaled_frames = 0;
+  for (size_t i = 0; i < buffered_.size() && frames > 0; ++i) {
+    int64_t min_frames = std::min(buffered_[i].frames, frames);
+    scaled_frames += min_frames * buffered_[i].playback_rate;
+    frames -= min_frames;
+  }
+
+  return base::TimeDelta::FromMicroseconds(scaled_frames *
+                                           microseconds_per_frame_);
 }
 
 }  // namespace media

media/filters/audio_clock.h

@@ -18,59 +18,75 @@ namespace media {
 // a playback pipeline with large delay.
 class MEDIA_EXPORT AudioClock {
  public:
-  explicit AudioClock(int sample_rate);
+  AudioClock(base::TimeDelta start_timestamp, int sample_rate);
   ~AudioClock();
 
-  // |frames| amount of audio data scaled to |playback_rate| was written.
+  // |frames_written| amount of audio data scaled to |playback_rate| written.
+  // |frames_requested| amount of audio data requested by hardware.
   // |delay_frames| is the current amount of hardware delay.
-  // |timestamp| is the endpoint media timestamp of the audio data written.
-  void WroteAudio(int frames,
+  void WroteAudio(int frames_written,
+                  int frames_requested,
                   int delay_frames,
-                  float playback_rate,
-                  base::TimeDelta timestamp);
-
-  // |frames| amount of silence was written.
-  // |delay_frames| is the current amount of hardware delay.
-  void WroteSilence(int frames, int delay_frames);
+                  float playback_rate);
 
   // Calculates the current media timestamp taking silence and changes in
   // playback rate into account.
-  //
+  base::TimeDelta current_media_timestamp() const {
+    return current_media_timestamp_;
+  }
+
   // Clients can provide |time_since_writing| to simulate the passage of time
   // since last writing audio to get a more accurate current media timestamp.
-  base::TimeDelta CurrentMediaTimestamp(
+  base::TimeDelta CurrentMediaTimestampSinceWriting(
       base::TimeDelta time_since_writing) const;
 
-  // Returns the last endpoint timestamp provided to WroteAudio().
-  base::TimeDelta last_endpoint_timestamp() const {
-    return last_endpoint_timestamp_;
+  // Returns the amount of contiguous media time buffered at the head of the
+  // audio hardware buffer. Silence introduced into the audio hardware buffer is
+  // treated as a break in media time.
+  base::TimeDelta contiguous_audio_data_buffered() const {
+    return contiguous_audio_data_buffered_;
   }
 
- private:
-  void TrimBufferedAudioToMatchDelay(int delay_frames);
-  void PushBufferedAudio(int frames,
-                         float playback_rate,
-                         base::TimeDelta endpoint_timestamp);
-
-  const int sample_rate_;
+  // Same as above, but also treats changes in playback rate as a break in media
+  // time.
+  base::TimeDelta contiguous_audio_data_buffered_at_same_rate() const {
+    return contiguous_audio_data_buffered_at_same_rate_;
+  }
 
-  // Initially set to kNoTimestamp(), otherwise is the last endpoint timestamp
-  // delivered to WroteAudio(). A copy is kept outside of |buffered_audio_| to
-  // handle the case where all of |buffered_audio_| has been replaced with
-  // silence.
-  base::TimeDelta last_endpoint_timestamp_;
+  // Returns true if there is any audio data buffered by the audio hardware,
+  // even if there is silence mixed in.
+  bool audio_data_buffered() const { return audio_data_buffered_; }
 
-  struct BufferedAudio {
-    BufferedAudio(int frames,
-                  float playback_rate,
-                  base::TimeDelta endpoint_timestamp);
+ private:
+  // Even with a ridiculously high sample rate of 256kHz, using 64 bits will
+  // permit tracking up to 416999965 days worth of time (that's 1141 millenia).
+  //
+  // 32 bits on the other hand would top out at measly 2 hours and 20 minutes.
+  struct AudioData {
+    AudioData(int64_t frames, float playback_rate);
 
-    int frames;
+    int64_t frames;
     float playback_rate;
-    base::TimeDelta endpoint_timestamp;
   };
 
-  std::deque<BufferedAudio> buffered_audio_;
+  // Helpers for operating on |buffered_|.
+  void PushBufferedAudioData(int64_t frames, float playback_rate);
+  void PopBufferedAudioData(int64_t frames);
+  base::TimeDelta ComputeBufferedMediaTime(int64_t frames) const;
+
+  const base::TimeDelta start_timestamp_;
+  const int sample_rate_;
+  const double microseconds_per_frame_;
+
+  std::deque<AudioData> buffered_;
+  int64_t total_buffered_frames_;
+
+  base::TimeDelta current_media_timestamp_;
+
+  // Cached results of last call to WroteAudio().
+  bool audio_data_buffered_;
+  base::TimeDelta contiguous_audio_data_buffered_;
+  base::TimeDelta contiguous_audio_data_buffered_at_same_rate_;
 
   DISALLOW_COPY_AND_ASSIGN(AudioClock);
 };

media/filters/audio_renderer_impl.cc

@@ -61,6 +61,7 @@ AudioRendererImpl::AudioRendererImpl(
       pending_read_(false),
       received_end_of_stream_(false),
       rendered_end_of_stream_(false),
+      last_timestamp_update_(kNoTimestamp()),
       weak_factory_(this) {
   audio_buffer_stream_->set_splice_observer(base::Bind(
       &AudioRendererImpl::OnNewSpliceBuffer, weak_factory_.GetWeakPtr()));
@@ -147,6 +148,7 @@ void AudioRendererImpl::SetMediaTime(base::TimeDelta time) {
   DCHECK_EQ(state_, kFlushed);
 
   start_timestamp_ = time;
+  audio_clock_.reset(new AudioClock(time, audio_parameters_.sample_rate()));
 }
 
 base::TimeDelta AudioRendererImpl::CurrentMediaTime() {
@@ -201,9 +203,10 @@ void AudioRendererImpl::ResetDecoderDone() {
     DCHECK_EQ(state_, kFlushed);
     DCHECK(!flush_cb_.is_null());
 
-    audio_clock_.reset(new AudioClock(audio_parameters_.sample_rate()));
+    audio_clock_.reset();
     received_end_of_stream_ = false;
     rendered_end_of_stream_ = false;
+    last_timestamp_update_ = kNoTimestamp();
 
     // Flush() may have been called while underflowed/not fully buffered.
     if (buffering_state_ != BUFFERING_HAVE_NOTHING)
@@ -294,7 +297,8 @@ void AudioRendererImpl::Initialize(DemuxerStream* stream,
         hardware_config_->GetHighLatencyBufferSize());
   }
 
-  audio_clock_.reset(new AudioClock(audio_parameters_.sample_rate()));
+  audio_clock_.reset(
+      new AudioClock(base::TimeDelta(), audio_parameters_.sample_rate()));
 
   audio_buffer_stream_->Initialize(
       stream,
@@ -549,18 +553,21 @@ int AudioRendererImpl::Render(AudioBus* audio_bus,
 
     // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
     if (!algorithm_) {
-      audio_clock_->WroteSilence(requested_frames, delay_frames);
+      audio_clock_->WroteAudio(
+          0, requested_frames, delay_frames, playback_rate_);
       return 0;
     }
 
     if (playback_rate_ == 0) {
-      audio_clock_->WroteSilence(requested_frames, delay_frames);
+      audio_clock_->WroteAudio(
+          0, requested_frames, delay_frames, playback_rate_);
       return 0;
     }
 
     // Mute audio by returning 0 when not playing.
     if (state_ != kPlaying) {
-      audio_clock_->WroteSilence(requested_frames, delay_frames);
+      audio_clock_->WroteAudio(
+          0, requested_frames, delay_frames, playback_rate_);
       return 0;
     }
 
@@ -576,20 +583,16 @@ int AudioRendererImpl::Render(AudioBus* audio_bus,
     //   3) We are in the kPlaying state
     //
     // Otherwise the buffer has data we can send to the device.
-    const base::TimeDelta media_timestamp_before_filling =
-        audio_clock_->CurrentMediaTimestamp(base::TimeDelta());
     if (algorithm_->frames_buffered() > 0) {
       frames_written =
           algorithm_->FillBuffer(audio_bus, requested_frames, playback_rate_);
-      audio_clock_->WroteAudio(
-          frames_written, delay_frames, playback_rate_, algorithm_->GetTime());
     }
-    audio_clock_->WroteSilence(requested_frames - frames_written, delay_frames);
+    audio_clock_->WroteAudio(
+        frames_written, requested_frames, delay_frames, playback_rate_);
 
     if (frames_written == 0) {
       if (received_end_of_stream_ && !rendered_end_of_stream_ &&
-          audio_clock_->CurrentMediaTimestamp(base::TimeDelta()) ==
-              audio_clock_->last_endpoint_timestamp()) {
+          !audio_clock_->audio_data_buffered()) {
         rendered_end_of_stream_ = true;
         task_runner_->PostTask(FROM_HERE, ended_cb_);
       } else if (!received_end_of_stream_ && state_ == kPlaying) {
@@ -606,15 +609,18 @@ int AudioRendererImpl::Render(AudioBus* audio_bus,
                                         weak_factory_.GetWeakPtr()));
     }
 
-    // We only want to execute |time_cb_| if time has progressed and we haven't
-    // signaled end of stream yet.
-    if (media_timestamp_before_filling !=
-            audio_clock_->CurrentMediaTimestamp(base::TimeDelta()) &&
-        !rendered_end_of_stream_) {
-      time_cb =
-          base::Bind(time_cb_,
-                     audio_clock_->CurrentMediaTimestamp(base::TimeDelta()),
-                     audio_clock_->last_endpoint_timestamp());
+    // Firing |ended_cb_| means we no longer need to run |time_cb_|.
+    if (!rendered_end_of_stream_ &&
+        last_timestamp_update_ != audio_clock_->current_media_timestamp()) {
+      // Since |max_time| uses linear interpolation, only provide an upper bound
+      // that is for audio data at the same playback rate. Failing to do so can
+      // make time jump backwards when the linear interpolated time advances
+      // past buffered regions of audio at different rates.
+      last_timestamp_update_ = audio_clock_->current_media_timestamp();
+      base::TimeDelta max_time =
+          last_timestamp_update_ +
+          audio_clock_->contiguous_audio_data_buffered_at_same_rate();
+      time_cb = base::Bind(time_cb_, last_timestamp_update_, max_time);
     }
   }
 

media/filters/audio_renderer_impl.h

@@ -249,6 +249,7 @@ class MEDIA_EXPORT AudioRendererImpl
   scoped_ptr<AudioClock> audio_clock_;
 
   base::TimeDelta start_timestamp_;
+  base::TimeDelta last_timestamp_update_;
 
   // End variables which must be accessed under |lock_|. ----------------------
 

media/filters/audio_renderer_impl_unittest.cc

@@ -62,6 +62,7 @@ class AudioRendererImplTest : public ::testing::Test {
         demuxer_stream_(DemuxerStream::AUDIO),
         decoder_(new MockAudioDecoder()),
         last_time_update_(kNoTimestamp()),
+        last_max_time_(kNoTimestamp()),
         ended_(false) {
     AudioDecoderConfig audio_config(kCodec,
                                     kSampleFormat,
@@ -117,6 +118,7 @@ class AudioRendererImplTest : public ::testing::Test {
   void OnAudioTimeCallback(TimeDelta current_time, TimeDelta max_time) {
     CHECK(current_time <= max_time);
     last_time_update_ = current_time;
+    last_max_time_ = max_time;
   }
 
   void InitializeRenderer(const PipelineStatusCB& pipeline_status_cb) {
@@ -334,6 +336,8 @@ class AudioRendererImplTest : public ::testing::Test {
     return last_time_update_;
   }
 
+  base::TimeDelta last_max_time() const { return last_max_time_; }
+
   bool ended() const { return ended_; }
 
   // Fixture members.
@@ -404,6 +408,7 @@ class AudioRendererImplTest : public ::testing::Test {
 
   PipelineStatusCB init_decoder_cb_;
   base::TimeDelta last_time_update_;
+  base::TimeDelta last_max_time_;
   bool ended_;
 
   DISALLOW_COPY_AND_ASSIGN(AudioRendererImplTest);
@@ -628,6 +633,7 @@ TEST_F(AudioRendererImplTest, TimeUpdatesOnFirstBuffer) {
 
   AudioTimestampHelper timestamp_helper(kOutputSamplesPerSecond);
   EXPECT_EQ(kNoTimestamp(), last_time_update());
+  EXPECT_EQ(kNoTimestamp(), last_max_time());
 
   // Preroll() should be buffered some data, consume half of it now.
   OutputFrames frames_to_consume(frames_buffered().value / 2);
@@ -639,17 +645,21 @@ TEST_F(AudioRendererImplTest, TimeUpdatesOnFirstBuffer) {
   // a time update that's equal to |kFramesToConsume| from above.
   timestamp_helper.SetBaseTimestamp(base::TimeDelta());
   timestamp_helper.AddFrames(frames_to_consume.value);
-  EXPECT_EQ(timestamp_helper.GetTimestamp(), last_time_update());
+  EXPECT_EQ(base::TimeDelta(), last_time_update());
+  EXPECT_EQ(timestamp_helper.GetTimestamp(), last_max_time());
 
   // The next time update should match the remaining frames_buffered(), but only
   // after running the message loop.
   frames_to_consume = frames_buffered();
   EXPECT_TRUE(ConsumeBufferedData(frames_to_consume));
-  EXPECT_EQ(timestamp_helper.GetTimestamp(), last_time_update());
+  EXPECT_EQ(base::TimeDelta(), last_time_update());
+  EXPECT_EQ(timestamp_helper.GetTimestamp(), last_max_time());
 
+  // Now the times should be updated.
   base::RunLoop().RunUntilIdle();
-  timestamp_helper.AddFrames(frames_to_consume.value);
   EXPECT_EQ(timestamp_helper.GetTimestamp(), last_time_update());
+  timestamp_helper.AddFrames(frames_to_consume.value);
+  EXPECT_EQ(timestamp_helper.GetTimestamp(), last_max_time());
 }
 
 TEST_F(AudioRendererImplTest, ImmediateEndOfStream) {