Android: Propagate sample rate change to audio decoder job

AudioDecoderJob maintains an object audio_timestamp_helper_ for use with AV sync. Previously, this object was configured with demuxer value for sample rate, whereas it should instead be synced with the decoder's reported sample rate.

This CL enables AudioDecoderJob to track decoder output rate, and ensures change in output format is propagated to this object.

BUG=internal b/18873488

Review URL: https://codereview.chromium.org/805273007

Cr-Commit-Position: refs/heads/master@{#313416}

media/base/android/audio_decoder_job.cc

@@ -40,9 +40,11 @@ AudioDecoderJob::AudioDecoderJob(
                       on_demuxer_config_changed_cb),
       audio_codec_(kUnknownAudioCodec),
       num_channels_(0),
-      sampling_rate_(0),
+      config_sampling_rate_(0),
       volume_(-1.0),
-      bytes_per_frame_(0) {
+      bytes_per_frame_(0),
+      output_sampling_rate_(0),
+      frame_count_(0) {
 }
 
 AudioDecoderJob::~AudioDecoderJob() {}
@@ -51,15 +53,22 @@ bool AudioDecoderJob::HasStream() const {
   return audio_codec_ != kUnknownAudioCodec;
 }
 
+void AudioDecoderJob::Flush() {
+  MediaDecoderJob::Flush();
+  frame_count_ = 0;
+}
+
 void AudioDecoderJob::SetDemuxerConfigs(const DemuxerConfigs& configs) {
   // TODO(qinmin): split DemuxerConfig for audio and video separately so we
   // can simply store the stucture here.
   audio_codec_ = configs.audio_codec;
   num_channels_ = configs.audio_channels;
-  sampling_rate_ = configs.audio_sampling_rate;
+  config_sampling_rate_ = configs.audio_sampling_rate;
   set_is_content_encrypted(configs.is_audio_encrypted);
   audio_extra_data_ = configs.audio_extra_data;
   bytes_per_frame_ = kBytesPerAudioOutputSample * num_channels_;
+  if (!media_codec_bridge_)
+    output_sampling_rate_ = config_sampling_rate_;
 }
 
 void AudioDecoderJob::SetVolume(double volume) {
@@ -74,6 +83,14 @@ void AudioDecoderJob::SetBaseTimestamp(base::TimeDelta base_timestamp) {
     audio_timestamp_helper_->SetBaseTimestamp(base_timestamp_);
 }
 
+void AudioDecoderJob::ResetTimestampHelper() {
+  if (audio_timestamp_helper_)
+    base_timestamp_ = audio_timestamp_helper_->GetTimestamp();
+  audio_timestamp_helper_.reset(
+      new AudioTimestampHelper(output_sampling_rate_));
+  audio_timestamp_helper_->SetBaseTimestamp(base_timestamp_);
+}
+
 void AudioDecoderJob::ReleaseOutputBuffer(
     int output_buffer_index,
     size_t size,
@@ -85,9 +102,10 @@ void AudioDecoderJob::ReleaseOutputBuffer(
     int64 head_position = (static_cast<AudioCodecBridge*>(
         media_codec_bridge_.get()))->PlayOutputBuffer(
             output_buffer_index, size);
-    audio_timestamp_helper_->AddFrames(size / bytes_per_frame_);
-    int64 frames_to_play =
-        audio_timestamp_helper_->frame_count() - head_position;
+    size_t new_frames_count = size / bytes_per_frame_;
+    frame_count_ += new_frames_count;
+    audio_timestamp_helper_->AddFrames(new_frames_count);
+    int64 frames_to_play = frame_count_ - head_position;
     DCHECK_GE(frames_to_play, 0);
     current_presentation_timestamp =
         audio_timestamp_helper_->GetTimestamp() -
@@ -109,7 +127,7 @@ bool AudioDecoderJob::AreDemuxerConfigsChanged(
     const DemuxerConfigs& configs) const {
   return audio_codec_ != configs.audio_codec ||
      num_channels_ != configs.audio_channels ||
-     sampling_rate_ != configs.audio_sampling_rate ||
+     config_sampling_rate_ != configs.audio_sampling_rate ||
      is_content_encrypted() != configs.is_audio_encrypted ||
      audio_extra_data_.size() != configs.audio_extra_data.size() ||
      !std::equal(audio_extra_data_.begin(),
@@ -123,7 +141,7 @@ bool AudioDecoderJob::CreateMediaCodecBridgeInternal() {
     return false;
 
   if (!(static_cast<AudioCodecBridge*>(media_codec_bridge_.get()))->Start(
-      audio_codec_, sampling_rate_, num_channels_, &audio_extra_data_[0],
+      audio_codec_, config_sampling_rate_, num_channels_, &audio_extra_data_[0],
       audio_extra_data_.size(), true, GetMediaCrypto().obj())) {
     media_codec_bridge_.reset();
     return false;
@@ -131,11 +149,10 @@ bool AudioDecoderJob::CreateMediaCodecBridgeInternal() {
 
   SetVolumeInternal();
 
-  // Need to pass the base timestamp to the new decoder.
-  if (audio_timestamp_helper_)
-    base_timestamp_ = audio_timestamp_helper_->GetTimestamp();
-  audio_timestamp_helper_.reset(new AudioTimestampHelper(sampling_rate_));
-  audio_timestamp_helper_->SetBaseTimestamp(base_timestamp_);
+  // Reset values used to track codec bridge output
+  frame_count_ = 0;
+  ResetTimestampHelper();
+
   return true;
 }
 
@@ -146,4 +163,13 @@ void AudioDecoderJob::SetVolumeInternal() {
   }
 }
 
+void AudioDecoderJob::OnOutputFormatChanged() {
+  DCHECK(media_codec_bridge_);
+
+  int old_sampling_rate = output_sampling_rate_;
+  output_sampling_rate_ = media_codec_bridge_->GetOutputSamplingRate();
+  if (output_sampling_rate_ != old_sampling_rate)
+    ResetTimestampHelper();
+}
+
 }  // namespace media

media/base/android/audio_decoder_job.h

@@ -29,6 +29,7 @@ class AudioDecoderJob : public MediaDecoderJob {
 
   // MediaDecoderJob implementation.
   virtual bool HasStream() const override;
+  virtual void Flush() override;
   virtual void SetDemuxerConfigs(const DemuxerConfigs& configs) override;
 
   // Sets the volume of the audio output.
@@ -49,18 +50,27 @@ class AudioDecoderJob : public MediaDecoderJob {
   virtual bool AreDemuxerConfigsChanged(
       const DemuxerConfigs& configs) const override;
   virtual bool CreateMediaCodecBridgeInternal() override;
+  virtual void OnOutputFormatChanged() override;
 
   // Helper method to set the audio output volume.
   void SetVolumeInternal();
 
+  void ResetTimestampHelper();
+
   // Audio configs from the demuxer.
   AudioCodec audio_codec_;
   int num_channels_;
-  int sampling_rate_;
+  int config_sampling_rate_;
   std::vector<uint8> audio_extra_data_;
   double volume_;
   int bytes_per_frame_;
 
+  // Audio output sample rate
+  int output_sampling_rate_;
+
+  // Frame count to sync with audio codec output
+  int64 frame_count_;
+
   // Base timestamp for the |audio_timestamp_helper_|.
   base::TimeDelta base_timestamp_;
 

media/base/android/java/src/org/chromium/media/MediaCodecBridge.java

@@ -77,7 +77,6 @@ class MediaCodecBridge {
     private long mLastPresentationTimeUs;
     private String mMime;
     private boolean mAdaptivePlaybackSupported;
-    private int mSampleRate;
 
     private static class DequeueInputResult {
         private final int mStatus;
@@ -448,6 +447,12 @@ class MediaCodecBridge {
                 : format.getInteger(MediaFormat.KEY_WIDTH);
     }
 
+    @CalledByNative
+    private int getOutputSamplingRate() {
+        MediaFormat format = mMediaCodec.getOutputFormat();
+        return format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
+    }
+
     @CalledByNative
     private ByteBuffer getInputBuffer(int index) {
         if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
@@ -569,7 +574,7 @@ class MediaCodecBridge {
                 MediaFormat newFormat = mMediaCodec.getOutputFormat();
                 if (mAudioTrack != null && newFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
                     int newSampleRate = newFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
-                    if (newSampleRate != mSampleRate && !reconfigureAudioTrack(newFormat)) {
+                    if (mAudioTrack.setPlaybackRate(newSampleRate) != AudioTrack.SUCCESS) {
                         status = MEDIA_CODEC_ERROR;
                     }
                 }
@@ -672,8 +677,20 @@ class MediaCodecBridge {
             boolean playAudio) {
         try {
             mMediaCodec.configure(format, null, crypto, flags);
-            if (playAudio && !reconfigureAudioTrack(format)) {
-                return false;
+            if (playAudio) {
+                int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
+                int channelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
+                int channelConfig = getAudioFormat(channelCount);
+                // Using 16bit PCM for output. Keep this value in sync with
+                // kBytesPerAudioOutputSample in media_codec_bridge.cc.
+                int minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig,
+                        AudioFormat.ENCODING_PCM_16BIT);
+                mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRate, channelConfig,
+                        AudioFormat.ENCODING_PCM_16BIT, minBufferSize, AudioTrack.MODE_STREAM);
+                if (mAudioTrack.getState() == AudioTrack.STATE_UNINITIALIZED) {
+                    mAudioTrack = null;
+                    return false;
+                }
             }
             return true;
         } catch (IllegalStateException e) {
@@ -682,35 +699,6 @@ class MediaCodecBridge {
         return false;
     }
 
-    /**
-     * Resets the AudioTrack instance, configured according to the given format.
-     * If a previous AudioTrack instance already exists, release it.
-     *
-     * @param format The format from which to get sample rate and channel count.
-     * @return Whether or not creating the AudioTrack succeeded.
-     */
-    private boolean reconfigureAudioTrack(MediaFormat format) {
-        if (mAudioTrack != null) {
-            mAudioTrack.release();
-        }
-
-        mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
-        int channelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
-        int channelConfig = getAudioFormat(channelCount);
-        // Using 16bit PCM for output. Keep this value in sync with
-        // kBytesPerAudioOutputSample in media_codec_bridge.cc.
-        int minBufferSize = AudioTrack.getMinBufferSize(mSampleRate, channelConfig,
-                AudioFormat.ENCODING_PCM_16BIT);
-        mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, mSampleRate, channelConfig,
-                AudioFormat.ENCODING_PCM_16BIT, minBufferSize, AudioTrack.MODE_STREAM);
-        if (mAudioTrack.getState() == AudioTrack.STATE_UNINITIALIZED) {
-            mAudioTrack = null;
-            Log.e(TAG, "Failed to initialize AudioTrack");
-            return false;
-        }
-        return true;
-    }
-
     /**
      *  Play the audio buffer that is passed in.
      *

media/base/android/media_codec_bridge.cc

@@ -291,6 +291,12 @@ void MediaCodecBridge::GetOutputFormat(int* width, int* height) {
   *height = Java_MediaCodecBridge_getOutputHeight(env, j_media_codec_.obj());
 }
 
+int MediaCodecBridge::GetOutputSamplingRate() {
+  JNIEnv* env = AttachCurrentThread();
+
+  return Java_MediaCodecBridge_getOutputSamplingRate(env, j_media_codec_.obj());
+}
+
 MediaCodecStatus MediaCodecBridge::QueueInputBuffer(
     int index,
     const uint8* data,

media/base/android/media_codec_bridge.h

@@ -107,6 +107,10 @@ class MEDIA_EXPORT MediaCodecBridge {
   // returns a format change by returning INFO_OUTPUT_FORMAT_CHANGED
   void GetOutputFormat(int* width, int* height);
 
+  // Used for checking for new sampling rate after DequeueInputBuffer() returns
+  // INFO_OUTPUT_FORMAT_CHANGED
+  int GetOutputSamplingRate();
+
   // Submits a byte array to the given input buffer. Call this after getting an
   // available buffer from DequeueInputBuffer().  If |data| is NULL, assume the
   // input buffer has already been populated (but still obey |size|).

media/base/android/media_decoder_job.cc

@@ -432,6 +432,7 @@ void MediaDecoderJob::DecodeInternal(
     if (status == MEDIA_CODEC_OUTPUT_FORMAT_CHANGED) {
       // TODO(qinmin): instead of waiting for the next output buffer to be
       // dequeued, post a task on the UI thread to signal the format change.
+      OnOutputFormatChanged();
       has_format_change = true;
     }
   } while (status != MEDIA_CODEC_OK && status != MEDIA_CODEC_ERROR &&
@@ -649,6 +650,8 @@ bool MediaDecoderJob::IsCodecReconfigureNeeded(
   return true;
 }
 
+void MediaDecoderJob::OnOutputFormatChanged() {}
+
 bool MediaDecoderJob::UpdateOutputFormat() {
   return false;
 }

media/base/android/media_decoder_job.h

@@ -231,6 +231,11 @@ class MediaDecoderJob {
   // new DemuxerConfigs, or false otherwise.
   virtual bool IsCodecReconfigureNeeded(const DemuxerConfigs& configs) const;
 
+  // Signals to decoder job that decoder has updated output format. Decoder job
+  // may need to do internal reconfiguration in order to correctly interpret
+  // incoming buffers
+  virtual void OnOutputFormatChanged();
+
   // Update the output format from the decoder, returns true if the output
   // format changes, or false otherwise.
   virtual bool UpdateOutputFormat();