Preserve timestamp in Android hw encoder

Android VEA currently doesn't preserve the timestamps in the input
frames, causing trouble for WebRTC. The reason is MediaCodec assumes
the presentation timestamp to be monotonically-growing value. But in
chromium the video capture maybe paused a while or drop some frames,
so the timestamps in input frames won't be continious as expected.
Then we have to generate a seperate monotonically-growing timestamp
for encoding instead.

Here we cache the timestamps in input frames, mapping to the generated
timestamp, and read them out after encoding. Then encoder can work
happily and we can preserve the input timestamps.

Bug: 785407
Cq-Include-Trybots: master.tryserver.chromium.android:android_optional_gpu_tests_rel;master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win_optional_gpu_tests_rel
Change-Id: I594190436f18437afb937c4f0667a5b1e3019280
Reviewed-on: https://chromium-review.googlesource.com/775144
Commit-Queue: Weiyong Yao <braveyao@chromium.org>
Reviewed-by: Emircan Uysaler <emircan@chromium.org>
Reviewed-by: Frank Liberato <liberato@chromium.org>
Cr-Commit-Position: refs/heads/master@{#517658}

content/renderer/media/gpu/rtc_video_encoder.cc

@@ -22,7 +22,6 @@
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
-#include "build/build_config.h"
 #include "content/public/common/content_features.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/webrtc/webrtc_video_frame_adapter.h"
@@ -523,11 +522,7 @@ void RTCVideoEncoder::Impl::BitstreamBufferReady(int32_t bitstream_buffer_id,
       }
       pending_timestamps_.pop_front();
     }
-#if !defined(OS_ANDROID)
-    // No capture timestamps available on Android at present. So generate rtp
-    // timestamps below.
     DCHECK(rtp_timestamp.has_value());
-#endif
   }
   if (!rtp_timestamp.has_value()) {
     failed_timestamp_match_ = true;

content/renderer/media/gpu/rtc_video_encoder_unittest.cc

@@ -292,15 +292,7 @@ TEST_F(RTCVideoEncoderTest, EncodeScaledFrame) {
             rtc_encoder_->Encode(rtc_frame, nullptr, &frame_types));
 }
 
-// We cannot run this test on Android because AndroidVideoEncodeAccelerator does
-// not preserve timestamps.
-#if defined(OS_ANDROID)
-#define MAYBE_PreserveTimestamps DISABLED_PreserveTimestamps
-#else
-#define MAYBE_PreserveTimestamps PreserveTimestamps
-#endif  // defined(OS_ANDROID)
-
-TEST_F(RTCVideoEncoderTest, MAYBE_PreserveTimestamps) {
+TEST_F(RTCVideoEncoderTest, PreserveTimestamps) {
   const webrtc::VideoCodecType codec_type = webrtc::kVideoCodecVP8;
   CreateEncoder(codec_type);
   webrtc::VideoCodec codec = GetDefaultCodec();

media/gpu/android/android_video_encode_accelerator.cc

@@ -356,16 +356,30 @@ void AndroidVideoEncodeAccelerator::QueueInput() {
       frame->coded_size().height());
   RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError);
 
-  input_timestamp_ += base::TimeDelta::FromMicroseconds(
+  // MediaCodec encoder assumes the presentation timestamps to be monotonically
+  // increasing at initialized framerate. But in Chromium, the video capture
+  // may be paused for a while or drop some frames, so the timestamp in input
+  // frames won't be continious. Here we cache the timestamps of input frames,
+  // mapping to the generated |presentation_timestamp_|, and will read them out
+  // after encoding. Then encoder can work happily always and we can preserve
+  // the timestamps in captured frames for other purpose.
+  presentation_timestamp_ += base::TimeDelta::FromMicroseconds(
       base::Time::kMicrosecondsPerSecond / INITIAL_FRAMERATE);
+  DCHECK(frame_timestamp_map_.find(presentation_timestamp_) ==
+         frame_timestamp_map_.end());
+  frame_timestamp_map_[presentation_timestamp_] = frame->timestamp();
+
   status = media_codec_->QueueInputBuffer(input_buf_index, nullptr, queued_size,
-                                          input_timestamp_);
+                                          presentation_timestamp_);
   UMA_HISTOGRAM_TIMES("Media.AVDA.InputQueueTime",
                       base::Time::Now() - std::get<2>(input));
   RETURN_ON_FAILURE(status == MEDIA_CODEC_OK,
                     "Failed to QueueInputBuffer: " << status,
                     kPlatformFailureError);
   ++num_buffers_at_codec_;
+  DCHECK(static_cast<int32_t>(frame_timestamp_map_.size()) ==
+         num_buffers_at_codec_);
+
   pending_frames_.pop();
 }
 
@@ -380,9 +394,10 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() {
   size_t size = 0;
   bool key_frame = false;
 
-  MediaCodecStatus status =
-      media_codec_->DequeueOutputBuffer(NoWaitTimeOut(), &buf_index, &offset,
-                                        &size, nullptr, nullptr, &key_frame);
+  base::TimeDelta presentaion_timestamp;
+  MediaCodecStatus status = media_codec_->DequeueOutputBuffer(
+      NoWaitTimeOut(), &buf_index, &offset, &size, &presentaion_timestamp,
+      nullptr, &key_frame);
   switch (status) {
     case MEDIA_CODEC_TRY_AGAIN_LATER:
       return;
@@ -407,6 +422,11 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() {
       break;
   }
 
+  const auto it = frame_timestamp_map_.find(presentaion_timestamp);
+  DCHECK(it != frame_timestamp_map_.end());
+  const base::TimeDelta frame_timestamp = it->second;
+  frame_timestamp_map_.erase(it);
+
   BitstreamBuffer bitstream_buffer = available_bitstream_buffers_.back();
   available_bitstream_buffers_.pop_back();
   std::unique_ptr<SharedMemoryRegion> shm(
@@ -427,7 +447,7 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() {
       FROM_HERE,
       base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady,
                  client_ptr_factory_->GetWeakPtr(), bitstream_buffer.id(), size,
-                 key_frame, base::TimeDelta()));
+                 key_frame, frame_timestamp));
 }
 
 }  // namespace media

media/gpu/android/android_video_encode_accelerator.h

@@ -9,6 +9,7 @@
 #include <stdint.h>
 
 #include <list>
+#include <map>
 #include <memory>
 #include <tuple>
 #include <vector>
@@ -96,7 +97,11 @@ class MEDIA_GPU_EXPORT AndroidVideoEncodeAccelerator
   int32_t num_buffers_at_codec_;
 
   // A monotonically-growing value.
-  base::TimeDelta input_timestamp_;
+  base::TimeDelta presentation_timestamp_;
+
+  std::map<base::TimeDelta /* presentation_timestamp */,
+           base::TimeDelta /* frame_timestamp */>
+      frame_timestamp_map_;
 
   // Resolution of input stream. Set once in initialization and not allowed to
   // change after.