[video-rAF] Add new VideoFrameMetada fields

This CL adds new keys to VideoFrameMetadata's dictionary. These keys
will be used to store new types of information, to be surfaced via the
video.requestAnimationFrame() API.

Bug: 1012063
Change-Id: Ib937fbf762d00afca46f6c1552ea818ecb1b6584
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1904336
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: Tommi <tommi@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#714046}

media/base/video_frame_metadata.h

@@ -162,9 +162,21 @@ class MEDIA_EXPORT VideoFrameMetadata {
     TOP_CONTROLS_SHOWN_RATIO,
 
     // If present, this field represents the local time at which the VideoFrame
-    // was decoded from whichever format it was encoded in.
-    // Use Get/SetTimeTicks() for this key.
-    DECODE_TIME,
+    // was decoded from whichever format it was encoded in. Sometimes only
+    // DECODE_END_TIME will be present. Use Get/SetTimeTicks() for this key.
+    DECODE_BEGIN_TIME,
+    DECODE_END_TIME,
+
+    // If present, this field represents the elapsed time from the submission of
+    // the encoded packet with the same PTS as this frame to the decoder until
+    // the decoded frame was ready for presentation. Stored as base::TimeDelta.
+    PROCESSING_TIME,
+
+    // The RTP timestamp associated with this video frame. Stored as a double
+    // since base::DictionaryValue doesn't have a uint32_t type.
+    //
+    // https://w3c.github.io/webrtc-pc/#dom-rtcrtpcontributingsource
+    RTP_TIMESTAMP,
 
     NUM_KEYS
   };

media/filters/decoder_stream.cc

@@ -630,6 +630,8 @@ void DecoderStream<StreamType>::OnDecodeOutputReady(
     return;
   }
 
+  traits_->OnOutputReady(output.get());
+
   if (read_cb_) {
     // If |ready_outputs_| was non-empty, the read would have already been
     // satisifed by Read().
@@ -987,6 +989,7 @@ void DecoderStream<StreamType>::OnPreparedOutputReady(
   DCHECK(!unprepared_outputs_.empty());
   DCHECK(preparing_output_);
 
+  traits_->OnOutputReady(output.get());
   CompletePrepare(output.get());
   unprepared_outputs_.pop_front();
   if (!read_cb_)

media/filters/decoder_stream_traits.cc

@@ -112,6 +112,9 @@ void DecoderStreamTraits<DemuxerStream::AUDIO>::OnConfigChanged(
   audio_ts_validator_.reset(new AudioTimestampValidator(config, media_log_));
 }
 
+void DecoderStreamTraits<DemuxerStream::AUDIO>::OnOutputReady(
+    OutputType* buffer) {}
+
 // Video decoder stream traits implementation.
 
 // static
@@ -201,6 +204,7 @@ void DecoderStreamTraits<DemuxerStream::VIDEO>::OnDecode(
   frame_metadata_[buffer.timestamp()] = {
       buffer.discard_padding().first == kInfiniteDuration,  // should_drop
       buffer.duration(),                                    // duration
+      base::TimeTicks::Now(),                               // decode_begin_time
   };
 
   if (!buffer.is_key_frame())
@@ -220,11 +224,6 @@ void DecoderStreamTraits<DemuxerStream::VIDEO>::OnDecode(
 
 PostDecodeAction DecoderStreamTraits<DemuxerStream::VIDEO>::OnDecodeDone(
     OutputType* buffer) {
-  // Add a timestamp here (after decoding completed) to enable buffering delay
-  // measurements down the line.
-  buffer->metadata()->SetTimeTicks(media::VideoFrameMetadata::DECODE_TIME,
-                                   base::TimeTicks::Now());
-
   auto it = frame_metadata_.find(buffer->timestamp());
 
   // If the frame isn't in |frame_metadata_| it probably was erased below on a
@@ -234,6 +233,12 @@ PostDecodeAction DecoderStreamTraits<DemuxerStream::VIDEO>::OnDecodeDone(
   if (it == frame_metadata_.end())
     return PostDecodeAction::DELIVER;
 
+  // Add a timestamp here to enable buffering delay measurements down the line.
+  buffer->metadata()->SetTimeTicks(VideoFrameMetadata::DECODE_BEGIN_TIME,
+                                   it->second.decode_begin_time);
+  buffer->metadata()->SetTimeTicks(VideoFrameMetadata::DECODE_END_TIME,
+                                   base::TimeTicks::Now());
+
   auto action = it->second.should_drop ? PostDecodeAction::DROP
                                        : PostDecodeAction::DELIVER;
 
@@ -252,4 +257,16 @@ PostDecodeAction DecoderStreamTraits<DemuxerStream::VIDEO>::OnDecodeDone(
   return action;
 }
 
+void DecoderStreamTraits<DemuxerStream::VIDEO>::OnOutputReady(
+    OutputType* buffer) {
+  base::TimeTicks decode_begin_time;
+  if (!buffer->metadata()->GetTimeTicks(VideoFrameMetadata::DECODE_BEGIN_TIME,
+                                        &decode_begin_time)) {
+    return;
+  }
+  // Tag buffer with elapsed time since creation.
+  buffer->metadata()->SetTimeDelta(VideoFrameMetadata::PROCESSING_TIME,
+                                   base::TimeTicks::Now() - decode_begin_time);
+}
+
 }  // namespace media

media/filters/decoder_stream_traits.h

@@ -62,6 +62,7 @@ class MEDIA_EXPORT DecoderStreamTraits<DemuxerStream::AUDIO> {
   void OnDecode(const DecoderBuffer& buffer);
   PostDecodeAction OnDecodeDone(OutputType* buffer);
   void OnStreamReset(DemuxerStream* stream);
+  void OnOutputReady(OutputType* output);
 
  private:
   void OnConfigChanged(const AudioDecoderConfig& config);
@@ -107,6 +108,7 @@ class MEDIA_EXPORT DecoderStreamTraits<DemuxerStream::VIDEO> {
   void OnDecode(const DecoderBuffer& buffer);
   PostDecodeAction OnDecodeDone(OutputType* buffer);
   void OnStreamReset(DemuxerStream* stream);
+  void OnOutputReady(OutputType* output);
 
  private:
   base::TimeDelta last_keyframe_timestamp_;
@@ -116,6 +118,7 @@ class MEDIA_EXPORT DecoderStreamTraits<DemuxerStream::VIDEO> {
   struct FrameMetadata {
     bool should_drop = false;
     base::TimeDelta duration = kNoTimestamp;
+    base::TimeTicks decode_begin_time;
   };
   base::flat_map<base::TimeDelta, FrameMetadata> frame_metadata_;
 

media/filters/video_decoder_stream_unittest.cc

@@ -43,6 +43,7 @@ namespace media {
 
 const int kNumConfigs = 4;
 const int kNumBuffersInOneConfig = 5;
+constexpr base::TimeDelta kPrepareDelay = base::TimeDelta::FromMilliseconds(5);
 
 static std::string GetDecoderName(int i) {
   return std::string("VideoDecoder") + base::NumberToString(i);
@@ -144,6 +145,15 @@ class VideoDecoderStreamTest
         base::BindOnce(std::move(output_ready_cb), std::move(frame)));
   }
 
+  void PrepareFrameWithDelay(
+      scoped_refptr<VideoFrame> frame,
+      VideoDecoderStream::OutputReadyCB output_ready_cb) {
+    task_environment_.FastForwardBy(kPrepareDelay);
+    task_environment_.GetMainThreadTaskRunner()->PostTask(
+        FROM_HERE,
+        base::BindOnce(std::move(output_ready_cb), std::move(frame)));
+  }
+
   void OnBytesDecoded(int count) { num_decoded_bytes_unreported_ += count; }
 
   // Callback to create a list of decoders for the DecoderSelector to select
@@ -463,7 +473,8 @@ class VideoDecoderStreamTest
     SatisfyPendingCallback(DECODER_REINIT);
   }
 
-  base::test::SingleThreadTaskEnvironment task_environment_;
+  base::test::SingleThreadTaskEnvironment task_environment_{
+      base::test::TaskEnvironment::TimeSource::MOCK_TIME};
 
   StrictMock<MockMediaLog> media_log_;
   std::unique_ptr<VideoDecoderStream> video_decoder_stream_;
@@ -571,6 +582,56 @@ TEST_P(VideoDecoderStreamTest, Read_AfterReset) {
   Read();
 }
 
+TEST_P(VideoDecoderStreamTest, Read_ProperMetadata) {
+  // For testing simplicity, omit parallel decode tests with a delay in frames.
+  if (GetParam().parallel_decoding > 1 && GetParam().decoding_delay > 0)
+    return;
+
+  if (GetParam().has_prepare) {
+    // Override the basic PrepareFrame() for a version that moves the MockTime
+    // by kPrepareDelay. This simulates real work done (e.g. YUV conversion).
+    video_decoder_stream_->SetPrepareCB(
+        base::BindRepeating(&VideoDecoderStreamTest::PrepareFrameWithDelay,
+                            base::Unretained(this)));
+  }
+
+  constexpr base::TimeDelta kDecodeDelay =
+      base::TimeDelta::FromMilliseconds(10);
+
+  Initialize();
+
+  // Simulate time elapsed by the decoder.
+  EnterPendingState(DECODER_DECODE);
+  task_environment_.FastForwardBy(kDecodeDelay);
+
+  SatisfyPendingCallback(DECODER_DECODE);
+
+  EXPECT_TRUE(frame_read_);
+
+  auto* metadata = frame_read_->metadata();
+
+  // Verify the decoding metadata is accurate.
+  base::TimeTicks decode_start;
+  EXPECT_TRUE(metadata->GetTimeTicks(VideoFrameMetadata::DECODE_BEGIN_TIME,
+                                     &decode_start));
+
+  base::TimeTicks decode_end;
+  EXPECT_TRUE(
+      metadata->GetTimeTicks(VideoFrameMetadata::DECODE_END_TIME, &decode_end));
+
+  EXPECT_EQ(decode_end - decode_start, kDecodeDelay);
+
+  // Verify the processing metadata is accurate.
+  const base::TimeDelta expected_processing_time =
+      GetParam().has_prepare ? (kDecodeDelay + kPrepareDelay) : kDecodeDelay;
+
+  base::TimeDelta processing_time;
+  EXPECT_TRUE(metadata->GetTimeDelta(VideoFrameMetadata::PROCESSING_TIME,
+                                     &processing_time));
+
+  EXPECT_EQ(processing_time, expected_processing_time);
+}
+
 TEST_P(VideoDecoderStreamTest, Read_BlockedDemuxer) {
   Initialize();
   demuxer_stream_->HoldNextRead();

third_party/blink/renderer/modules/peerconnection/media_stream_remote_video_source.cc

@@ -100,9 +100,10 @@ void DoNothing(const scoped_refptr<rtc::RefCountInterface>& ref) {}
 void MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate::OnFrame(
     const webrtc::VideoFrame& incoming_frame) {
   const bool render_immediately = incoming_frame.timestamp_us() == 0;
+  const base::TimeTicks current_time = base::TimeTicks::Now();
   const base::TimeDelta incoming_timestamp =
       render_immediately
-          ? base::TimeTicks::Now() - base::TimeTicks()
+          ? current_time - base::TimeTicks()
           : base::TimeDelta::FromMicroseconds(incoming_frame.timestamp_us());
   const base::TimeTicks render_time =
       render_immediately ? base::TimeTicks() + incoming_timestamp
@@ -208,8 +209,12 @@ void MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate::OnFrame(
     video_frame->metadata()->SetTimeTicks(
         media::VideoFrameMetadata::REFERENCE_TIME, render_time);
   }
-  video_frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::DECODE_TIME,
-                                        base::TimeTicks::Now());
+  video_frame->metadata()->SetTimeTicks(
+      media::VideoFrameMetadata::DECODE_END_TIME, current_time);
+
+  video_frame->metadata()->SetDouble(
+      media::VideoFrameMetadata::RTP_TIMESTAMP,
+      static_cast<double>(incoming_frame.timestamp()));
 
   PostCrossThreadTask(
       *io_task_runner_, FROM_HERE,

third_party/blink/renderer/platform/graphics/video_frame_submitter.cc

@@ -525,7 +525,7 @@ viz::CompositorFrame VideoFrameSubmitter::CreateCompositorFrame(
 
   base::TimeTicks value;
   if (video_frame && video_frame->metadata()->GetTimeTicks(
-                         media::VideoFrameMetadata::DECODE_TIME, &value)) {
+                         media::VideoFrameMetadata::DECODE_END_TIME, &value)) {
     TRACE_EVENT_ASYNC_BEGIN_WITH_TIMESTAMP0("media", "VideoFrameSubmitter",
                                             *next_frame_token_, value);
     TRACE_EVENT_ASYNC_STEP_PAST0("media", "VideoFrameSubmitter",