[RTCInsertableStreams] Expose metadata fields for video frames

Also enable the GFD extension in tests so that all the metadata and type
fields are reported correctly.

Drive-by: test cleanups.

Bug: 1069295
Change-Id: I9ffb5c9874c9f8556785a0664ee010d3912f866a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2207162Reviewed-by: Marina Ciocea <marinaciocea@chromium.org>
Reviewed-by: Harald Alvestrand <hta@chromium.org>
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#771727}

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.cc

@@ -41,13 +41,27 @@ RTCEncodedVideoFrameMetadata* RTCEncodedVideoFrame::getMetadata() const {
   RTCEncodedVideoFrameMetadata* metadata =
       RTCEncodedVideoFrameMetadata::Create();
   metadata->setSynchronizationSource(delegate_->Ssrc());
+  const auto* webrtc_metadata = delegate_->GetMetadata();
+  if (!webrtc_metadata)
+    return metadata;
+
+  if (webrtc_metadata->GetFrameId())
+    metadata->setFrameId(*webrtc_metadata->GetFrameId());
+
+  Vector<int64_t> dependencies;
+  for (const auto& dependency : webrtc_metadata->GetFrameDependencies())
+    dependencies.push_back(dependency);
+  metadata->setDependencies(dependencies);
+  metadata->setWidth(webrtc_metadata->GetWidth());
+  metadata->setHeight(webrtc_metadata->GetHeight());
+  metadata->setSpatialIndex(webrtc_metadata->GetSpatialIndex());
+  metadata->setTemporalIndex(webrtc_metadata->GetTemporalIndex());
   return metadata;
 }
 
 DOMArrayBuffer* RTCEncodedVideoFrame::additionalData() const {
   if (!additional_data_)
     additional_data_ = delegate_->CreateAdditionalDataBuffer();
-
   return additional_data_;
 }
 

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame_delegate.cc

@@ -80,6 +80,12 @@ uint32_t RTCEncodedVideoFrameDelegate::Ssrc() const {
   return webrtc_frame_ ? webrtc_frame_->GetSsrc() : 0;
 }
 
+const webrtc::VideoFrameMetadata* RTCEncodedVideoFrameDelegate::GetMetadata()
+    const {
+  MutexLocker lock(mutex_);
+  return webrtc_frame_ ? &webrtc_frame_->GetMetadata() : nullptr;
+}
+
 std::unique_ptr<webrtc::TransformableVideoFrameInterface>
 RTCEncodedVideoFrameDelegate::PassWebRtcFrame() {
   MutexLocker lock(mutex_);

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame_delegate.h

@@ -15,6 +15,7 @@
 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
 #include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
 #include "third_party/webrtc/api/frame_transformer_interface.h"
+#include "third_party/webrtc/api/video/video_frame_metadata.h"
 
 namespace blink {
 
@@ -35,6 +36,7 @@ class RTCEncodedVideoFrameDelegate
   void SetData(const DOMArrayBuffer* data);
   DOMArrayBuffer* CreateAdditionalDataBuffer() const;
   uint32_t Ssrc() const;
+  const webrtc::VideoFrameMetadata* GetMetadata() const;
   std::unique_ptr<webrtc::TransformableVideoFrameInterface> PassWebRtcFrame();
 
  private:

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame_metadata.idl

@@ -7,12 +7,12 @@
 
 [Serializable]
 dictionary RTCEncodedVideoFrameMetadata {
-    long long frame_id;
+    long long frameId;
     sequence<long long> dependencies;
     unsigned short width;
     unsigned short height;
-    long spatial_index;
-    long temporal_index;
+    long spatialIndex;
+    long temporalIndex;
     unsigned long synchronizationSource;
     sequence<unsigned long> contributingSources;
 };

third_party/blink/web_tests/fast/peerconnection/RTCPeerConnection-insertable-streams.html

@@ -68,11 +68,14 @@ async function testVideoFlow(t, negotiationFunction) {
 
   // Pass frames as they come from the encoder.
   for (let i = 0; i < numFramesPassthrough; i++) {
-    const result = await senderReader.read()
+    const result = await senderReader.read();
+    const metadata = result.value.getMetadata();
+    assert_true(containsVideoMetadata(metadata));
     frameInfos.push({
       timestamp: result.value.timestamp,
+      type: result.value.type,
       data: result.value.data,
-      metadata: result.value.getMetadata(),
+      metadata: metadata,
       getMetadata() { return this.metadata; }
     });
     senderWriter.write(result.value);
@@ -80,7 +83,9 @@ async function testVideoFlow(t, negotiationFunction) {
 
   // Replace frame data with arbitrary buffers.
   for (let i = 0; i < numFramesReplaceData; i++) {
-    const result = await senderReader.read()
+    const result = await senderReader.read();
+    const metadata = result.value.getMetadata();
+    assert_true(containsVideoMetadata(metadata));
     const buffer = new ArrayBuffer(100);
     const int8View = new Int8Array(buffer);
     int8View.fill(i);
@@ -88,8 +93,9 @@ async function testVideoFlow(t, negotiationFunction) {
     result.value.data = buffer;
     frameInfos.push({
       timestamp: result.value.timestamp,
+      type: result.value.type,
       data: result.value.data,
-      metadata: result.value.getMetadata(),
+      metadata: metadata,
       getMetadata() { return this.metadata; }
     });
     senderWriter.write(result.value);
@@ -97,14 +103,17 @@ async function testVideoFlow(t, negotiationFunction) {
 
   // Modify frame data.
   for (let i = 0; i < numFramesReplaceData; i++) {
-    const result = await senderReader.read()
+    const result = await senderReader.read();
+    const metadata = result.value.getMetadata();
+    assert_true(containsVideoMetadata(metadata));
     const int8View = new Int8Array(result.value.data);
     int8View.fill(i);
 
     frameInfos.push({
       timestamp: result.value.timestamp,
+      type: result.value.type,
       data: result.value.data,
-      metadata: result.value.getMetadata(),
+      metadata: metadata,
       getMetadata() { return this.metadata; }
     });
     senderWriter.write(result.value);
@@ -391,6 +400,7 @@ promise_test(async t => {
   const result = await senderReader.read()
   sentFrameInfo = {
     timestamp: result.value.timestamp,
+    type: result.value.type,
     data: result.value.data,
     metadata: result.value.getMetadata(),
     getMetadata() { return this.metadata; }

third_party/blink/web_tests/fast/peerconnection/RTCPeerConnection-insertable-streams.js

@@ -13,36 +13,85 @@ function areArrayBuffersEqual(buffer1, buffer2)
   return true;
 }
 
-function areMetadataEqual(metadata1, metadata2) {
-  return metadata1.synchronizationSource === metadata2.synchronizationSource;
+function areArraysEqual(a1, a2) {
+  if (a1 === a1)
+    return true;
+  if (a1.length != a2.length)
+    return false;
+  for (let i = 0; i < a1.length; i++) {
+    if (a1[i] != a2[i])
+      return false;
+  }
+  return true;
+}
+
+function areMetadataEqual(metadata1, metadata2, type) {
+  return metadata1.synchronizationSource === metadata2.synchronizationSource &&
+         areArraysEqual(metadata1.contributingSources, metadata2.contributingSources) &&
+         metadata1.frameId === metadata2.frameId &&
+         areArraysEqual(metadata1.dependencies, metadata2.dependencies) &&
+         metadata1.spatialIndex === metadata2.spatialIndex &&
+         metadata1.temporalIndex === metadata2.temporalIndex &&
+         // Width and height are reported only for key frames on the receiver side.
+         type == "key"
+           ? metadata1.width === metadata2.width && metadata1.height === metadata2.height
+           : true;
 }
 
 function areFrameInfosEqual(frame1, frame2) {
   return frame1.timestamp === frame2.timestamp &&
-         areMetadataEqual(frame1.getMetadata(), frame2.getMetadata()) &&
+         frame1.type === frame2.type &&
+         areMetadataEqual(frame1.getMetadata(), frame2.getMetadata(), frame1.type) &&
          areArrayBuffersEqual(frame1.data, frame2.data);
 }
 
+function containsVideoMetadata(metadata) {
+  return metadata.synchronizationSource !== undefined &&
+         metadata.width !== undefined &&
+         metadata.height !== undefined &&
+         metadata.spatialIndex !== undefined &&
+         metadata.temporalIndex !== undefined &&
+         metadata.dependencies !== undefined;
+}
+
+function enableGFD(sdp) {
+  const FRAME_MARKER_EXTENSION =
+      'http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07';
+  const GFD_V00_EXTENSION =
+      'http://www.webrtc.org/experiments/rtp-hdrext/generic-frame-descriptor-00';
+  if (sdp.indexOf(GFD_V00_EXTENSION) !== -1)
+    return sdp;
+
+  // Replace the frame marker extension, which is unused with the GFD extension.
+  return sdp.split(FRAME_MARKER_EXTENSION).join(GFD_V00_EXTENSION);
+}
+
 async function exchangeOfferAnswer(pc1, pc2) {
-  const offer = await pc2.createOffer({offerToReceiveAudio: true, offerToReceiveVideo: true});
-  // TODO(crbug.com/1066819): remove this hack when we do not receive duplicates from RTX
-  //   anymore.
+  const offer = await pc1.createOffer();
+  // Munge the SDP to enable the GFD extension in order to get correct metadata.
+  const sdpGFD = enableGFD(offer.sdp);
+  await pc1.setLocalDescription({type: offer.type, sdp: sdpGFD});
   // Munge the SDP to disable bandwidth probing via RTX.
-  const sdp = offer.sdp.replace(new RegExp('rtx', 'g'), 'invalid');
-  await pc2.setLocalDescription(offer);
-  await pc1.setRemoteDescription({type: 'offer', sdp});
+  // TODO(crbug.com/1066819): remove this hack when we do not receive duplicates from RTX
+  // anymore.
+  const sdpRTX = sdpGFD.replace(new RegExp('rtx', 'g'), 'invalid');
+  await pc2.setRemoteDescription({type: 'offer', sdp: sdpRTX});
 
-  const answer = await pc1.createAnswer();
-  await pc1.setLocalDescription(answer);
-  await pc2.setRemoteDescription(answer);
+  const answer = await pc2.createAnswer();
+  await pc2.setLocalDescription(answer);
+  await pc1.setRemoteDescription(answer);
 }
 
 async function exchangeOfferAnswerReverse(pc1, pc2) {
   const offer = await pc2.createOffer({offerToReceiveAudio: true, offerToReceiveVideo: true});
+  // Munge the SDP to enable the GFD extension in order to get correct metadata.
+  const sdpGFD = enableGFD(offer.sdp);
   // Munge the SDP to disable bandwidth probing via RTX.
-  const sdp = offer.sdp.replace(new RegExp('rtx', 'g'), 'invalid');
-  await pc1.setRemoteDescription({type: 'offer', sdp});
-  await pc2.setLocalDescription(offer);
+  // TODO(crbug.com/1066819): remove this hack when we do not receive duplicates from RTX
+  // anymore.
+  const sdpRTX = sdpGFD.replace(new RegExp('rtx', 'g'), 'invalid');
+  await pc1.setRemoteDescription({type: 'offer', sdp: sdpRTX});
+  await pc2.setLocalDescription({type: 'offer', sdp: sdpGFD});
 
   const answer = await pc1.createAnswer();
   await pc2.setRemoteDescription(answer);

third_party/blink/web_tests/fast/peerconnection/RTCPeerConnection-sender-worker-single-frame.js

@@ -2,7 +2,6 @@ onmessage = async (event) => {
   const readableStream = event.data.readableStream;
   const reader = readableStream.getReader();
   const result = await reader.read();
-  console.log('WORKER metadata = ', result.value.getMetadata().synchronizationSource);
 
   // Post an object with individual fields so that the test side has
   // values to verify the serialization of the RTCEncodedVideoFrame.