[RTCInsertableStreams] Add WebRTC Insertable Streams API

This CL adds:
* RTCInsertableStreams Blink runtime-enabled flag
* IDL files with the API
* Implementation of encoded video frames based on webrtc::EncodedFrame
* Skeleton implementation of audio frames
* Skeleton implementation of methods in RTCRtpSender and RTCRtpReceiver

Real implementations will be added in follow-up CLs.

Bug: 1052765
Change-Id: I07089c60678c1987a1996bf782aaed1a0f3cc996
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2066765Reviewed-by: Harald Alvestrand <hta@chromium.org>
Reviewed-by: Nate Chapin <japhet@chromium.org>
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#744614}

third_party/blink/renderer/bindings/idl_in_modules.gni

@@ -444,6 +444,8 @@ static_idl_files_in_modules = get_path_info(
           "//third_party/blink/renderer/modules/peerconnection/rtc_dtmf_sender.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_dtmf_tone_change_event.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_dtmf_tone_change_event_init.idl",
+          "//third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.idl",
+          "//third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_error.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_error_event.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_error_event_init.idl",
@@ -455,6 +457,7 @@ static_idl_files_in_modules = get_path_info(
           "//third_party/blink/renderer/modules/peerconnection/rtc_ice_parameters.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_ice_server.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_ice_transport.idl",
+          "//third_party/blink/renderer/modules/peerconnection/rtc_insertable_streams.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_legacy_stats_report.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_offer_answer_options.idl",
           "//third_party/blink/renderer/modules/peerconnection/rtc_offer_options.idl",

third_party/blink/renderer/modules/peerconnection/BUILD.gn

@@ -76,6 +76,10 @@ blink_modules_sources("peerconnection") {
     "rtc_dtmf_sender.h",
     "rtc_dtmf_tone_change_event.cc",
     "rtc_dtmf_tone_change_event.h",
+    "rtc_encoded_audio_frame.cc",
+    "rtc_encoded_audio_frame.h",
+    "rtc_encoded_video_frame.cc",
+    "rtc_encoded_video_frame.h",
     "rtc_error.cc",
     "rtc_error.h",
     "rtc_error_event.cc",

third_party/blink/renderer/modules/peerconnection/idls.gni

@@ -9,6 +9,8 @@ modules_idl_files = [
   "rtc_dtls_transport.idl",
   "rtc_dtmf_sender.idl",
   "rtc_dtmf_tone_change_event.idl",
+  "rtc_encoded_video_frame.idl",
+  "rtc_encoded_audio_frame.idl",
   "rtc_error.idl",
   "rtc_error_event.idl",
   "rtc_ice_candidate.idl",
@@ -44,6 +46,7 @@ modules_dictionary_idl_files = [
   "rtc_ice_gather_options.idl",
   "rtc_ice_parameters.idl",
   "rtc_ice_server.idl",
+  "rtc_insertable_streams.idl",
   "rtc_offer_answer_options.idl",
   "rtc_offer_options.idl",
   "rtc_peer_connection_ice_error_event_init.idl",

third_party/blink/renderer/modules/peerconnection/rtc_configuration.idl

@@ -54,5 +54,6 @@ dictionary RTCConfiguration {
     [RuntimeEnabled=RtcAudioJitterBufferMaxPackets] long rtcAudioJitterBufferMaxPackets;
     [RuntimeEnabled=RtcAudioJitterBufferMaxPackets] boolean rtcAudioJitterBufferFastAccelerate;
     [RuntimeEnabled=RtcAudioJitterBufferMaxPackets] long rtcAudioJitterBufferMinDelayMs;
+    [RuntimeEnabled=RTCInsertableStreams] boolean forceEncodedAudioInsertableStreams = false;
+    [RuntimeEnabled=RTCInsertableStreams] boolean forceEncodedVideoInsertableStreams = false;
 };
-

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h"
+
+#include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
+#include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
+
+namespace blink {
+
+uint64_t RTCEncodedAudioFrame::timestamp() const {
+  return 0;
+}
+
+DOMArrayBuffer* RTCEncodedAudioFrame::data() const {
+  return frame_data_;
+}
+
+DOMArrayBuffer* RTCEncodedAudioFrame::additionalData() const {
+  return nullptr;
+}
+
+void RTCEncodedAudioFrame::setData(DOMArrayBuffer* data) {
+  frame_data_ = data;
+}
+
+String RTCEncodedAudioFrame::toString() const {
+  StringBuilder sb;
+  sb.Append("RTCEncodedAudioFrame{timestamp: ");
+  sb.AppendNumber(timestamp());
+  sb.Append("us, size: ");
+  sb.AppendNumber(data() ? data()->ByteLengthAsSizeT() : 0);
+  sb.Append("}");
+  return sb.ToString();
+}
+
+void RTCEncodedAudioFrame::Trace(Visitor* visitor) {
+  ScriptWrappable::Trace(visitor);
+  visitor->Trace(frame_data_);
+}
+
+}  // namespace blink

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_H_
+
+#include <stdint.h>
+
+#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
+#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
+
+namespace blink {
+
+class DOMArrayBuffer;
+
+class RTCEncodedAudioFrame final : public ScriptWrappable {
+  DEFINE_WRAPPERTYPEINFO();
+
+ public:
+  RTCEncodedAudioFrame() = default;
+
+  // rtc_encoded_audio_frame.idl implementation.
+  uint64_t timestamp() const;
+  DOMArrayBuffer* data() const;
+  DOMArrayBuffer* additionalData() const;
+  void setData(DOMArrayBuffer*);
+  String toString() const;
+
+  void Trace(Visitor*) override;
+
+ private:
+  mutable Member<DOMArrayBuffer> frame_data_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_H_

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md#api
+// TODO(crbug.com/1052765): Add link to standard when available.
+
+// TODO(crbug.com/1052765): Align with WebCodecs definition once it is stable.
+[RuntimeEnabled=RTCInsertableStreams]
+interface RTCEncodedAudioFrame {
+    readonly attribute unsigned long long timestamp;  // microseconds
+    attribute ArrayBuffer data;
+    // TODO(crbug.com/1052765): Replace this ArrayBuffer with structured data
+    // once we have decided what will be exposed.
+    readonly attribute ArrayBuffer additionalData;
+    stringifier;
+};

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.h"
+
+#include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
+#include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
+#include "third_party/webrtc/api/video/encoded_frame.h"
+#include "third_party/webrtc/api/video/encoded_image.h"
+
+namespace blink {
+
+RTCEncodedVideoFrame::RTCEncodedVideoFrame(
+    std::unique_ptr<webrtc::video_coding::EncodedFrame> delegate,
+    std::vector<uint8_t> additional_data)
+    : delegate_(std::move(delegate)),
+      additional_data_vector_(std::move(additional_data)) {}
+
+String RTCEncodedVideoFrame::type() const {
+  if (!delegate_)
+    return "empty";
+
+  return delegate_->is_keyframe() ? "key" : "delta";
+}
+
+uint64_t RTCEncodedVideoFrame::timestamp() const {
+  return delegate_->ReceivedTime();
+}
+
+DOMArrayBuffer* RTCEncodedVideoFrame::data() const {
+  if (delegate_ && !frame_data_) {
+    frame_data_ = DOMArrayBuffer::Create(delegate_->EncodedImage().data(),
+                                         delegate_->EncodedImage().size());
+  }
+  return frame_data_;
+}
+
+DOMArrayBuffer* RTCEncodedVideoFrame::additionalData() const {
+  if (!additional_data_) {
+    additional_data_ = DOMArrayBuffer::Create(additional_data_vector_.data(),
+                                              additional_data_vector_.size());
+  }
+  return additional_data_;
+}
+
+void RTCEncodedVideoFrame::setData(DOMArrayBuffer* data) {
+  frame_data_ = data;
+  replaced_frame_data_ = true;
+}
+
+String RTCEncodedVideoFrame::toString() const {
+  if (!delegate_)
+    return "empty";
+
+  StringBuilder sb;
+  sb.Append("RTCEncodedVideoFrame{timestamp: ");
+  sb.AppendNumber(timestamp());
+  sb.Append("us, size: ");
+  sb.AppendNumber(data()->ByteLengthAsSizeT());
+  sb.Append(" bytes, type: ");
+  sb.Append(type());
+  sb.Append("}");
+  return sb.ToString();
+}
+
+std::unique_ptr<webrtc::video_coding::EncodedFrame>
+RTCEncodedVideoFrame::PassDelegate() {
+  // Sync the delegate data with |frame_data_| if necessary.
+  if (replaced_frame_data_) {
+    rtc::scoped_refptr<webrtc::EncodedImageBuffer> webrtc_image =
+        webrtc::EncodedImageBuffer::Create(
+            static_cast<const uint8_t*>(frame_data_->Data()),
+            frame_data_->ByteLengthAsSizeT());
+    delegate_->SetEncodedData(std::move(webrtc_image));
+  }
+  return std::move(delegate_);
+}
+
+void RTCEncodedVideoFrame::Trace(Visitor* visitor) {
+  ScriptWrappable::Trace(visitor);
+  visitor->Trace(frame_data_);
+  visitor->Trace(additional_data_);
+}
+
+}  // namespace blink

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_VIDEO_FRAME_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_VIDEO_FRAME_H_
+
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
+#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
+
+namespace webrtc {
+namespace video_coding {
+class EncodedFrame;
+}  // namespace video_coding
+}  // namespace webrtc
+
+namespace blink {
+
+class DOMArrayBuffer;
+
+class RTCEncodedVideoFrame final : public ScriptWrappable {
+  DEFINE_WRAPPERTYPEINFO();
+
+ public:
+  explicit RTCEncodedVideoFrame(
+      std::unique_ptr<webrtc::video_coding::EncodedFrame> delegate,
+      std::vector<uint8_t> generic_descriptor);
+
+  // rtc_encoded_video_frame.idl implementation.
+  String type() const;
+  uint64_t timestamp() const;
+  DOMArrayBuffer* data() const;
+  DOMArrayBuffer* additionalData() const;
+  void setData(DOMArrayBuffer*);
+  String toString() const;
+
+  // Internal API
+  bool HasDelegate() { return !!delegate_; }
+  // Returns and transfers ownership of the internal WebRTC frame
+  // backing this RTCEncodedVideoFrame, leaving the RTCEncodedVideoFrame
+  // without a delegate WebRTC frame.
+  std::unique_ptr<webrtc::video_coding::EncodedFrame> PassDelegate();
+
+  void Trace(Visitor*) override;
+
+ private:
+  std::unique_ptr<webrtc::video_coding::EncodedFrame> delegate_;
+  std::vector<uint8_t> additional_data_vector_;
+  bool replaced_frame_data_ = false;
+  // Exposes encoded frame data from |delegate_|.
+  mutable Member<DOMArrayBuffer> frame_data_;
+  // Exposes data from |additional_data_vector_|.
+  mutable Member<DOMArrayBuffer> additional_data_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_VIDEO_FRAME_H_

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md#api
+// TODO(crbug.com/1052765): Add link to standard when available.
+
+enum RTCCodedVideoFrameType {
+    "empty",
+    "key",
+    "delta",
+};
+
+// TODO(crbug.com/1052765): Align with WebCodecs definition once it is stable.
+[RuntimeEnabled=RTCInsertableStreams]
+interface RTCEncodedVideoFrame {
+    readonly attribute RTCCodedVideoFrameType type;
+    readonly attribute unsigned long long timestamp;  // microseconds
+    attribute ArrayBuffer data;
+    // TODO(crbug.com/1052765): Replace this ArrayBuffer with structured data
+    // once we have decided what will be exposed.
+    readonly attribute ArrayBuffer additionalData;
+    stringifier;
+};

third_party/blink/renderer/modules/peerconnection/rtc_insertable_streams.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md#api
+// TODO(guidou): Add standards link when available.
+
+[Serializable]
+dictionary RTCInsertableStreams {
+    ReadableStream readableStream;
+    WritableStream writableStream;
+};

third_party/blink/renderer/modules/peerconnection/rtc_rtp_receiver.cc

@@ -134,6 +134,22 @@ ScriptPromise RTCRtpReceiver::getStats(ScriptState* script_state) {
   return promise;
 }
 
+RTCInsertableStreams* RTCRtpReceiver::createEncodedAudioStreams(
+    ScriptState* script_state,
+    ExceptionState& exception_state) {
+  exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+                                    "Not supported");
+  return nullptr;
+}
+
+RTCInsertableStreams* RTCRtpReceiver::createEncodedVideoStreams(
+    ScriptState* script_state,
+    ExceptionState& exception_state) {
+  exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+                                    "Not supported");
+  return nullptr;
+}
+
 RTCRtpReceiverPlatform* RTCRtpReceiver::platform_receiver() {
   return receiver_.get();
 }

third_party/blink/renderer/modules/peerconnection/rtc_rtp_receiver.h

@@ -23,6 +23,7 @@
 
 namespace blink {
 class RTCDtlsTransport;
+class RTCInsertableStreams;
 class RTCPeerConnection;
 class RTCRtpCapabilities;
 class RTCRtpTransceiver;
@@ -49,6 +50,10 @@ class RTCRtpReceiver final : public ScriptWrappable {
   HeapVector<Member<RTCRtpSynchronizationSource>> getSynchronizationSources();
   HeapVector<Member<RTCRtpContributingSource>> getContributingSources();
   ScriptPromise getStats(ScriptState*);
+  RTCInsertableStreams* createEncodedAudioStreams(ScriptState*,
+                                                  ExceptionState&);
+  RTCInsertableStreams* createEncodedVideoStreams(ScriptState*,
+                                                  ExceptionState&);
 
   RTCRtpReceiverPlatform* platform_receiver();
   MediaStreamVector streams() const;

third_party/blink/renderer/modules/peerconnection/rtc_rtp_receiver.idl

@@ -15,4 +15,6 @@ interface RTCRtpReceiver {
     sequence<RTCRtpSynchronizationSource> getSynchronizationSources();
     sequence<RTCRtpContributingSource> getContributingSources();
     [CallWith=ScriptState] Promise<RTCStatsReport> getStats();
+    [RuntimeEnabled=RTCInsertableStreams, CallWith=ScriptState, RaisesException] RTCInsertableStreams createEncodedAudioStreams();
+    [RuntimeEnabled=RTCInsertableStreams, CallWith=ScriptState, RaisesException] RTCInsertableStreams createEncodedVideoStreams();
 };

third_party/blink/renderer/modules/peerconnection/rtc_rtp_sender.cc

@@ -561,6 +561,22 @@ void RTCRtpSender::setStreams(HeapVector<Member<MediaStream>> streams,
   sender_->SetStreams(stream_ids);
 }
 
+RTCInsertableStreams* RTCRtpSender::createEncodedAudioStreams(
+    ScriptState* script_state,
+    ExceptionState& exception_state) {
+  exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+                                    "Not supported");
+  return nullptr;
+}
+
+RTCInsertableStreams* RTCRtpSender::createEncodedVideoStreams(
+    ScriptState* script_state,
+    ExceptionState& exception_state) {
+  exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+                                    "Not supported");
+  return nullptr;
+}
+
 void RTCRtpSender::Trace(Visitor* visitor) {
   visitor->Trace(pc_);
   visitor->Trace(track_);

third_party/blink/renderer/modules/peerconnection/rtc_rtp_sender.h

@@ -25,6 +25,7 @@ class ExceptionState;
 class MediaStreamTrack;
 class RTCDtlsTransport;
 class RTCDTMFSender;
+class RTCInsertableStreams;
 class RTCPeerConnection;
 class RTCRtpCapabilities;
 class RTCRtpTransceiver;
@@ -59,6 +60,10 @@ class RTCRtpSender final : public ScriptWrappable {
   ScriptPromise setParameters(ScriptState*, const RTCRtpSendParameters*);
   ScriptPromise getStats(ScriptState*);
   void setStreams(HeapVector<Member<MediaStream>> streams, ExceptionState&);
+  RTCInsertableStreams* createEncodedAudioStreams(ScriptState*,
+                                                  ExceptionState&);
+  RTCInsertableStreams* createEncodedVideoStreams(ScriptState*,
+                                                  ExceptionState&);
 
   RTCRtpSenderPlatform* web_sender();
   // Sets the track. This must be called when the |RTCRtpSenderPlatform| has its

third_party/blink/renderer/modules/peerconnection/rtc_rtp_sender.idl

@@ -15,5 +15,7 @@ interface RTCRtpSender {
     [Measure] readonly attribute RTCDTMFSender? dtmf;
     [Measure, RaisesException] void setStreams(MediaStream... streams);
     [CallWith=ScriptState] Promise<RTCStatsReport> getStats();
+    [RuntimeEnabled=RTCInsertableStreams, CallWith=ScriptState, RaisesException] RTCInsertableStreams createEncodedAudioStreams();
+    [RuntimeEnabled=RTCInsertableStreams, CallWith=ScriptState, RaisesException] RTCInsertableStreams createEncodedVideoStreams();
     // TODO(hbos): Implement the rest of RTCRtpSender, https://crbug.com/700916.
 };

third_party/blink/renderer/platform/runtime_enabled_features.json5

@@ -1454,6 +1454,10 @@
       origin_trial_feature_name: "RTCQuicTransport",
       status: "experimental",
     },
+    // Enables the use of Insertable Streams.
+    {
+      name: "RTCInsertableStreams",
+    },
     // Enables the use of the RTCQuicTransport object.
     {
       name: "RTCQuicTransport",