[RTCInsertableStreams] Make RTCEncodedAudioFrame serializable

The use case to be supported is efficiently running Streams of
RTCEncodedVideoFrame in a Worker. In this case, RTCEncodedVideoFrame
are serialized to the Worker.

Serialization is supported only in the same process as a shallow copy
where the underlying webrtc frame is shared among the original and
the deserialized object.
See https://github.com/alvestrand/webrtc-media-streams/issues/5

Bug: 1052765
Change-Id: Ie236cb3d0e0537ac55b07f030eee028cb5706217
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2132156
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Jeremy Roman <jbroman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#755880}

third_party/blink/renderer/bindings/core/v8/serialization/serialization_tag.h

@@ -102,6 +102,7 @@ enum SerializationTag {
   //                     namedCurve:uint32_t
   kRTCCertificateTag = 'k',  // length:uint32_t, pemPrivateKey:WebCoreString,
                              // pemCertificate:WebCoreString
+  kRTCEncodedAudioFrameTag = 'A',  // uint32_t -> transferred audio frame ID
   kRTCEncodedVideoFrameTag = 'V',  // uint32_t -> transferred video frame ID
 
   // The following tags were used by the Shape Detection API implementation

third_party/blink/renderer/bindings/modules/v8/serialization/v8_script_value_deserializer_for_modules.cc

@@ -18,6 +18,8 @@
 #include "third_party/blink/renderer/modules/native_file_system/native_file_system_file_handle.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_certificate.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_certificate_generator.h"
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h"
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame_delegate.h"
 
@@ -67,6 +69,8 @@ ScriptWrappable* V8ScriptValueDeserializerForModules::ReadDOMObject(
         return nullptr;
       return MakeGarbageCollected<RTCCertificate>(std::move(certificate));
     }
+    case kRTCEncodedAudioFrameTag:
+      return ReadRTCEncodedAudioFrame();
     case kRTCEncodedVideoFrameTag:
       return ReadRTCEncodedVideoFrame();
     default:
@@ -373,6 +377,30 @@ V8ScriptValueDeserializerForModules::ReadNativeFileSystemHandle(
   }
 }
 
+RTCEncodedAudioFrame*
+V8ScriptValueDeserializerForModules::ReadRTCEncodedAudioFrame() {
+  if (!RuntimeEnabledFeatures::RTCInsertableStreamsEnabled(
+          ExecutionContext::From(GetScriptState()))) {
+    return nullptr;
+  }
+
+  uint32_t index;
+  if (!ReadUint32(&index))
+    return nullptr;
+
+  const auto* attachment =
+      GetSerializedScriptValue()
+          ->GetAttachmentIfExists<RTCEncodedAudioFramesAttachment>();
+  if (!attachment)
+    return nullptr;
+
+  const auto& frames = attachment->EncodedAudioFrames();
+  if (index >= frames.size())
+    return nullptr;
+
+  return MakeGarbageCollected<RTCEncodedAudioFrame>(frames[index]);
+}
+
 RTCEncodedVideoFrame*
 V8ScriptValueDeserializerForModules::ReadRTCEncodedVideoFrame() {
   if (!RuntimeEnabledFeatures::RTCInsertableStreamsEnabled(

third_party/blink/renderer/bindings/modules/v8/serialization/v8_script_value_deserializer_for_modules.h

@@ -12,6 +12,7 @@ namespace blink {
 
 class CryptoKey;
 class NativeFileSystemHandle;
+class RTCEncodedAudioFrame;
 class RTCEncodedVideoFrame;
 
 // Extends V8ScriptValueSerializer with support for modules/ types.
@@ -45,6 +46,7 @@ class MODULES_EXPORT V8ScriptValueDeserializerForModules final
   }
   CryptoKey* ReadCryptoKey();
   NativeFileSystemHandle* ReadNativeFileSystemHandle(SerializationTag tag);
+  RTCEncodedAudioFrame* ReadRTCEncodedAudioFrame();
   RTCEncodedVideoFrame* ReadRTCEncodedVideoFrame();
 };
 

third_party/blink/renderer/bindings/modules/v8/serialization/v8_script_value_serializer_for_modules.cc

@@ -18,7 +18,10 @@
 #include "third_party/blink/renderer/bindings/modules/v8/v8_landmark.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_point_2d.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_rtc_certificate.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_rtc_encoded_audio_frame.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_rtc_encoded_video_frame.h"
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h"
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame.h"
 #include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_frame_delegate.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
@@ -77,6 +80,17 @@ bool V8ScriptValueSerializerForModules::WriteDOMObject(
     WriteUTF8String(pem.certificate().c_str());
     return true;
   }
+  if (wrapper_type_info == V8RTCEncodedAudioFrame::GetWrapperTypeInfo() &&
+      RuntimeEnabledFeatures::RTCInsertableStreamsEnabled(
+          ExecutionContext::From(GetScriptState()))) {
+    if (IsForStorage()) {
+      exception_state.ThrowDOMException(DOMExceptionCode::kDataCloneError,
+                                        "An RTCEncodedAudioFrame cannot be "
+                                        "serialized for storage.");
+      return false;
+    }
+    return WriteRTCEncodedAudioFrame(wrappable->ToImpl<RTCEncodedAudioFrame>());
+  }
   if (wrapper_type_info == V8RTCEncodedVideoFrame::GetWrapperTypeInfo() &&
       RuntimeEnabledFeatures::RTCInsertableStreamsEnabled(
           ExecutionContext::From(GetScriptState()))) {
@@ -286,6 +300,20 @@ bool V8ScriptValueSerializerForModules::WriteNativeFileSystemHandle(
   return true;
 }
 
+bool V8ScriptValueSerializerForModules::WriteRTCEncodedAudioFrame(
+    RTCEncodedAudioFrame* audio_frame) {
+  auto* attachment =
+      GetSerializedScriptValue()
+          ->GetOrCreateAttachment<RTCEncodedAudioFramesAttachment>();
+  auto& frames = attachment->EncodedAudioFrames();
+  frames.push_back(audio_frame->Delegate());
+  const uint32_t index = static_cast<uint32_t>(frames.size() - 1);
+
+  WriteTag(kRTCEncodedAudioFrameTag);
+  WriteUint32(index);
+  return true;
+}
+
 bool V8ScriptValueSerializerForModules::WriteRTCEncodedVideoFrame(
     RTCEncodedVideoFrame* video_frame) {
   auto* attachment =

third_party/blink/renderer/bindings/modules/v8/serialization/v8_script_value_serializer_for_modules.h

@@ -12,6 +12,7 @@
 namespace blink {
 
 class NativeFileSystemHandle;
+class RTCEncodedAudioFrame;
 class RTCEncodedVideoFrame;
 class WebCryptoKey;
 
@@ -33,6 +34,7 @@ class MODULES_EXPORT V8ScriptValueSerializerForModules final
   bool WriteNativeFileSystemHandle(
       SerializationTag tag,
       NativeFileSystemHandle* native_file_system_handle);
+  bool WriteRTCEncodedAudioFrame(RTCEncodedAudioFrame*);
   bool WriteRTCEncodedVideoFrame(RTCEncodedVideoFrame*);
 };
 

third_party/blink/renderer/modules/peerconnection/BUILD.gn

@@ -78,6 +78,8 @@ blink_modules_sources("peerconnection") {
     "rtc_dtmf_tone_change_event.h",
     "rtc_encoded_audio_frame.cc",
     "rtc_encoded_audio_frame.h",
+    "rtc_encoded_audio_frame_delegate.cc",
+    "rtc_encoded_audio_frame_delegate.h",
     "rtc_encoded_audio_underlying_sink.cc",
     "rtc_encoded_audio_underlying_sink.h",
     "rtc_encoded_audio_underlying_source.cc",

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.cc

@@ -4,7 +4,10 @@
 
 #include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h"
 
+#include <utility>
+
 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.h"
 #include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
 #include "third_party/webrtc/api/frame_transformer_interface.h"
 
@@ -12,30 +15,37 @@ namespace blink {
 
 RTCEncodedAudioFrame::RTCEncodedAudioFrame(
     std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame)
-    : webrtc_frame_(std::move(webrtc_frame)) {}
+    : delegate_(base::MakeRefCounted<RTCEncodedAudioFrameDelegate>(
+          std::move(webrtc_frame),
+          Vector<uint32_t>())) {}
 
 RTCEncodedAudioFrame::RTCEncodedAudioFrame(
     std::unique_ptr<webrtc::TransformableAudioFrameInterface>
         webrtc_audio_frame) {
+  Vector<uint32_t> contributing_sources;
   if (webrtc_audio_frame) {
     wtf_size_t num_csrcs = webrtc_audio_frame->GetHeader().numCSRCs;
-    contributing_sources_.ReserveInitialCapacity(num_csrcs);
+    contributing_sources.ReserveInitialCapacity(num_csrcs);
     for (wtf_size_t i = 0; i < num_csrcs; i++) {
-      contributing_sources_.push_back(
+      contributing_sources.push_back(
           webrtc_audio_frame->GetHeader().arrOfCSRCs[i]);
     }
   }
-  webrtc_frame_ = std::move(webrtc_audio_frame);
+  delegate_ = base::MakeRefCounted<RTCEncodedAudioFrameDelegate>(
+      std::move(webrtc_audio_frame), std::move(contributing_sources));
 }
 
+RTCEncodedAudioFrame::RTCEncodedAudioFrame(
+    scoped_refptr<RTCEncodedAudioFrameDelegate> delegate)
+    : delegate_(std::move(delegate)) {}
+
 uint64_t RTCEncodedAudioFrame::timestamp() const {
-  return webrtc_frame_ ? webrtc_frame_->GetTimestamp() : 0;
+  return delegate_->Timestamp();
 }
 
 DOMArrayBuffer* RTCEncodedAudioFrame::data() const {
-  if (webrtc_frame_ && !frame_data_) {
-    frame_data_ = DOMArrayBuffer::Create(webrtc_frame_->GetData().data(),
-                                         webrtc_frame_->GetData().size());
+  if (!frame_data_) {
+    frame_data_ = delegate_->CreateDataBuffer();
   }
   return frame_data_;
 }
@@ -49,11 +59,11 @@ void RTCEncodedAudioFrame::setData(DOMArrayBuffer* data) {
 }
 
 uint32_t RTCEncodedAudioFrame::synchronizationSource() const {
-  return webrtc_frame_ ? webrtc_frame_->GetSsrc() : 0;
+  return delegate_->Ssrc();
 }
 
 Vector<uint32_t> RTCEncodedAudioFrame::contributingSources() const {
-  return webrtc_frame_ ? contributing_sources_ : Vector<uint32_t>();
+  return delegate_->ContributingSources();
 }
 
 String RTCEncodedAudioFrame::toString() const {
@@ -66,15 +76,20 @@ String RTCEncodedAudioFrame::toString() const {
   return sb.ToString();
 }
 
+void RTCEncodedAudioFrame::SyncDelegate() const {
+  delegate_->SetData(frame_data_);
+}
+
+scoped_refptr<RTCEncodedAudioFrameDelegate> RTCEncodedAudioFrame::Delegate()
+    const {
+  SyncDelegate();
+  return delegate_;
+}
+
 std::unique_ptr<webrtc::TransformableFrameInterface>
-RTCEncodedAudioFrame::PassDelegate() {
-  // Sync the delegate data with |frame_data_| if necessary.
-  if (webrtc_frame_ && frame_data_) {
-    webrtc_frame_->SetData(rtc::ArrayView<const uint8_t>(
-        static_cast<const uint8_t*>(frame_data_->Data()),
-        frame_data_->ByteLengthAsSizeT()));
-  }
-  return std::move(webrtc_frame_);
+RTCEncodedAudioFrame::PassWebRtcFrame() {
+  SyncDelegate();
+  return delegate_->PassWebRtcFrame();
 }
 
 void RTCEncodedAudioFrame::Trace(Visitor* visitor) {

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame.h

@@ -20,6 +20,7 @@ class TransformableFrameInterface;
 namespace blink {
 
 class DOMArrayBuffer;
+class RTCEncodedAudioFrameDelegate;
 
 class MODULES_EXPORT RTCEncodedAudioFrame final : public ScriptWrappable {
   DEFINE_WRAPPERTYPEINFO();
@@ -29,6 +30,8 @@ class MODULES_EXPORT RTCEncodedAudioFrame final : public ScriptWrappable {
       std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame);
   explicit RTCEncodedAudioFrame(
       std::unique_ptr<webrtc::TransformableAudioFrameInterface> webrtc_frame);
+  explicit RTCEncodedAudioFrame(
+      scoped_refptr<RTCEncodedAudioFrameDelegate> delegate);
 
   // rtc_encoded_audio_frame.idl implementation.
   uint64_t timestamp() const;
@@ -39,17 +42,20 @@ class MODULES_EXPORT RTCEncodedAudioFrame final : public ScriptWrappable {
   Vector<uint32_t> contributingSources() const;
   String toString() const;
 
+  scoped_refptr<RTCEncodedAudioFrameDelegate> Delegate() const;
+  void SyncDelegate() const;
+
   // Returns and transfers ownership of the internal WebRTC frame
-  // backing this RTCEncodedVideoFrame, leaving the RTCEncodedVideoFrame
-  // without a delegate WebRTC frame.
-  std::unique_ptr<webrtc::TransformableFrameInterface> PassDelegate();
+  // backing this RTCEncodedAudioFrame, neutering all RTCEncodedAudioFrames
+  // backed by that internal WebRTC frame.
+  std::unique_ptr<webrtc::TransformableFrameInterface> PassWebRtcFrame();
 
   void Trace(Visitor*) override;
 
  private:
-  mutable Member<DOMArrayBuffer> frame_data_;
+  scoped_refptr<RTCEncodedAudioFrameDelegate> delegate_;
   Vector<uint32_t> contributing_sources_;
-  std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame_;
+  mutable Member<DOMArrayBuffer> frame_data_;
 };
 
 }  // namespace blink

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.h"
+
+#include <utility>
+
+#include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h"
+
+namespace blink {
+
+const void* RTCEncodedAudioFramesAttachment::kAttachmentKey;
+
+RTCEncodedAudioFrameDelegate::RTCEncodedAudioFrameDelegate(
+    std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame,
+    Vector<uint32_t> contributing_sources)
+    : webrtc_frame_(std::move(webrtc_frame)),
+      contributing_sources_(std::move(contributing_sources)) {}
+
+uint64_t RTCEncodedAudioFrameDelegate::Timestamp() const {
+  MutexLocker lock(mutex_);
+  return webrtc_frame_ ? webrtc_frame_->GetTimestamp() : 0;
+}
+
+DOMArrayBuffer* RTCEncodedAudioFrameDelegate::CreateDataBuffer() const {
+  ArrayBufferContents contents;
+  {
+    MutexLocker lock(mutex_);
+    if (!webrtc_frame_)
+      return nullptr;
+
+    auto data = webrtc_frame_->GetData();
+    contents =
+        ArrayBufferContents(data.size(), 1, ArrayBufferContents::kNotShared,
+                            ArrayBufferContents::kDontInitialize);
+    if (UNLIKELY(!contents.Data()))
+      OOM_CRASH(data.size());
+    memcpy(contents.Data(), data.data(), data.size());
+  }
+  return DOMArrayBuffer::Create(std::move(contents));
+}
+
+void RTCEncodedAudioFrameDelegate::SetData(const DOMArrayBuffer* data) {
+  MutexLocker lock(mutex_);
+  if (webrtc_frame_ && data) {
+    webrtc_frame_->SetData(rtc::ArrayView<const uint8_t>(
+        static_cast<const uint8_t*>(data->Data()), data->ByteLengthAsSizeT()));
+  }
+}
+
+uint32_t RTCEncodedAudioFrameDelegate::Ssrc() const {
+  MutexLocker lock(mutex_);
+  return webrtc_frame_ ? webrtc_frame_->GetSsrc() : 0;
+}
+
+Vector<uint32_t> RTCEncodedAudioFrameDelegate::ContributingSources() const {
+  MutexLocker lock(mutex_);
+  return contributing_sources_;
+}
+
+std::unique_ptr<webrtc::TransformableFrameInterface>
+RTCEncodedAudioFrameDelegate::PassWebRtcFrame() {
+  MutexLocker lock(mutex_);
+  return std::move(webrtc_frame_);
+}
+
+}  // namespace blink

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_frame_delegate.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_DELEGATE_H_
+#define THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_DELEGATE_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "third_party/blink/renderer/bindings/core/v8/serialization/serialized_script_value.h"
+#include "third_party/blink/renderer/modules/modules_export.h"
+#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
+#include "third_party/blink/renderer/platform/wtf/threading_primitives.h"
+#include "third_party/webrtc/api/frame_transformer_interface.h"
+
+namespace blink {
+
+class DOMArrayBuffer;
+
+// This class wraps a WebRTC audio frame and allows making shallow
+// copies. Its purpose is to support making RTCEncodedVideoFrames
+// serializable in the same process.
+class RTCEncodedAudioFrameDelegate
+    : public WTF::ThreadSafeRefCounted<RTCEncodedAudioFrameDelegate> {
+ public:
+  explicit RTCEncodedAudioFrameDelegate(
+      std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame,
+      Vector<uint32_t> contributing_sources);
+
+  uint64_t Timestamp() const;
+  DOMArrayBuffer* CreateDataBuffer() const;
+  void SetData(const DOMArrayBuffer* data);
+  uint32_t Ssrc() const;
+  Vector<uint32_t> ContributingSources() const;
+  std::unique_ptr<webrtc::TransformableFrameInterface> PassWebRtcFrame();
+
+ private:
+  mutable Mutex mutex_;
+  std::unique_ptr<webrtc::TransformableFrameInterface> webrtc_frame_
+      GUARDED_BY(mutex_);
+  Vector<uint32_t> contributing_sources_;
+};
+
+class MODULES_EXPORT RTCEncodedAudioFramesAttachment
+    : public SerializedScriptValue::Attachment {
+ public:
+  static const void* kAttachmentKey;
+  RTCEncodedAudioFramesAttachment() = default;
+  ~RTCEncodedAudioFramesAttachment() override = default;
+
+  bool IsLockedToAgentCluster() const override {
+    return !encoded_audio_frames_.IsEmpty();
+  }
+
+  Vector<scoped_refptr<RTCEncodedAudioFrameDelegate>>& EncodedAudioFrames() {
+    return encoded_audio_frames_;
+  }
+
+  const Vector<scoped_refptr<RTCEncodedAudioFrameDelegate>>&
+  EncodedAudioFrames() const {
+    return encoded_audio_frames_;
+  }
+
+ private:
+  Vector<scoped_refptr<RTCEncodedAudioFrameDelegate>> encoded_audio_frames_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_PEERCONNECTION_RTC_ENCODED_AUDIO_FRAME_DELEGATE_H_

third_party/blink/renderer/modules/peerconnection/rtc_encoded_audio_underlying_sink.cc

@@ -48,7 +48,8 @@ ScriptPromise RTCEncodedAudioUnderlyingSink::write(
     return ScriptPromise();
   }
 
-  transformer_callback_.Run()->SendFrameToSink(encoded_frame->PassDelegate());
+  transformer_callback_.Run()->SendFrameToSink(
+      encoded_frame->PassWebRtcFrame());
   return ScriptPromise::CastUndefined(script_state);
 }