[RTCInsertableStreams] Handle simulcast layers in video transformer.

Test in
https://chromium-review.googlesource.com/c/chromium/src/+/2144276


Bug: 1065838
Change-Id: I8804f187dec8f889993c757fc8466faab032c107
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2142033
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Marina Ciocea <marinaciocea@chromium.org>
Cr-Commit-Position: refs/heads/master@{#758394}

third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_underlying_sink_test.cc

@@ -27,6 +27,8 @@ namespace blink {
 
 namespace {
 
+const uint32_t kSSRC = 1;
+
 class MockWebRtcTransformedFrameCallback
     : public webrtc::TransformedFrameCallback {
  public:
@@ -34,6 +36,27 @@ class MockWebRtcTransformedFrameCallback
                void(std::unique_ptr<webrtc::TransformableFrameInterface>));
 };
 
+class FakeVideoFrame : public webrtc::TransformableVideoFrameInterface {
+ public:
+  explicit FakeVideoFrame(uint32_t ssrc) : ssrc_(ssrc) {}
+
+  rtc::ArrayView<const uint8_t> GetData() const override {
+    return rtc::ArrayView<const uint8_t>();
+  }
+
+  // Copies |data| into the owned frame payload data.
+  void SetData(rtc::ArrayView<const uint8_t> data) override {}
+  uint32_t GetTimestamp() const override { return 0; }
+  uint32_t GetSsrc() const override { return ssrc_; }
+  bool IsKeyFrame() const override { return true; }
+  std::vector<uint8_t> GetAdditionalData() const override {
+    return std::vector<uint8_t>();
+  }
+
+ private:
+  uint32_t ssrc_;
+};
+
 }  // namespace
 
 class RTCEncodedVideoUnderlyingSinkTest : public testing::Test {
@@ -46,15 +69,15 @@ class RTCEncodedVideoUnderlyingSinkTest : public testing::Test {
         transformer_(main_task_runner_) {}
 
   void SetUp() override {
-    EXPECT_FALSE(transformer_.HasTransformedFrameCallback());
-    transformer_.RegisterTransformedFrameCallback(webrtc_callback_);
-    EXPECT_TRUE(transformer_.HasTransformedFrameCallback());
+    EXPECT_FALSE(transformer_.HasTransformedFrameSinkCallback(kSSRC));
+    transformer_.RegisterTransformedFrameSinkCallback(webrtc_callback_, kSSRC);
+    EXPECT_TRUE(transformer_.HasTransformedFrameSinkCallback(kSSRC));
   }
 
   void TearDown() override {
     platform_->RunUntilIdle();
-    transformer_.UnregisterTransformedFrameCallback();
-    EXPECT_FALSE(transformer_.HasTransformedFrameCallback());
+    transformer_.UnregisterTransformedFrameSinkCallback(kSSRC);
+    EXPECT_FALSE(transformer_.HasTransformedFrameSinkCallback(kSSRC));
   }
 
   RTCEncodedVideoUnderlyingSink* CreateSink(ScriptState* script_state) {
@@ -68,7 +91,7 @@ class RTCEncodedVideoUnderlyingSinkTest : public testing::Test {
 
   ScriptValue CreateEncodedVideoFrameChunk(ScriptState* script_state) {
     RTCEncodedVideoFrame* frame = MakeGarbageCollected<RTCEncodedVideoFrame>(
-        std::unique_ptr<webrtc::TransformableVideoFrameInterface>());
+        std::make_unique<FakeVideoFrame>(kSSRC));
     return ScriptValue(script_state->GetIsolate(),
                        ToV8(frame, script_state->GetContext()->Global(),
                             script_state->GetIsolate()));

third_party/blink/renderer/platform/peerconnection/rtc_encoded_video_stream_transformer.cc

@@ -36,22 +36,45 @@ class RTCEncodedVideoStreamTransformerDelegate
   }
 
   // webrtc::FrameTransformerInterface
+  // TODO(crbug.com/1065838): Remove the non-ssrc version of the registration
+  // and unregistration methods once WebRTC uses the ssrc version in all cases.
   void RegisterTransformedFrameCallback(
       rtc::scoped_refptr<webrtc::TransformedFrameCallback>
           send_frame_to_sink_callback) override {
     PostCrossThreadTask(
         *main_task_runner_, FROM_HERE,
-        CrossThreadBindOnce(
-            &RTCEncodedVideoStreamTransformer::RegisterTransformedFrameCallback,
-            transformer_, std::move(send_frame_to_sink_callback)));
+        CrossThreadBindOnce(&RTCEncodedVideoStreamTransformer::
+                                RegisterTransformedFrameSinkCallback,
+                            transformer_,
+                            std::move(send_frame_to_sink_callback), 0));
   }
 
   void UnregisterTransformedFrameCallback() override {
     PostCrossThreadTask(
         *main_task_runner_, FROM_HERE,
         CrossThreadBindOnce(&RTCEncodedVideoStreamTransformer::
-                                UnregisterTransformedFrameCallback,
-                            transformer_));
+                                UnregisterTransformedFrameSinkCallback,
+                            transformer_, 0));
+  }
+
+  void RegisterTransformedFrameSinkCallback(
+      rtc::scoped_refptr<webrtc::TransformedFrameCallback>
+          send_frame_to_sink_callback,
+      uint32_t ssrc) override {
+    PostCrossThreadTask(
+        *main_task_runner_, FROM_HERE,
+        CrossThreadBindOnce(&RTCEncodedVideoStreamTransformer::
+                                RegisterTransformedFrameSinkCallback,
+                            transformer_,
+                            std::move(send_frame_to_sink_callback), ssrc));
+  }
+
+  void UnregisterTransformedFrameSinkCallback(uint32_t ssrc) override {
+    PostCrossThreadTask(
+        *main_task_runner_, FROM_HERE,
+        CrossThreadBindOnce(&RTCEncodedVideoStreamTransformer::
+                                UnregisterTransformedFrameSinkCallback,
+                            transformer_, ssrc));
   }
 
   void Transform(
@@ -80,15 +103,28 @@ RTCEncodedVideoStreamTransformer::RTCEncodedVideoStreamTransformer(
           weak_factory_.GetWeakPtr(), std::move(main_task_runner));
 }
 
-void RTCEncodedVideoStreamTransformer::RegisterTransformedFrameCallback(
-    rtc::scoped_refptr<webrtc::TransformedFrameCallback> callback) {
+void RTCEncodedVideoStreamTransformer::RegisterTransformedFrameSinkCallback(
+    rtc::scoped_refptr<webrtc::TransformedFrameCallback> callback,
+    uint32_t ssrc) {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  send_frame_to_sink_cb_ = callback;
+  for (auto& sink_callback : send_frame_to_sink_callbacks_) {
+    if (sink_callback.first == ssrc) {
+      sink_callback.second = std::move(callback);
+      return;
+    }
+  }
+  send_frame_to_sink_callbacks_.push_back(std::make_pair(ssrc, callback));
 }
 
-void RTCEncodedVideoStreamTransformer::UnregisterTransformedFrameCallback() {
+void RTCEncodedVideoStreamTransformer::UnregisterTransformedFrameSinkCallback(
+    uint32_t ssrc) {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  send_frame_to_sink_cb_ = nullptr;
+  for (wtf_size_t i = 0; i < send_frame_to_sink_callbacks_.size(); ++i) {
+    if (send_frame_to_sink_callbacks_[i].first == ssrc) {
+      send_frame_to_sink_callbacks_.EraseAt(i);
+      return;
+    }
+  }
 }
 
 void RTCEncodedVideoStreamTransformer::TransformFrame(
@@ -104,8 +140,20 @@ void RTCEncodedVideoStreamTransformer::TransformFrame(
 void RTCEncodedVideoStreamTransformer::SendFrameToSink(
     std::unique_ptr<webrtc::TransformableVideoFrameInterface> frame) {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  if (send_frame_to_sink_cb_)
-    send_frame_to_sink_cb_->OnTransformedFrame(std::move(frame));
+  // TODO(crbug.com/1069275): Remove this section once WebRTC reports ssrc in
+  // all sink callback registrations.
+  if (send_frame_to_sink_callbacks_.size() == 1 &&
+      send_frame_to_sink_callbacks_[0].first == 0) {
+    send_frame_to_sink_callbacks_[0].second->OnTransformedFrame(
+        std::move(frame));
+    return;
+  }
+  for (const auto& sink_callback : send_frame_to_sink_callbacks_) {
+    if (sink_callback.first == frame->GetSsrc()) {
+      sink_callback.second->OnTransformedFrame(std::move(frame));
+      return;
+    }
+  }
 }
 
 void RTCEncodedVideoStreamTransformer::SetTransformerCallback(
@@ -124,9 +172,14 @@ bool RTCEncodedVideoStreamTransformer::HasTransformerCallback() const {
   return !transformer_callback_.is_null();
 }
 
-bool RTCEncodedVideoStreamTransformer::HasTransformedFrameCallback() const {
+bool RTCEncodedVideoStreamTransformer::HasTransformedFrameSinkCallback(
+    uint32_t ssrc) const {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  return !!send_frame_to_sink_cb_;
+  for (const auto& sink_callbacks : send_frame_to_sink_callbacks_) {
+    if (sink_callbacks.first == ssrc)
+      return true;
+  }
+  return false;
 }
 
 rtc::scoped_refptr<webrtc::FrameTransformerInterface>

third_party/blink/renderer/platform/peerconnection/rtc_encoded_video_stream_transformer.h

@@ -8,12 +8,13 @@
 #include <stdint.h>
 
 #include <memory>
-#include <vector>
+#include <utility>
 
 #include "base/callback.h"
 #include "base/memory/weak_ptr.h"
 #include "base/threading/thread_checker.h"
 #include "third_party/blink/renderer/platform/platform_export.h"
+#include "third_party/blink/renderer/platform/wtf/vector.h"
 #include "third_party/webrtc/api/scoped_refptr.h"
 
 namespace base {
@@ -38,13 +39,17 @@ class PLATFORM_EXPORT RTCEncodedVideoStreamTransformer {
   // Called by WebRTC to let us know about a callback object to send transformed
   // frames to the WebRTC decoder. Runs on an internal WebRTC thread.
   // The callback can run on any thread.
-  void RegisterTransformedFrameCallback(
-      rtc::scoped_refptr<webrtc::TransformedFrameCallback>);
+  void RegisterTransformedFrameSinkCallback(
+      rtc::scoped_refptr<webrtc::TransformedFrameCallback>,
+      uint32_t ssrc);
 
   // Called by WebRTC to let us know that any reference to the callback object
   // reported by RegisterTransformedFrameCallback() should be released since
   // the callback is no longer useful and is intended for destruction.
-  void UnregisterTransformedFrameCallback();
+  // TODO(crbug.com/1065838): Remove the non-ssrc version once WebRTC uses the
+  // ssrc version in all cases.
+  //   void UnregisterTransformedFrameCallback();
+  void UnregisterTransformedFrameSinkCallback(uint32_t ssrc);
 
   // Called by WebRTC to notify of new untransformed frames from the WebRTC
   // stack. Runs on an internal WebRTC thread.
@@ -67,15 +72,18 @@ class PLATFORM_EXPORT RTCEncodedVideoStreamTransformer {
   // false otherwise. Must run on the main thread.
   bool HasTransformerCallback() const;
 
-  // Returns true if a webrtc::TransformedFrameCallback is registered.
-  bool HasTransformedFrameCallback() const;
+  // Returns true if a webrtc::TransformedFrameCallback is registered for
+  // the given ssrc.
+  bool HasTransformedFrameSinkCallback(uint32_t ssrc) const;
 
   rtc::scoped_refptr<webrtc::FrameTransformerInterface> Delegate();
 
  private:
   THREAD_CHECKER(thread_checker_);
   rtc::scoped_refptr<webrtc::FrameTransformerInterface> delegate_;
-  rtc::scoped_refptr<webrtc::TransformedFrameCallback> send_frame_to_sink_cb_;
+  Vector<
+      std::pair<uint32_t, rtc::scoped_refptr<webrtc::TransformedFrameCallback>>>
+      send_frame_to_sink_callbacks_;
   TransformerCallback transformer_callback_;
   base::WeakPtrFactory<RTCEncodedVideoStreamTransformer> weak_factory_{this};
 };

third_party/blink/renderer/platform/peerconnection/rtc_encoded_video_stream_transformer_test.cc

@@ -26,6 +26,9 @@ namespace blink {
 
 namespace {
 
+const uint32_t kSSRC = 1;
+const uint32_t kNonexistentSSRC = 0;
+
 class MockWebRtcTransformedFrameCallback
     : public webrtc::TransformedFrameCallback {
  public:
@@ -39,6 +42,31 @@ class MockTransformerCallbackHolder {
                void(std::unique_ptr<webrtc::TransformableVideoFrameInterface>));
 };
 
+class FakeVideoFrame : public webrtc::TransformableVideoFrameInterface {
+ public:
+  explicit FakeVideoFrame(uint32_t ssrc) : ssrc_(ssrc) {}
+
+  rtc::ArrayView<const uint8_t> GetData() const override {
+    return rtc::ArrayView<const uint8_t>();
+  }
+
+  // Copies |data| into the owned frame payload data.
+  void SetData(rtc::ArrayView<const uint8_t> data) override {}
+  uint32_t GetTimestamp() const override { return 0; }
+  uint32_t GetSsrc() const override { return ssrc_; }
+  bool IsKeyFrame() const override { return true; }
+  std::vector<uint8_t> GetAdditionalData() const override {
+    return std::vector<uint8_t>();
+  }
+
+ private:
+  uint32_t ssrc_;
+};
+
+std::unique_ptr<webrtc::TransformableVideoFrameInterface> CreateFakeFrame() {
+  return std::make_unique<FakeVideoFrame>(kSSRC);
+}
+
 }  // namespace
 
 class RTCEncodedVideoStreamTransformerTest : public ::testing::Test {
@@ -53,17 +81,24 @@ class RTCEncodedVideoStreamTransformerTest : public ::testing::Test {
 
   void SetUp() override {
     EXPECT_FALSE(
-        encoded_video_stream_transformer_.HasTransformedFrameCallback());
-    encoded_video_stream_transformer_.RegisterTransformedFrameCallback(
-        webrtc_callback_);
+        encoded_video_stream_transformer_.HasTransformedFrameSinkCallback(
+            kSSRC));
+    encoded_video_stream_transformer_.RegisterTransformedFrameSinkCallback(
+        webrtc_callback_, kSSRC);
     EXPECT_TRUE(
-        encoded_video_stream_transformer_.HasTransformedFrameCallback());
+        encoded_video_stream_transformer_.HasTransformedFrameSinkCallback(
+            kSSRC));
+    EXPECT_FALSE(
+        encoded_video_stream_transformer_.HasTransformedFrameSinkCallback(
+            kNonexistentSSRC));
   }
 
   void TearDown() override {
-    encoded_video_stream_transformer_.UnregisterTransformedFrameCallback();
+    encoded_video_stream_transformer_.UnregisterTransformedFrameSinkCallback(
+        kSSRC);
     EXPECT_FALSE(
-        encoded_video_stream_transformer_.HasTransformedFrameCallback());
+        encoded_video_stream_transformer_.HasTransformedFrameSinkCallback(
+            kSSRC));
   }
 
  protected:
@@ -92,13 +127,13 @@ TEST_F(RTCEncodedVideoStreamTransformerTest,
       *webrtc_task_runner_, FROM_HERE,
       CrossThreadBindOnce(&webrtc::FrameTransformerInterface::Transform,
                           encoded_video_stream_transformer_.Delegate(),
-                          nullptr));
+                          CreateFakeFrame()));
   task_environment_.RunUntilIdle();
 }
 
 TEST_F(RTCEncodedVideoStreamTransformerTest, TransformerForwardsFrameToWebRTC) {
   EXPECT_CALL(*webrtc_callback_, OnTransformedFrame);
-  encoded_video_stream_transformer_.SendFrameToSink(nullptr);
+  encoded_video_stream_transformer_.SendFrameToSink(CreateFakeFrame());
   task_environment_.RunUntilIdle();
 }