MediaRecorder: enable passthrough mode.

Passthrough recording is enabled for video iff
1. mimeType is empty
2. the video source supports encoded output.

When passthrough recording is enabled, a key frame is requested
from the video source and recording starts when it appears.

Bug: 1013590
Change-Id: I8ecd0970d59146336af001232545be42b840e4d2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1893992Reviewed-by: Miguel Casas <mcasas@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Commit-Queue: Markus Handell <handellm@google.com>
Cr-Commit-Position: refs/heads/master@{#721921}

third_party/blink/renderer/modules/mediarecorder/media_recorder.cc

@@ -24,8 +24,6 @@ namespace blink {
 
 namespace {
 
-const char kDefaultMimeType[] = "video/webm";
-
 // Boundaries of Opus bitrate from https://www.opus-codec.org/.
 const int kSmallestPossibleOpusBitRate = 6000;
 const int kLargestAutoAllocatedOpusBitRate = 128000;
@@ -161,8 +159,7 @@ MediaRecorder::MediaRecorder(ExecutionContext* context,
                              ExceptionState& exception_state)
     : ContextLifecycleObserver(context),
       stream_(stream),
-      mime_type_(options->hasMimeType() ? options->mimeType()
-                                        : kDefaultMimeType),
+      mime_type_(options->mimeType()),
       stopped_(true),
       audio_bits_per_second_(0),
       video_bits_per_second_(0),

third_party/blink/renderer/modules/mediarecorder/media_recorder_handler.cc

@@ -46,7 +46,24 @@ namespace {
 const float kNumPixelsPerSecondSmoothnessThresholdLow = 640 * 480 * 30.0;
 const float kNumPixelsPerSecondSmoothnessThresholdHigh = 1280 * 720 * 30.0;
 
-media::VideoCodec CodecIdToMediaVideoCodec(VideoTrackRecorder::CodecId id) {
+VideoTrackRecorder::CodecId CodecIdFromMediaVideoCodec(media::VideoCodec id) {
+  switch (id) {
+    case media::kCodecVP8:
+      return VideoTrackRecorder::CodecId::VP8;
+    case media::kCodecVP9:
+      return VideoTrackRecorder::CodecId::VP9;
+#if BUILDFLAG(RTC_USE_H264)
+    case media::kCodecH264:
+      return VideoTrackRecorder::CodecId::H264;
+#endif
+    default:
+      return VideoTrackRecorder::CodecId::LAST;
+  }
+  NOTREACHED() << "Unsupported video codec";
+  return VideoTrackRecorder::CodecId::LAST;
+}
+
+media::VideoCodec MediaVideoCodecFromCodecId(VideoTrackRecorder::CodecId id) {
   switch (id) {
     case VideoTrackRecorder::CodecId::VP8:
       return media::kCodecVP8;
@@ -108,7 +125,8 @@ AudioTrackRecorder::CodecId AudioStringToCodecId(const String& codecs) {
 
 MediaRecorderHandler::MediaRecorderHandler(
     scoped_refptr<base::SingleThreadTaskRunner> task_runner)
-    : video_bits_per_second_(0),
+    : passthrough_enabled_(false),
+      video_bits_per_second_(0),
       audio_bits_per_second_(0),
       video_codec_id_(VideoTrackRecorder::CodecId::LAST),
       audio_codec_id_(AudioTrackRecorder::CodecId::LAST),
@@ -183,6 +201,8 @@ bool MediaRecorderHandler::Initialize(MediaRecorder* recorder,
     return false;
   }
 
+  passthrough_enabled_ = type.IsEmpty();
+
   // Once established that we support the codec(s), hunt then individually.
   const VideoTrackRecorder::CodecId video_codec_id =
       VideoStringToCodecId(codecs);
@@ -258,13 +278,29 @@ bool MediaRecorderHandler::Start(int timeslice) {
     if (!video_tracks_[0])
       return false;
 
-    const VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb =
-        media::BindToCurrentLoop(WTF::BindRepeating(
-            &MediaRecorderHandler::OnEncodedVideo, WrapWeakPersistent(this)));
-
-    video_recorders_.emplace_back(MakeGarbageCollected<VideoTrackRecorderImpl>(
-        video_codec_id_, video_tracks_[0], on_encoded_video_cb,
-        video_bits_per_second_, task_runner_));
+    MediaStreamVideoTrack* const video_track =
+        static_cast<MediaStreamVideoTrack*>(
+            video_tracks_[0]->GetPlatformTrack());
+    DCHECK(video_track->source());
+    const bool use_encoded_source_output =
+        video_track->source()->SupportsEncodedOutput();
+    if (passthrough_enabled_ && use_encoded_source_output) {
+      const VideoTrackRecorder::OnEncodedVideoCB on_passthrough_video_cb =
+          media::BindToCurrentLoop(
+              WTF::BindRepeating(&MediaRecorderHandler::OnPassthroughVideo,
+                                 WrapWeakPersistent(this)));
+      video_recorders_.emplace_back(
+          MakeGarbageCollected<VideoTrackRecorderPassthrough>(
+              video_tracks_[0], on_passthrough_video_cb, task_runner_));
+    } else {
+      const VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb =
+          media::BindToCurrentLoop(WTF::BindRepeating(
+              &MediaRecorderHandler::OnEncodedVideo, WrapWeakPersistent(this)));
+      video_recorders_.emplace_back(
+          MakeGarbageCollected<VideoTrackRecorderImpl>(
+              video_codec_id_, video_tracks_[0], on_encoded_video_cb,
+              video_bits_per_second_, task_runner_));
+    }
   }
 
   if (use_audio_tracks) {
@@ -454,6 +490,33 @@ void MediaRecorderHandler::OnEncodedVideo(
     base::TimeTicks timestamp,
     bool is_key_frame) {
   DCHECK(IsMainThread());
+  auto params_with_codec = params;
+  params_with_codec.codec = MediaVideoCodecFromCodecId(video_codec_id_);
+  HandleEncodedVideo(params_with_codec, std::move(encoded_data),
+                     std::move(encoded_alpha), timestamp, is_key_frame);
+}
+
+void MediaRecorderHandler::OnPassthroughVideo(
+    const media::WebmMuxer::VideoParameters& params,
+    std::string encoded_data,
+    std::string encoded_alpha,
+    base::TimeTicks timestamp,
+    bool is_key_frame) {
+  DCHECK(IsMainThread());
+
+  // Update |video_codec_id_| so that ActualMimeType() works.
+  video_codec_id_ = CodecIdFromMediaVideoCodec(params.codec);
+  HandleEncodedVideo(params, std::move(encoded_data), std::move(encoded_alpha),
+                     timestamp, is_key_frame);
+}
+
+void MediaRecorderHandler::HandleEncodedVideo(
+    const media::WebmMuxer::VideoParameters& params,
+    std::string encoded_data,
+    std::string encoded_alpha,
+    base::TimeTicks timestamp,
+    bool is_key_frame) {
+  DCHECK(IsMainThread());
 
   if (video_recorders_.IsEmpty())
     return;
@@ -464,9 +527,7 @@ void MediaRecorderHandler::OnEncodedVideo(
   }
   if (!webm_muxer_)
     return;
-  media::WebmMuxer::VideoParameters params_with_codec = params;
-  params_with_codec.codec = CodecIdToMediaVideoCodec(video_codec_id_);
-  if (!webm_muxer_->OnEncodedVideo(params_with_codec, std::move(encoded_data),
+  if (!webm_muxer_->OnEncodedVideo(params, std::move(encoded_data),
                                    std::move(encoded_alpha), timestamp,
                                    is_key_frame)) {
     DLOG(ERROR) << "Error muxing video data";
@@ -559,6 +620,13 @@ void MediaRecorderHandler::OnVideoFrameForTesting(
     recorder->OnVideoFrameForTesting(frame, timestamp);
 }
 
+void MediaRecorderHandler::OnEncodedVideoFrameForTesting(
+    scoped_refptr<EncodedVideoFrame> frame,
+    const base::TimeTicks& timestamp) {
+  for (const auto& recorder : video_recorders_)
+    recorder->OnEncodedVideoFrameForTesting(frame, timestamp);
+}
+
 void MediaRecorderHandler::OnAudioBusForTesting(
     const media::AudioBus& audio_bus,
     const base::TimeTicks& timestamp) {

third_party/blink/renderer/modules/mediarecorder/media_recorder_handler.h

@@ -11,6 +11,7 @@
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_piece.h"
 #include "base/threading/thread_checker.h"
+#include "third_party/blink/public/web/modules/mediastream/encoded_video_frame.h"
 #include "third_party/blink/renderer/modules/mediarecorder/audio_track_recorder.h"
 #include "third_party/blink/renderer/modules/mediarecorder/video_track_recorder.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
@@ -80,6 +81,7 @@ class MODULES_EXPORT MediaRecorderHandler final
 
  private:
   friend class MediaRecorderHandlerTest;
+  friend class MediaRecorderHandlerPassthroughTest;
 
   // Called to indicate there is encoded video data available. |encoded_alpha|
   // represents the encode output of alpha channel when available, can be
@@ -89,6 +91,16 @@ class MODULES_EXPORT MediaRecorderHandler final
                       std::string encoded_alpha,
                       base::TimeTicks timestamp,
                       bool is_key_frame);
+  void OnPassthroughVideo(const media::WebmMuxer::VideoParameters& params,
+                          std::string encoded_data,
+                          std::string encoded_alpha,
+                          base::TimeTicks timestamp,
+                          bool is_key_frame);
+  void HandleEncodedVideo(const media::WebmMuxer::VideoParameters& params,
+                          std::string encoded_data,
+                          std::string encoded_alpha,
+                          base::TimeTicks timestamp,
+                          bool is_key_frame);
   void OnEncodedAudio(const media::AudioParameters& params,
                       std::string encoded_data,
                       base::TimeTicks timestamp);
@@ -99,10 +111,17 @@ class MODULES_EXPORT MediaRecorderHandler final
 
   void OnVideoFrameForTesting(scoped_refptr<media::VideoFrame> frame,
                               const base::TimeTicks& timestamp);
+  void OnEncodedVideoFrameForTesting(scoped_refptr<EncodedVideoFrame> frame,
+                                     const base::TimeTicks& timestamp);
   void OnAudioBusForTesting(const media::AudioBus& audio_bus,
                             const base::TimeTicks& timestamp);
   void SetAudioFormatForTesting(const media::AudioParameters& params);
 
+  // Set to true if there is no MIME type configured upon Initialize()
+  // and the video track's source supports encoded output, giving
+  // this class the freedom to provide whatever it chooses to produce.
+  bool passthrough_enabled_;
+
   // Sanitized video and audio bitrate settings passed on initialize().
   int32_t video_bits_per_second_;
   int32_t audio_bits_per_second_;

third_party/blink/renderer/modules/mediarecorder/media_recorder_handler_unittest.cc

@@ -11,14 +11,17 @@
 #include "base/time/time.h"
 #include "media/audio/simple_sources.h"
 #include "media/base/audio_bus.h"
+#include "media/base/video_color_space.h"
 #include "media/base/video_frame.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h"
 #include "third_party/blink/renderer/bindings/core/v8/v8_binding_for_testing.h"
+#include "third_party/blink/renderer/modules/mediarecorder/fake_encoded_video_frame.h"
 #include "third_party/blink/renderer/modules/mediarecorder/media_recorder.h"
 #include "third_party/blink/renderer/modules/mediarecorder/media_recorder_handler.h"
 #include "third_party/blink/renderer/modules/mediastream/mock_media_stream_registry.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_source.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/heap/heap.h"
 #include "third_party/blink/renderer/platform/heap/thread_state.h"
@@ -27,10 +30,12 @@
 
 using ::testing::_;
 using ::testing::AtLeast;
+using ::testing::Ge;
 using ::testing::Gt;
 using ::testing::InSequence;
 using ::testing::Lt;
 using ::testing::Mock;
+using ::testing::Return;
 using ::testing::TestWithParam;
 using ::testing::ValuesIn;
 
@@ -142,10 +147,14 @@ class MediaRecorderHandlerTest : public TestWithParam<MediaRecorderTestParams> {
     media_recorder_handler_->SetAudioFormatForTesting(params);
   }
 
+  void AddVideoTrack() {
+    video_source_ = registry_.AddVideoTrack(kTestVideoTrackId);
+  }
+
   void AddTracks() {
     // Avoid issues with non-parameterized tests by calling this outside of ctr.
     if (GetParam().has_video)
-      registry_.AddVideoTrack(kTestVideoTrackId);
+      AddVideoTrack();
     if (GetParam().has_audio)
       registry_.AddAudioTrack(kTestAudioTrackId);
   }
@@ -173,6 +182,8 @@ class MediaRecorderHandlerTest : public TestWithParam<MediaRecorderTestParams> {
   // For generating test AudioBuses
   media::SineWaveAudioSource audio_source_;
 
+  MockMediaStreamVideoSource* video_source_ = 0;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(MediaRecorderHandlerTest);
 };
@@ -503,4 +514,90 @@ TEST_P(MediaRecorderHandlerTest, ActualMimeType) {
   media_recorder_handler_ = nullptr;
 }
 
+struct MediaRecorderPassthroughTestParams {
+  const char* mime_type;
+  media::VideoCodec codec;
+};
+
+static const MediaRecorderPassthroughTestParams
+    kMediaRecorderPassthroughTestParams[] = {
+        {"video/webm;codecs=vp8", media::kCodecVP8},
+        {"video/webm;codecs=vp9", media::kCodecVP9},
+#if BUILDFLAG(RTC_USE_H264)
+        {"video/x-matroska;codecs=avc1", media::kCodecH264},
+#endif
+};
+
+class MediaRecorderHandlerPassthroughTest
+    : public TestWithParam<MediaRecorderPassthroughTestParams> {
+ public:
+  MediaRecorderHandlerPassthroughTest() {
+    registry_.Init();
+    video_source_ = registry_.AddVideoTrack(kTestVideoTrackId);
+    ON_CALL(*video_source_, SupportsEncodedOutput).WillByDefault(Return(true));
+    media_recorder_handler_ = MakeGarbageCollected<MediaRecorderHandler>(
+        scheduler::GetSingleThreadTaskRunnerForTesting());
+    EXPECT_FALSE(media_recorder_handler_->recording_);
+  }
+
+  ~MediaRecorderHandlerPassthroughTest() {
+    registry_.reset();
+    ThreadState::Current()->CollectAllGarbageForTesting();
+  }
+
+  void OnVideoFrameForTesting(scoped_refptr<EncodedVideoFrame> frame) {
+    media_recorder_handler_->OnEncodedVideoFrameForTesting(
+        std::move(frame), base::TimeTicks::Now());
+  }
+
+  ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_;
+  MockMediaStreamRegistry registry_;
+  MockMediaStreamVideoSource* video_source_ = 0;
+  Persistent<MediaRecorderHandler> media_recorder_handler_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MediaRecorderHandlerPassthroughTest);
+};
+
+TEST_P(MediaRecorderHandlerPassthroughTest, PassesThrough) {
+  // Setup the mock video source to allow for passthrough recording.
+  EXPECT_CALL(*video_source_, OnEncodedSinkEnabled);
+  EXPECT_CALL(*video_source_, OnEncodedSinkDisabled);
+
+  V8TestingScope scope;
+  auto* recorder = MakeGarbageCollected<MockMediaRecorder>(scope);
+  media_recorder_handler_->Initialize(recorder, registry_.test_stream(), "", "",
+                                      0, 0);
+  media_recorder_handler_->Start(0);
+
+  const size_t kFrameSize = 42;
+  auto frame = FakeEncodedVideoFrame::Builder()
+                   .WithKeyFrame(true)
+                   .WithCodec(GetParam().codec)
+                   .WithData(std::string(kFrameSize, 'P'))
+                   .BuildRefPtr();
+  {
+    base::RunLoop run_loop;
+    base::Closure quit_closure = run_loop.QuitClosure();
+    EXPECT_CALL(*recorder, WriteData(_, _, _, _)).Times(AtLeast(1));
+    EXPECT_CALL(*recorder, WriteData(_, Ge(kFrameSize), _, _))
+        .Times(1)
+        .WillOnce(RunClosure5(std::move(quit_closure)));
+    OnVideoFrameForTesting(frame);
+    run_loop.Run();
+  }
+  EXPECT_EQ(media_recorder_handler_->ActualMimeType(), GetParam().mime_type);
+  Mock::VerifyAndClearExpectations(this);
+
+  media_recorder_handler_->Stop();
+
+  // Expect a last call on destruction.
+  EXPECT_CALL(*recorder, WriteData(_, _, true, _)).Times(1);
+  media_recorder_handler_ = nullptr;
+}
+
+INSTANTIATE_TEST_SUITE_P(,
+                         MediaRecorderHandlerPassthroughTest,
+                         ValuesIn(kMediaRecorderPassthroughTestParams));
+
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/mock_media_stream_registry.cc

@@ -55,7 +55,7 @@ void MockMediaStreamRegistry::Init() {
   test_stream_.Initialize(label, webkit_audio_tracks, webkit_video_tracks);
 }
 
-void MockMediaStreamRegistry::AddVideoTrack(
+MockMediaStreamVideoSource* MockMediaStreamRegistry::AddVideoTrack(
     const std::string& track_id,
     const VideoTrackAdapterSettings& adapter_settings,
     const base::Optional<bool>& noise_reduction,
@@ -75,11 +75,14 @@ void MockMediaStreamRegistry::AddVideoTrack(
       min_frame_rate, MediaStreamVideoSource::ConstraintsOnceCallback(),
       true /* enabled */));
   test_stream_.AddTrack(blink_track);
+  return native_source;
 }
 
-void MockMediaStreamRegistry::AddVideoTrack(const std::string& track_id) {
-  AddVideoTrack(track_id, VideoTrackAdapterSettings(), base::Optional<bool>(),
-                false /* is_screncast */, 0.0 /* min_frame_rate */);
+MockMediaStreamVideoSource* MockMediaStreamRegistry::AddVideoTrack(
+    const std::string& track_id) {
+  return AddVideoTrack(track_id, VideoTrackAdapterSettings(),
+                       base::Optional<bool>(), false /* is_screncast */,
+                       0.0 /* min_frame_rate */);
 }
 
 void MockMediaStreamRegistry::AddAudioTrack(const std::string& track_id) {

third_party/blink/renderer/modules/mediastream/mock_media_stream_registry.h

@@ -9,6 +9,7 @@
 
 #include "base/optional.h"
 #include "third_party/blink/public/platform/web_media_stream.h"
+#include "third_party/blink/renderer/modules/mediastream/mock_media_stream_video_source.h"
 
 namespace blink {
 
@@ -22,12 +23,15 @@ class MockMediaStreamRegistry final {
   MockMediaStreamRegistry();
 
   void Init();
-  void AddVideoTrack(const std::string& track_id,
-                     const VideoTrackAdapterSettings& adapter_settings,
-                     const base::Optional<bool>& noise_reduction,
-                     bool is_screen_cast,
-                     double min_frame_rate);
-  void AddVideoTrack(const std::string& track_id);
+
+  // Returns the native mock vidoe source for optional use in tests.
+  MockMediaStreamVideoSource* AddVideoTrack(
+      const std::string& track_id,
+      const VideoTrackAdapterSettings& adapter_settings,
+      const base::Optional<bool>& noise_reduction,
+      bool is_screen_cast,
+      double min_frame_rate);
+  MockMediaStreamVideoSource* AddVideoTrack(const std::string& track_id);
   void AddAudioTrack(const std::string& track_id);
 
   const WebMediaStream test_stream() const { return test_stream_; }