[webcodecs] Offloading software video encoding from the main thread

Bug: 1132057
Change-Id: Ie30034adb37fa9d105c98be78f01143da4608469
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2504736
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#822502}

media/base/BUILD.gn

@@ -230,6 +230,8 @@ source_set("base") {
     "multi_channel_resampler.h",
     "null_video_sink.cc",
     "null_video_sink.h",
+    "offloading_video_encoder.cc",
+    "offloading_video_encoder.h",
     "output_device_info.cc",
     "output_device_info.h",
     "overlay_info.cc",
@@ -576,6 +578,7 @@ source_set("unit_tests") {
     "moving_average_unittest.cc",
     "multi_channel_resampler_unittest.cc",
     "null_video_sink_unittest.cc",
+    "offloading_video_encoder_unittest.cc",
     "pipeline_impl_unittest.cc",
     "ranges_unittest.cc",
     "reentrancy_checker_unittest.cc",

media/base/mock_filters.cc

@@ -100,6 +100,11 @@ std::string MockVideoDecoder::GetDisplayName() const {
   return decoder_name_;
 }
 
+MockVideoEncoder::MockVideoEncoder() = default;
+MockVideoEncoder::~MockVideoEncoder() {
+  Dtor();
+}
+
 MockAudioDecoder::MockAudioDecoder() : MockAudioDecoder("MockAudioDecoder") {}
 
 MockAudioDecoder::MockAudioDecoder(std::string decoder_name)

media/base/mock_filters.h

@@ -42,6 +42,7 @@
 #include "media/base/time_source.h"
 #include "media/base/video_decoder.h"
 #include "media/base/video_decoder_config.h"
+#include "media/base/video_encoder.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_renderer.h"
 #include "testing/gmock/include/gmock/gmock.h"
@@ -261,6 +262,42 @@ class MockVideoDecoder : public VideoDecoder {
   DISALLOW_COPY_AND_ASSIGN(MockVideoDecoder);
 };
 
+class MockVideoEncoder : public VideoEncoder {
+ public:
+  MockVideoEncoder();
+  ~MockVideoEncoder() override;
+
+  // VideoEncoder implementation.
+  MOCK_METHOD(void,
+              Initialize,
+              (VideoCodecProfile profile,
+               const VideoEncoder::Options& options,
+               VideoEncoder::OutputCB output_cb,
+               VideoEncoder::StatusCB done_cb),
+              (override));
+
+  MOCK_METHOD(void,
+              Encode,
+              (scoped_refptr<VideoFrame> frame,
+               bool key_frame,
+               VideoEncoder::StatusCB done_cb),
+              (override));
+
+  MOCK_METHOD(void,
+              ChangeOptions,
+              (const VideoEncoder::Options& options,
+               VideoEncoder::StatusCB done_cb),
+              (override));
+
+  MOCK_METHOD(void, Flush, (VideoEncoder::StatusCB done_cb), (override));
+
+  // A function for mocking destructor calls
+  MOCK_METHOD(void, Dtor, ());
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MockVideoEncoder);
+};
+
 class MockAudioDecoder : public AudioDecoder {
  public:
   MockAudioDecoder();

media/base/offloading_video_encoder.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/offloading_video_encoder.h"
+
+#include "base/sequenced_task_runner.h"
+#include "base/task/task_traits.h"
+#include "base/task/thread_pool.h"
+#include "media/base/bind_to_current_loop.h"
+#include "media/base/video_frame.h"
+
+namespace media {
+
+OffloadingVideoEncoder::OffloadingVideoEncoder(
+    std::unique_ptr<VideoEncoder> wrapped_encoder,
+    const scoped_refptr<base::SequencedTaskRunner> work_runner,
+    const scoped_refptr<base::SequencedTaskRunner> callback_runner)
+    : wrapped_encoder_(std::move(wrapped_encoder)),
+      work_runner_(std::move(work_runner)),
+      callback_runner_(std::move(callback_runner)) {
+  DCHECK(wrapped_encoder_);
+  DCHECK(work_runner_);
+  DCHECK(callback_runner_);
+  DCHECK_NE(callback_runner_, work_runner_);
+}
+
+OffloadingVideoEncoder::OffloadingVideoEncoder(
+    std::unique_ptr<VideoEncoder> wrapped_encoder)
+    : OffloadingVideoEncoder(std::move(wrapped_encoder),
+                             base::ThreadPool::CreateSequencedTaskRunner(
+                                 {base::TaskPriority::USER_VISIBLE}),
+                             base::SequencedTaskRunnerHandle::Get()) {}
+
+void OffloadingVideoEncoder::Initialize(VideoCodecProfile profile,
+                                        const Options& options,
+                                        OutputCB output_cb,
+                                        StatusCB done_cb) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  work_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&VideoEncoder::Initialize,
+                     base::Unretained(wrapped_encoder_.get()), profile, options,
+                     WrapCallback(std::move(output_cb)),
+                     WrapCallback(std::move(done_cb))));
+}
+
+void OffloadingVideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
+                                    bool key_frame,
+                                    StatusCB done_cb) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  work_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&VideoEncoder::Encode,
+                     base::Unretained(wrapped_encoder_.get()), std::move(frame),
+                     key_frame, WrapCallback(std::move(done_cb))));
+}
+
+void OffloadingVideoEncoder::ChangeOptions(const Options& options,
+                                           StatusCB done_cb) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  work_runner_->PostTask(
+      FROM_HERE, base::BindOnce(&VideoEncoder::ChangeOptions,
+                                base::Unretained(wrapped_encoder_.get()),
+                                options, WrapCallback(std::move(done_cb))));
+}
+
+void OffloadingVideoEncoder::Flush(StatusCB done_cb) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  work_runner_->PostTask(
+      FROM_HERE, base::BindOnce(&VideoEncoder::Flush,
+                                base::Unretained(wrapped_encoder_.get()),
+                                WrapCallback(std::move(done_cb))));
+}
+
+OffloadingVideoEncoder::~OffloadingVideoEncoder() {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  work_runner_->DeleteSoon(FROM_HERE, std::move(wrapped_encoder_));
+}
+
+template <class T>
+T OffloadingVideoEncoder::WrapCallback(T cb) {
+  DCHECK(callback_runner_);
+  return media::BindToLoop(callback_runner_.get(), std::move(cb));
+}
+
+}  // namespace media
\ No newline at end of file

media/base/offloading_video_encoder.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_BASE_OFFLOADING_VIDEO_ENCODER_H_
+#define MEDIA_BASE_OFFLOADING_VIDEO_ENCODER_H_
+
+#include <memory>
+#include <type_traits>
+
+#include "base/sequence_checker.h"
+#include "media/base/video_encoder.h"
+
+namespace base {
+class SequencedTaskRunner;
+}
+
+namespace media {
+
+// A wrapper around video encoder that offloads all the calls to a dedicated
+// task runner. It's used to move synchronous software encoding work off the
+// current (main) thread.
+class MEDIA_EXPORT OffloadingVideoEncoder final : public VideoEncoder {
+ public:
+  // |work_runner| - task runner for encoding work
+  // |callback_runner| - all encoder's callbacks will be executed on this task
+  // runner.
+  OffloadingVideoEncoder(
+      std::unique_ptr<VideoEncoder> wrapped_encoder,
+      const scoped_refptr<base::SequencedTaskRunner> work_runner,
+      const scoped_refptr<base::SequencedTaskRunner> callback_runner);
+
+  // Uses current task runner for callbacks and asks thread pool for a new task
+  // runner to do actual encoding work.
+  explicit OffloadingVideoEncoder(
+      std::unique_ptr<VideoEncoder> wrapped_encoder);
+
+  ~OffloadingVideoEncoder() override;
+
+  void Initialize(VideoCodecProfile profile,
+                  const Options& options,
+                  OutputCB output_cb,
+                  StatusCB done_cb) override;
+
+  void Encode(scoped_refptr<VideoFrame> frame,
+              bool key_frame,
+              StatusCB done_cb) override;
+
+  void ChangeOptions(const Options& options, StatusCB done_cb) override;
+
+  void Flush(StatusCB done_cb) override;
+
+ private:
+  template <class T>
+  T WrapCallback(T cb);
+
+  std::unique_ptr<VideoEncoder> wrapped_encoder_;
+  const scoped_refptr<base::SequencedTaskRunner> work_runner_;
+  const scoped_refptr<base::SequencedTaskRunner> callback_runner_;
+  SEQUENCE_CHECKER(sequence_checker_);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_BASE_OFFLOADING_VIDEO_ENCODER_H_

media/base/offloading_video_encoder_unittest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <memory>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/run_loop.h"
+#include "base/sequenced_task_runner.h"
+#include "base/test/bind_test_util.h"
+#include "base/test/gmock_callback_support.h"
+#include "base/test/task_environment.h"
+#include "media/base/media_util.h"
+#include "media/base/mock_filters.h"
+#include "media/base/offloading_video_encoder.h"
+#include "media/base/video_types.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using ::base::test::RunCallback;
+using ::base::test::RunOnceCallback;
+using ::testing::_;
+using ::testing::DoAll;
+using ::testing::Invoke;
+using ::testing::Return;
+
+namespace media {
+
+class OffloadingVideoEncoderTest : public testing::Test {
+ protected:
+  void SetUp() override {
+    auto mock_video_encoder = std::make_unique<MockVideoEncoder>();
+    mock_video_encoder_ = mock_video_encoder.get();
+    work_runner_ = base::ThreadPool::CreateSequencedTaskRunner({});
+    callback_runner_ = base::SequencedTaskRunnerHandle::Get();
+    offloading_encoder_ = std::make_unique<OffloadingVideoEncoder>(
+        std::move(mock_video_encoder), work_runner_, callback_runner_);
+    EXPECT_CALL(*mock_video_encoder_, Dtor()).WillOnce(Invoke([this]() {
+      EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence());
+    }));
+  }
+
+  void RunLoop() { task_environment_.RunUntilIdle(); }
+
+  base::test::TaskEnvironment task_environment_;
+  scoped_refptr<base::SequencedTaskRunner> work_runner_;
+  scoped_refptr<base::SequencedTaskRunner> callback_runner_;
+  MockVideoEncoder* mock_video_encoder_;
+  std::unique_ptr<OffloadingVideoEncoder> offloading_encoder_;
+};
+
+TEST_F(OffloadingVideoEncoderTest, Initialize) {
+  bool called_done = false;
+  bool called_output = false;
+  VideoEncoder::Options options;
+  VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
+  VideoEncoder::OutputCB output_cb = base::BindLambdaForTesting(
+      [&](VideoEncoderOutput, base::Optional<VideoEncoder::CodecDescription>) {
+        EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
+        called_output = true;
+      });
+  VideoEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) {
+    EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
+    called_done = true;
+  });
+
+  EXPECT_CALL(*mock_video_encoder_, Initialize(_, _, _, _))
+      .WillOnce(Invoke([this](VideoCodecProfile profile,
+                              const VideoEncoder::Options& options,
+                              VideoEncoder::OutputCB output_cb,
+                              VideoEncoder::StatusCB done_cb) {
+        EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence());
+        std::move(done_cb).Run(Status());
+        std::move(output_cb).Run(VideoEncoderOutput(), {});
+      }));
+
+  offloading_encoder_->Initialize(profile, options, std::move(output_cb),
+                                  std::move(done_cb));
+  RunLoop();
+  EXPECT_TRUE(called_done);
+  EXPECT_TRUE(called_output);
+}
+
+TEST_F(OffloadingVideoEncoderTest, Encode) {
+  bool called_done = false;
+  VideoEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) {
+    EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
+    called_done = true;
+  });
+
+  EXPECT_CALL(*mock_video_encoder_, Encode(_, _, _))
+      .WillOnce(Invoke([this](scoped_refptr<VideoFrame> frame, bool key_frame,
+                              VideoEncoder::StatusCB done_cb) {
+        EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence());
+        std::move(done_cb).Run(Status());
+      }));
+
+  offloading_encoder_->Encode(nullptr, false, std::move(done_cb));
+  RunLoop();
+  EXPECT_TRUE(called_done);
+}
+
+TEST_F(OffloadingVideoEncoderTest, ChangeOptions) {
+  bool called_done = false;
+  VideoEncoder::Options options;
+  VideoEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) {
+    EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
+    called_done = true;
+  });
+
+  EXPECT_CALL(*mock_video_encoder_, ChangeOptions(_, _))
+      .WillOnce(Invoke([this](const VideoEncoder::Options& options,
+                              VideoEncoder::StatusCB done_cb) {
+        EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence());
+        std::move(done_cb).Run(Status());
+      }));
+
+  offloading_encoder_->ChangeOptions(options, std::move(done_cb));
+  RunLoop();
+  EXPECT_TRUE(called_done);
+}
+
+TEST_F(OffloadingVideoEncoderTest, Flush) {
+  bool called_done = false;
+  VideoEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) {
+    EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
+    called_done = true;
+  });
+
+  EXPECT_CALL(*mock_video_encoder_, Flush(_))
+      .WillOnce(Invoke([this](VideoEncoder::StatusCB done_cb) {
+        EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence());
+        std::move(done_cb).Run(Status());
+      }));
+
+  offloading_encoder_->Flush(std::move(done_cb));
+  RunLoop();
+  EXPECT_TRUE(called_done);
+}
+
+}  // namespace media

third_party/blink/renderer/modules/webcodecs/video_encoder.cc

@@ -11,7 +11,9 @@
 #include "base/logging.h"
 #include "base/macros.h"
 #include "build/build_config.h"
+#include "media/base/async_destroy_video_encoder.h"
 #include "media/base/mime_util.h"
+#include "media/base/offloading_video_encoder.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_color_space.h"
 #include "media/base/video_encoder.h"
@@ -21,7 +23,6 @@
 #if BUILDFLAG(ENABLE_LIBVPX)
 #include "media/video/vpx_video_encoder.h"
 #endif
-#include "media/base/async_destroy_video_encoder.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "media/video/video_encode_accelerator_adapter.h"
 #include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
@@ -288,31 +289,27 @@ std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateMediaVideoEncoder(
   switch (config.acc_pref) {
     case AccelerationPreference::kRequire:
       return CreateAcceleratedVideoEncoder(config.profile, config.options);
-    case AccelerationPreference::kAllow: {
-      auto result =
-          CreateAcceleratedVideoEncoder(config.profile, config.options);
-      if (result)
+    case AccelerationPreference::kAllow:
+      if (auto result =
+              CreateAcceleratedVideoEncoder(config.profile, config.options))
         return result;
-      switch (config.codec) {
-        case media::kCodecVP8:
-        case media::kCodecVP9:
-          return CreateVpxVideoEncoder();
-        case media::kCodecH264:
-          return CreateOpenH264VideoEncoder();
-        default:
-          return nullptr;
-      }
-    }
+      FALLTHROUGH;
     case AccelerationPreference::kDeny: {
+      std::unique_ptr<media::VideoEncoder> result;
       switch (config.codec) {
         case media::kCodecVP8:
         case media::kCodecVP9:
-          return CreateVpxVideoEncoder();
+          result = CreateVpxVideoEncoder();
+          break;
         case media::kCodecH264:
-          return CreateOpenH264VideoEncoder();
+          result = CreateOpenH264VideoEncoder();
+          break;
         default:
           return nullptr;
       }
+      if (!result)
+        return nullptr;
+      return std::make_unique<media::OffloadingVideoEncoder>(std::move(result));
     }
 
     default: