Add RTCVideoDecoderAdapter, which can use any media::VideoDecoder.

This is the main body of code required to support MojoVideoDecoder for
hardware decode of WebRTC streams. It tries to be as similar as possible
to RTCVideoDecoder so that a transition can be managed slowly.

The implementation actually supports any media::VideoDecoder, in
particular FFmpegVideoDecoder works well for testing.

Bug: 857111
Change-Id: I0a4dc37a0a133b4c112a55a215aba8058cca598b
Reviewed-on: https://chromium-review.googlesource.com/1117837
Commit-Queue: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Henrik Boström <hbos@chromium.org>
Cr-Commit-Position: refs/heads/master@{#582291}

content/renderer/BUILD.gn

@@ -407,6 +407,8 @@ target(link_target_type, "renderer") {
     "media/webrtc/rtc_stats.h",
     "media/webrtc/rtc_video_decoder.cc",
     "media/webrtc/rtc_video_decoder.h",
+    "media/webrtc/rtc_video_decoder_adapter.cc",
+    "media/webrtc/rtc_video_decoder_adapter.h",
     "media/webrtc/rtc_video_decoder_factory.cc",
     "media/webrtc/rtc_video_decoder_factory.h",
     "media/webrtc/rtc_video_encoder.cc",

content/renderer/media/webrtc/rtc_video_decoder_adapter.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_RENDERER_MEDIA_WEBRTC_RTC_VIDEO_DECODER_ADAPTER_H_
+#define CONTENT_RENDERER_MEDIA_WEBRTC_RTC_VIDEO_DECODER_ADAPTER_H_
+
+#include <memory>
+
+#include "base/callback_forward.h"
+#include "base/containers/circular_deque.h"
+#include "base/macros.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/synchronization/lock.h"
+#include "base/threading/thread_checker.h"
+#include "content/common/content_export.h"
+#include "media/base/decode_status.h"
+#include "media/base/video_codecs.h"
+#include "media/base/video_decoder.h"
+#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
+#include "ui/gfx/geometry/size.h"
+
+namespace base {
+class SingleThreadTaskRunner;
+}  // namespace base
+
+namespace media {
+class DecoderBuffer;
+class GpuVideoAcceleratorFactories;
+class MediaLog;
+class VideoFrame;
+}  // namespace media
+
+namespace content {
+
+// This class decodes video for WebRTC using a media::VideoDecoder. In
+// particular, either GpuVideoDecoder or MojoVideoDecoder are used to provide
+// access to hardware decoding in the GPU process.
+//
+// Lifecycle methods are called on the WebRTC worker thread. Decoding happens on
+// a WebRTC DecodingThread, which is an rtc::PlatformThread owend by WebRTC; it
+// does not have a TaskRunner.
+//
+// To stop decoding, WebRTC stops the DecodingThread and then calls Release() on
+// the worker. Calling the DecodedImageCallback after the DecodingThread is
+// stopped is illegal but, because we decode on the media thread, there is no
+// way to synchronize this correctly.
+class CONTENT_EXPORT RTCVideoDecoderAdapter : public webrtc::VideoDecoder {
+ public:
+  using CreateVideoDecoderCB =
+      base::RepeatingCallback<std::unique_ptr<media::VideoDecoder>(
+          media::MediaLog*)>;
+
+  // Creates and initializes an RTCVideoDecoderAdapter. Returns nullptr if
+  // |video_codec_type| cannot be supported.
+  // Called on the worker thread.
+  static std::unique_ptr<RTCVideoDecoderAdapter> Create(
+      webrtc::VideoCodecType video_codec_type,
+      media::GpuVideoAcceleratorFactories* gpu_factories,
+      CreateVideoDecoderCB create_video_decoder_cb);
+
+  // Called on the worker thread.
+  static void DeleteSoonOnMediaThread(
+      std::unique_ptr<webrtc::VideoDecoder> rtc_video_decoder_adapter,
+      media::GpuVideoAcceleratorFactories* gpu_factories);
+
+  // Called on |media_task_runner_|.
+  ~RTCVideoDecoderAdapter() override;
+
+  // webrtc::VideoDecoder implementation.
+  // Called on the DecodingThread.
+  int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
+                     int32_t number_of_cores) override;
+  // Called on the DecodingThread.
+  int32_t RegisterDecodeCompleteCallback(
+      webrtc::DecodedImageCallback* callback) override;
+  // Called on the DecodingThread.
+  int32_t Decode(const webrtc::EncodedImage& input_image,
+                 bool missing_frames,
+                 const webrtc::CodecSpecificInfo* codec_specific_info,
+                 int64_t render_time_ms) override;
+  // Called on the worker thread.
+  int32_t Release() override;
+  // Called on the worker thread and on the DecodingThread.
+  const char* ImplementationName() const override;
+
+ private:
+  // |create_video_decoder_cb| will always be called on |media_task_runner|.
+  // Called on the worker thread.
+  RTCVideoDecoderAdapter(
+      scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
+      CreateVideoDecoderCB create_video_decoder_cb,
+      webrtc::VideoCodecType video_codec_type);
+
+  bool InitializeSync();
+  void InitializeOnMediaThread(media::VideoDecoder::InitCB init_cb);
+  void DecodeOnMediaThread();
+  void OnDecodeDone(media::DecodeStatus status);
+  void OnOutput(const scoped_refptr<media::VideoFrame>& frame);
+
+  // Construction parameters.
+  scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
+  CreateVideoDecoderCB create_video_decoder_cb_;
+  webrtc::VideoCodecType video_codec_type_;
+
+  // Media thread members.
+  // |media_log_| must outlive |video_decoder_| because it is passed as a raw
+  // pointer.
+  std::unique_ptr<media::MediaLog> media_log_;
+  std::unique_ptr<media::VideoDecoder> video_decoder_;
+  int32_t outstanding_decode_requests_ = 0;
+
+  // Shared members.
+  base::Lock lock_;
+  int32_t consecutive_error_count_ = 0;
+  bool has_error_ = false;
+  webrtc::DecodedImageCallback* decode_complete_callback_ = nullptr;
+  // Requests that have not been submitted to the decoder yet.
+  base::circular_deque<scoped_refptr<media::DecoderBuffer>> pending_buffers_;
+  // Record of timestamps that have been sent to be decoded. Removing a
+  // timestamp will cause the frame to be dropped when it is output.
+  base::circular_deque<base::TimeDelta> decode_timestamps_;
+
+  // Thread management.
+  THREAD_CHECKER(worker_thread_checker_);
+  THREAD_CHECKER(decoding_thread_checker_);
+
+  base::WeakPtr<RTCVideoDecoderAdapter> weak_this_;
+  base::WeakPtrFactory<RTCVideoDecoderAdapter> weak_this_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(RTCVideoDecoderAdapter);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_RENDERER_MEDIA_WEBRTC_RTC_VIDEO_DECODER_ADAPTER_H_

content/renderer/media/webrtc/rtc_video_decoder_adapter_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <memory>
+#include <vector>
+
+#include <stdint.h>
+
+#include "base/callback_forward.h"
+#include "base/logging.h"
+#include "base/macros.h"
+#include "base/memory/ptr_util.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/test/mock_callback.h"
+#include "base/test/scoped_task_environment.h"
+#include "base/threading/thread.h"
+#include "base/time/time.h"
+#include "content/renderer/media/webrtc/rtc_video_decoder_adapter.h"
+#include "gpu/command_buffer/common/mailbox.h"
+#include "media/base/decode_status.h"
+#include "media/base/gmock_callback_support.h"
+#include "media/base/media_util.h"
+#include "media/base/video_decoder.h"
+#include "media/base/video_decoder_config.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_types.h"
+#include "media/video/mock_gpu_video_accelerator_factories.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/size.h"
+
+using ::testing::_;
+using ::testing::AtLeast;
+using ::testing::DoAll;
+using ::testing::Mock;
+using ::testing::Return;
+using ::testing::SaveArg;
+using ::testing::StrictMock;
+
+namespace content {
+
+namespace {
+
+class MockVideoDecoder : public media::VideoDecoder {
+ public:
+  std::string GetDisplayName() const override { return "MockVideoDecoder"; }
+  MOCK_METHOD6(
+      Initialize,
+      void(const media::VideoDecoderConfig& config,
+           bool low_delay,
+           media::CdmContext* cdm_context,
+           const InitCB& init_cb,
+           const OutputCB& output_cb,
+           const WaitingForDecryptionKeyCB& waiting_for_decryption_key_cb));
+  MOCK_METHOD2(Decode,
+               void(scoped_refptr<media::DecoderBuffer> buffer,
+                    const DecodeCB&));
+  MOCK_METHOD1(Reset, void(const base::RepeatingClosure&));
+  bool NeedsBitstreamConversion() const override { return false; }
+  bool CanReadWithoutStalling() const override { return true; }
+  int GetMaxDecodeRequests() const override { return 1; }
+};
+
+// Wraps a callback as a webrtc::DecodedImageCallback.
+class DecodedImageCallback : public webrtc::DecodedImageCallback {
+ public:
+  DecodedImageCallback(
+      base::RepeatingCallback<void(const webrtc::VideoFrame&)> callback)
+      : callback_(callback) {}
+
+  int32_t Decoded(webrtc::VideoFrame& decodedImage) override {
+    callback_.Run(decodedImage);
+    // TODO(sandersd): Does the return value matter? RTCVideoDecoder
+    // ignores it.
+    return 0;
+  }
+
+ private:
+  base::RepeatingCallback<void(const webrtc::VideoFrame&)> callback_;
+
+  DISALLOW_COPY_AND_ASSIGN(DecodedImageCallback);
+};
+
+}  // namespace
+
+class RTCVideoDecoderAdapterTest : public ::testing::Test {
+ public:
+  RTCVideoDecoderAdapterTest()
+      : media_thread_("Media Thread"),
+        gpu_factories_(nullptr),
+        decoded_image_callback_(decoded_cb_.Get()) {
+    media_thread_.Start();
+    ON_CALL(gpu_factories_, GetTaskRunner())
+        .WillByDefault(Return(media_thread_.task_runner()));
+    EXPECT_CALL(gpu_factories_, GetTaskRunner()).Times(AtLeast(0));
+    owned_video_decoder_ = std::make_unique<StrictMock<MockVideoDecoder>>();
+    video_decoder_ = owned_video_decoder_.get();
+  }
+
+  ~RTCVideoDecoderAdapterTest() {
+    if (!rtc_video_decoder_adapter_)
+      return;
+
+    RTCVideoDecoderAdapter::DeleteSoonOnMediaThread(
+        std::move(rtc_video_decoder_adapter_), &gpu_factories_);
+    media_thread_.FlushForTesting();
+  }
+
+ protected:
+  std::unique_ptr<media::VideoDecoder> CreateVideoDecoder(
+      media::MediaLog* media_log) {
+    DCHECK(owned_video_decoder_);
+    return std::move(owned_video_decoder_);
+  }
+
+  bool BasicSetup() {
+    if (!CreateAndInitialize())
+      return false;
+    if (InitDecode() != WEBRTC_VIDEO_CODEC_OK)
+      return false;
+    if (RegisterDecodeCompleteCallback() != WEBRTC_VIDEO_CODEC_OK)
+      return false;
+    return true;
+  }
+
+  bool BasicTeardown() {
+    if (Release() != WEBRTC_VIDEO_CODEC_OK)
+      return false;
+    return true;
+  }
+
+  bool CreateAndInitialize(bool init_cb_result = true) {
+    EXPECT_CALL(*video_decoder_, Initialize(_, _, _, _, _, _))
+        .WillOnce(DoAll(SaveArg<4>(&output_cb_),
+                        media::RunCallback<3>(init_cb_result)));
+    rtc_video_decoder_adapter_ = RTCVideoDecoderAdapter::Create(
+        webrtc::kVideoCodecVP9, &gpu_factories_,
+        base::BindRepeating(&RTCVideoDecoderAdapterTest::CreateVideoDecoder,
+                            base::Unretained(this)));
+    return !!rtc_video_decoder_adapter_;
+  }
+
+  int32_t InitDecode() {
+    webrtc::VideoCodec codec_settings;
+    codec_settings.codecType = webrtc::kVideoCodecVP9;
+    return rtc_video_decoder_adapter_->InitDecode(&codec_settings, 1);
+  }
+
+  int32_t RegisterDecodeCompleteCallback() {
+    return rtc_video_decoder_adapter_->RegisterDecodeCompleteCallback(
+        &decoded_image_callback_);
+  }
+
+  int32_t Decode(uint32_t timestamp) {
+    uint8_t buf[] = {0};
+    webrtc::EncodedImage input_image(&buf[0], 1, 1);
+    input_image._completeFrame = true;
+    input_image._timeStamp = timestamp;
+    return rtc_video_decoder_adapter_->Decode(input_image, false, nullptr, 0);
+  }
+
+  void FinishDecode(uint32_t timestamp) {
+    media_thread_.task_runner()->PostTask(
+        FROM_HERE,
+        base::BindOnce(&RTCVideoDecoderAdapterTest::FinishDecodeOnMediaThread,
+                       base::Unretained(this), timestamp));
+  }
+
+  void FinishDecodeOnMediaThread(uint32_t timestamp) {
+    DCHECK(media_thread_.task_runner()->BelongsToCurrentThread());
+    gpu::MailboxHolder mailbox_holders[media::VideoFrame::kMaxPlanes];
+    mailbox_holders[0].mailbox = gpu::Mailbox::Generate();
+    scoped_refptr<media::VideoFrame> frame =
+        media::VideoFrame::WrapNativeTextures(
+            media::PIXEL_FORMAT_ARGB, mailbox_holders,
+            media::VideoFrame::ReleaseMailboxCB(), gfx::Size(640, 360),
+            gfx::Rect(640, 360), gfx::Size(640, 360),
+            base::TimeDelta::FromMicroseconds(timestamp));
+    output_cb_.Run(std::move(frame));
+  }
+
+  int32_t Release() { return rtc_video_decoder_adapter_->Release(); }
+
+  base::test::ScopedTaskEnvironment scoped_task_environment_;
+  base::Thread media_thread_;
+
+  // Owned by |rtc_video_decoder_adapter_|.
+  StrictMock<MockVideoDecoder>* video_decoder_ = nullptr;
+
+  StrictMock<base::MockCallback<
+      base::RepeatingCallback<void(const webrtc::VideoFrame&)>>>
+      decoded_cb_;
+
+ private:
+  StrictMock<media::MockGpuVideoAcceleratorFactories> gpu_factories_;
+  std::unique_ptr<RTCVideoDecoderAdapter> rtc_video_decoder_adapter_;
+  std::unique_ptr<StrictMock<MockVideoDecoder>> owned_video_decoder_;
+  DecodedImageCallback decoded_image_callback_;
+  media::VideoDecoder::OutputCB output_cb_;
+
+  DISALLOW_COPY_AND_ASSIGN(RTCVideoDecoderAdapterTest);
+};
+
+TEST_F(RTCVideoDecoderAdapterTest, Lifecycle) {
+  ASSERT_TRUE(BasicSetup());
+  ASSERT_TRUE(BasicTeardown());
+}
+
+TEST_F(RTCVideoDecoderAdapterTest, InitializationFailure) {
+  ASSERT_FALSE(CreateAndInitialize(false));
+}
+
+TEST_F(RTCVideoDecoderAdapterTest, Decode) {
+  ASSERT_TRUE(BasicSetup());
+
+  EXPECT_CALL(*video_decoder_, Decode(_, _))
+      .WillOnce(media::RunCallback<1>(media::DecodeStatus::OK));
+
+  ASSERT_EQ(Decode(0), WEBRTC_VIDEO_CODEC_OK);
+
+  EXPECT_CALL(decoded_cb_, Run(_));
+  FinishDecode(0);
+  media_thread_.FlushForTesting();
+}
+
+TEST_F(RTCVideoDecoderAdapterTest, Decode_Error) {
+  ASSERT_TRUE(BasicSetup());
+
+  EXPECT_CALL(*video_decoder_, Decode(_, _))
+      .WillOnce(media::RunCallback<1>(media::DecodeStatus::DECODE_ERROR));
+
+  ASSERT_EQ(Decode(0), WEBRTC_VIDEO_CODEC_OK);
+  media_thread_.FlushForTesting();
+
+  ASSERT_EQ(Decode(1), WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
+}
+
+TEST_F(RTCVideoDecoderAdapterTest, Decode_Hang_Short) {
+  ASSERT_TRUE(BasicSetup());
+
+  // Ignore Decode() calls.
+  EXPECT_CALL(*video_decoder_, Decode(_, _)).Times(AtLeast(1));
+
+  for (int counter = 0; counter < 10; counter++) {
+    int32_t result = Decode(counter);
+    if (result == WEBRTC_VIDEO_CODEC_ERROR) {
+      ASSERT_GT(counter, 2);
+      return;
+    }
+    media_thread_.FlushForTesting();
+  }
+
+  FAIL();
+}
+
+TEST_F(RTCVideoDecoderAdapterTest, Decode_Hang_Long) {
+  ASSERT_TRUE(BasicSetup());
+
+  // Ignore Decode() calls.
+  EXPECT_CALL(*video_decoder_, Decode(_, _)).Times(AtLeast(1));
+
+  for (int counter = 0; counter < 100; counter++) {
+    int32_t result = Decode(counter);
+    if (result == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
+      ASSERT_GT(counter, 10);
+      return;
+    }
+    media_thread_.FlushForTesting();
+  }
+
+  FAIL();
+}
+
+}  // namespace content

content/test/BUILD.gn

@@ -1724,6 +1724,7 @@ test("content_unittests") {
     "../renderer/media/webrtc/rtc_rtp_sender_unittest.cc",
     "../renderer/media/webrtc/rtc_rtp_transceiver_unittest.cc",
     "../renderer/media/webrtc/rtc_stats_unittest.cc",
+    "../renderer/media/webrtc/rtc_video_decoder_adapter_unittest.cc",
     "../renderer/media/webrtc/rtc_video_decoder_unittest.cc",
     "../renderer/media/webrtc/rtc_video_encoder_unittest.cc",
     "../renderer/media/webrtc/stun_field_trial_unittest.cc",