Pass webrtc::SdpVideoFormat to HW decoders for parsing profiles

WebRTC currenty passes only the codec selection to the decoders but not
the profile info. This has been working fine so far because we only had
different profiles under H264 and initializing the decoder with MAIN was
enough to cover all of them. However, this became problematic when we
introduce VP9.2, as it is a 10-bit codec that decoder needs to know and
initialize in advance.

This CL changes the decoder creation methods such that
webrtc::SdpVideoFormat is passed and parsed for profile information. This
will allow us to enabled VP9.2 HW decoder usage.

Bug: 908945,webrtc:9376
Change-Id: I6959c85fe4074ca1644e6fb02d58d10df2ff0cff
Reviewed-on: https://chromium-review.googlesource.com/c/1363804
Commit-Queue: Emircan Uysaler <emircan@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#615739}

content/renderer/BUILD.gn

@@ -713,6 +713,7 @@ target(link_target_type, "renderer") {
     "//third_party/webrtc/media:rtc_media",
     "//third_party/webrtc/media:rtc_media_base",
     "//third_party/webrtc/media:rtc_simulcast_encoder_adapter",
+    "//third_party/webrtc/media:rtc_vp9_profile",
     "//third_party/webrtc/modules/audio_device",
     "//third_party/webrtc/modules/audio_processing",
     "//third_party/webrtc/modules/audio_processing:api",

content/renderer/media/webrtc/rtc_video_decoder.cc

@@ -20,6 +20,7 @@
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/webrtc/api/video/video_frame.h"
+#include "third_party/webrtc/media/base/vp9_profile.h"
 #include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "third_party/webrtc/rtc_base/bind.h"
 #include "third_party/webrtc/rtc_base/refcount.h"
@@ -91,8 +92,10 @@ RTCVideoDecoder::~RTCVideoDecoder() {
 
 // static
 std::unique_ptr<RTCVideoDecoder> RTCVideoDecoder::Create(
-    webrtc::VideoCodecType type,
+    const webrtc::SdpVideoFormat& format,
     media::GpuVideoAcceleratorFactories* factories) {
+  const webrtc::VideoCodecType type =
+      webrtc::PayloadStringToCodecType(format.name);
   std::unique_ptr<RTCVideoDecoder> decoder;
 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5717.
 #if defined(OS_WIN)
@@ -110,9 +113,21 @@ std::unique_ptr<RTCVideoDecoder> RTCVideoDecoder::Create(
     case webrtc::kVideoCodecVP8:
       profile = media::VP8PROFILE_ANY;
       break;
-    case webrtc::kVideoCodecVP9:
-      profile = media::VP9PROFILE_MIN;
+    case webrtc::kVideoCodecVP9: {
+      const webrtc::VP9Profile vp9_profile =
+          webrtc::ParseSdpForVP9Profile(format.parameters)
+              .value_or(webrtc::VP9Profile::kProfile0);
+      switch (vp9_profile) {
+        case webrtc::VP9Profile::kProfile2:
+          profile = media::VP9PROFILE_PROFILE2;
+          break;
+        case webrtc::VP9Profile::kProfile0:
+        default:
+          profile = media::VP9PROFILE_PROFILE0;
+          break;
+      }
       break;
+    }
     case webrtc::kVideoCodecH264:
       profile = media::H264PROFILE_MAIN;
       break;

content/renderer/media/webrtc/rtc_video_decoder.h

@@ -25,6 +25,7 @@
 #include "media/base/video_decoder.h"
 #include "media/video/picture.h"
 #include "media/video/video_decode_accelerator.h"
+#include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
 #include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "ui/gfx/geometry/rect.h"
 
@@ -57,7 +58,7 @@ class CONTENT_EXPORT RTCVideoDecoder
   // Creates a RTCVideoDecoder on the message loop of |factories|. Returns NULL
   // if failed. The video decoder will run on the message loop of |factories|.
   static std::unique_ptr<RTCVideoDecoder> Create(
-      webrtc::VideoCodecType type,
+      const webrtc::SdpVideoFormat& format,
       media::GpuVideoAcceleratorFactories* factories);
   // Destroys |decoder| on the loop of |factories|
   static void Destroy(webrtc::VideoDecoder* decoder,
@@ -116,6 +117,7 @@ class CONTENT_EXPORT RTCVideoDecoder
                            GetVDAErrorCounterForRunningOutOfPendingBuffers);
   FRIEND_TEST_ALL_PREFIXES(RTCVideoDecoderTest,
                            GetVDAErrorCounterForSendingFramesWithoutSize);
+  FRIEND_TEST_ALL_PREFIXES(RTCVideoDecoderTest, ParsesVP9CodecProfile);
 
   RTCVideoDecoder(webrtc::VideoCodecType type,
                   media::GpuVideoAcceleratorFactories* factories);

content/renderer/media/webrtc/rtc_video_decoder_adapter.cc

@@ -26,6 +26,7 @@
 #include "media/base/video_types.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "third_party/webrtc/api/video/video_frame.h"
+#include "third_party/webrtc/media/base/vp9_profile.h"
 #include "third_party/webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "third_party/webrtc/rtc_base/bind.h"
 #include "third_party/webrtc/rtc_base/refcount.h"
@@ -75,14 +76,27 @@ media::VideoCodec ToVideoCodec(webrtc::VideoCodecType video_codec_type) {
   }
 }
 
-// Map webrtc::VideoCodecType to a guess for media::VideoCodecProfile.
+// Map webrtc::SdpVideoFormat to a guess for media::VideoCodecProfile.
 media::VideoCodecProfile GuessVideoCodecProfile(
-    webrtc::VideoCodecType video_codec_type) {
+    const webrtc::SdpVideoFormat& format) {
+  const webrtc::VideoCodecType video_codec_type =
+      webrtc::PayloadStringToCodecType(format.name);
   switch (video_codec_type) {
     case webrtc::kVideoCodecVP8:
       return media::VP8PROFILE_ANY;
-    case webrtc::kVideoCodecVP9:
+    case webrtc::kVideoCodecVP9: {
+      const webrtc::VP9Profile vp9_profile =
+          webrtc::ParseSdpForVP9Profile(format.parameters)
+              .value_or(webrtc::VP9Profile::kProfile0);
+      switch (vp9_profile) {
+        case webrtc::VP9Profile::kProfile2:
+          return media::VP9PROFILE_PROFILE2;
+        case webrtc::VP9Profile::kProfile0:
+        default:
+          return media::VP9PROFILE_PROFILE0;
+      }
       return media::VP9PROFILE_PROFILE0;
+    }
     case webrtc::kVideoCodecH264:
       return media::H264PROFILE_BASELINE;
     default:
@@ -108,9 +122,11 @@ void OnRequestOverlayInfo(bool decoder_requires_restart_for_overlay,
 // static
 std::unique_ptr<RTCVideoDecoderAdapter> RTCVideoDecoderAdapter::Create(
     media::GpuVideoAcceleratorFactories* gpu_factories,
-    webrtc::VideoCodecType video_codec_type) {
-  DVLOG(1) << __func__ << "(" << video_codec_type << ")";
+    const webrtc::SdpVideoFormat& format) {
+  DVLOG(1) << __func__ << "(" << format.name << ")";
 
+  const webrtc::VideoCodecType video_codec_type =
+      webrtc::PayloadStringToCodecType(format.name);
 #if defined(OS_WIN)
   // Do not use hardware decoding for H.264 on Win7, due to high latency.
   // See https://crbug.com/webrtc/5717.
@@ -128,8 +144,7 @@ std::unique_ptr<RTCVideoDecoderAdapter> RTCVideoDecoderAdapter::Create(
     return nullptr;
 
   std::unique_ptr<RTCVideoDecoderAdapter> rtc_video_decoder_adapter =
-      base::WrapUnique(
-          new RTCVideoDecoderAdapter(gpu_factories, video_codec_type));
+      base::WrapUnique(new RTCVideoDecoderAdapter(gpu_factories, format));
 
   // Synchronously verify that the decoder can be initialized.
   if (!rtc_video_decoder_adapter->InitializeSync()) {
@@ -143,10 +158,10 @@ std::unique_ptr<RTCVideoDecoderAdapter> RTCVideoDecoderAdapter::Create(
 
 RTCVideoDecoderAdapter::RTCVideoDecoderAdapter(
     media::GpuVideoAcceleratorFactories* gpu_factories,
-    webrtc::VideoCodecType video_codec_type)
+    const webrtc::SdpVideoFormat& format)
     : media_task_runner_(gpu_factories->GetTaskRunner()),
       gpu_factories_(gpu_factories),
-      video_codec_type_(video_codec_type),
+      format_(format),
       weak_this_factory_(this) {
   DVLOG(1) << __func__;
   DETACH_FROM_THREAD(decoding_thread_checker_);
@@ -181,7 +196,8 @@ int32_t RTCVideoDecoderAdapter::InitDecode(
     int32_t number_of_cores) {
   DVLOG(1) << __func__;
   DCHECK_CALLED_ON_VALID_THREAD(decoding_thread_checker_);
-  DCHECK_EQ(video_codec_type_, codec_settings->codecType);
+  DCHECK_EQ(webrtc::PayloadStringToCodecType(format_.name),
+            codec_settings->codecType);
 
   base::AutoLock auto_lock(lock_);
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoDecoderInitDecodeSuccess", !has_error_);
@@ -292,8 +308,8 @@ void RTCVideoDecoderAdapter::InitializeOnMediaThread(
 
   // We don't know much about the media that is coming.
   media::VideoDecoderConfig config(
-      ToVideoCodec(video_codec_type_),
-      GuessVideoCodecProfile(video_codec_type_), kDefaultPixelFormat,
+      ToVideoCodec(webrtc::PayloadStringToCodecType(format_.name)),
+      GuessVideoCodecProfile(format_), kDefaultPixelFormat,
       media::VideoColorSpace(), media::VIDEO_ROTATION_0, kDefaultSize,
       gfx::Rect(kDefaultSize), kDefaultSize, media::EmptyExtraData(),
       media::Unencrypted());

content/renderer/media/webrtc/rtc_video_decoder_adapter.h

@@ -18,6 +18,7 @@
 #include "media/base/decode_status.h"
 #include "media/base/video_codecs.h"
 #include "media/base/video_decoder.h"
+#include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
 #include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -49,11 +50,11 @@ namespace content {
 class CONTENT_EXPORT RTCVideoDecoderAdapter : public webrtc::VideoDecoder {
  public:
   // Creates and initializes an RTCVideoDecoderAdapter. Returns nullptr if
-  // |video_codec_type| cannot be supported.
+  // |format| cannot be supported.
   // Called on the worker thread.
   static std::unique_ptr<RTCVideoDecoderAdapter> Create(
       media::GpuVideoAcceleratorFactories* gpu_factories,
-      webrtc::VideoCodecType video_codec_type);
+      const webrtc::SdpVideoFormat& format);
 
   // Called on |media_task_runner_|.
   ~RTCVideoDecoderAdapter() override;
@@ -82,7 +83,7 @@ class CONTENT_EXPORT RTCVideoDecoderAdapter : public webrtc::VideoDecoder {
 
   // Called on the worker thread.
   RTCVideoDecoderAdapter(media::GpuVideoAcceleratorFactories* gpu_factories,
-                         webrtc::VideoCodecType video_codec_type);
+                         const webrtc::SdpVideoFormat& format);
 
   bool InitializeSync();
   void InitializeOnMediaThread(media::VideoDecoder::InitCB init_cb);
@@ -93,7 +94,7 @@ class CONTENT_EXPORT RTCVideoDecoderAdapter : public webrtc::VideoDecoder {
   // Construction parameters.
   scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
   media::GpuVideoAcceleratorFactories* gpu_factories_;
-  webrtc::VideoCodecType video_codec_type_;
+  webrtc::SdpVideoFormat format_;
 
   // Media thread members.
   // |media_log_| must outlive |video_decoder_| because it is passed as a raw

content/renderer/media/webrtc/rtc_video_decoder_adapter_unittest.cc

@@ -28,6 +28,7 @@
 #include "media/video/mock_gpu_video_accelerator_factories.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/webrtc/api/video_codecs/video_codec.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -141,8 +142,10 @@ class RTCVideoDecoderAdapterTest : public ::testing::Test {
     EXPECT_CALL(*video_decoder_, Initialize(_, _, _, _, _, _))
         .WillOnce(DoAll(SaveArg<4>(&output_cb_),
                         media::RunCallback<3>(init_cb_result)));
-    rtc_video_decoder_adapter_ =
-        RTCVideoDecoderAdapter::Create(&gpu_factories_, webrtc::kVideoCodecVP9);
+    rtc_video_decoder_adapter_ = RTCVideoDecoderAdapter::Create(
+        &gpu_factories_,
+        webrtc::SdpVideoFormat(
+            webrtc::CodecTypeToPayloadString(webrtc::kVideoCodecVP9)));
     return !!rtc_video_decoder_adapter_;
   }
 

content/renderer/media/webrtc/rtc_video_decoder_factory.cc

@@ -16,6 +16,7 @@
 #include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
 #include "third_party/webrtc/common_video/h264/profile_level_id.h"
 #include "third_party/webrtc/media/base/codec.h"
+#include "third_party/webrtc/media/base/vp9_profile.h"
 
 namespace content {
 
@@ -32,7 +33,21 @@ base::Optional<webrtc::SdpVideoFormat> VDAToWebRTCFormat(
     return webrtc::SdpVideoFormat("VP8");
   } else if (profile.profile >= media::VP9PROFILE_MIN &&
              profile.profile <= media::VP9PROFILE_MAX) {
-    return webrtc::SdpVideoFormat("VP9");
+    webrtc::VP9Profile vp9_profile;
+    switch (profile.profile) {
+      case media::VP9PROFILE_PROFILE0:
+        vp9_profile = webrtc::VP9Profile::kProfile0;
+        break;
+      case media::VP9PROFILE_PROFILE2:
+        vp9_profile = webrtc::VP9Profile::kProfile2;
+        break;
+      default:
+        // Unsupported H264 profile in WebRTC.
+        return base::nullopt;
+    }
+    return webrtc::SdpVideoFormat(
+        "VP9",
+        {{webrtc::kVP9FmtpProfileId, webrtc::VP9ProfileToString(vp9_profile)}});
   } else if (profile.profile >= media::H264PROFILE_MIN &&
              profile.profile <= media::H264PROFILE_MAX) {
     webrtc::H264::Profile h264_profile;
@@ -168,11 +183,9 @@ RTCVideoDecoderFactory::CreateVideoDecoder(
   DVLOG(2) << __func__;
   std::unique_ptr<webrtc::VideoDecoder> decoder;
   if (base::FeatureList::IsEnabled(media::kRTCVideoDecoderAdapter)) {
-    decoder = RTCVideoDecoderAdapter::Create(
-        gpu_factories_, webrtc::PayloadStringToCodecType(format.name));
+    decoder = RTCVideoDecoderAdapter::Create(gpu_factories_, format);
   } else {
-    decoder = RTCVideoDecoder::Create(
-        webrtc::PayloadStringToCodecType(format.name), gpu_factories_);
+    decoder = RTCVideoDecoder::Create(format, gpu_factories_);
   }
   // ScopedVideoDecoder uses the task runner to make sure the decoder is
   // destructed on the correct thread.

content/renderer/media/webrtc/rtc_video_decoder_unittest.cc

@@ -17,6 +17,7 @@
 #include "media/video/mock_video_decode_accelerator.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h"
+#include "third_party/webrtc/media/base/vp9_profile.h"
 
 #if defined(OS_WIN)
 #include "base/command_line.h"
@@ -70,7 +71,9 @@ class RTCVideoDecoderTest
     capabilities_.supported_profiles.push_back(supported_profile);
     supported_profile.profile = media::VP8PROFILE_ANY;
     capabilities_.supported_profiles.push_back(supported_profile);
-    supported_profile.profile = media::VP9PROFILE_MIN;
+    supported_profile.profile = media::VP9PROFILE_PROFILE0;
+    capabilities_.supported_profiles.push_back(supported_profile);
+    supported_profile.profile = media::VP9PROFILE_PROFILE2;
     capabilities_.supported_profiles.push_back(supported_profile);
 
     EXPECT_CALL(*mock_gpu_factories_.get(), GetTaskRunner())
@@ -108,11 +111,15 @@ class RTCVideoDecoderTest
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
-  void CreateDecoder(webrtc::VideoCodecType codec_type) {
+  void CreateDecoder(const webrtc::SdpVideoFormat& format) {
     DVLOG(2) << "CreateDecoder";
-    codec_.codecType = codec_type;
-    rtc_decoder_ =
-        RTCVideoDecoder::Create(codec_type, mock_gpu_factories_.get());
+    codec_.codecType = webrtc::PayloadStringToCodecType(format.name);
+    rtc_decoder_ = RTCVideoDecoder::Create(format, mock_gpu_factories_.get());
+  }
+
+  void CreateDecoder(webrtc::VideoCodecType codec_type) {
+    CreateDecoder(
+        webrtc::SdpVideoFormat(webrtc::CodecTypeToPayloadString(codec_type)));
   }
 
   void Initialize() {
@@ -196,7 +203,9 @@ class RTCVideoDecoderTest
 TEST_F(RTCVideoDecoderTest, CreateReturnsNullOnUnsupportedCodec) {
   CreateDecoder(webrtc::kVideoCodecVP8);
   std::unique_ptr<RTCVideoDecoder> null_rtc_decoder(RTCVideoDecoder::Create(
-      webrtc::kVideoCodecI420, mock_gpu_factories_.get()));
+      webrtc::SdpVideoFormat(
+          webrtc::CodecTypeToPayloadString(webrtc::kVideoCodecI420)),
+      mock_gpu_factories_.get()));
   EXPECT_EQ(nullptr, null_rtc_decoder.get());
 }
 
@@ -342,6 +351,14 @@ TEST_F(RTCVideoDecoderTest, MultipleTexturesPerBuffer) {
   }
 }
 
+TEST_F(RTCVideoDecoderTest, ParsesVP9CodecProfile) {
+  webrtc::SdpVideoFormat sdp_format(
+      "VP9", {{webrtc::kVP9FmtpProfileId,
+               webrtc::VP9ProfileToString(webrtc::VP9Profile::kProfile2)}});
+  CreateDecoder(sdp_format);
+  EXPECT_EQ(media::VP9PROFILE_PROFILE2, rtc_decoder_->vda_codec_profile_);
+}
+
 // Tests/Verifies that |rtc_encoder_| drops incoming frames and its error
 // counter is increased when decoder implementation calls NotifyError().
 TEST_P(RTCVideoDecoderTest, GetVDAErrorCounterForNotifyError) {

content/test/BUILD.gn

@@ -1945,6 +1945,7 @@ test("content_unittests") {
     "//third_party/webrtc/api/video:video_frame_i420",
     "//third_party/webrtc/api/video_codecs:video_codecs_api",
     "//third_party/webrtc/media:rtc_media",
+    "//third_party/webrtc/media:rtc_vp9_profile",
     "//third_party/webrtc/modules/desktop_capture:primitives",
     "//third_party/webrtc/modules/video_capture",
     "//third_party/webrtc/modules/video_coding:video_codec_interface",