VideoEncodeAccelerator: Each VEA selects H264 level if it is unspecified or invalid

A VideoEncodeAccelerator (VEA) client needs to specify a proper h264 level in
the original VEA API. If the h264 level is unspecified, Level 4.1 is used by
default. This design causes a failure of recording 4k video with media recorder
because most VEA clients don't specify the level.

This CL modifies the VEA API design. A VEA client is still able to specify h264
level. However, if it is invalid (or unspecified), VideoEncodeAccelerator s
selects one of proper h264 levels for the specified resolution, bitrate and
framerate.

Bug: b:139788862
Test: video_encode_accelerator_unittest on hana and eve
Test: Record 4k video
Change-Id: I807a6fce10929739fe6ab161d2faddfd9e39d3a0
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1765131Reviewed-by: Dan Sanders <sandersd@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#691468}

media/gpu/v4l2/v4l2_video_encode_accelerator.cc

@@ -16,6 +16,7 @@
 #include <utility>
 
 #include "base/bind.h"
+#include "base/bits.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/command_line.h"
@@ -33,6 +34,7 @@
 #include "media/gpu/gpu_video_encode_accelerator_helpers.h"
 #include "media/gpu/image_processor_factory.h"
 #include "media/gpu/macros.h"
+#include "media/video/h264_level_limits.h"
 #include "media/video/h264_parser.h"
 
 #define NOTIFY_ERROR(x)                      \
@@ -1498,13 +1500,42 @@ bool V4L2VideoEncodeAccelerator::InitControls(const Config& config) {
     ctrls.push_back(ctrl);
 
     // Set H.264 output level from config. Use Level 4.0 as fallback default.
-    int32_t level_value = V4L2Device::H264LevelIdcToV4L2H264Level(
-        config.h264_output_level.value_or(
-            VideoEncodeAccelerator::kDefaultH264Level));
-    if (level_value < 0) {
-      NOTIFY_ERROR(kInvalidArgumentError);
-      return false;
+    uint8_t h264_level =
+        config.h264_output_level.value_or(H264SPS::kLevelIDC4p0);
+    constexpr size_t kH264MacroblockSizeInPixels = 16;
+    const uint32_t framerate = config.initial_framerate.value_or(
+        VideoEncodeAccelerator::kDefaultFramerate);
+    const uint32_t mb_width =
+        base::bits::Align(config.input_visible_size.width(),
+                          kH264MacroblockSizeInPixels) /
+        kH264MacroblockSizeInPixels;
+    const uint32_t mb_height =
+        base::bits::Align(config.input_visible_size.height(),
+                          kH264MacroblockSizeInPixels) /
+        kH264MacroblockSizeInPixels;
+    const uint32_t framesize_in_mbs = mb_width * mb_height;
+
+    // Check whether the h264 level is valid.
+    if (!CheckH264LevelLimits(config.output_profile, h264_level,
+                              config.initial_bitrate, framerate,
+                              framesize_in_mbs)) {
+      base::Optional<uint8_t> valid_level =
+          FindValidH264Level(config.output_profile, config.initial_bitrate,
+                             framerate, framesize_in_mbs);
+      if (!valid_level) {
+        VLOGF(1) << "Could not find a valid h264 level for"
+                 << " profile=" << config.output_profile
+                 << " bitrate=" << config.initial_bitrate
+                 << " framerate=" << framerate
+                 << " size=" << config.input_visible_size.ToString();
+        NOTIFY_ERROR(kInvalidArgumentError);
+        return false;
+      }
+
+      h264_level = *valid_level;
     }
+
+    int32_t level_value = V4L2Device::H264LevelIdcToV4L2H264Level(h264_level);
     memset(&ctrl, 0, sizeof(ctrl));
     ctrl.id = V4L2_CID_MPEG_VIDEO_H264_LEVEL;
     ctrl.value = level_value;

media/gpu/vaapi/BUILD.gn

@@ -166,6 +166,7 @@ source_set("vaapi_test_utils") {
 source_set("unit_test") {
   testonly = true
   sources = [
+    "h264_encoder_unittest.cc",
     "vaapi_image_decode_accelerator_worker_unittest.cc",
     "vaapi_video_decode_accelerator_unittest.cc",
   ]

media/gpu/vaapi/h264_encoder.cc

@@ -93,13 +93,28 @@ bool H264Encoder::Initialize(
   mb_height_ = coded_size_.height() / kH264MacroblockSizeInPixels;
 
   profile_ = config.output_profile;
-  level_ = config.h264_output_level.value_or(
-      VideoEncodeAccelerator::kDefaultH264Level);
+  level_ = config.h264_output_level.value_or(H264SPS::kLevelIDC4p0);
   uint32_t initial_framerate = config.initial_framerate.value_or(
       VideoEncodeAccelerator::kDefaultFramerate);
+
+  // Checks if |level_| is valid. If it is invalid, set |level_| to a minimum
+  // level that comforts Table A-1 in H.264 spec with specified bitrate,
+  // framerate and dimension.
   if (!CheckH264LevelLimits(profile_, level_, config.initial_bitrate,
-                            initial_framerate, mb_width_ * mb_height_))
-    return false;
+                            initial_framerate, mb_width_ * mb_height_)) {
+    base::Optional<uint8_t> valid_level =
+        FindValidH264Level(profile_, config.initial_bitrate, initial_framerate,
+                           mb_width_ * mb_height_);
+    if (!valid_level) {
+      VLOGF(1) << "Could not find a valid h264 level for"
+               << " profile=" << profile_
+               << " bitrate=" << config.initial_bitrate
+               << " framerate=" << initial_framerate
+               << " size=" << config.input_visible_size.ToString();
+      return false;
+    }
+    level_ = *valid_level;
+  }
 
   curr_params_.max_ref_pic_list0_size =
       std::min(kMaxRefIdxL0Size, ave_config.max_num_ref_frames & 0xffff);

media/gpu/vaapi/h264_encoder.h

@@ -113,6 +113,8 @@ class H264Encoder : public AcceleratedVideoEncoder {
   bool PrepareEncodeJob(EncodeJob* encode_job) override;
 
  private:
+  friend class H264EncoderTest;
+
   // Fill current_sps_ and current_pps_ with current encoding state parameters.
   void UpdateSPS();
   void UpdatePPS();

media/gpu/vaapi/h264_encoder_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/gpu/vaapi/h264_encoder.h"
+
+#include <memory>
+
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Return;
+
+namespace media {
+namespace {
+
+AcceleratedVideoEncoder::Config kDefaultAVEConfig{10};
+
+VideoEncodeAccelerator::Config kDefaultVEAConfig(
+    PIXEL_FORMAT_I420,
+    gfx::Size(1280, 720),
+    H264PROFILE_BASELINE,
+    14000000 /* = maximum bitrate in bits per second for level 3.1 */,
+    VideoEncodeAccelerator::kDefaultFramerate,
+    base::nullopt /* gop_length */,
+    base::nullopt /* h264 output level*/,
+    VideoEncodeAccelerator::Config::StorageType::kShmem,
+    VideoEncodeAccelerator::Config::ContentType::kCamera);
+
+class MockH264Accelerator : public H264Encoder::Accelerator {
+ public:
+  MockH264Accelerator() = default;
+  MOCK_METHOD1(
+      GetPicture,
+      scoped_refptr<H264Picture>(AcceleratedVideoEncoder::EncodeJob* job));
+  MOCK_METHOD3(SubmitPackedHeaders,
+               bool(AcceleratedVideoEncoder::EncodeJob*,
+                    scoped_refptr<H264BitstreamBuffer>,
+                    scoped_refptr<H264BitstreamBuffer>));
+  MOCK_METHOD7(SubmitFrameParameters,
+               bool(AcceleratedVideoEncoder::EncodeJob*,
+                    const H264Encoder::EncodeParams&,
+                    const H264SPS&,
+                    const H264PPS&,
+                    scoped_refptr<H264Picture>,
+                    const std::list<scoped_refptr<H264Picture>>&,
+                    const std::list<scoped_refptr<H264Picture>>&));
+};
+}  // namespace
+
+class H264EncoderTest : public ::testing::Test {
+ public:
+  H264EncoderTest() = default;
+  void SetUp() override;
+
+  void ExpectLevel(uint8_t level) { EXPECT_EQ(encoder_->level_, level); }
+
+ protected:
+  std::unique_ptr<H264Encoder> encoder_;
+  MockH264Accelerator* accelerator_;
+};
+
+void H264EncoderTest::SetUp() {
+  auto mock_accelerator = std::make_unique<MockH264Accelerator>();
+  accelerator_ = mock_accelerator.get();
+  encoder_ = std::make_unique<H264Encoder>(std::move(mock_accelerator));
+
+  // Set default behaviors for mock methods for convenience.
+  ON_CALL(*accelerator_, GetPicture(_))
+      .WillByDefault(Invoke([](AcceleratedVideoEncoder::EncodeJob*) {
+        return new H264Picture();
+      }));
+  ON_CALL(*accelerator_, SubmitPackedHeaders(_, _, _))
+      .WillByDefault(Return(true));
+  ON_CALL(*accelerator_, SubmitFrameParameters(_, _, _, _, _, _, _))
+      .WillByDefault(Return(true));
+}
+
+TEST_F(H264EncoderTest, Initialize) {
+  VideoEncodeAccelerator::Config vea_config = kDefaultVEAConfig;
+  AcceleratedVideoEncoder::Config ave_config = kDefaultAVEConfig;
+  EXPECT_TRUE(encoder_->Initialize(vea_config, ave_config));
+  // Profile is unspecified, H264Encoder will select the default level, 4.0.
+  // 4.0 will be proper with |vea_config|'s values.
+  ExpectLevel(H264SPS::kLevelIDC4p0);
+
+  // Initialize with 4k size. The level will be adjusted to 5.1 by H264Encoder.
+  vea_config.input_visible_size.SetSize(3840, 2160);
+  EXPECT_TRUE(encoder_->Initialize(vea_config, ave_config));
+  ExpectLevel(H264SPS::kLevelIDC5p1);
+}
+
+}  // namespace media

media/video/h264_level_limits.cc

@@ -5,6 +5,7 @@
 #include "media/video/h264_level_limits.h"
 
 #include "base/logging.h"
+#include "base/stl_util.h"
 #include "media/video/h264_parser.h"
 
 namespace media {
@@ -141,4 +142,27 @@ bool CheckH264LevelLimits(VideoCodecProfile profile,
   return true;
 }
 
+base::Optional<uint8_t> FindValidH264Level(VideoCodecProfile profile,
+                                           uint32_t bitrate,
+                                           uint32_t framerate,
+                                           uint32_t framesize_in_mbs) {
+  constexpr uint8_t kH264Levels[] = {
+      H264SPS::kLevelIDC1p0, H264SPS::kLevelIDC1B,  H264SPS::kLevelIDC1p1,
+      H264SPS::kLevelIDC1p2, H264SPS::kLevelIDC1p3, H264SPS::kLevelIDC2p0,
+      H264SPS::kLevelIDC2p1, H264SPS::kLevelIDC2p2, H264SPS::kLevelIDC3p0,
+      H264SPS::kLevelIDC3p1, H264SPS::kLevelIDC3p2, H264SPS::kLevelIDC4p0,
+      H264SPS::kLevelIDC4p1, H264SPS::kLevelIDC4p2, H264SPS::kLevelIDC5p0,
+      H264SPS::kLevelIDC5p1, H264SPS::kLevelIDC5p2, H264SPS::kLevelIDC6p0,
+      H264SPS::kLevelIDC6p1, H264SPS::kLevelIDC6p2,
+  };
+
+  for (const uint8_t level : kH264Levels) {
+    if (CheckH264LevelLimits(profile, level, bitrate, framerate,
+                             framesize_in_mbs)) {
+      return level;
+    }
+  }
+  return base::nullopt;
+}
+
 }  // namespace media

media/video/h264_level_limits.h

@@ -7,6 +7,7 @@
 
 #include <stddef.h>
 
+#include "base/optional.h"
 #include "media/base/media_export.h"
 #include "media/base/video_codecs.h"
 
@@ -37,6 +38,14 @@ bool MEDIA_EXPORT CheckH264LevelLimits(VideoCodecProfile profile,
                                        uint32_t framerate,
                                        uint32_t framesize_in_mbs);
 
+// Return a minimum level that comforts Table A-1 in spec with |profile|,
+// |bitrate|, |framerate| and |framesize_in_mbs|. If there is no proper level,
+// returns base::nullopt.
+base::Optional<uint8_t> MEDIA_EXPORT
+FindValidH264Level(VideoCodecProfile profile,
+                   uint32_t bitrate,
+                   uint32_t framerate,
+                   uint32_t framesize_in_mbs);
 }  // namespace media
 
 #endif  // MEDIA_VIDEO_H264_LEVEL_LIMITS_H_

media/video/video_encode_accelerator.cc

@@ -52,8 +52,7 @@ VideoEncodeAccelerator::Config::Config(
       initial_framerate(initial_framerate.value_or(
           VideoEncodeAccelerator::kDefaultFramerate)),
       gop_length(gop_length),
-      h264_output_level(h264_output_level.value_or(
-          VideoEncodeAccelerator::kDefaultH264Level)),
+      h264_output_level(h264_output_level),
       storage_type(storage_type),
       content_type(content_type) {}
 

media/video/video_encode_accelerator.h

@@ -98,11 +98,8 @@ class MEDIA_EXPORT VideoEncodeAccelerator {
     kErrorMax = kPlatformFailureError
   };
 
-  // Unified default values for all VEA implementations.
-  enum {
-    kDefaultFramerate = 30,
-    kDefaultH264Level = H264SPS::kLevelIDC4p0,
-  };
+  // A default framerate for all VEA implementations.
+  enum { kDefaultFramerate = 30 };
 
   // Parameters required for VEA initialization.
   struct MEDIA_EXPORT Config {
@@ -154,11 +151,9 @@ class MEDIA_EXPORT VideoEncodeAccelerator {
     base::Optional<uint32_t> gop_length;
 
     // Codec level of encoded output stream for H264 only. This value should
-    // be aligned to the H264 standard definition of SPS.level_idc. The only
-    // exception is in Main and Baseline profile we still use
-    // |h264_output_level|=9 for Level 1b, which should set level_idc to 11 and
-    // constraint_set3_flag to 1 (Spec A.3.1 and A.3.2). This is optional and
-    // use |kDefaultH264Level| if not given.
+    // be aligned to the H264 standard definition of SPS.level_idc.
+    // If this is not given, VideoEncodeAccelerator selects one of proper H.264
+    // levels for |input_visible_size| and |initial_framerate|.
     base::Optional<uint8_t> h264_output_level;
 
     // The storage type of video frame provided on Encode().