Implement MediaCapabilities: spatialRendering decoding query

This CL allows for spatialRendering to be queried with
MediaCapabilities.decodingInfo() for clear and EME. Changes required to blink
and media were relatively small so they were combined into this CL. This CL
does not add the ability to render spatial audio.

This functionality is behind blink feature flag MediaCapabilitiesSpatialAudio.
Note that if the flag is disabled, querying for spatialRendering will return
supported because decodingInfo ignores extraneous parameters by design.

For the media implementation, AudioType was because spatialRendering is
strictly tied to the codec.

Bug: 1020943

Change-Id: I01c9853b88f72ae8cb6291f9ca49e1d82cb220d8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1928077Reviewed-by: Dave Tapuska <dtapuska@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Yuchen Liu <yucliu@chromium.org>
Reviewed-by: Mounir Lamouri <mlamouri@chromium.org>
Commit-Queue: Vi Nguyen <ving@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#721185}

chromecast/renderer/cast_content_renderer_client.cc

@@ -238,6 +238,9 @@ void CastContentRendererClient::AddSupportedKeySystems(
 
 bool CastContentRendererClient::IsSupportedAudioType(
     const ::media::AudioType& type) {
+  if (type.spatialRendering)
+    return false;
+
 #if defined(OS_ANDROID)
   // No ATV device we know of has (E)AC3 decoder, so it relies on the audio sink
   // device.

content/browser/media/media_capabilities_browsertest.cc

@@ -34,7 +34,7 @@ const char* kPropSupported = kSupported;
 const char* kPropSupported = kUnsupported;
 #endif  // USE_PROPRIETARY_CODECS
 
-enum StreamType { kAudio, kVideo };
+enum StreamType { kAudio, kVideo, kAudioWithSpatialRendering };
 
 enum ConfigType { kFile, kMediaSource };
 
@@ -49,6 +49,8 @@ class MediaCapabilitiesTest : public ContentBrowserTest {
   void SetUpCommandLine(base::CommandLine* command_line) override {
     command_line->AppendSwitchASCII(switches::kEnableBlinkFeatures,
                                     "MediaCapabilities");
+    command_line->AppendSwitchASCII(switches::kEnableBlinkFeatures,
+                                    "MediaCapabilitiesSpatialAudio");
   }
 
   std::string CanDecodeAudio(const std::string& config_type,
@@ -61,12 +63,24 @@ class MediaCapabilitiesTest : public ContentBrowserTest {
     return CanDecode(config_type, content_type, StreamType::kVideo);
   }
 
+  std::string CanDecodeAudioWithSpatialRendering(
+      const std::string& config_type,
+      const std::string& content_type,
+      bool spatial_rendering) {
+    return CanDecode(config_type, content_type,
+                     StreamType::kAudioWithSpatialRendering, spatial_rendering);
+  }
+
   std::string CanDecode(const std::string& config_type,
                         const std::string& content_type,
-                        StreamType stream_type) {
+                        StreamType stream_type,
+                        const bool& spatial_rendering = false) {
     std::string command;
     if (stream_type == StreamType::kAudio) {
       command.append("testAudioConfig(");
+    } else if (stream_type == StreamType::kAudioWithSpatialRendering) {
+      command.append("testAudioConfigWithSpatialRendering(");
+      command.append(spatial_rendering ? "true, " : "false, ");
     } else {
       command.append("testVideoConfig(");
     }
@@ -229,6 +243,79 @@ IN_PROC_BROWSER_TEST_P(MediaCapabilitiesTestWithConfigType,
             CanDecodeAudio(config_type, "'audio/ogg; codecs=\"vorbis\"'"));
 }
 
+// Cover basic spatial rendering support.
+IN_PROC_BROWSER_TEST_P(MediaCapabilitiesTestWithConfigType,
+                       AudioTypesWithSpatialRendering) {
+  base::FilePath file_path = media::GetTestDataFilePath(kDecodeTestFile);
+
+  const std::string& config_type = GetTypeString();
+
+  EXPECT_TRUE(
+      NavigateToURL(shell(), content::GetFileUrlWithQuery(file_path, "")));
+
+  // These common codecs are not associated with a spatial audio format.
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/webm; codecs=\"opus\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/webm; codecs=\"vorbis\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/mp4; codecs=\"flac\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported,
+            CanDecodeAudioWithSpatialRendering(config_type, "'audio/mpeg'",
+                                               /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/mp4; codecs=\"mp4a.40.02\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported,
+            CanDecodeAudioWithSpatialRendering(config_type, "'audio/aac'",
+                                               /*spatial_rendering*/ true));
+
+  // Supported codecs should remain supported when querying with
+  // spatialRendering set to false.
+  EXPECT_EQ(kSupported, CanDecodeAudioWithSpatialRendering(
+                            config_type, "'audio/webm; codecs=\"opus\"'",
+                            /*spatial_rendering*/ false));
+  EXPECT_EQ(kSupported, CanDecodeAudioWithSpatialRendering(
+                            config_type, "'audio/webm; codecs=\"vorbis\"'",
+                            /*spatial_rendering*/ false));
+  EXPECT_EQ(kSupported, CanDecodeAudioWithSpatialRendering(
+                            config_type, "'audio/mp4; codecs=\"flac\"'",
+                            /*spatial_rendering*/ false));
+  EXPECT_EQ(kSupported,
+            CanDecodeAudioWithSpatialRendering(config_type, "'audio/mpeg'",
+                                               /*spatial_rendering*/ false));
+  EXPECT_EQ(kPropSupported,
+            CanDecodeAudioWithSpatialRendering(
+                config_type, "'audio/mp4; codecs=\"mp4a.40.02\"'",
+                /*spatial_rendering*/ false));
+  EXPECT_EQ(kPropSupported,
+            CanDecodeAudioWithSpatialRendering(config_type, "'audio/aac'",
+                                               /*spatial_rendering*/ false));
+
+  // Test a handful of invalid strings.
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/wav; codecs=\"mp3\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/webm; codecs=\"vp8\"'",
+                              /*spatial_rendering*/ true));
+
+  // Dolby Atmos = Dolby Digital Plus + Spatial Rendering. Currently not
+  // supported.
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/mp4; codecs=\"ec-3\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/mp4; codecs=\"mp4a.a6\"'",
+                              /*spatial_rendering*/ true));
+  EXPECT_EQ(kUnsupported, CanDecodeAudioWithSpatialRendering(
+                              config_type, "'audio/mp4; codecs=\"mp4a.A6\"'",
+                              /*spatial_rendering*/ true));
+}
+
 INSTANTIATE_TEST_SUITE_P(File,
                          MediaCapabilitiesTestWithConfigType,
                          ::testing::Values(ConfigType::kFile));

media/base/media_types.h

@@ -22,6 +22,7 @@ struct MEDIA_EXPORT AudioType {
   static AudioType FromDecoderConfig(const AudioDecoderConfig& config);
 
   AudioCodec codec;
+  bool spatialRendering;
 };
 
 struct MEDIA_EXPORT VideoType {

media/base/supported_types.cc

@@ -185,6 +185,9 @@ bool IsAudioCodecProprietary(AudioCodec codec) {
 }
 
 bool IsDefaultSupportedAudioType(const AudioType& type) {
+  if (type.spatialRendering)
+    return false;
+
 #if !BUILDFLAG(USE_PROPRIETARY_CODECS)
   if (IsAudioCodecProprietary(type.codec))
     return false;

media/base/supported_types_unittest.cc

@@ -164,4 +164,40 @@ TEST(SupportedTypesTest, IsSupportedVideoType_VP9Matrix) {
   EXPECT_EQ(kSupportedMatrix.size(), num_found);
 }
 
+TEST(SupportedTypesTest, IsSupportedAudioTypeWithSpatialRenderingBasics) {
+  const bool is_spatial_rendering = true;
+  // Dolby Atmos = E-AC3 (Dolby Digital Plus) + spatialRendering. Currently not
+  // supported.
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecEAC3, is_spatial_rendering}));
+
+  // Expect non-support for codecs with which there is no spatial audio format.
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecAAC, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecMP3, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecPCM, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecVorbis, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecFLAC, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecAMR_NB, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecAMR_WB, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecPCM_MULAW, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecGSM_MS, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecPCM_S16BE, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecPCM_S24BE, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecOpus, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecPCM_ALAW, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecALAC, is_spatial_rendering}));
+  EXPECT_FALSE(IsSupportedAudioType({media::kCodecAC3, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kCodecMpegHAudio, is_spatial_rendering}));
+  EXPECT_FALSE(
+      IsSupportedAudioType({media::kUnknownAudioCodec, is_spatial_rendering}));
+}
+
 }  // namespace media

media/test/data/decode_capabilities_test.html

@@ -67,4 +67,21 @@
 
     runTest(configuration);
   }
+
+  function testAudioConfigWithSpatialRendering(spatialRendering, queryType, contentType) {
+    // Clear previous test result from title.
+    document.title = '';
+
+    log("Testing audio content type: " + contentType + ", spatialRendering: " + spatialRendering);
+
+    const configuration = {
+      type : queryType,
+      audio : {
+        contentType : contentType,
+        spatialRendering : spatialRendering
+      }
+    };
+
+    runTest(configuration);
+  }
 </script>

third_party/blink/renderer/modules/media_capabilities/audio_configuration.idl

@@ -10,4 +10,5 @@ dictionary AudioConfiguration {
     DOMString channels;
     unsigned long bitrate;
     unsigned long samplerate;
+    [RuntimeEnabled=MediaCapabilitiesSpatialAudio] boolean spatialRendering;
 };

third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc

@@ -396,6 +396,7 @@ bool IsAudioConfigurationSupported(
     const String& codec) {
   media::AudioCodec audio_codec = media::kUnknownAudioCodec;
   bool is_audio_codec_ambiguous = true;
+  bool is_spatial_rendering = false;
 
   // Must succeed as IsAudioCodecValid() should have been called before.
   bool parsed =
@@ -403,7 +404,10 @@ bool IsAudioConfigurationSupported(
                                    &is_audio_codec_ambiguous, &audio_codec);
   DCHECK(parsed && !is_audio_codec_ambiguous);
 
-  return media::IsSupportedAudioType({audio_codec});
+  if (audio_config->hasSpatialRendering())
+    is_spatial_rendering = audio_config->spatialRendering();
+
+  return media::IsSupportedAudioType({audio_codec, is_spatial_rendering});
 }
 
 // Returns whether the VideoConfiguration is supported.

third_party/blink/renderer/platform/media_capabilities/web_audio_configuration.h

@@ -21,6 +21,7 @@ struct WebAudioConfiguration {
   WebString channels;
   base::Optional<unsigned> bitrate;
   base::Optional<unsigned> samplerate;
+  base::Optional<bool> spatialRendering;
 };
 
 }  // namespace blink

third_party/blink/renderer/platform/runtime_enabled_features.json5

@@ -944,6 +944,10 @@
       name: "MediaCapabilitiesEncryptedMedia",
       status: "stable",
     },
+    {
+      name: "MediaCapabilitiesSpatialAudio",
+      status: "test",
+    },
     {
       name: "MediaCapture",
       status: {"Android": "stable"},

third_party/blink/web_tests/external/wpt/media-capabilities/decodingInfo.any.js

@@ -17,6 +17,12 @@ var minimalAudioConfiguration = {
   contentType: 'audio/webm; codecs="opus"',
 };
 
+// AudioConfiguration with optional spatialRendering param.
+var audioConfigurationWithSpatialRendering = {
+  contentType: 'audio/webm; codecs="opus"',
+  spatialRendering: true,
+};
+
 promise_test(t => {
   return promise_rejects_js(t, TypeError, navigator.mediaCapabilities.decodingInfo());
 }, "Test that decodingInfo rejects if it doesn't get a configuration");
@@ -296,3 +302,15 @@ async_test(t => {
     }
   }), t.unreached_func('Promise.all should not reject for valid types'));
 }, "Test that decodingInfo rejects if the MediaConfiguration does not have a valid type");
+
+promise_test(t => {
+  return navigator.mediaCapabilities.decodingInfo({
+    type: 'file',
+    audio: audioConfigurationWithSpatialRendering,
+  }).then(ability => {
+    assert_equals(typeof ability.supported, "boolean");
+    assert_equals(typeof ability.smooth, "boolean");
+    assert_equals(typeof ability.powerEfficient, "boolean");
+    assert_equals(typeof ability.keySystemAccess, "object");
+  });
+}, "Test that decodingInfo with spatialRendering set returns a valid MediaCapabilitiesInfo objects");