Add method to WebMediaPlayerClient to get metadata for text tracks

There is an upload warning for use of std::vector, with an attached
note saying that it's acceptable at the blink/chromium layer, which
I think this falls under. This CL should only be merged after the
devtools frontend CL here:
https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2336204

This also adds a "Chapters" type to the enum used in TextTrackConfig
because apparently that is a thing blink supports.

Bug: 1090022
Change-Id: I19cd27b2c926bef1c36fed19efe8599c9d168386
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2337402Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Commit-Queue: Ted Meyer <tmathmeyer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#795465}

media/base/media_log_properties.cc

@@ -31,6 +31,7 @@ std::string MediaLogPropertyKeyToString(MediaLogProperty property) {
     STRINGIFY(kAudioDecoderName);
     STRINGIFY(kIsPlatformAudioDecoder);
     STRINGIFY(kAudioTracks);
+    STRINGIFY(kTextTracks);
     STRINGIFY(kVideoTracks);
     STRINGIFY(kFramerate);
     STRINGIFY(kVideoPlaybackRoughness);

media/base/media_log_properties.h

@@ -11,6 +11,7 @@
 #include "media/base/audio_decoder_config.h"
 #include "media/base/media_export.h"
 #include "media/base/media_log_type_enforcement.h"
+#include "media/base/text_track_config.h"
 #include "media/base/video_decoder_config.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -74,6 +75,7 @@ enum class MediaLogProperty {
 
   // Track metadata.
   kAudioTracks,
+  kTextTracks,
   kVideoTracks,
 
   // Effective video playback frame rate adjusted for the playback speed.
@@ -103,6 +105,7 @@ MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioDecoderName, std::string);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsPlatformAudioDecoder, bool);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsAudioDecryptingDemuxerStream, bool);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioTracks, std::vector<AudioDecoderConfig>);
+MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kTextTracks, std::vector<TextTrackConfig>);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoTracks, std::vector<VideoDecoderConfig>);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kFramerate, double);
 MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoPlaybackRoughness, double);

media/base/media_serializers.h

@@ -15,6 +15,7 @@
 #include "media/base/media_serializers_base.h"
 #include "media/base/status.h"
 #include "media/base/status_codes.h"
+#include "media/base/text_track_config.h"
 #include "media/base/video_decoder_config.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -293,6 +294,40 @@ struct MediaSerializer<VideoDecoderConfig> {
   }
 };
 
+// Class (complex)
+template <>
+struct MediaSerializer<TextTrackConfig> {
+  static base::Value Serialize(const TextTrackConfig& value) {
+    base::Value result(base::Value::Type::DICTIONARY);
+    FIELD_SERIALIZE("kind", value.kind());
+    FIELD_SERIALIZE("language", value.language());
+    if (value.label().length())
+      FIELD_SERIALIZE("label", value.label());
+    return result;
+  }
+};
+
+// enum (simple)
+template <>
+struct MediaSerializer<TextKind> {
+  static base::Value Serialize(const TextKind value) {
+    switch (value) {
+      case kTextSubtitles:
+        return base::Value("Subtitles");
+      case kTextCaptions:
+        return base::Value("Captions");
+      case kTextDescriptions:
+        return base::Value("Descriptions");
+      case kTextMetadata:
+        return base::Value("Metadata");
+      case kTextChapters:
+        return base::Value("Chapters");
+      case kTextNone:
+        return base::Value("None");
+    }
+  }
+};
+
 // enum (simple)
 template <>
 struct MediaSerializer<BufferingState> {

media/base/text_track_config.cc

@@ -29,4 +29,19 @@ bool TextTrackConfig::Matches(const TextTrackConfig& config) const {
          config.id() == id_;
 }
 
+// static
+TextKind TextTrackConfig::ConvertKind(const std::string& str) {
+  if (str == "subtitles")
+    return kTextSubtitles;
+  if (str == "captions")
+    return kTextCaptions;
+  if (str == "descriptions")
+    return kTextDescriptions;
+  if (str == "chapters")
+    return kTextChapters;
+  if (str == "metadata")
+    return kTextMetadata;
+  return kTextNone;
+}
+
 }  // namespace media

media/base/text_track_config.h

@@ -17,6 +17,7 @@ enum TextKind {
   kTextCaptions,
   kTextDescriptions,
   kTextMetadata,
+  kTextChapters,
   kTextNone
 };
 
@@ -37,6 +38,8 @@ class MEDIA_EXPORT TextTrackConfig {
   const std::string& language() const { return language_; }
   const std::string& id() const { return id_; }
 
+  static TextKind ConvertKind(const std::string& kind);
+
  private:
   TextKind kind_;
   std::string label_;

media/blink/webmediaplayer_impl.cc

@@ -71,6 +71,7 @@
 #include "third_party/blink/public/platform/web_size.h"
 #include "third_party/blink/public/platform/web_string.h"
 #include "third_party/blink/public/platform/web_surface_layer_bridge.h"
+#include "third_party/blink/public/platform/web_texttrack_metadata.h"
 #include "third_party/blink/public/platform/web_url.h"
 #include "third_party/blink/public/platform/webaudiosourceprovider_impl.h"
 #include "third_party/blink/public/strings/grit/blink_strings.h"
@@ -756,6 +757,14 @@ void WebMediaPlayerImpl::DoLoad(LoadType load_type,
       url.GetString().Substring(0, kMaxUrlLength + 1).Utf8());
   load_start_time_ = base::TimeTicks::Now();
 
+  std::vector<TextTrackConfig> text_configs;
+  for (const auto& metadata : client_->GetTextTrackMetadata()) {
+    text_configs.emplace_back(TextTrackConfig::ConvertKind(metadata.kind()),
+                              metadata.label(), metadata.language(),
+                              metadata.id());
+  }
+  media_log_->SetProperty<MediaLogProperty::kTextTracks>(text_configs);
+
   // If we're adapting, then restart the smoothness experiment.
   if (smoothness_helper_)
     smoothness_helper_.reset();

media/blink/webmediaplayer_impl_unittest.cc

@@ -161,6 +161,7 @@ class MockWebMediaPlayerClient : public blink::WebMediaPlayerClient {
   MOCK_METHOD0(RequestExitPictureInPicture, void());
   MOCK_METHOD0(GetFeatures, Features(void));
   MOCK_METHOD0(OnRequestVideoFrameCallback, void());
+  MOCK_METHOD0(GetTextTrackMetadata, std::vector<blink::TextTrackMetadata>());
 
   bool was_always_muted_ = false;
 

third_party/blink/public/BUILD.gn

@@ -255,6 +255,7 @@ source_set("blink_headers") {
     "platform/web_text_input_mode.h",
     "platform/web_text_input_type.h",
     "platform/web_text_run.h",
+    "platform/web_texttrack_metadata.h",
     "platform/web_theme_engine.h",
     "platform/web_thread_safe_data.h",
     "platform/web_time_range.h",

third_party/blink/public/platform/web_media_player_client.h

@@ -36,6 +36,8 @@
 #include "third_party/blink/public/platform/web_media_player.h"
 #include "ui/gfx/color_space.h"
 
+#include "third_party/blink/public/platform/web_texttrack_metadata.h"
+
 namespace cc {
 class Layer;
 }
@@ -136,6 +138,9 @@ class BLINK_PLATFORM_EXPORT WebMediaPlayerClient {
   // any.
   virtual WebRemotePlaybackClient* RemotePlaybackClient() { return nullptr; }
 
+  // Returns metadata for out-of-band text tracks declared as <track> elements.
+  virtual std::vector<TextTrackMetadata> GetTextTrackMetadata() = 0;
+
   // Returns the color space to render media into if.
   // Rendering media into this color space may avoid some conversions.
   virtual gfx::ColorSpace TargetColorSpace() { return gfx::ColorSpace(); }

third_party/blink/public/platform/web_texttrack_metadata.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_TEXTTRACK_METADATA_H_
+#define THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_TEXTTRACK_METADATA_H_
+
+#include "third_party/blink/public/platform/web_common.h"
+
+namespace blink {
+
+class BLINK_PLATFORM_EXPORT TextTrackMetadata {
+ public:
+  TextTrackMetadata(const std::string& lang,
+                    const std::string& kind,
+                    const std::string& label,
+                    const std::string& id)
+      : language_(lang), kind_(kind), label_(label), id_(id) {}
+  ~TextTrackMetadata() = default;
+
+  const std::string& language() const { return language_; }
+  const std::string& kind() const { return kind_; }
+  const std::string& label() const { return label_; }
+  const std::string& id() const { return id_; }
+
+ private:
+  std::string language_;
+  std::string kind_;
+  std::string label_;
+  std::string id_;
+};
+
+}  // namespace blink
+
+#endif  // THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_TEXTTRACK_METADATA_H_

third_party/blink/renderer/core/html/media/html_media_element.cc

@@ -3039,6 +3039,18 @@ TextTrack* HTMLMediaElement::addTextTrack(const AtomicString& kind,
   return text_track;
 }
 
+std::vector<TextTrackMetadata> HTMLMediaElement::GetTextTrackMetadata() {
+  TextTrackList* tracks = textTracks();
+  std::vector<TextTrackMetadata> result;
+  for (unsigned i = 0; i < tracks->length(); i++) {
+    TextTrack* track = tracks->AnonymousIndexedGetter(i);
+    result.emplace_back(track->language().GetString().Utf8(),
+                        track->kind().GetString().Utf8(),
+                        track->label().GetString().Utf8(), track->id().Utf8());
+  }
+  return result;
+}
+
 TextTrackList* HTMLMediaElement::textTracks() {
   if (!text_tracks_) {
     UseCounter::Count(GetDocument(), WebFeature::kMediaElementTextTrackList);

third_party/blink/renderer/core/html/media/html_media_element.h

@@ -441,6 +441,7 @@ class CORE_EXPORT HTMLMediaElement
   WebRemotePlaybackClient* RemotePlaybackClient() final {
     return remote_playback_client_;
   }
+  std::vector<TextTrackMetadata> GetTextTrackMetadata() override;
   gfx::ColorSpace TargetColorSpace() override;
   bool WasAutoplayInitiated() override;
   bool IsInAutoPIP() const override { return false; }

third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc

@@ -627,6 +627,10 @@ class WebMediaPlayerMSTest
     background_rendering_ = background_rendering;
   }
 
+  std::vector<blink::TextTrackMetadata> GetTextTrackMetadata() override {
+    return {};
+  }
+
   void SetGpuMemoryBufferVideoForTesting() {
 #if defined(OS_WIN)
     render_factory_->provider()->set_standard_size(

third_party/blink/web_tests/inspector-protocol/media/media-player-expected.txt

@@ -5,6 +5,6 @@ Received creation event for 1 player
 All IDs match
 events length: 2
 messages length: 0
-properties length: 2
+properties length: 3
 errors length: 0