MediaCapabilities: setup machine learning predictions

Adds predictors for non-network rebuffer (NNR) and consecutive "bad
windows" with high counts of dropped frames. The predictions will be
compared against configurable thresholds for determining
smooth=true/false for a given MediaDecodingConfiguration.

Bug: 1049339

Change-Id: Ia80a9cdc887c070f8156b58a0da64d02318fda40
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2021950
Commit-Queue: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Mounir Lamouri <mlamouri@chromium.org>
Reviewed-by: Frank Liberato <liberato@chromium.org>
Reviewed-by: Ken Rockot <rockot@google.com>
Cr-Commit-Position: refs/heads/master@{#749364}

media/learning/common/BUILD.gn

@@ -10,6 +10,7 @@ component("common") {
     "//media/learning/impl:test_support",
     "//media/learning/mojo/public/cpp:*",
     "//media/learning/mojo/public/mojom:mojom",
+    "//media/learning/mojo/public/mojom:mojom_blink",
     "//media/learning/mojo:*",
     "//media/learning/common:unit_tests",
 

media/learning/common/target_histogram.h

@@ -23,6 +23,13 @@ namespace mojom {
 class TargetHistogramDataView;
 }
 
+// Intermediate type for mojom struct traits translation.
+// See learning_types.mojom.
+struct COMPONENT_EXPORT(LEARNING_COMMON) TargetHistogramPair {
+  TargetValue target_value;
+  double count;
+};
+
 // Histogram of target values that allows fractional counts.
 class COMPONENT_EXPORT(LEARNING_COMMON) TargetHistogram {
  public:

media/learning/mojo/public/cpp/BUILD.gn

@@ -23,6 +23,26 @@ component("cpp") {
   ]
 }
 
+# Normally typemap traits sources should be build directly into mojom targets
+# via the typemap file. This target is for typemapped mojo_base types whose
+# traits are shared between chromium and blink variants.
+component("shared_typemap_traits") {
+  output_name = "media_learning_shared_typemap_traits"
+
+  defines = [ "IS_MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS_IMPL" ]
+
+  sources = [
+    "//media/learning/mojo/public/cpp/learning_mojom_traits.cc",
+    "//media/learning/mojo/public/cpp/learning_mojom_traits.h",
+  ]
+
+  deps = [
+    "//base",
+    "//media/learning/common",
+    "//media/learning/mojo/public/mojom:mojom_shared",
+  ]
+}
+
 source_set("unit_tests") {
   testonly = true
 

media/learning/mojo/public/cpp/learning_mojom_traits.cc

@@ -50,14 +50,37 @@ bool StructTraits<media::learning::mojom::ObservationCompletionDataView,
   return true;
 }
 
+// static
+bool StructTraits<media::learning::mojom::TargetHistogramPairDataView,
+                  media::learning::TargetHistogramPair>::
+    Read(media::learning::mojom::TargetHistogramPairDataView data,
+         media::learning::TargetHistogramPair* out_pair) {
+  if (!data.ReadTargetValue(&out_pair->target_value))
+    return false;
+  out_pair->count = data.count();
+  return true;
+}
+
 // static
 bool StructTraits<media::learning::mojom::TargetHistogramDataView,
                   media::learning::TargetHistogram>::
     Read(media::learning::mojom::TargetHistogramDataView data,
          media::learning::TargetHistogram* out_target_histogram) {
-  if (!data.ReadCounts(&out_target_histogram->counts_))
+  ArrayDataView<media::learning::mojom::TargetHistogramPairDataView> pairs;
+  data.GetPairsDataView(&pairs);
+  if (pairs.is_null())
     return false;
 
+  for (size_t i = 0; i < pairs.size(); ++i) {
+    media::learning::mojom::TargetHistogramPairDataView pair_data;
+    pairs.GetDataView(i, &pair_data);
+    media::learning::TargetValue value;
+    if (!pair_data.ReadTargetValue(&value))
+      return false;
+
+    out_target_histogram->counts_.emplace(std::move(value), pair_data.count());
+  }
+
   return true;
 }
 

media/learning/mojo/public/cpp/learning_mojom_traits.h

@@ -7,16 +7,18 @@
 
 #include <vector>
 
+#include "base/component_export.h"
 #include "media/learning/common/learning_task_controller.h"
 #include "media/learning/common/value.h"
-#include "media/learning/mojo/public/mojom/learning_types.mojom.h"
+#include "media/learning/mojo/public/mojom/learning_types.mojom-shared.h"
 #include "mojo/public/cpp/bindings/struct_traits.h"
 
 namespace mojo {
 
 template <>
-struct StructTraits<media::learning::mojom::LabelledExampleDataView,
-                    media::learning::LabelledExample> {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::LabelledExampleDataView,
+                 media::learning::LabelledExample> {
   static const std::vector<media::learning::FeatureValue>& features(
       const media::learning::LabelledExample& e) {
     return e.features;
@@ -31,9 +33,10 @@ struct StructTraits<media::learning::mojom::LabelledExampleDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::FeatureValueDataView,
-                    media::learning::FeatureValue> {
-  static int64_t value(const media::learning::FeatureValue& e) {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::FeatureValueDataView,
+                 media::learning::FeatureValue> {
+  static double value(const media::learning::FeatureValue& e) {
     return e.value();
   }
   static bool Read(media::learning::mojom::FeatureValueDataView data,
@@ -41,9 +44,10 @@ struct StructTraits<media::learning::mojom::FeatureValueDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::TargetValueDataView,
-                    media::learning::TargetValue> {
-  static int64_t value(const media::learning::TargetValue& e) {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetValueDataView,
+                 media::learning::TargetValue> {
+  static double value(const media::learning::TargetValue& e) {
     return e.value();
   }
   static bool Read(media::learning::mojom::TargetValueDataView data,
@@ -51,8 +55,9 @@ struct StructTraits<media::learning::mojom::TargetValueDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::ObservationCompletionDataView,
-                    media::learning::ObservationCompletion> {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::ObservationCompletionDataView,
+                 media::learning::ObservationCompletion> {
   static media::learning::TargetValue target_value(
       const media::learning::ObservationCompletion& e) {
     return e.target_value;
@@ -67,11 +72,32 @@ struct StructTraits<media::learning::mojom::ObservationCompletionDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::TargetHistogramDataView,
-                    media::learning::TargetHistogram> {
-  static media::learning::TargetHistogram::CountMap counts(
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetHistogramPairDataView,
+                 media::learning::TargetHistogramPair> {
+  static media::learning::TargetValue target_value(
+      const media::learning::TargetHistogramPair& e) {
+    return e.target_value;
+  }
+  static double count(const media::learning::TargetHistogramPair& e) {
+    return e.count;
+  }
+  static bool Read(media::learning::mojom::TargetHistogramPairDataView data,
+                   media::learning::TargetHistogramPair* out_pair);
+};
+
+template <>
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetHistogramDataView,
+                 media::learning::TargetHistogram> {
+  static std::vector<media::learning::TargetHistogramPair> pairs(
       const media::learning::TargetHistogram& e) {
-    return e.counts();
+    std::vector<media::learning::TargetHistogramPair> pairs;
+    for (auto const& entry : e.counts_) {
+      pairs.push_back({entry.first, entry.second});
+    }
+
+    return pairs;
   }
 
   static bool Read(media::learning::mojom::TargetHistogramDataView data,

media/learning/mojo/public/mojom/BUILD.gn

@@ -13,4 +13,8 @@ mojom("mojom") {
   ]
 
   public_deps = [ "//mojo/public/mojom/base" ]
+
+  export_class_attribute_blink = "BLINK_PLATFORM_EXPORT"
+  export_define_blink = "BLINK_PLATFORM_IMPLEMENTATION=1"
+  export_header_blink = "third_party/blink/public/platform/web_common.h"
 }

media/learning/mojo/public/mojom/learning_types.mojom

@@ -6,12 +6,12 @@ module media.learning.mojom;
 
 // learning::FeatureValue (common/value.h)
 struct FeatureValue {
-  int64 value;
+  double value;
 };
 
 // learning::TargetValue (common/value.h)
 struct TargetValue {
-  int64 value;
+  double value;
 };
 
 // learning::LabelledExample (common/training_example.h)
@@ -26,7 +26,19 @@ struct ObservationCompletion {
   uint64 weight = 1;
 };
 
+// Hack for TargetHistogram. Would ideally be a map<TargetValue, double>, but
+// this causes pain in the translation to WTF::HashMap. HashMap requires
+// reservations for "deleted" and "empty" sentinel values. This is especially
+// undesirable in our case because TargetValue (the map key) is designed to be
+// completely generic, supporting any possible value for double. We instead
+// use a list of pairs (key, value) to avoid having to reserve any values as
+// "empty" or "deleted".
+struct TargetHistogramPair {
+  TargetValue target_value;
+  double count;
+};
+
 // learning::TargetHistogram (common/target_histogram.h)
 struct TargetHistogram {
-  map<TargetValue, double> counts;
+  array<TargetHistogramPair> pairs;
 };

media/learning/mojo/public/mojom/learning_types.typemap

@@ -5,15 +5,15 @@ public_headers = [
   "//media/learning/common/value.h",
 ]
 traits_headers = [ "//media/learning/mojo/public/cpp/learning_mojom_traits.h" ]
-sources = [
-  "//media/learning/mojo/public/cpp/learning_mojom_traits.cc",
-  "//media/learning/mojo/public/cpp/learning_mojom_traits.h",
+public_deps = [
+  "//media/learning/common",
+  "//media/learning/mojo/public/cpp:shared_typemap_traits",
 ]
-public_deps = [ "//media/learning/common" ]
 type_mappings = [
   "media.learning.mojom.LabelledExample=::media::learning::LabelledExample",
   "media.learning.mojom.FeatureValue=::media::learning::FeatureValue",
   "media.learning.mojom.TargetValue=::media::learning::TargetValue",
   "media.learning.mojom.ObservationCompletion=::media::learning::ObservationCompletion",
+  "media.learning.mojom.TargetHistogramPair=::media::learning::TargetHistogramPair",
   "media.learning.mojom.TargetHistogram=::media::learning::TargetHistogram",
 ]

mojo/public/tools/bindings/blink_bindings_configuration.gni

@@ -10,6 +10,7 @@ _typemap_imports = [
   "//cc/typemaps.gni",
   "//device/gamepad/public/cpp/typemaps.gni",
   "//device/vr/public/mojom/blink_typemaps.gni",
+  "//media/learning/mojo/public/cpp/typemaps.gni",
   "//mojo/public/cpp/bindings/tests/blink_typemaps.gni",
   "//skia/public/mojom/typemaps.gni",
   "//third_party/blink/renderer/platform/mojo/blink_typemaps.gni",

third_party/blink/renderer/modules/media_capabilities/BUILD.gn

@@ -15,7 +15,11 @@ blink_modules_sources("media_capabilities") {
     "worker_navigator_media_capabilities.cc",
     "worker_navigator_media_capabilities.h",
   ]
-  deps = [ "//third_party/blink/renderer/modules/mediarecorder" ]
+  deps = [
+    "//media",
+    "//media/learning/mojo:impl",
+    "//third_party/blink/renderer/modules/mediarecorder",
+  ]
 }
 
 fuzzable_proto_library("fuzzer_media_configuration_proto") {

third_party/blink/renderer/modules/media_capabilities/DEPS

 include_rules = [
-    "+media/base/mime_util.h",
-    "+media/base/supported_types.h",
-    "+media/base/video_codecs.h",
+    "+media/base",
     "+media/filters/stream_parser_factory.h",
+    "+media/learning/common",
+    "+media/learning/mojo/public/cpp/mojo_learning_task_controller.h",
+    "+media/learning/mojo/public/mojom/learning_task_controller.mojom-blink.h",
     "+media/mojo/mojom/media_types.mojom-blink.h",
     "+media/mojo/mojom/video_decode_perf_history.mojom-blink.h",
+    "+media/mojo/mojom/media_metrics_provider.mojom-blink.h",
     "-third_party/blink/renderer/modules",
     "+third_party/blink/renderer/modules/encryptedmedia",
     "+third_party/blink/renderer/modules/media_capabilities",

third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc

@@ -5,12 +5,20 @@
 #include "third_party/blink/renderer/modules/media_capabilities/media_capabilities.h"
 
 #include <memory>
+#include <sstream>
 #include <utility>
 
+#include "base/feature_list.h"
+#include "base/metrics/field_trial_params.h"
 #include "base/optional.h"
+#include "media/base/media_switches.h"
 #include "media/base/mime_util.h"
 #include "media/base/supported_types.h"
 #include "media/filters/stream_parser_factory.h"
+#include "media/learning/common/media_learning_tasks.h"
+#include "media/learning/common/target_histogram.h"
+#include "media/learning/mojo/public/mojom/learning_task_controller.mojom-blink.h"
+#include "media/mojo/mojom/media_metrics_provider.mojom-blink.h"
 #include "media/mojo/mojom/media_types.mojom-blink.h"
 #include "third_party/blink/public/common/browser_interface_broker_proxy.h"
 #include "third_party/blink/public/mojom/feature_policy/feature_policy.mojom-blink.h"
@@ -59,11 +67,34 @@ namespace blink {
 
 namespace {
 
+const double kLearningBadWindowThresholdDefault = 2;
+const double kLearningNnrThresholdDefault = 3;
+
 constexpr const char* kApplicationMimeTypePrefix = "application/";
 constexpr const char* kAudioMimeTypePrefix = "audio/";
 constexpr const char* kVideoMimeTypePrefix = "video/";
 constexpr const char* kCodecsMimeTypeParam = "codecs";
 
+// Gets parameters for kMediaLearningSmoothnessExperiment field trial. Will
+// provide sane defaults when field trial not enabled. Values of -1 indicate
+// predictions from a given task should be ignored.
+
+// static
+double GetLearningBadWindowThreshold() {
+  return base::GetFieldTrialParamByFeatureAsDouble(
+      media::kMediaLearningSmoothnessExperiment,
+      MediaCapabilities::kLearningBadWindowThresholdParamName,
+      kLearningBadWindowThresholdDefault);
+}
+
+// static
+double GetLearningNnrThreshold() {
+  return base::GetFieldTrialParamByFeatureAsDouble(
+      media::kMediaLearningSmoothnessExperiment,
+      MediaCapabilities::kLearningNnrThresholdParamName,
+      kLearningNnrThresholdDefault);
+}
+
 // Utility function that will create a MediaCapabilitiesDecodingInfo object with
 // all the values set to either true or false.
 MediaCapabilitiesDecodingInfo* CreateDecodingInfoWith(bool value) {
@@ -111,7 +142,7 @@ class MediaCapabilitiesKeySystemAccessInitializer final
     // Query the client for smoothness and power efficiency of the video. It
     // will resolve the promise.
     std::move(get_perf_callback_)
-        .Run(std::move(resolver_),
+        .Run(resolver_.Get(),
              MakeGarbageCollected<MediaKeySystemAccess>(std::move(access)));
   }
 
@@ -363,16 +394,17 @@ bool IsAudioCodecValid(const String& mime_type,
 //                   console.
 bool IsVideoCodecValid(const String& mime_type,
                        const String& codec,
+                       media::VideoCodec* out_video_codec,
                        media::VideoCodecProfile* out_video_profile,
                        String* console_warning) {
-  media::VideoCodec video_codec = media::kUnknownVideoCodec;
   uint8_t video_level = 0;
   media::VideoColorSpace video_color_space;
   bool is_video_codec_ambiguous = true;
 
-  if (!media::ParseVideoCodecString(
-          mime_type.Ascii(), codec.Ascii(), &is_video_codec_ambiguous,
-          &video_codec, out_video_profile, &video_level, &video_color_space)) {
+  if (!media::ParseVideoCodecString(mime_type.Ascii(), codec.Ascii(),
+                                    &is_video_codec_ambiguous, out_video_codec,
+                                    out_video_profile, &video_level,
+                                    &video_color_space)) {
     *console_warning = StringView("Failed to parse video contentType: ") +
                        String{mime_type} + StringView("; codecs=") +
                        String{codec};
@@ -471,8 +503,29 @@ bool ParseContentType(const String& content_type,
 
 }  // anonymous namespace
 
+const char MediaCapabilities::kLearningBadWindowThresholdParamName[] =
+    "bad_window_threshold";
+
+const char MediaCapabilities::kLearningNnrThresholdParamName[] =
+    "nnr_threshold";
+
 MediaCapabilities::MediaCapabilities() = default;
 
+void MediaCapabilities::Trace(blink::Visitor* visitor) {
+  visitor->Trace(pending_cb_map_);
+  ScriptWrappable::Trace(visitor);
+}
+
+MediaCapabilities::PendingCallbackState::PendingCallbackState(
+    ScriptPromiseResolver* resolver,
+    MediaKeySystemAccess* access)
+    : resolver(resolver), key_system_access(access) {}
+
+void MediaCapabilities::PendingCallbackState::Trace(blink::Visitor* visitor) {
+  visitor->Trace(key_system_access);
+  visitor->Trace(resolver);
+}
+
 ScriptPromise MediaCapabilities::decodingInfo(
     ScriptState* script_state,
     const MediaDecodingConfiguration* config,
@@ -504,21 +557,21 @@ ScriptPromise MediaCapabilities::decodingInfo(
   // Validation errors should return above.
   DCHECK(message.IsEmpty());
 
-  String audio_mime;
-  String audio_codec;
+  String audio_mime_str;
+  String audio_codec_str;
   if (config->hasAudio()) {
     DCHECK(config->audio()->hasContentType());
-    bool valid_content_type = ParseContentType(config->audio()->contentType(),
-                                               &audio_mime, &audio_codec);
+    bool valid_content_type = ParseContentType(
+        config->audio()->contentType(), &audio_mime_str, &audio_codec_str);
     DCHECK(valid_content_type);
   }
 
-  String video_mime;
-  String video_codec;
+  String video_mime_str;
+  String video_codec_str;
   if (config->hasVideo()) {
     DCHECK(config->video()->hasContentType());
-    bool valid_content_type = ParseContentType(config->video()->contentType(),
-                                               &video_mime, &video_codec);
+    bool valid_content_type = ParseContentType(
+        config->video()->contentType(), &video_mime_str, &video_codec_str);
     DCHECK(valid_content_type);
   }
 
@@ -526,8 +579,10 @@ ScriptPromise MediaCapabilities::decodingInfo(
   // that MSE support is not implied by EME support, so do it irrespective of
   // whether we have a KeySystem configuration.
   if (config->type() == "media-source") {
-    if ((config->hasAudio() && !CheckMseSupport(audio_mime, audio_codec)) ||
-        (config->hasVideo() && !CheckMseSupport(video_mime, video_codec))) {
+    if ((config->hasAudio() &&
+         !CheckMseSupport(audio_mime_str, audio_codec_str)) ||
+        (config->hasVideo() &&
+         !CheckMseSupport(video_mime_str, video_codec_str))) {
       // Unsupported EME queries should resolve with a null
       // MediaKeySystemAccess.
       return ScriptPromise::Cast(
@@ -536,12 +591,14 @@ ScriptPromise MediaCapabilities::decodingInfo(
     }
   }
 
+  media::VideoCodec video_codec = media::kUnknownVideoCodec;
   media::VideoCodecProfile video_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
 
   if ((config->hasAudio() &&
-       !IsAudioCodecValid(audio_mime, audio_codec, &message)) ||
+       !IsAudioCodecValid(audio_mime_str, audio_codec_str, &message)) ||
       (config->hasVideo() &&
-       !IsVideoCodecValid(video_mime, video_codec, &video_profile, &message))) {
+       !IsVideoCodecValid(video_mime_str, video_codec_str, &video_codec,
+                          &video_profile, &message))) {
     DCHECK(!message.IsEmpty());
     if (ExecutionContext* execution_context =
             ExecutionContext::From(script_state)) {
@@ -560,14 +617,15 @@ ScriptPromise MediaCapabilities::decodingInfo(
   if (config->hasKeySystemConfiguration()) {
     // GetEmeSupport() will call the VideoDecodePerfHistory service after
     // receiving info about support for the configuration for encrypted content.
-    return GetEmeSupport(script_state, video_profile, config, exception_state);
+    return GetEmeSupport(script_state, video_codec, video_profile, config,
+                         exception_state);
   }
 
   bool audio_supported = true;
 
   if (config->hasAudio()) {
-    audio_supported =
-        IsAudioConfigurationSupported(config->audio(), audio_mime, audio_codec);
+    audio_supported = IsAudioConfigurationSupported(
+        config->audio(), audio_mime_str, audio_codec_str);
   }
 
   // No need to check video capabilities if video not included in configuration
@@ -582,8 +640,8 @@ ScriptPromise MediaCapabilities::decodingInfo(
   DCHECK(config->hasVideo());
 
   // Return early for unsupported configurations.
-  if (!IsVideoConfigurationSupported(config->video(), video_mime,
-                                     video_codec)) {
+  if (!IsVideoConfigurationSupported(config->video(), video_mime_str,
+                                     video_codec_str)) {
     return ScriptPromise::Cast(
         script_state, ToV8(CreateDecodingInfoWith(false), script_state));
   }
@@ -595,7 +653,8 @@ ScriptPromise MediaCapabilities::decodingInfo(
   // undefined. See comment above Promise() in script_promise_resolver.h
   ScriptPromise promise = resolver->Promise();
 
-  GetPerfInfo(video_profile, config->video(), resolver, nullptr /* access */);
+  GetPerfInfo(video_codec, video_profile, config->video(), resolver,
+              nullptr /* access */);
 
   return promise;
 }
@@ -669,7 +728,49 @@ ScriptPromise MediaCapabilities::encodingInfo(
   return promise;
 }
 
-bool MediaCapabilities::EnsureService(ExecutionContext* execution_context) {
+bool MediaCapabilities::EnsureLearningPredictors(
+    ExecutionContext* execution_context) {
+  DCHECK(execution_context && !execution_context->IsContextDestroyed());
+
+  // One or both of these will have been bound in an earlier pass.
+  if (bad_window_predictor_ || nnr_predictor_)
+    return true;
+
+  // MediaMetricsProvider currently only exposed via render frame.
+  // TODO(chcunningham): Expose in worker contexts pending outcome of
+  // media-learning experiments.
+  if (execution_context->IsWorkerGlobalScope())
+    return false;
+
+  scoped_refptr<base::SingleThreadTaskRunner> task_runner =
+      execution_context->GetTaskRunner(TaskType::kMediaElementEvent);
+
+  mojo::Remote<media::mojom::blink::MediaMetricsProvider> metrics_provider;
+  execution_context->GetBrowserInterfaceBroker().GetInterface(
+      metrics_provider.BindNewPipeAndPassReceiver(task_runner));
+
+  if (!metrics_provider)
+    return false;
+
+  if (GetLearningBadWindowThreshold() != -1.0) {
+    DCHECK_GE(GetLearningBadWindowThreshold(), 0);
+    metrics_provider->AcquireLearningTaskController(
+        media::learning::tasknames::kConsecutiveBadWindows,
+        bad_window_predictor_.BindNewPipeAndPassReceiver());
+  }
+
+  if (GetLearningNnrThreshold() != -1.0) {
+    DCHECK_GE(GetLearningNnrThreshold(), 0);
+    metrics_provider->AcquireLearningTaskController(
+        media::learning::tasknames::kConsecutiveNNRs,
+        nnr_predictor_.BindNewPipeAndPassReceiver());
+  }
+
+  return bad_window_predictor_ || nnr_predictor_;
+}
+
+bool MediaCapabilities::EnsurePerfHistoryService(
+    ExecutionContext* execution_context) {
   if (decode_history_service_)
     return true;
 
@@ -686,6 +787,7 @@ bool MediaCapabilities::EnsureService(ExecutionContext* execution_context) {
 
 ScriptPromise MediaCapabilities::GetEmeSupport(
     ScriptState* script_state,
+    media::VideoCodec video_codec,
     media::VideoCodecProfile video_profile,
     const MediaDecodingConfiguration* configuration,
     ExceptionState& exception_state) {
@@ -811,7 +913,8 @@ ScriptPromise MediaCapabilities::GetEmeSupport(
       MakeGarbageCollected<MediaCapabilitiesKeySystemAccessInitializer>(
           script_state, key_system_config->keySystem(), config_vector,
           WTF::Bind(&MediaCapabilities::GetPerfInfo, WrapPersistent(this),
-                    video_profile, WrapPersistent(configuration->video())));
+                    video_codec, video_profile,
+                    WrapPersistent(configuration->video())));
 
   // IMPORTANT: Acquire the promise before potentially synchronously resolving
   // it in the code that follows. Otherwise the promise returned to JS will be
@@ -825,7 +928,8 @@ ScriptPromise MediaCapabilities::GetEmeSupport(
   return promise;
 }
 
-void MediaCapabilities::GetPerfInfo(media::VideoCodecProfile video_profile,
+void MediaCapabilities::GetPerfInfo(media::VideoCodec video_codec,
+                                    media::VideoCodecProfile video_profile,
                                     const VideoConfiguration* video_config,
                                     ScriptPromiseResolver* resolver,
                                     MediaKeySystemAccess* access) {
@@ -849,11 +953,22 @@ void MediaCapabilities::GetPerfInfo(media::VideoCodecProfile video_profile,
     use_hw_secure_codecs = access->UseHardwareSecureCodecs();
   }
 
-  if (!EnsureService(execution_context)) {
-    resolver->Resolve(WrapPersistent(CreateDecodingInfoWith(false)));
+  if (!EnsurePerfHistoryService(execution_context)) {
+    resolver->Resolve(WrapPersistent(CreateDecodingInfoWith(true)));
     return;
   }
 
+  const int callback_id = CreateCallbackId();
+  pending_cb_map_.insert(
+      callback_id,
+      MakeGarbageCollected<MediaCapabilities::PendingCallbackState>(resolver,
+                                                                    access));
+
+  if (base::FeatureList::IsEnabled(media::kMediaLearningSmoothnessExperiment)) {
+    GetPerfInfo_ML(execution_context, callback_id, video_codec, video_profile,
+                   video_config->width(), video_config->framerate());
+  }
+
   media::mojom::blink::PredictionFeaturesPtr features =
       media::mojom::blink::PredictionFeatures::New(
           static_cast<media::mojom::blink::VideoCodecProfile>(video_profile),
@@ -861,28 +976,153 @@ void MediaCapabilities::GetPerfInfo(media::VideoCodecProfile video_profile,
           video_config->framerate(), key_system, use_hw_secure_codecs);
 
   decode_history_service_->GetPerfInfo(
-      std::move(features),
-      WTF::Bind(&MediaCapabilities::OnPerfInfo, WrapPersistent(this),
-                WrapPersistent(resolver), WrapPersistent(access)));
+      std::move(features), WTF::Bind(&MediaCapabilities::OnPerfHistoryInfo,
+                                     WrapPersistent(this), callback_id));
 }
 
-void MediaCapabilities::OnPerfInfo(ScriptPromiseResolver* resolver,
-                                   MediaKeySystemAccess* access,
-                                   bool is_smooth,
-                                   bool is_power_efficient) {
-  if (!resolver->GetExecutionContext() ||
-      resolver->GetExecutionContext()->IsContextDestroyed()) {
+void MediaCapabilities::GetPerfInfo_ML(ExecutionContext* execution_context,
+                                       int callback_id,
+                                       media::VideoCodec video_codec,
+                                       media::VideoCodecProfile video_profile,
+                                       int width,
+                                       double framerate) {
+  DCHECK(execution_context && !execution_context->IsContextDestroyed());
+  DCHECK(pending_cb_map_.Contains(callback_id));
+
+  if (!EnsureLearningPredictors(execution_context)) {
+    return;
+  }
+
+  // FRAGILE: Order here MUST match order in
+  // WebMediaPlayerImpl::UpdateSmoothnessHelper().
+  // TODO(chcunningham): refactor into something more robust.
+  Vector<media::learning::FeatureValue> ml_features(
+      {media::learning::FeatureValue(video_codec),
+       media::learning::FeatureValue(video_profile),
+       media::learning::FeatureValue(width),
+       media::learning::FeatureValue(framerate)});
+
+  if (bad_window_predictor_) {
+    bad_window_predictor_->PredictDistribution(
+        ml_features, WTF::Bind(&MediaCapabilities::OnBadWindowPrediction,
+                               WrapPersistent(this), callback_id));
+  }
+
+  if (nnr_predictor_) {
+    nnr_predictor_->PredictDistribution(
+        ml_features, WTF::Bind(&MediaCapabilities::OnNnrPrediction,
+                               WrapPersistent(this), callback_id));
+  }
+}
+
+void MediaCapabilities::ResolveCallbackIfReady(int callback_id) {
+  DCHECK(pending_cb_map_.Contains(callback_id));
+  PendingCallbackState* pending_cb = pending_cb_map_.at(callback_id);
+
+  if (!pending_cb->db_is_power_efficient.has_value())
+    return;
+  // Both db_* fields should be set simultaneously by the DB callback.
+  DCHECK(pending_cb->db_is_smooth.has_value());
+
+  if (nnr_predictor_ && !pending_cb->is_nnr_prediction_smooth.has_value())
+    return;
+
+  if (bad_window_predictor_ &&
+      !pending_cb->is_bad_window_prediction_smooth.has_value())
+    return;
+
+  if (!pending_cb->resolver->GetExecutionContext() ||
+      pending_cb->resolver->GetExecutionContext()->IsContextDestroyed()) {
+    // We're too late! Now that all the callbacks have provided state, its safe
+    // to erase the entry in the map.
+    pending_cb_map_.erase(callback_id);
     return;
   }
 
   Persistent<MediaCapabilitiesDecodingInfo> info(
       MediaCapabilitiesDecodingInfo::Create());
   info->setSupported(true);
-  info->setSmooth(is_smooth);
-  info->setPowerEfficient(is_power_efficient);
-  info->setKeySystemAccess(access);
+  info->setKeySystemAccess(pending_cb->key_system_access);
+  info->setPowerEfficient(*pending_cb->db_is_power_efficient);
+
+  // If ML experiment is running: AND available ML signals.
+  if (pending_cb->is_bad_window_prediction_smooth.has_value() ||
+      pending_cb->is_nnr_prediction_smooth.has_value()) {
+    info->setSmooth(
+        pending_cb->is_bad_window_prediction_smooth.value_or(true) &&
+        pending_cb->is_nnr_prediction_smooth.value_or(true));
+  } else {
+    // Use DB when ML experiment not running.
+    info->setSmooth(*pending_cb->db_is_smooth);
+  }
 
-  resolver->Resolve(std::move(info));
+  pending_cb->resolver->Resolve(std::move(info));
+  pending_cb_map_.erase(callback_id);
+}
+
+void MediaCapabilities::OnBadWindowPrediction(
+    int callback_id,
+    const base::Optional<::media::learning::TargetHistogram>& histogram) {
+  DCHECK(pending_cb_map_.Contains(callback_id));
+  PendingCallbackState* pending_cb = pending_cb_map_.at(callback_id);
+
+  std::stringstream histogram_log;
+  if (!histogram) {
+    // No data, so optimistically assume zero bad windows.
+    pending_cb->is_bad_window_prediction_smooth = true;
+    histogram_log << "none";
+  } else {
+    double histogram_average = histogram->Average();
+    pending_cb->is_bad_window_prediction_smooth =
+        histogram_average <= GetLearningBadWindowThreshold();
+    histogram_log << histogram_average;
+  }
+
+  DVLOG(2) << __func__ << " bad_win_avg:" << histogram_log.str()
+           << " smooth_threshold (<=):" << GetLearningBadWindowThreshold();
+
+  ResolveCallbackIfReady(callback_id);
+}
+
+void MediaCapabilities::OnNnrPrediction(
+    int callback_id,
+    const base::Optional<::media::learning::TargetHistogram>& histogram) {
+  DCHECK(pending_cb_map_.Contains(callback_id));
+  PendingCallbackState* pending_cb = pending_cb_map_.at(callback_id);
+
+  std::stringstream histogram_log;
+  if (!histogram) {
+    // No data, so optimistically assume zero NNRs
+    pending_cb->is_nnr_prediction_smooth = true;
+    histogram_log << "none";
+  } else {
+    double histogram_average = histogram->Average();
+    pending_cb->is_nnr_prediction_smooth =
+        histogram_average <= GetLearningNnrThreshold();
+    histogram_log << histogram_average;
+  }
+
+  DVLOG(2) << __func__ << " nnr_avg:" << histogram_log.str()
+           << " smooth_threshold (<=):" << GetLearningNnrThreshold();
+
+  ResolveCallbackIfReady(callback_id);
+}
+
+void MediaCapabilities::OnPerfHistoryInfo(int callback_id,
+                                          bool is_smooth,
+                                          bool is_power_efficient) {
+  DCHECK(pending_cb_map_.Contains(callback_id));
+  PendingCallbackState* pending_cb = pending_cb_map_.at(callback_id);
+
+  pending_cb->db_is_smooth = is_smooth;
+  pending_cb->db_is_power_efficient = is_power_efficient;
+
+  ResolveCallbackIfReady(callback_id);
+}
+
+int MediaCapabilities::CreateCallbackId() {
+  ++last_callback_id_;
+  return last_callback_id_;
 }
 
 }  // namespace blink

third_party/blink/renderer/modules/media_capabilities/media_capabilities.h

@@ -6,12 +6,15 @@
 #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIA_CAPABILITIES_MEDIA_CAPABILITIES_H_
 
 #include "media/base/video_codecs.h"  // for media::VideoCodecProfile
+#include "media/learning/mojo/public/cpp/mojo_learning_task_controller.h"
+#include "media/learning/mojo/public/mojom/learning_task_controller.mojom-blink.h"
 #include "media/mojo/mojom/video_decode_perf_history.mojom-blink.h"
 #include "mojo/public/cpp/bindings/remote.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_configuration.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
 #include "third_party/blink/renderer/platform/heap/visitor.h"
+#include "third_party/blink/renderer/platform/wtf/hash_map.h"
 
 namespace blink {
 
@@ -28,34 +31,108 @@ class MODULES_EXPORT MediaCapabilities final : public ScriptWrappable {
   DEFINE_WRAPPERTYPEINFO();
 
  public:
+  static const char kLearningBadWindowThresholdParamName[];
+  static const char kLearningNnrThresholdParamName[];
+
   MediaCapabilities();
 
+  void Trace(blink::Visitor* visitor) override;
+
   ScriptPromise decodingInfo(ScriptState*,
                              const MediaDecodingConfiguration*,
                              ExceptionState&);
   ScriptPromise encodingInfo(ScriptState*, const MediaEncodingConfiguration*);
 
  private:
-  // Binds to the VideoDecodePerfHistory service. Returns whether it was
+  // Stores pending callback state from and intermediate prediction values while
+  // we wait for all predictions to arrive.
+  class PendingCallbackState : public GarbageCollected<PendingCallbackState> {
+   public:
+    PendingCallbackState(ScriptPromiseResolver* resolver,
+                         MediaKeySystemAccess* access);
+    virtual void Trace(blink::Visitor* visitor);
+
+    Member<ScriptPromiseResolver> resolver;
+    Member<MediaKeySystemAccess> key_system_access;
+    base::Optional<bool> is_bad_window_prediction_smooth;
+    base::Optional<bool> is_nnr_prediction_smooth;
+    base::Optional<bool> db_is_smooth;
+    base::Optional<bool> db_is_power_efficient;
+  };
+
+  // Lazily binds remote LearningTaskControllers for ML smoothness predictions
+  // and returns whether binding succeeds. Returns true if it was already bound.
+  bool EnsureLearningPredictors(ExecutionContext*);
+
+  // Lazily binds to the VideoDecodePerfHistory service. Returns whether it was
   // successful. Returns true if it was already bound.
-  bool EnsureService(ExecutionContext*);
+  bool EnsurePerfHistoryService(ExecutionContext*);
 
   ScriptPromise GetEmeSupport(ScriptState*,
+                              media::VideoCodec,
                               media::VideoCodecProfile,
                               const MediaDecodingConfiguration*,
                               ExceptionState&);
-  void GetPerfInfo(media::VideoCodecProfile,
+  // Gets perf info from VideoDecodePerrHistory DB. Will optionally kick off
+  // parallel request to GetPerfInfo_ML() when learning experiment is enabled.
+  void GetPerfInfo(media::VideoCodec,
+                   media::VideoCodecProfile,
                    const VideoConfiguration*,
                    ScriptPromiseResolver*,
                    MediaKeySystemAccess*);
 
-  void OnPerfInfo(ScriptPromiseResolver*,
-                  MediaKeySystemAccess*,
-                  bool is_smooth,
-                  bool is_power_efficient);
+  // Gets ML perf predictions from remote LearingTaskControllers.
+  void GetPerfInfo_ML(ExecutionContext* execution_context,
+                      int callback_id,
+                      media::VideoCodec video_codec,
+                      media::VideoCodecProfile video_profile,
+                      int width,
+                      double framerate);
+
+  // Callback for perf info from the VideoDecodePerfHistory service.
+  void OnPerfHistoryInfo(int callback_id,
+                         bool is_smooth,
+                         bool is_power_efficient);
+
+  // Callback for predictions from |bad_window_predictor_|.
+  void OnBadWindowPrediction(
+      int callback_id,
+      const base::Optional<::media::learning::TargetHistogram>& histogram);
+
+  // Callback for predictions from |nnr_predictor_|.
+  void OnNnrPrediction(
+      int callback_id,
+      const base::Optional<::media::learning::TargetHistogram>& histogram);
+
+  // Resolves the callback with associated |callback_id| and removes it from the
+  // |pending_callback_map_|.
+  void ResolveCallbackIfReady(int callback_id);
+
+  // Creates a new (incremented) callback ID from |last_callback_id_| for
+  // mapping in |pending_cb_map_|.
+  int CreateCallbackId();
 
   mojo::Remote<media::mojom::blink::VideoDecodePerfHistory>
       decode_history_service_;
+
+  // Connection to a browser-process LearningTaskController for predicting the
+  // number of consecutive "bad" dropped frame windows during a playback. See
+  // media::SmoothnessHelper.
+  mojo::Remote<media::learning::mojom::blink::LearningTaskController>
+      bad_window_predictor_;
+
+  // Connects to a browser-process LearningTaskController for predicting the
+  // number of consecutive non-network re-buffers (NNRs). See
+  // media::SmoothnessHelper.
+  mojo::Remote<media::learning::mojom::blink::LearningTaskController>
+      nnr_predictor_;
+
+  // Holds the last key for callbacks in the map below. Incremented for each
+  // usage.
+  int last_callback_id_ = 0;
+
+  // Maps a callback ID to state for pending callbacks.
+  HeapHashMap<int, Member<PendingCallbackState>> pending_cb_map_;
 };
 
 }  // namespace blink