MediaCapabilities: setup machine learning predictions

Adds predictors for non-network rebuffer (NNR) and consecutive "bad
windows" with high counts of dropped frames. The predictions will be
compared against configurable thresholds for determining
smooth=true/false for a given MediaDecodingConfiguration.

Bug: 1049339

Change-Id: Ia80a9cdc887c070f8156b58a0da64d02318fda40
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2021950
Commit-Queue: Chrome Cunningham <chcunningham@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Mounir Lamouri <mlamouri@chromium.org>
Reviewed-by: Frank Liberato <liberato@chromium.org>
Reviewed-by: Ken Rockot <rockot@google.com>
Cr-Commit-Position: refs/heads/master@{#749364}

media/learning/common/BUILD.gn

@@ -10,6 +10,7 @@ component("common") {
     "//media/learning/impl:test_support",
     "//media/learning/mojo/public/cpp:*",
     "//media/learning/mojo/public/mojom:mojom",
+    "//media/learning/mojo/public/mojom:mojom_blink",
     "//media/learning/mojo:*",
     "//media/learning/common:unit_tests",
 

media/learning/common/target_histogram.h

@@ -23,6 +23,13 @@ namespace mojom {
 class TargetHistogramDataView;
 }
 
+// Intermediate type for mojom struct traits translation.
+// See learning_types.mojom.
+struct COMPONENT_EXPORT(LEARNING_COMMON) TargetHistogramPair {
+  TargetValue target_value;
+  double count;
+};
+
 // Histogram of target values that allows fractional counts.
 class COMPONENT_EXPORT(LEARNING_COMMON) TargetHistogram {
  public:

media/learning/mojo/public/cpp/BUILD.gn

@@ -23,6 +23,26 @@ component("cpp") {
   ]
 }
 
+# Normally typemap traits sources should be build directly into mojom targets
+# via the typemap file. This target is for typemapped mojo_base types whose
+# traits are shared between chromium and blink variants.
+component("shared_typemap_traits") {
+  output_name = "media_learning_shared_typemap_traits"
+
+  defines = [ "IS_MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS_IMPL" ]
+
+  sources = [
+    "//media/learning/mojo/public/cpp/learning_mojom_traits.cc",
+    "//media/learning/mojo/public/cpp/learning_mojom_traits.h",
+  ]
+
+  deps = [
+    "//base",
+    "//media/learning/common",
+    "//media/learning/mojo/public/mojom:mojom_shared",
+  ]
+}
+
 source_set("unit_tests") {
   testonly = true
 

media/learning/mojo/public/cpp/learning_mojom_traits.cc

@@ -50,14 +50,37 @@ bool StructTraits<media::learning::mojom::ObservationCompletionDataView,
   return true;
 }
 
+// static
+bool StructTraits<media::learning::mojom::TargetHistogramPairDataView,
+                  media::learning::TargetHistogramPair>::
+    Read(media::learning::mojom::TargetHistogramPairDataView data,
+         media::learning::TargetHistogramPair* out_pair) {
+  if (!data.ReadTargetValue(&out_pair->target_value))
+    return false;
+  out_pair->count = data.count();
+  return true;
+}
+
 // static
 bool StructTraits<media::learning::mojom::TargetHistogramDataView,
                   media::learning::TargetHistogram>::
     Read(media::learning::mojom::TargetHistogramDataView data,
          media::learning::TargetHistogram* out_target_histogram) {
-  if (!data.ReadCounts(&out_target_histogram->counts_))
+  ArrayDataView<media::learning::mojom::TargetHistogramPairDataView> pairs;
+  data.GetPairsDataView(&pairs);
+  if (pairs.is_null())
     return false;
 
+  for (size_t i = 0; i < pairs.size(); ++i) {
+    media::learning::mojom::TargetHistogramPairDataView pair_data;
+    pairs.GetDataView(i, &pair_data);
+    media::learning::TargetValue value;
+    if (!pair_data.ReadTargetValue(&value))
+      return false;
+
+    out_target_histogram->counts_.emplace(std::move(value), pair_data.count());
+  }
+
   return true;
 }
 

media/learning/mojo/public/cpp/learning_mojom_traits.h

@@ -7,16 +7,18 @@
 
 #include <vector>
 
+#include "base/component_export.h"
 #include "media/learning/common/learning_task_controller.h"
 #include "media/learning/common/value.h"
-#include "media/learning/mojo/public/mojom/learning_types.mojom.h"
+#include "media/learning/mojo/public/mojom/learning_types.mojom-shared.h"
 #include "mojo/public/cpp/bindings/struct_traits.h"
 
 namespace mojo {
 
 template <>
-struct StructTraits<media::learning::mojom::LabelledExampleDataView,
-                    media::learning::LabelledExample> {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::LabelledExampleDataView,
+                 media::learning::LabelledExample> {
   static const std::vector<media::learning::FeatureValue>& features(
       const media::learning::LabelledExample& e) {
     return e.features;
@@ -31,9 +33,10 @@ struct StructTraits<media::learning::mojom::LabelledExampleDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::FeatureValueDataView,
-                    media::learning::FeatureValue> {
-  static int64_t value(const media::learning::FeatureValue& e) {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::FeatureValueDataView,
+                 media::learning::FeatureValue> {
+  static double value(const media::learning::FeatureValue& e) {
     return e.value();
   }
   static bool Read(media::learning::mojom::FeatureValueDataView data,
@@ -41,9 +44,10 @@ struct StructTraits<media::learning::mojom::FeatureValueDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::TargetValueDataView,
-                    media::learning::TargetValue> {
-  static int64_t value(const media::learning::TargetValue& e) {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetValueDataView,
+                 media::learning::TargetValue> {
+  static double value(const media::learning::TargetValue& e) {
     return e.value();
   }
   static bool Read(media::learning::mojom::TargetValueDataView data,
@@ -51,8 +55,9 @@ struct StructTraits<media::learning::mojom::TargetValueDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::ObservationCompletionDataView,
-                    media::learning::ObservationCompletion> {
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::ObservationCompletionDataView,
+                 media::learning::ObservationCompletion> {
   static media::learning::TargetValue target_value(
       const media::learning::ObservationCompletion& e) {
     return e.target_value;
@@ -67,11 +72,32 @@ struct StructTraits<media::learning::mojom::ObservationCompletionDataView,
 };
 
 template <>
-struct StructTraits<media::learning::mojom::TargetHistogramDataView,
-                    media::learning::TargetHistogram> {
-  static media::learning::TargetHistogram::CountMap counts(
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetHistogramPairDataView,
+                 media::learning::TargetHistogramPair> {
+  static media::learning::TargetValue target_value(
+      const media::learning::TargetHistogramPair& e) {
+    return e.target_value;
+  }
+  static double count(const media::learning::TargetHistogramPair& e) {
+    return e.count;
+  }
+  static bool Read(media::learning::mojom::TargetHistogramPairDataView data,
+                   media::learning::TargetHistogramPair* out_pair);
+};
+
+template <>
+struct COMPONENT_EXPORT(MEDIA_LEARNING_SHARED_TYPEMAP_TRAITS)
+    StructTraits<media::learning::mojom::TargetHistogramDataView,
+                 media::learning::TargetHistogram> {
+  static std::vector<media::learning::TargetHistogramPair> pairs(
       const media::learning::TargetHistogram& e) {
-    return e.counts();
+    std::vector<media::learning::TargetHistogramPair> pairs;
+    for (auto const& entry : e.counts_) {
+      pairs.push_back({entry.first, entry.second});
+    }
+
+    return pairs;
   }
 
   static bool Read(media::learning::mojom::TargetHistogramDataView data,

media/learning/mojo/public/mojom/BUILD.gn

@@ -13,4 +13,8 @@ mojom("mojom") {
   ]
 
   public_deps = [ "//mojo/public/mojom/base" ]
+
+  export_class_attribute_blink = "BLINK_PLATFORM_EXPORT"
+  export_define_blink = "BLINK_PLATFORM_IMPLEMENTATION=1"
+  export_header_blink = "third_party/blink/public/platform/web_common.h"
 }

media/learning/mojo/public/mojom/learning_types.mojom

@@ -6,12 +6,12 @@ module media.learning.mojom;
 
 // learning::FeatureValue (common/value.h)
 struct FeatureValue {
-  int64 value;
+  double value;
 };
 
 // learning::TargetValue (common/value.h)
 struct TargetValue {
-  int64 value;
+  double value;
 };
 
 // learning::LabelledExample (common/training_example.h)
@@ -26,7 +26,19 @@ struct ObservationCompletion {
   uint64 weight = 1;
 };
 
+// Hack for TargetHistogram. Would ideally be a map<TargetValue, double>, but
+// this causes pain in the translation to WTF::HashMap. HashMap requires
+// reservations for "deleted" and "empty" sentinel values. This is especially
+// undesirable in our case because TargetValue (the map key) is designed to be
+// completely generic, supporting any possible value for double. We instead
+// use a list of pairs (key, value) to avoid having to reserve any values as
+// "empty" or "deleted".
+struct TargetHistogramPair {
+  TargetValue target_value;
+  double count;
+};
+
 // learning::TargetHistogram (common/target_histogram.h)
 struct TargetHistogram {
-  map<TargetValue, double> counts;
+  array<TargetHistogramPair> pairs;
 };

media/learning/mojo/public/mojom/learning_types.typemap

@@ -5,15 +5,15 @@ public_headers = [
   "//media/learning/common/value.h",
 ]
 traits_headers = [ "//media/learning/mojo/public/cpp/learning_mojom_traits.h" ]
-sources = [
-  "//media/learning/mojo/public/cpp/learning_mojom_traits.cc",
-  "//media/learning/mojo/public/cpp/learning_mojom_traits.h",
+public_deps = [
+  "//media/learning/common",
+  "//media/learning/mojo/public/cpp:shared_typemap_traits",
 ]
-public_deps = [ "//media/learning/common" ]
 type_mappings = [
   "media.learning.mojom.LabelledExample=::media::learning::LabelledExample",
   "media.learning.mojom.FeatureValue=::media::learning::FeatureValue",
   "media.learning.mojom.TargetValue=::media::learning::TargetValue",
   "media.learning.mojom.ObservationCompletion=::media::learning::ObservationCompletion",
+  "media.learning.mojom.TargetHistogramPair=::media::learning::TargetHistogramPair",
   "media.learning.mojom.TargetHistogram=::media::learning::TargetHistogram",
 ]

mojo/public/tools/bindings/blink_bindings_configuration.gni

@@ -10,6 +10,7 @@ _typemap_imports = [
   "//cc/typemaps.gni",
   "//device/gamepad/public/cpp/typemaps.gni",
   "//device/vr/public/mojom/blink_typemaps.gni",
+  "//media/learning/mojo/public/cpp/typemaps.gni",
   "//mojo/public/cpp/bindings/tests/blink_typemaps.gni",
   "//skia/public/mojom/typemaps.gni",
   "//third_party/blink/renderer/platform/mojo/blink_typemaps.gni",

third_party/blink/renderer/modules/media_capabilities/BUILD.gn

@@ -15,7 +15,11 @@ blink_modules_sources("media_capabilities") {
     "worker_navigator_media_capabilities.cc",
     "worker_navigator_media_capabilities.h",
   ]
-  deps = [ "//third_party/blink/renderer/modules/mediarecorder" ]
+  deps = [
+    "//media",
+    "//media/learning/mojo:impl",
+    "//third_party/blink/renderer/modules/mediarecorder",
+  ]
 }
 
 fuzzable_proto_library("fuzzer_media_configuration_proto") {

third_party/blink/renderer/modules/media_capabilities/DEPS

 include_rules = [
-    "+media/base/mime_util.h",
-    "+media/base/supported_types.h",
-    "+media/base/video_codecs.h",
+    "+media/base",
     "+media/filters/stream_parser_factory.h",
+    "+media/learning/common",
+    "+media/learning/mojo/public/cpp/mojo_learning_task_controller.h",
+    "+media/learning/mojo/public/mojom/learning_task_controller.mojom-blink.h",
     "+media/mojo/mojom/media_types.mojom-blink.h",
     "+media/mojo/mojom/video_decode_perf_history.mojom-blink.h",
+    "+media/mojo/mojom/media_metrics_provider.mojom-blink.h",
     "-third_party/blink/renderer/modules",
     "+third_party/blink/renderer/modules/encryptedmedia",
     "+third_party/blink/renderer/modules/media_capabilities",

third_party/blink/renderer/modules/media_capabilities/media_capabilities.h

@@ -6,12 +6,15 @@
 #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIA_CAPABILITIES_MEDIA_CAPABILITIES_H_
 
 #include "media/base/video_codecs.h"  // for media::VideoCodecProfile
+#include "media/learning/mojo/public/cpp/mojo_learning_task_controller.h"
+#include "media/learning/mojo/public/mojom/learning_task_controller.mojom-blink.h"
 #include "media/mojo/mojom/video_decode_perf_history.mojom-blink.h"
 #include "mojo/public/cpp/bindings/remote.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_configuration.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
 #include "third_party/blink/renderer/platform/heap/visitor.h"
+#include "third_party/blink/renderer/platform/wtf/hash_map.h"
 
 namespace blink {
 
@@ -28,34 +31,108 @@ class MODULES_EXPORT MediaCapabilities final : public ScriptWrappable {
   DEFINE_WRAPPERTYPEINFO();
 
  public:
+  static const char kLearningBadWindowThresholdParamName[];
+  static const char kLearningNnrThresholdParamName[];
+
   MediaCapabilities();
 
+  void Trace(blink::Visitor* visitor) override;
+
   ScriptPromise decodingInfo(ScriptState*,
                              const MediaDecodingConfiguration*,
                              ExceptionState&);
   ScriptPromise encodingInfo(ScriptState*, const MediaEncodingConfiguration*);
 
  private:
-  // Binds to the VideoDecodePerfHistory service. Returns whether it was
+  // Stores pending callback state from and intermediate prediction values while
+  // we wait for all predictions to arrive.
+  class PendingCallbackState : public GarbageCollected<PendingCallbackState> {
+   public:
+    PendingCallbackState(ScriptPromiseResolver* resolver,
+                         MediaKeySystemAccess* access);
+    virtual void Trace(blink::Visitor* visitor);
+
+    Member<ScriptPromiseResolver> resolver;
+    Member<MediaKeySystemAccess> key_system_access;
+    base::Optional<bool> is_bad_window_prediction_smooth;
+    base::Optional<bool> is_nnr_prediction_smooth;
+    base::Optional<bool> db_is_smooth;
+    base::Optional<bool> db_is_power_efficient;
+  };
+
+  // Lazily binds remote LearningTaskControllers for ML smoothness predictions
+  // and returns whether binding succeeds. Returns true if it was already bound.
+  bool EnsureLearningPredictors(ExecutionContext*);
+
+  // Lazily binds to the VideoDecodePerfHistory service. Returns whether it was
   // successful. Returns true if it was already bound.
-  bool EnsureService(ExecutionContext*);
+  bool EnsurePerfHistoryService(ExecutionContext*);
 
   ScriptPromise GetEmeSupport(ScriptState*,
+                              media::VideoCodec,
                               media::VideoCodecProfile,
                               const MediaDecodingConfiguration*,
                               ExceptionState&);
-  void GetPerfInfo(media::VideoCodecProfile,
+  // Gets perf info from VideoDecodePerrHistory DB. Will optionally kick off
+  // parallel request to GetPerfInfo_ML() when learning experiment is enabled.
+  void GetPerfInfo(media::VideoCodec,
+                   media::VideoCodecProfile,
                    const VideoConfiguration*,
                    ScriptPromiseResolver*,
                    MediaKeySystemAccess*);
 
-  void OnPerfInfo(ScriptPromiseResolver*,
-                  MediaKeySystemAccess*,
-                  bool is_smooth,
-                  bool is_power_efficient);
+  // Gets ML perf predictions from remote LearingTaskControllers.
+  void GetPerfInfo_ML(ExecutionContext* execution_context,
+                      int callback_id,
+                      media::VideoCodec video_codec,
+                      media::VideoCodecProfile video_profile,
+                      int width,
+                      double framerate);
+
+  // Callback for perf info from the VideoDecodePerfHistory service.
+  void OnPerfHistoryInfo(int callback_id,
+                         bool is_smooth,
+                         bool is_power_efficient);
+
+  // Callback for predictions from |bad_window_predictor_|.
+  void OnBadWindowPrediction(
+      int callback_id,
+      const base::Optional<::media::learning::TargetHistogram>& histogram);
+
+  // Callback for predictions from |nnr_predictor_|.
+  void OnNnrPrediction(
+      int callback_id,
+      const base::Optional<::media::learning::TargetHistogram>& histogram);
+
+  // Resolves the callback with associated |callback_id| and removes it from the
+  // |pending_callback_map_|.
+  void ResolveCallbackIfReady(int callback_id);
+
+  // Creates a new (incremented) callback ID from |last_callback_id_| for
+  // mapping in |pending_cb_map_|.
+  int CreateCallbackId();
 
   mojo::Remote<media::mojom::blink::VideoDecodePerfHistory>
       decode_history_service_;
+
+  // Connection to a browser-process LearningTaskController for predicting the
+  // number of consecutive "bad" dropped frame windows during a playback. See
+  // media::SmoothnessHelper.
+  mojo::Remote<media::learning::mojom::blink::LearningTaskController>
+      bad_window_predictor_;
+
+  // Connects to a browser-process LearningTaskController for predicting the
+  // number of consecutive non-network re-buffers (NNRs). See
+  // media::SmoothnessHelper.
+  mojo::Remote<media::learning::mojom::blink::LearningTaskController>
+      nnr_predictor_;
+
+  // Holds the last key for callbacks in the map below. Incremented for each
+  // usage.
+  int last_callback_id_ = 0;
+
+  // Maps a callback ID to state for pending callbacks.
+  HeapHashMap<int, Member<PendingCallbackState>> pending_cb_map_;
 };
 
 }  // namespace blink