Add MediaLog to WebCodecs

This CL adds the necessary plumbing to use MediaLog from WebCodecs.

Future CLs will use MediaLogs to improve the logging reported in the
chrome://media-internals tab and the DevTools media tab.

Bug: 1139089
Change-Id: I7730fb388c354a4bdd07d14e9768a39e298e5da7
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2465583
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: danakj <danakj@chromium.org>
Reviewed-by: David Bokan <bokan@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#819932}

content/renderer/media/batching_media_log.cc

@@ -51,11 +51,9 @@ constexpr char duration_changed_message[] = "kDurationChanged";
 namespace content {
 
 BatchingMediaLog::BatchingMediaLog(
-    const GURL& security_origin,
     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
     std::vector<std::unique_ptr<EventHandler>> event_handlers)
-    : security_origin_(security_origin),
-      task_runner_(std::move(task_runner)),
+    : task_runner_(std::move(task_runner)),
       event_handlers_(std::move(event_handlers)),
       tick_clock_(base::DefaultTickClock::GetInstance()),
       last_ipc_send_time_(tick_clock_->NowTicks()),

content/renderer/media/batching_media_log.h

@@ -38,8 +38,7 @@ class CONTENT_EXPORT BatchingMediaLog : public media::MediaLog {
     virtual void OnWebMediaPlayerDestroyed() = 0;
   };
 
-  BatchingMediaLog(const GURL& security_origin,
-                   scoped_refptr<base::SingleThreadTaskRunner> task_runner,
+  BatchingMediaLog(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
                    std::vector<std::unique_ptr<EventHandler>> impl);
   ~BatchingMediaLog() override;
 
@@ -60,9 +59,6 @@ class CONTENT_EXPORT BatchingMediaLog : public media::MediaLog {
 
   std::string MediaEventToMessageString(const media::MediaLogRecord& event);
 
-  // Security origin of the current frame.
-  const GURL security_origin_;
-
   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 
   // impl for sending queued events.

content/renderer/media/batching_media_log_unittest.cc

@@ -40,9 +40,7 @@ class BatchingMediaLogTest : public testing::Test {
  public:
   BatchingMediaLogTest()
       : task_runner_(new base::TestMockTimeTaskRunner()),
-        log_(GURL("http://foo.com"),
-             task_runner_,
-             TestEventHandler::Create(this)) {
+        log_(task_runner_, TestEventHandler::Create(this)) {
     log_.SetTickClockForTesting(&tick_clock_);
   }
 

content/renderer/media/media_factory.cc

@@ -358,12 +358,10 @@ blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
     viz::FrameSinkId parent_frame_sink_id,
     const cc::LayerTreeSettings& settings) {
   blink::WebLocalFrame* web_frame = render_frame_->GetWebFrame();
-  blink::WebSecurityOrigin security_origin =
-      render_frame_->GetWebFrame()->GetSecurityOrigin();
   if (source.IsMediaStream()) {
-    return CreateWebMediaPlayerForMediaStream(
-        client, inspector_context, sink_id, security_origin, web_frame,
-        parent_frame_sink_id, settings);
+    return CreateWebMediaPlayerForMediaStream(client, inspector_context,
+                                              sink_id, web_frame,
+                                              parent_frame_sink_id, settings);
   }
 
   // If |source| was not a MediaStream, it must be a URL.
@@ -408,7 +406,6 @@ blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
   // This must be created for every new WebMediaPlayer, each instance generates
   // a new player id which is used to collate logs on the browser side.
   auto media_log = std::make_unique<BatchingMediaLog>(
-      url::Origin(security_origin).GetURL(),
       render_frame_->GetTaskRunner(blink::TaskType::kInternalMedia),
       std::move(handlers));
 
@@ -685,7 +682,6 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
     blink::WebMediaPlayerClient* client,
     blink::MediaInspectorContext* inspector_context,
     const blink::WebString& sink_id,
-    const blink::WebSecurityOrigin& security_origin,
     blink::WebLocalFrame* frame,
     viz::FrameSinkId parent_frame_sink_id,
     const cc::LayerTreeSettings& settings) {
@@ -702,7 +698,6 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
   // This must be created for every new WebMediaPlayer, each instance generates
   // a new player id which is used to collate logs on the browser side.
   auto media_log = std::make_unique<BatchingMediaLog>(
-      url::Origin(security_origin).GetURL(),
       render_frame_->GetTaskRunner(blink::TaskType::kInternalMedia),
       std::move(handlers));
 

content/renderer/media/media_factory.h

@@ -21,7 +21,6 @@
 #include "media/mojo/mojom/interface_factory.mojom.h"
 #include "mojo/public/cpp/bindings/remote.h"
 #include "third_party/blink/public/platform/web_media_player_source.h"
-#include "third_party/blink/public/platform/web_security_origin.h"
 #include "third_party/blink/public/platform/web_set_sink_id_callbacks.h"
 #include "third_party/blink/public/platform/web_string.h"
 #include "third_party/blink/public/web/web_media_inspector.h"
@@ -129,7 +128,6 @@ class MediaFactory {
       blink::WebMediaPlayerClient* client,
       blink::MediaInspectorContext* inspector_context,
       const blink::WebString& sink_id,
-      const blink::WebSecurityOrigin& security_origin,
       blink::WebLocalFrame* frame,
       viz::FrameSinkId parent_frame_sink_id,
       const cc::LayerTreeSettings& settings);

content/renderer/renderer_blink_platform_impl.cc

@@ -45,6 +45,9 @@
 #include "content/renderer/loader/resource_dispatcher.h"
 #include "content/renderer/loader/web_url_loader_impl.h"
 #include "content/renderer/media/audio_decoder.h"
+#include "content/renderer/media/batching_media_log.h"
+#include "content/renderer/media/inspector_media_event_handler.h"
+#include "content/renderer/media/render_media_event_handler.h"
 #include "content/renderer/media/renderer_webaudiodevice_impl.h"
 #include "content/renderer/render_frame_impl.h"
 #include "content/renderer/render_thread_impl.h"
@@ -91,6 +94,7 @@
 #include "third_party/blink/public/platform/web_vector.h"
 #include "third_party/blink/public/web/modules/media/audio/web_audio_device_factory.h"
 #include "third_party/blink/public/web/web_local_frame.h"
+#include "third_party/blink/public/web/web_media_inspector.h"
 #include "third_party/sqlite/sqlite3.h"
 #include "ui/base/ui_base_switches.h"
 #include "ui/gl/buildflags.h"
@@ -918,6 +922,27 @@ void RendererBlinkPlatformImpl::RecordMetricsForBackgroundedRendererPurge() {
   render_thread->RecordMetricsForBackgroundedRendererPurge();
 }
 
+//------------------------------------------------------------------------------
+
+// The returned BatchingMediaLog can be used on any thread, but must be
+// destroyed on |owner_task_runner|. The aggregated MediaLogRecords will be
+// sent back to the Browser via Mojo objects bound to |owner_task_runner|.
+std::unique_ptr<media::MediaLog> RendererBlinkPlatformImpl::GetMediaLog(
+    blink::MediaInspectorContext* inspector_context,
+    scoped_refptr<base::SingleThreadTaskRunner> owner_task_runner) {
+  std::vector<std::unique_ptr<BatchingMediaLog::EventHandler>> handlers;
+
+  // For chrome://media-internals.
+  handlers.push_back(std::make_unique<RenderMediaEventHandler>());
+
+  // For devtools' media tab.
+  handlers.push_back(
+      std::make_unique<InspectorMediaEventHandler>(inspector_context));
+
+  return std::make_unique<BatchingMediaLog>(owner_task_runner,
+                                            std::move(handlers));
+}
+
 //------------------------------------------------------------------------------
 media::GpuVideoAcceleratorFactories*
 RendererBlinkPlatformImpl::GetGpuFactories() {

content/renderer/renderer_blink_platform_impl.h

@@ -213,6 +213,9 @@ class CONTENT_EXPORT RendererBlinkPlatformImpl : public BlinkPlatformImpl {
       override;
   std::unique_ptr<blink::WebURLLoaderFactory> WrapSharedURLLoaderFactory(
       scoped_refptr<network::SharedURLLoaderFactory> factory) override;
+  std::unique_ptr<media::MediaLog> GetMediaLog(
+      blink::MediaInspectorContext* inspector_context,
+      scoped_refptr<base::SingleThreadTaskRunner> owner_task_runner) override;
 
   media::GpuVideoAcceleratorFactories* GetGpuFactories() override;
   void SetRenderingColorSpace(const gfx::ColorSpace& color_space) override;

third_party/blink/public/platform/DEPS

@@ -28,6 +28,7 @@ include_rules = [
     "+media/base/audio_capturer_source.h",
     "+media/base/audio_renderer_sink.h",
     "+media/base/eme_constants.h",
+    "+media/base/media_log.h",
     "+media/base/video_frame_metadata.h",
     "+media/base/video_transformation.h",
     "+mojo/public",

third_party/blink/public/platform/platform.h

@@ -43,6 +43,7 @@
 #include "components/viz/common/surfaces/frame_sink_id.h"
 #include "media/base/audio_capturer_source.h"
 #include "media/base/audio_renderer_sink.h"
+#include "media/base/media_log.h"
 #include "mojo/public/cpp/base/big_buffer.h"
 #include "mojo/public/cpp/bindings/generic_pending_receiver.h"
 #include "mojo/public/cpp/system/message_pipe.h"
@@ -108,6 +109,7 @@ class RasterContextProvider;
 namespace blink {
 
 class BrowserInterfaceBrokerProxy;
+class MediaInspectorContext;
 class ThreadSafeBrowserInterfaceBrokerProxy;
 class Thread;
 struct ThreadCreationParams;
@@ -707,6 +709,23 @@ class BLINK_PLATFORM_EXPORT Platform {
     return nullptr;
   }
 
+  // Media Log -----------------------------------------------------------
+
+  // MediaLog is used by WebCodecs to report events and errors up to the
+  // chrome://media-internals page and the DevTools media tab.
+  // |owner_task_runner| must be bound to the main thead or the worker thread
+  // on which WebCodecs will using the MediaLog. It is safe to add logs to
+  // MediaLog from any thread, but it must be destroyed on |owner_task_runner|.
+  // MediaLog owners should destroy the MediaLog if the ExecutionContext is
+  // destroyed, since |inspector_context| may no longer be valid at that point.
+  // Note: |inspector_context| is only used on |owner_task_runner|, so
+  // destroying the MediaLog on |owner_task_runner| should avoid races.
+  virtual std::unique_ptr<media::MediaLog> GetMediaLog(
+      MediaInspectorContext* inspector_context,
+      scoped_refptr<base::SingleThreadTaskRunner> owner_task_runner) {
+    return nullptr;
+  }
+
   // GpuVideoAcceleratorFactories --------------------------------------
 
   virtual media::GpuVideoAcceleratorFactories* GetGpuFactories() {

third_party/blink/renderer/core/inspector/inspector_media_agent.cc

@@ -7,6 +7,7 @@
 
 #include <utility>
 
+#include "third_party/blink/renderer/core/frame/local_dom_window.h"
 #include "third_party/blink/renderer/core/frame/local_frame.h"
 #include "third_party/blink/renderer/core/page/page.h"
 
@@ -93,7 +94,8 @@ void InspectorMediaAgent::Restore() {
 
 void InspectorMediaAgent::RegisterAgent() {
   instrumenting_agents_->AddInspectorMediaAgent(this);
-  auto* cache = MediaInspectorContextImpl::From(*local_frame_->DomWindow());
+  auto* cache = MediaInspectorContextImpl::From(
+      *local_frame_->DomWindow()->GetExecutionContext());
   Vector<WebString> players = cache->AllPlayerIds();
   PlayersCreated(players);
   for (const auto& player_id : players) {

third_party/blink/renderer/core/inspector/inspector_media_context_impl.cc

@@ -8,7 +8,7 @@
 #include <utility>
 
 #include "base/unguessable_token.h"
-#include "third_party/blink/renderer/core/frame/local_dom_window.h"
+#include "third_party/blink/renderer/core/execution_context/execution_context.h"
 #include "third_party/blink/renderer/core/probe/core_probes.h"
 
 namespace blink {
@@ -19,18 +19,21 @@ const char MediaInspectorContextImpl::kSupplementName[] =
 
 // static
 MediaInspectorContextImpl* MediaInspectorContextImpl::From(
-    LocalDOMWindow& window) {
-  auto* context =
-      Supplement<LocalDOMWindow>::From<MediaInspectorContextImpl>(window);
+    ExecutionContext& execution_context) {
+  auto* context = Supplement<ExecutionContext>::From<MediaInspectorContextImpl>(
+      execution_context);
   if (!context) {
-    context = MakeGarbageCollected<MediaInspectorContextImpl>(window);
-    Supplement<LocalDOMWindow>::ProvideTo(window, context);
+    context =
+        MakeGarbageCollected<MediaInspectorContextImpl>(execution_context);
+    Supplement<ExecutionContext>::ProvideTo(execution_context, context);
   }
   return context;
 }
 
-MediaInspectorContextImpl::MediaInspectorContextImpl(LocalDOMWindow& frame)
-    : Supplement<LocalDOMWindow>(frame) {}
+MediaInspectorContextImpl::MediaInspectorContextImpl(ExecutionContext& context)
+    : Supplement<ExecutionContext>(context) {
+  DCHECK(context.IsWindow() || context.IsWorkerGlobalScope());
+}
 
 // Local to cc file for converting
 template <typename T, typename Iterable>
@@ -42,7 +45,7 @@ static Vector<T> Iter2Vector(const Iterable& iterable) {
 
 // Garbage collection method.
 void MediaInspectorContextImpl::Trace(Visitor* visitor) const {
-  Supplement<LocalDOMWindow>::Trace(visitor);
+  Supplement<ExecutionContext>::Trace(visitor);
   visitor->Trace(players_);
 }
 

third_party/blink/renderer/core/inspector/inspector_media_context_impl.h

@@ -13,7 +13,7 @@
 
 namespace blink {
 
-class LocalDOMWindow;
+class ExecutionContext;
 
 struct MediaPlayer final : public GarbageCollected<MediaPlayer> {
   void Trace(Visitor*) const {}
@@ -27,14 +27,14 @@ struct MediaPlayer final : public GarbageCollected<MediaPlayer> {
 
 class CORE_EXPORT MediaInspectorContextImpl final
     : public GarbageCollected<MediaInspectorContextImpl>,
-      public Supplement<LocalDOMWindow>,
+      public Supplement<ExecutionContext>,
       public MediaInspectorContext {
  public:
   static const char kSupplementName[];
 
-  static MediaInspectorContextImpl* From(LocalDOMWindow&);
+  static MediaInspectorContextImpl* From(ExecutionContext&);
 
-  explicit MediaInspectorContextImpl(LocalDOMWindow&);
+  explicit MediaInspectorContextImpl(ExecutionContext&);
 
   // MediaInspectorContext methods.
   WebString CreatePlayer() override;

third_party/blink/renderer/modules/webcodecs/decoder_template.cc

@@ -13,6 +13,7 @@
 #include "base/time/time.h"
 #include "media/base/media_util.h"
 #include "media/media_buildflags.h"
+#include "third_party/blink/public/platform/platform.h"
 #include "third_party/blink/public/platform/task_type.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_audio_decoder_init.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_audio_chunk.h"
@@ -22,6 +23,7 @@
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_init.h"
 #include "third_party/blink/renderer/core/dom/dom_exception.h"
 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
+#include "third_party/blink/renderer/core/inspector/inspector_media_context_impl.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/modules/webcodecs/audio_decoder.h"
 #include "third_party/blink/renderer/modules/webcodecs/audio_frame.h"
@@ -43,10 +45,28 @@ template <typename Traits>
 DecoderTemplate<Traits>::DecoderTemplate(ScriptState* script_state,
                                          const InitType* init,
                                          ExceptionState& exception_state)
-    : script_state_(script_state), state_(V8CodecState::Enum::kUnconfigured) {
+    : ExecutionContextLifecycleObserver(ExecutionContext::From(script_state)),
+      script_state_(script_state),
+      state_(V8CodecState::Enum::kUnconfigured) {
   DVLOG(1) << __func__;
   DCHECK(init->hasOutput());
   DCHECK(init->hasError());
+
+  ExecutionContext* context = GetExecutionContext();
+
+  DCHECK(context);
+
+  parent_media_log_ = Platform::Current()->GetMediaLog(
+      MediaInspectorContextImpl::From(*context),
+      context->GetTaskRunner(TaskType::kInternalMedia));
+
+  if (!parent_media_log_)
+    parent_media_log_ = std::make_unique<media::NullMediaLog>();
+
+  // This allows us to destroy |parent_media_log_| and stop logging,
+  // without causing problems to |media_log_| users.
+  media_log_ = parent_media_log_->Clone();
+
   output_cb_ = init->output();
   error_cb_ = init->error();
 }
@@ -204,7 +224,6 @@ bool DecoderTemplate<Traits>::ProcessConfigureRequest(Request* request) {
   // until there is a decode request.
 
   if (!decoder_) {
-    media_log_ = std::make_unique<media::NullMediaLog>();
     decoder_ = Traits::CreateDecoder(*ExecutionContext::From(script_state_),
                                      media_log_.get());
     if (!decoder_) {
@@ -491,6 +510,12 @@ void DecoderTemplate<Traits>::Trace(Visitor* visitor) const {
   visitor->Trace(pending_request_);
   visitor->Trace(pending_decodes_);
   ScriptWrappable::Trace(visitor);
+  ExecutionContextLifecycleObserver::Trace(visitor);
+}
+
+template <typename Traits>
+void DecoderTemplate<Traits>::ContextDestroyed() {
+  parent_media_log_ = nullptr;
 }
 
 template <typename Traits>

third_party/blink/renderer/modules/webcodecs/decoder_template.h

@@ -18,6 +18,7 @@
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/modules/webcodecs/codec_config_eval.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
+#include "third_party/blink/renderer/platform/context_lifecycle_observer.h"
 #include "third_party/blink/renderer/platform/heap/garbage_collected.h"
 #include "third_party/blink/renderer/platform/heap/heap.h"
 #include "third_party/blink/renderer/platform/heap/heap_allocator.h"
@@ -26,7 +27,9 @@
 namespace blink {
 
 template <typename Traits>
-class MODULES_EXPORT DecoderTemplate : public ScriptWrappable {
+class MODULES_EXPORT DecoderTemplate
+    : public ScriptWrappable,
+      public ExecutionContextLifecycleObserver {
  public:
   typedef typename Traits::ConfigType ConfigType;
   typedef typename Traits::MediaConfigType MediaConfigType;
@@ -48,6 +51,9 @@ class MODULES_EXPORT DecoderTemplate : public ScriptWrappable {
   void close(ExceptionState&);
   String state() const { return state_; }
 
+  // ExecutionContextLifecycleObserver override.
+  void ContextDestroyed() override;
+
   // GarbageCollected override.
   void Trace(Visitor*) const override;
 
@@ -123,6 +129,14 @@ class MODULES_EXPORT DecoderTemplate : public ScriptWrappable {
   // Could be a configure, flush, or reset. Decodes go in |pending_decodes_|.
   Member<Request> pending_request_;
 
+  // |parent_media_log_| must be destroyed if ever the ExecutionContext is
+  // destroyed, since the blink::MediaInspectorContext* pointer given to
+  // InspectorMediaEventHandler might no longer be valid.
+  // |parent_media_log_| should not be used directly. Use |media_log_| instead.
+  std::unique_ptr<media::MediaLog> parent_media_log_;
+
+  // We might destroy |parent_media_log_| at any point, so keep a clone which
+  // can be safely accessed, and whose raw pointer can be given to |decoder_|.
   std::unique_ptr<media::MediaLog> media_log_;
 
   // TODO(sandersd): Store the last config, flush, and reset so that