MSE-in-Workers: Do MSE media element track mgmt via attachments

Instead of manipulating the media element's audio and video tracks
directly from the SourceBuffer, this change moves that into the
MediaSourceAttachmentSupplement interface.

Currently, TrackBase and TrackListBase usage by HTMLMediaElement and
SourceBuffer require ownership and operation by the main thread, and
further require knowledge of the main-thread-owned media element for
use in scheduling events on the main thread. For cross-thread MSE-in-
Workers attachments, to prevent regressing BackgroundVideoOptimization,
the media element will need knowledge of the worker-MSE audio and video
track identifications used by the underlying media pipeline.

This change supports future signalling of the media element to add or
remove such audio and video tracks via new methods in
MediaSourceAttachmentSupplement. Existing same-thread behavior is
maintained by new implementations of these methods in the
SameThreadMediaSourceAttachment. SourceBuffer is updated to understand
which of the methods to use, as until track ownership and operation
in worker context is supported, workarounds in SourceBuffer and the
attachment will be needed.

Later changes will include the cross-thread attachment implementation
of the media-element track addition and removal to actually perform
the signalling.

This change retains the pre-existing experimental AudioVideoTracks
ability of MSE in Chrome for same-thread-attachments, but prevents
(with a CHECK failure, since IDL prohibits alternatives) retrieval by
the app of a SourceBuffer's audioTracks or videoTracks when the MSE is
in a worker context.

BUG=878133

Change-Id: I209ce61915de2e4182807539013606322ece8f67
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2427566Reviewed-by: Will Cassella <cassew@google.com>
Commit-Queue: Matthew Wolenetz <wolenetz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#812984}

third_party/blink/renderer/modules/mediasource/cross_thread_media_source_attachment.cc

@@ -64,6 +64,67 @@ bool CrossThreadMediaSourceAttachment::GetElementError(
   return true;
 }
 
+void CrossThreadMediaSourceAttachment::AddAudioTrackToMediaElement(
+    MediaSourceTracer* /* tracer */,
+    AudioTrack* /* track */) {
+  // TODO(https://crbug.com/878133): Implement this once worker thread can
+  // create tracks.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::AddVideoTrackToMediaElement(
+    MediaSourceTracer* /* tracer */,
+    VideoTrack* /* track */) {
+  // TODO(https://crbug.com/878133): Implement this once worker thread can
+  // create tracks.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::RemoveAudioTracksFromMediaElement(
+    MediaSourceTracer* tracer,
+    Vector<String> audio_ids,
+    bool enqueue_change_event) {
+  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::RemoveVideoTracksFromMediaElement(
+    MediaSourceTracer* tracer,
+    Vector<String> video_ids,
+    bool enqueue_change_event) {
+  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::AddMainThreadAudioTrackToMediaElement(
+    String id,
+    String kind,
+    String label,
+    String language,
+    bool enabled) {
+  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::AddMainThreadVideoTrackToMediaElement(
+    String id,
+    String kind,
+    String label,
+    String language,
+    bool selected) {
+  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
+  NOTIMPLEMENTED();
+}
+
+void CrossThreadMediaSourceAttachment::OnMediaSourceContextDestroyed() {
+  // Called only by the MSE API on worker thread.
+  DCHECK(!IsMainThread());
+
+  DVLOG(3) << __func__ << " this=" << this;
+  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
+  NOTIMPLEMENTED();
+}
+
 void CrossThreadMediaSourceAttachment::Unregister() {
   DVLOG(1) << __func__ << " this=" << this
            << ", IsMainThread=" << IsMainThread();
@@ -178,13 +239,4 @@ void CrossThreadMediaSourceAttachment::OnElementContextDestroyed() {
   NOTIMPLEMENTED();
 }
 
-void CrossThreadMediaSourceAttachment::OnMediaSourceContextDestroyed() {
-  // Called only by the MSE API on worker thread.
-  DCHECK(!IsMainThread());
-
-  DVLOG(3) << __func__ << " this=" << this;
-  // TODO(https://crbug.com/878133): Implement cross-thread behavior for this.
-  NOTIMPLEMENTED();
-}
-
 }  // namespace blink

third_party/blink/renderer/modules/mediasource/cross_thread_media_source_attachment.h

@@ -9,6 +9,10 @@
 
 #include "base/util/type_safety/pass_key.h"
 #include "third_party/blink/public/platform/web_time_range.h"
+#include "third_party/blink/renderer/core/html/track/audio_track.h"
+#include "third_party/blink/renderer/core/html/track/audio_track_list.h"
+#include "third_party/blink/renderer/core/html/track/video_track.h"
+#include "third_party/blink/renderer/core/html/track/video_track_list.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.h"
 #include "third_party/blink/renderer/modules/mediasource/url_media_source.h"
@@ -35,6 +39,26 @@ class CrossThreadMediaSourceAttachment final
   void NotifyDurationChanged(MediaSourceTracer* tracer, double duration) final;
   double GetRecentMediaTime(MediaSourceTracer* tracer) final;
   bool GetElementError(MediaSourceTracer* tracer) final;
+  void AddAudioTrackToMediaElement(MediaSourceTracer* tracer,
+                                   AudioTrack* track) final;
+  void AddVideoTrackToMediaElement(MediaSourceTracer* tracer,
+                                   VideoTrack* track) final;
+  void RemoveAudioTracksFromMediaElement(MediaSourceTracer* tracer,
+                                         Vector<String> audio_ids,
+                                         bool enqueue_change_event) final;
+  void RemoveVideoTracksFromMediaElement(MediaSourceTracer* tracer,
+                                         Vector<String> video_ids,
+                                         bool enqueue_change_event) final;
+  void AddMainThreadAudioTrackToMediaElement(String id,
+                                             String kind,
+                                             String label,
+                                             String language,
+                                             bool enabled) final;
+  void AddMainThreadVideoTrackToMediaElement(String id,
+                                             String kind,
+                                             String label,
+                                             String language,
+                                             bool selected) final;
   void OnMediaSourceContextDestroyed() final;
 
   // MediaSourceAttachment methods called on main thread by media element,

third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.cc

@@ -10,4 +10,26 @@ MediaSourceAttachmentSupplement::MediaSourceAttachmentSupplement() = default;
 
 MediaSourceAttachmentSupplement::~MediaSourceAttachmentSupplement() = default;
 
+void MediaSourceAttachmentSupplement::AddMainThreadAudioTrackToMediaElement(
+    String /* id */,
+    String /* kind */,
+    String /* label */,
+    String /* language */,
+    bool /* enabled */) {
+  // TODO(https::/crbug.com/878133): Remove this once cross-thread
+  // implementation supports creation of worker-thread tracks.
+  NOTIMPLEMENTED();
+}
+
+void MediaSourceAttachmentSupplement::AddMainThreadVideoTrackToMediaElement(
+    String /* id */,
+    String /* kind */,
+    String /* label */,
+    String /* language */,
+    bool /* selected */) {
+  // TODO(https::/crbug.com/878133): Remove this once cross-thread
+  // implementation supports creation of worker-thread tracks.
+  NOTIMPLEMENTED();
+}
+
 }  // namespace blink

third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.h

@@ -7,6 +7,8 @@
 
 #include "third_party/blink/renderer/core/html/media/media_source_attachment.h"
 #include "third_party/blink/renderer/core/html/media/media_source_tracer.h"
+#include "third_party/blink/renderer/core/html/track/audio_track.h"
+#include "third_party/blink/renderer/core/html/track/video_track.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source.h"
 #include "third_party/blink/renderer/platform/heap/handle.h"
 #include "third_party/blink/renderer/platform/wtf/forward.h"
@@ -45,6 +47,38 @@ class MediaSourceAttachmentSupplement : public MediaSourceAttachment {
   // attachment (in a cross-thread implementation).
   virtual bool GetElementError(MediaSourceTracer* tracer) = 0;
 
+  // Add/Remove tracks to/from the media Element's audioTracks() or
+  // videoTracks() list. Note that this is synchronous in
+  // SameThreadMediaSourceAttachment, but the CrossThreadMediaSourceAttachment
+  // does a cross-thread task post and performs the operations on the main
+  // thread to enable correct context ownership of created tracks and correct
+  // context for track list operations.
+  virtual void AddAudioTrackToMediaElement(MediaSourceTracer* tracer,
+                                           AudioTrack* track) = 0;
+  virtual void AddVideoTrackToMediaElement(MediaSourceTracer* tracer,
+                                           VideoTrack* track) = 0;
+  virtual void RemoveAudioTracksFromMediaElement(MediaSourceTracer* tracer,
+                                                 Vector<String> audio_ids,
+                                                 bool enqueue_change_event) = 0;
+  virtual void RemoveVideoTracksFromMediaElement(MediaSourceTracer* tracer,
+                                                 Vector<String> video_ids,
+                                                 bool enqueue_change_event) = 0;
+  // TODO(https://crbug.com/878133): Update the implementations to remove these
+  // short-term cross-thread helpers and instead use the methods, above, once
+  // track creation outside of main thread is supported. These helpers create
+  // the tracks on the main thread from parameters (not from a currently-
+  // uncreatable worker thread track).
+  virtual void AddMainThreadAudioTrackToMediaElement(String id,
+                                                     String kind,
+                                                     String label,
+                                                     String language,
+                                                     bool enabled);
+  virtual void AddMainThreadVideoTrackToMediaElement(String id,
+                                                     String kind,
+                                                     String label,
+                                                     String language,
+                                                     bool selected);
+
   virtual void OnMediaSourceContextDestroyed() = 0;
 
  protected:

third_party/blink/renderer/modules/mediasource/same_thread_media_source_attachment.cc

@@ -95,6 +95,87 @@ bool SameThreadMediaSourceAttachment::GetElementError(
   return current_element_error_state;
 }
 
+void SameThreadMediaSourceAttachment::AddAudioTrackToMediaElement(
+    MediaSourceTracer* tracer,
+    AudioTrack* track) {
+  DVLOG(3) << __func__ << " this=" << this;
+
+  VerifyCalledWhileContextsAliveForDebugging();
+
+  HTMLMediaElement* element = GetMediaElement(tracer);
+  element->audioTracks().Add(track);
+}
+
+void SameThreadMediaSourceAttachment::AddVideoTrackToMediaElement(
+    MediaSourceTracer* tracer,
+    VideoTrack* track) {
+  DVLOG(3) << __func__ << " this=" << this;
+
+  VerifyCalledWhileContextsAliveForDebugging();
+
+  HTMLMediaElement* element = GetMediaElement(tracer);
+  element->videoTracks().Add(track);
+}
+
+void SameThreadMediaSourceAttachment::RemoveAudioTracksFromMediaElement(
+    MediaSourceTracer* tracer,
+    Vector<String> audio_ids,
+    bool enqueue_change_event) {
+  DVLOG(3) << __func__ << " this=" << this << ", ids size=" << audio_ids.size()
+           << ", enqueue_change_event=" << enqueue_change_event;
+
+  if (element_context_destroyed_ || media_source_context_destroyed_) {
+    DVLOG(3) << __func__ << " this=" << this
+             << " -> skipping due to context(s) destroyed";
+    return;
+  }
+
+  HTMLMediaElement* element = GetMediaElement(tracer);
+  for (auto& audio_id : audio_ids) {
+    element->audioTracks().Remove(audio_id);
+  }
+
+  if (enqueue_change_event) {
+    Event* event = Event::Create(event_type_names::kChange);
+    event->SetTarget(&element->audioTracks());
+    element->ScheduleEvent(event);
+  }
+}
+
+void SameThreadMediaSourceAttachment::RemoveVideoTracksFromMediaElement(
+    MediaSourceTracer* tracer,
+    Vector<String> video_ids,
+    bool enqueue_change_event) {
+  DVLOG(3) << __func__ << " this=" << this << ", ids size=" << video_ids.size()
+           << ", enqueue_change_event=" << enqueue_change_event;
+
+  if (element_context_destroyed_ || media_source_context_destroyed_) {
+    DVLOG(3) << __func__ << " this=" << this
+             << " -> skipping due to context(s) destroyed";
+    return;
+  }
+
+  HTMLMediaElement* element = GetMediaElement(tracer);
+  for (auto& video_id : video_ids) {
+    element->videoTracks().Remove(video_id);
+  }
+
+  if (enqueue_change_event) {
+    Event* event = Event::Create(event_type_names::kChange);
+    event->SetTarget(&element->videoTracks());
+    element->ScheduleEvent(event);
+  }
+}
+
+void SameThreadMediaSourceAttachment::OnMediaSourceContextDestroyed() {
+  DVLOG(3) << __func__ << " this=" << this;
+
+  // We should only be notified once.
+  DCHECK(!media_source_context_destroyed_);
+
+  media_source_context_destroyed_ = true;
+}
+
 void SameThreadMediaSourceAttachment::Unregister() {
   DVLOG(1) << __func__ << " this=" << this;
 
@@ -202,15 +283,6 @@ void SameThreadMediaSourceAttachment::OnElementContextDestroyed() {
   element_context_destroyed_ = true;
 }
 
-void SameThreadMediaSourceAttachment::OnMediaSourceContextDestroyed() {
-  DVLOG(3) << __func__ << " this=" << this;
-
-  // We should only be notified once.
-  DCHECK(!element_context_destroyed_);
-
-  media_source_context_destroyed_ = true;
-}
-
 void SameThreadMediaSourceAttachment::
     VerifyCalledWhileContextsAliveForDebugging() const {
   DCHECK(!element_context_destroyed_);

third_party/blink/renderer/modules/mediasource/same_thread_media_source_attachment.h

@@ -9,6 +9,10 @@
 
 #include "base/util/type_safety/pass_key.h"
 #include "third_party/blink/public/platform/web_time_range.h"
+#include "third_party/blink/renderer/core/html/track/audio_track.h"
+#include "third_party/blink/renderer/core/html/track/audio_track_list.h"
+#include "third_party/blink/renderer/core/html/track/video_track.h"
+#include "third_party/blink/renderer/core/html/track/video_track_list.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source_attachment_supplement.h"
 #include "third_party/blink/renderer/modules/mediasource/url_media_source.h"
@@ -31,6 +35,16 @@ class SameThreadMediaSourceAttachment final
   void NotifyDurationChanged(MediaSourceTracer* tracer, double duration) final;
   double GetRecentMediaTime(MediaSourceTracer* tracer) final;
   bool GetElementError(MediaSourceTracer* tracer) final;
+  void AddAudioTrackToMediaElement(MediaSourceTracer* tracer,
+                                   AudioTrack* track) final;
+  void AddVideoTrackToMediaElement(MediaSourceTracer* tracer,
+                                   VideoTrack* track) final;
+  void RemoveAudioTracksFromMediaElement(MediaSourceTracer* tracer,
+                                         Vector<String> audio_ids,
+                                         bool enqueue_change_event) final;
+  void RemoveVideoTracksFromMediaElement(MediaSourceTracer* tracer,
+                                         Vector<String> video_ids,
+                                         bool enqueue_change_event) final;
   void OnMediaSourceContextDestroyed() final;
 
   // MediaSourceAttachment

third_party/blink/renderer/modules/mediasource/source_buffer.h

@@ -32,6 +32,8 @@
 #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASOURCE_SOURCE_BUFFER_H_
 
 #include <memory>
+
+#include "base/memory/scoped_refptr.h"
 #include "third_party/blink/public/platform/web_source_buffer_client.h"
 #include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
 #include "third_party/blink/renderer/core/execution_context/execution_context_lifecycle_observer.h"
@@ -50,6 +52,8 @@ class DOMArrayBufferView;
 class EventQueue;
 class ExceptionState;
 class MediaSource;
+class MediaSourceTracer;
+class MediaSourceAttachmentSupplement;
 class TimeRanges;
 class VideoTrackList;
 class WebSourceBuffer;
@@ -151,6 +155,17 @@ class SourceBuffer final : public EventTargetWithInlineData,
       const AtomicString& track_type,
       const AtomicString& byte_stream_track_id) const;
 
+  // TODO(https://crbug.com/878133): Remove these once worker thread track
+  // creation and tracklist modifications are supported. These are needed for
+  // now to retain stable BackgroundVideoOptimization support with experimental
+  // MSE-in-Workers.
+  void AddPlaceholderCrossThreadTracks(
+      const WebVector<MediaTrackInfo>& new_tracks,
+      scoped_refptr<MediaSourceAttachmentSupplement> attachment);
+  void RemovePlaceholderCrossThreadTracks(
+      scoped_refptr<MediaSourceAttachmentSupplement> attachment,
+      MediaSourceTracer* tracer);
+
   std::unique_ptr<WebSourceBuffer> web_source_buffer_;
 
   // If any portion of an attached HTMLMediaElement (HTMLME) and the MediaSource
@@ -180,6 +195,14 @@ class SourceBuffer final : public EventTargetWithInlineData,
   double pending_remove_start_;
   double pending_remove_end_;
   TaskHandle remove_async_task_handle_;
+
+  // Temporary vectors used for MSE-in-Workers removal of the audio and video
+  // tracks from the media element when needed by this SourceBuffer.
+  // TODO(https://crbug.com/878133): Refactor to remove these once
+  // CrossThreadMediaSourceAttachments, TrackBase and TrackListBase support
+  // track creation off-the-main thread.
+  Vector<String> audio_track_ids_for_crossthread_removal_;
+  Vector<String> video_track_ids_for_crossthread_removal_;
 };
 
 }  // namespace blink