Reland of Make rendering of MediaStreams reflect changes to its set of tracks....

Reland of Make rendering of MediaStreams reflect changes to its set of tracks. (patchset #1 id:1 of https://codereview.chromium.org/2985393002/ )

Reason for revert:
BUILD files have been fixed in a separate CL, so this is safe to reland.

Original issue's description:
> Revert of Make rendering of MediaStreams reflect changes to its set of tracks. (patchset #6 id:120001 of https://codereview.chromium.org/2969093002/ )
>
> Reason for revert:
> Breaking build https://uberchromegw.corp.google.com/i/chromium.linux/builders/Cast%20Audio%20Linux/builds/3356
>
> Original issue's description:
> > Make rendering of MediaStreams reflect changes to its set of tracks.
> >
> > Before this CL, MediaStreams assigned to a media element required
> > reassignment of the stream to the element in order to make changes
> > to the set of tracks visible.
> >
> > This CL fixes this problem by making WebMediaPlayerMS subscribe to
> > changes in the set of tracks of a MediaStream, and correspondingly
> > update audio and video renderers.
> >
> > BUG=720258
> >
> > Review-Url: https://codereview.chromium.org/2969093002
> > Cr-Commit-Position: refs/heads/master@{#490906}
> > Committed: https://chromium.googlesource.com/chromium/src/+/c9612ba8bf66e48a32301e35425dd3384f48a3d8
>
> TBR=dalecurtis@chromium.org,emircan@chromium.org,foolip@chromium.org,mkwst@chromium.org,guidou@chromium.org
> # Skipping CQ checks because original CL landed less than 1 days ago.
> NOPRESUBMIT=true
> NOTREECHECKS=true
> NOTRY=true
> BUG=720258
>
> Review-Url: https://codereview.chromium.org/2985393002
> Cr-Commit-Position: refs/heads/master@{#490920}
> Committed: https://chromium.googlesource.com/chromium/src/+/b35b4c271b7819e3661942ab2c72083673722910

TBR=dalecurtis@chromium.org,emircan@chromium.org,foolip@chromium.org,mkwst@chromium.org,jkrcal@chromium.org
# Not skipping CQ checks because original CL landed more than 1 days ago.
BUG=720258

Review-Url: https://codereview.chromium.org/2994473002
Cr-Commit-Position: refs/heads/master@{#491341}

content/browser/webrtc/webrtc_getusermedia_browsertest.cc

@@ -1504,4 +1504,39 @@ IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
                 "failTestAfterTimeout('Got no mute event', 1500);"));
 }
 
+IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest, SrcObjectAddVideoTrack) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+  GURL url(embedded_test_server()->GetURL("/media/getusermedia.html"));
+  NavigateToURL(shell(), url);
+  ExecuteJavascriptAndWaitForOk("srcObjectAddVideoTrack()");
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
+                       SrcObjectRemoveVideoTrack) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+  GURL url(embedded_test_server()->GetURL("/media/getusermedia.html"));
+  NavigateToURL(shell(), url);
+  ExecuteJavascriptAndWaitForOk("srcObjectRemoveVideoTrack()");
+}
+
+IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
+                       SrcObjectRemoveFirstOfTwoVideoTracks) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+  GURL url(embedded_test_server()->GetURL("/media/getusermedia.html"));
+  NavigateToURL(shell(), url);
+  ExecuteJavascriptAndWaitForOk("srcObjectRemoveFirstOfTwoVideoTracks()");
+}
+
+// TODO(guidou): Add SrcObjectAddAudioTrack and SrcObjectRemoveAudioTrack tests
+// when a straightforward mechanism to detect the presence/absence of audio in a
+// media element with an assigned MediaStream becomes available.
+
+IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
+                       SrcObjectReassignSameObject) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+  GURL url(embedded_test_server()->GetURL("/media/getusermedia.html"));
+  NavigateToURL(shell(), url);
+  ExecuteJavascriptAndWaitForOk("srcObjectReassignSameObject()");
+}
+
 }  // namespace content

content/renderer/media/webmediaplayer_ms.cc

@@ -20,6 +20,8 @@
 #include "content/public/renderer/media_stream_audio_renderer.h"
 #include "content/public/renderer/media_stream_renderer_factory.h"
 #include "content/public/renderer/media_stream_video_renderer.h"
+#include "content/renderer/media/media_stream_audio_track.h"
+#include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/web_media_element_source_utils.h"
 #include "content/renderer/media/webmediaplayer_ms_compositor.h"
 #include "content/renderer/media/webrtc_logging.h"
@@ -39,6 +41,14 @@
 #include "third_party/WebKit/public/platform/WebSize.h"
 #include "third_party/WebKit/public/web/WebLocalFrame.h"
 
+namespace {
+enum class RendererReloadAction {
+  KEEP_RENDERER,
+  REMOVE_RENDERER,
+  NEW_RENDERER
+};
+}  // namespace
+
 namespace content {
 
 // FrameDeliverer is responsible for delivering frames received on
@@ -191,6 +201,12 @@ WebMediaPlayerMS::~WebMediaPlayerMS() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
 
+  if (!web_stream_.IsNull()) {
+    MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_);
+    if (native_stream)
+      native_stream->RemoveObserver(this);
+  }
+
   // Destruct compositor resources in the proper order.
   get_client()->SetWebLayer(nullptr);
   if (video_weblayer_)
@@ -224,24 +240,29 @@ void WebMediaPlayerMS::Load(LoadType load_type,
   // TODO(acolwell): Change this to DCHECK_EQ(load_type, LoadTypeMediaStream)
   // once Blink-side changes land.
   DCHECK_NE(load_type, kLoadTypeMediaSource);
-  blink::WebMediaStream web_stream =
-      GetWebMediaStreamFromWebMediaPlayerSource(source);
+  web_stream_ = GetWebMediaStreamFromWebMediaPlayerSource(source);
+  if (!web_stream_.IsNull()) {
+    MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_);
+    if (native_stream)
+      native_stream->AddObserver(this);
+  }
 
   compositor_ = new WebMediaPlayerMSCompositor(
-      compositor_task_runner_, io_task_runner_, web_stream, AsWeakPtr());
+      compositor_task_runner_, io_task_runner_, web_stream_, AsWeakPtr());
 
   SetNetworkState(WebMediaPlayer::kNetworkStateLoading);
   SetReadyState(WebMediaPlayer::kReadyStateHaveNothing);
   std::string stream_id =
-      web_stream.IsNull() ? std::string() : web_stream.Id().Utf8();
+      web_stream_.IsNull() ? std::string() : web_stream_.Id().Utf8();
   media_log_->AddEvent(media_log_->CreateLoadEvent(stream_id));
 
   frame_deliverer_.reset(new WebMediaPlayerMS::FrameDeliverer(
       AsWeakPtr(),
       base::Bind(&WebMediaPlayerMSCompositor::EnqueueFrame, compositor_)));
   video_frame_provider_ = renderer_factory_->GetVideoRenderer(
-      web_stream, media::BindToCurrentLoop(base::Bind(
-                      &WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
+      web_stream_,
+      media::BindToCurrentLoop(
+          base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
       frame_deliverer_->GetRepaintCallback(), io_task_runner_,
       media_task_runner_, worker_task_runner_, gpu_factories_);
 
@@ -258,7 +279,7 @@ void WebMediaPlayerMS::Load(LoadType load_type,
   }
 
   audio_renderer_ = renderer_factory_->GetAudioRenderer(
-      web_stream, routing_id, initial_audio_output_device_id_,
+      web_stream_, routing_id, initial_audio_output_device_id_,
       initial_security_origin_);
 
   if (!audio_renderer_)
@@ -272,10 +293,27 @@ void WebMediaPlayerMS::Load(LoadType load_type,
   if (audio_renderer_) {
     audio_renderer_->SetVolume(volume_);
     audio_renderer_->Start();
+
+    // Store the ID of audio track being played in |current_video_track_id_|
+    blink::WebVector<blink::WebMediaStreamTrack> audio_tracks;
+    if (!web_stream_.IsNull()) {
+      web_stream_.AudioTracks(audio_tracks);
+      DCHECK_GT(audio_tracks.size(), 0U);
+      current_audio_track_id_ = audio_tracks[0].Id();
+    }
   }
-  if (video_frame_provider_)
+
+  if (video_frame_provider_) {
     video_frame_provider_->Start();
 
+    // Store the ID of video track being played in |current_video_track_id_|
+    if (!web_stream_.IsNull()) {
+      blink::WebVector<blink::WebMediaStreamTrack> video_tracks;
+      web_stream_.VideoTracks(video_tracks);
+      DCHECK_GT(video_tracks.size(), 0U);
+      current_video_track_id_ = video_tracks[0].Id();
+    }
+  }
   // When associated with an <audio> element, we don't want to wait for the
   // first video fram to become available as we do for <video> elements
   // (<audio> elements can also be assigned video tracks).
@@ -288,6 +326,113 @@ void WebMediaPlayerMS::Load(LoadType load_type,
   }
 }
 
+void WebMediaPlayerMS::TrackAdded(const blink::WebMediaStreamTrack& track) {
+  Reload();
+}
+
+void WebMediaPlayerMS::TrackRemoved(const blink::WebMediaStreamTrack& track) {
+  Reload();
+}
+
+void WebMediaPlayerMS::Reload() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (web_stream_.IsNull())
+    return;
+
+  ReloadVideo();
+  ReloadAudio();
+}
+
+void WebMediaPlayerMS::ReloadVideo() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!web_stream_.IsNull());
+  blink::WebVector<blink::WebMediaStreamTrack> video_tracks;
+  // VideoTracks() is a getter.
+  web_stream_.VideoTracks(video_tracks);
+
+  RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER;
+  if (video_tracks.IsEmpty()) {
+    if (video_frame_provider_)
+      renderer_action = RendererReloadAction::REMOVE_RENDERER;
+    current_video_track_id_ = blink::WebString();
+  } else if (video_tracks[0].Id() != current_video_track_id_) {
+    renderer_action = RendererReloadAction::NEW_RENDERER;
+    current_video_track_id_ = video_tracks[0].Id();
+  }
+
+  switch (renderer_action) {
+    case RendererReloadAction::NEW_RENDERER:
+      if (video_frame_provider_)
+        video_frame_provider_->Stop();
+
+      video_frame_provider_ = renderer_factory_->GetVideoRenderer(
+          web_stream_,
+          media::BindToCurrentLoop(
+              base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
+          frame_deliverer_->GetRepaintCallback(), io_task_runner_,
+          media_task_runner_, worker_task_runner_, gpu_factories_);
+      DCHECK(video_frame_provider_);
+      video_frame_provider_->Start();
+      break;
+
+    case RendererReloadAction::REMOVE_RENDERER:
+      video_frame_provider_->Stop();
+      video_frame_provider_ = nullptr;
+      break;
+
+    default:
+      return;
+  }
+
+  DCHECK_NE(renderer_action, RendererReloadAction::KEEP_RENDERER);
+  if (!paused_)
+    delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize());
+}
+
+void WebMediaPlayerMS::ReloadAudio() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!web_stream_.IsNull());
+  RenderFrame* const frame = RenderFrame::FromWebFrame(frame_);
+  if (!frame)
+    return;
+
+  blink::WebVector<blink::WebMediaStreamTrack> audio_tracks;
+  // AudioTracks() is a getter.
+  web_stream_.AudioTracks(audio_tracks);
+
+  RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER;
+  if (audio_tracks.IsEmpty()) {
+    if (audio_renderer_)
+      renderer_action = RendererReloadAction::REMOVE_RENDERER;
+    current_audio_track_id_ = blink::WebString();
+  } else if (audio_tracks[0].Id() != current_video_track_id_) {
+    renderer_action = RendererReloadAction::NEW_RENDERER;
+    current_audio_track_id_ = audio_tracks[0].Id();
+  }
+
+  switch (renderer_action) {
+    case RendererReloadAction::NEW_RENDERER:
+      if (audio_renderer_)
+        audio_renderer_->Stop();
+
+      audio_renderer_ = renderer_factory_->GetAudioRenderer(
+          web_stream_, frame->GetRoutingID(), initial_audio_output_device_id_,
+          initial_security_origin_);
+      audio_renderer_->SetVolume(volume_);
+      audio_renderer_->Start();
+      audio_renderer_->Play();
+      break;
+
+    case RendererReloadAction::REMOVE_RENDERER:
+      audio_renderer_->Stop();
+      audio_renderer_ = nullptr;
+      break;
+
+    default:
+      break;
+  }
+}
+
 void WebMediaPlayerMS::Play() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
@@ -394,6 +539,9 @@ bool WebMediaPlayerMS::HasAudio() const {
 
 blink::WebSize WebMediaPlayerMS::NaturalSize() const {
   DCHECK(thread_checker_.CalledOnValidThread());
+  if (!video_frame_provider_)
+    return blink::WebSize();
+
   if (video_rotation_ == media::VIDEO_ROTATION_90 ||
       video_rotation_ == media::VideoRotation::VIDEO_ROTATION_270) {
     const gfx::Size& current_size = compositor_->GetCurrentSize();

content/renderer/media/webmediaplayer_ms.h

@@ -14,6 +14,7 @@
 #include "base/synchronization/lock.h"
 #include "base/threading/thread_checker.h"
 #include "content/common/content_export.h"
+#include "content/renderer/media/media_stream.h"
 #include "media/blink/webmediaplayer_delegate.h"
 #include "media/blink/webmediaplayer_util.h"
 #include "media/renderers/gpu_video_accelerator_factories.h"
@@ -62,7 +63,8 @@ class WebMediaPlayerMSCompositor;
 // blink::WebMediaPlayerClient
 //   WebKit client of this media player object.
 class CONTENT_EXPORT WebMediaPlayerMS
-    : public NON_EXPORTED_BASE(blink::WebMediaPlayer),
+    : public NON_EXPORTED_BASE(MediaStreamObserver),
+      public NON_EXPORTED_BASE(blink::WebMediaPlayer),
       public NON_EXPORTED_BASE(media::WebMediaPlayerDelegate::Observer),
       public NON_EXPORTED_BASE(base::SupportsWeakPtr<WebMediaPlayerMS>) {
  public:
@@ -179,6 +181,10 @@ class CONTENT_EXPORT WebMediaPlayerMS
                     bool flip_y,
                     bool premultiply_alpha) override;
 
+  // MediaStreamObserver implementation
+  void TrackAdded(const blink::WebMediaStreamTrack& track) override;
+  void TrackRemoved(const blink::WebMediaStreamTrack& track) override;
+
  private:
   friend class WebMediaPlayerMSTest;
 
@@ -201,6 +207,11 @@ class CONTENT_EXPORT WebMediaPlayerMS
   // Getter method to |client_|.
   blink::WebMediaPlayerClient* get_client() { return client_; }
 
+  // To be run when tracks are added or removed.
+  void Reload();
+  void ReloadVideo();
+  void ReloadAudio();
+
   blink::WebLocalFrame* const frame_;
 
   blink::WebMediaPlayer::NetworkState network_state_;
@@ -266,6 +277,10 @@ class CONTENT_EXPORT WebMediaPlayerMS
   // True if playback should be started upon the next call to OnShown(). Only
   // used on Android.
   bool should_play_upon_shown_;
+  blink::WebMediaStream web_stream_;
+  // IDs of the tracks currently played.
+  blink::WebString current_video_track_id_;
+  blink::WebString current_audio_track_id_;
 
   DISALLOW_COPY_AND_ASSIGN(WebMediaPlayerMS);
 };

content/shell/test_runner/mock_web_user_media_client.cc

@@ -33,8 +33,6 @@ using blink::WebVector;
 
 namespace test_runner {
 
-class MockExtraData : public WebMediaStream::ExtraData {};
-
 MockWebUserMediaClient::MockWebUserMediaClient(WebTestDelegate* delegate)
     : delegate_(delegate),
       should_enumerate_extra_device_(false),
@@ -57,7 +55,6 @@ void MockWebUserMediaClient::RequestUserMedia(
   WebMediaStream stream;
   stream.Initialize(WebVector<WebMediaStreamTrack>(),
                     WebVector<WebMediaStreamTrack>());
-  stream.SetExtraData(new MockExtraData());
 
   if (request.Audio() &&
       !delegate_->AddMediaStreamAudioSourceAndTrack(&stream)) {

content/test/data/media/getusermedia.html

@@ -612,6 +612,144 @@
         failTest("Unexpected error: " + e)
       });
   }
+
+  function srcObjectAddVideoTrack() {
+    var video = document.createElement('video');
+    video.autoplay = true;
+    assertEquals(video.srcObject, null);
+    navigator.mediaDevices.getUserMedia({audio: true, video: true})
+      .then(stream => {
+        video.onplaying = function() {
+          video.onplaying = null;
+          video.onloadstart = function() {
+            failTest("loadstart should not be called");
+          }
+
+          video.onresize = function() {
+            assertNotEquals(video.srcObject, null);
+            assertTrue(video.videoHeight > 0);
+            assertTrue(video.videoWidth > 0);
+            reportTestSuccess();
+          }
+
+          assertNotEquals(video.srcObject, null);
+          assertEquals(video.videoWidth, 0);
+          assertEquals(video.videoHeight, 0);
+          video.srcObject.addTrack(stream.getVideoTracks()[0]);
+        }
+        video.srcObject = new MediaStream(stream.getAudioTracks());
+      })
+    .catch(e => {
+      failTest("Unexpected error: " + e)
+    });
+  }
+
+  function srcObjectRemoveVideoTrack() {
+    var video = document.createElement('video');
+    video.autoplay = true;
+    assertEquals(video.srcObject, null);
+    navigator.mediaDevices.getUserMedia({audio: true, video: true})
+      .then(stream => {
+        video.onplaying = function() {
+          video.onplaying = null;
+          video.onloadstart = function() {
+            failTest("loadstart should not be called");
+          }
+
+          video.onresize = function() {
+            assertNotEquals(video.srcObject, null);
+            assertEquals(0, video.videoHeight);
+            assertEquals(0, video.videoWidth);
+            reportTestSuccess();
+          }
+
+          assertNotEquals(video.srcObject, null);
+          assertTrue(video.videoWidth > 0);
+          assertTrue(video.videoHeight > 0);
+          stream.removeTrack(stream.getVideoTracks()[0]);
+        }
+        video.srcObject = stream;
+      })
+    .catch(e => {
+      failTest("Unexpected error: " + e)
+    });
+  }
+
+  function srcObjectRemoveFirstOfTwoVideoTracks() {
+    var canvas = document.createElement('canvas');
+    var canvas_stream = canvas.captureStream();
+    var canvas_width = canvas_stream.getVideoTracks()[0].getSettings().width;
+    var canvas_height = canvas_stream.getVideoTracks()[0].getSettings().height;
+    assertTrue(canvas_width > 1);
+    assertTrue(canvas_height > 1);
+
+    // Paint something on the canvas, so that it produces frames.
+    var ctx = canvas.getContext("2d");
+    ctx.moveTo(0,0);
+    ctx.lineTo(200,100);
+    ctx.stroke();
+
+    var video = document.createElement('video');
+    video.autoplay = true;
+    assertEquals(video.srcObject, null);
+    var gum_width = canvas_width + 1;
+    var gum_height = canvas_height + 1;
+    navigator.mediaDevices.getUserMedia({
+        video: {
+          width: {exact: gum_width},
+          height: {exact: gum_height}
+        }
+      }).then(gum_stream => {
+        var gum_settings = gum_stream.getVideoTracks()[0].getSettings();
+        assertEquals(gum_width, gum_settings.width)
+        assertEquals(gum_height, gum_settings.height)
+        var big_stream = new MediaStream();
+        big_stream.addTrack(canvas_stream.getVideoTracks()[0]);
+        big_stream.addTrack(gum_stream.getVideoTracks()[0]);
+        video.onprogress = function() {
+          assertEquals(canvas_width, video.videoWidth);
+          assertEquals(canvas_height, video.videoHeight);
+          assertNotEquals(video.videoWidth, gum_width)
+          assertNotEquals(video.videoHeight, gum_height)
+          video.onprogress = function() {
+            assertEquals(gum_width, video.videoWidth);
+            assertEquals(gum_height, video.videoHeight);
+            assertNotEquals(video.videoWidth, canvas_width)
+            assertNotEquals(video.videoHeight, canvas_height)
+            reportTestSuccess();
+          }
+          big_stream.removeTrack(big_stream.getVideoTracks()[0]);
+        }
+        video.srcObject = big_stream;
+    })
+    .catch(e => {
+      failTest("Unexpected error: " + e)
+    });
+  }
+
+  function srcObjectReassignSameObject() {
+    var video = document.createElement('video');
+    video.autoplay = true;
+    assertEquals(video.srcObject, null);
+    navigator.mediaDevices.getUserMedia({audio: true, video: true})
+      .then(stream => {
+        video.onplaying = function() {
+          video.onplaying = null;
+          video.onloadstart = function() {
+            reportTestSuccess();
+          }
+          assertNotEquals(video.srcObject, null);
+          assertTrue(video.videoWidth > 0);
+          assertTrue(video.videoHeight > 0);
+          // Reassigning the same object should trigger the load algorithm.
+          video.srcObject = video.srcObject;
+        }
+        video.srcObject = stream;
+      })
+    .catch(e => {
+      failTest("Unexpected error: " + e)
+    });
+  }
   </script>
 </head>
 <body>