Allows for on demand switching to surfaces for PiP for MediaStreams

This change allows the kOnDemand option to apply for MediaStreams, meaning
that we can switch over to using Surfaces when it is needed for PiP, deferring
to a VideoLayer in the normal non-PiP case.

Bug: 889857
Change-Id: I56019885d24e9d36f74a279d31f8259081edd1d3
Reviewed-on: https://chromium-review.googlesource.com/c/1271591Reviewed-by: Frank Liberato <liberato@chromium.org>
Reviewed-by: Emircan Uysaler <emircan@chromium.org>
Reviewed-by: Chris Harrelson <chrishtr@chromium.org>
Reviewed-by: François Beaufort <beaufort.francois@gmail.com>
Commit-Queue: CJ DiMeglio <lethalantidote@chromium.org>
Cr-Commit-Position: refs/heads/master@{#601738}

content/renderer/media/media_factory.cc

@@ -194,6 +194,54 @@ bool UseMediaPlayerRenderer(const GURL& url) {
 }
 #endif  // defined(OS_ANDROID)
 
+std::unique_ptr<blink::WebVideoFrameSubmitter> MediaFactory::CreateSubmitter(
+    scoped_refptr<base::SingleThreadTaskRunner>*
+        video_frame_compositor_task_runner,
+    const cc::LayerTreeSettings& settings) {
+  blink::WebMediaPlayer::SurfaceLayerMode use_surface_layer_for_video =
+      GetVideoSurfaceLayerMode();
+  content::RenderThreadImpl* render_thread =
+      content::RenderThreadImpl::current();
+  *video_frame_compositor_task_runner = nullptr;
+
+  if (!render_thread)
+    return nullptr;
+
+  bool use_sync_primitives = false;
+  if (use_surface_layer_for_video ==
+      blink::WebMediaPlayer::SurfaceLayerMode::kAlways) {
+    // Run the compositor / frame submitter on its own thread.
+    *video_frame_compositor_task_runner =
+        render_thread->CreateVideoFrameCompositorTaskRunner();
+    // We must use sync primitives on this thread.
+    use_sync_primitives = true;
+  } else {
+    // Run on the cc thread, even if we may switch to SurfaceLayer mode later
+    // if we're in kOnDemand mode.  We do this to avoid switching threads when
+    // switching to SurfaceLayer.
+    *video_frame_compositor_task_runner =
+        render_thread->compositor_task_runner()
+            ? render_thread->compositor_task_runner()
+            : render_frame_->GetTaskRunner(
+                  blink::TaskType::kInternalMediaRealTime);
+  }
+
+  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter;
+
+  if (use_surface_layer_for_video !=
+      blink::WebMediaPlayer::SurfaceLayerMode::kNever) {
+    submitter = blink::WebVideoFrameSubmitter::Create(
+        base::BindRepeating(
+            &PostContextProviderToCallback,
+            RenderThreadImpl::current()->GetCompositorMainThreadTaskRunner()),
+        settings, use_sync_primitives);
+  }
+
+  DCHECK(*video_frame_compositor_task_runner);
+
+  return submitter;
+}
+
 blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
     const blink::WebMediaPlayerSource& source,
     blink::WebMediaPlayerClient* client,
@@ -276,36 +324,8 @@ blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
 
   scoped_refptr<base::SingleThreadTaskRunner>
       video_frame_compositor_task_runner;
-  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter;
-  blink::WebMediaPlayer::SurfaceLayerMode use_surface_layer_for_video =
-      GetVideoSurfaceLayerMode();
-  bool use_sync_primitives = false;
-  if (use_surface_layer_for_video ==
-      blink::WebMediaPlayer::SurfaceLayerMode::kAlways) {
-    // Run the compositor / frame submitter on its own thread.
-    video_frame_compositor_task_runner =
-        render_thread->CreateVideoFrameCompositorTaskRunner();
-    // We must use sync primitives on this thread.
-    use_sync_primitives = true;
-  } else {
-    // Run on the cc thread, even if we may switch to SurfaceLayer mode later
-    // if we're in kOnDemand mode.  We do this to avoid switching threads when
-    // switching to SurfaceLayer.
-    video_frame_compositor_task_runner =
-        render_thread->compositor_task_runner()
-            ? render_thread->compositor_task_runner()
-            : render_frame_->GetTaskRunner(
-                  blink::TaskType::kInternalMediaRealTime);
-  }
-
-  if (use_surface_layer_for_video !=
-      blink::WebMediaPlayer::SurfaceLayerMode::kNever) {
-    submitter = blink::WebVideoFrameSubmitter::Create(
-        base::BindRepeating(
-            &PostContextProviderToCallback,
-            RenderThreadImpl::current()->GetCompositorMainThreadTaskRunner()),
-        settings, use_sync_primitives);
-  }
+  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter =
+      CreateSubmitter(&video_frame_compositor_task_runner, settings);
 
   DCHECK(layer_tree_view);
   scoped_refptr<base::SingleThreadTaskRunner> media_task_runner =
@@ -337,7 +357,7 @@ blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
           base::BindOnce(&blink::WebSurfaceLayerBridge::Create,
                          layer_tree_view),
           RenderThreadImpl::current()->SharedMainThreadContextProvider(),
-          use_surface_layer_for_video));
+          GetVideoSurfaceLayerMode()));
 
   std::unique_ptr<media::VideoFrameCompositor> vfc =
       std::make_unique<media::VideoFrameCompositor>(
@@ -517,33 +537,8 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
 
   scoped_refptr<base::SingleThreadTaskRunner>
       video_frame_compositor_task_runner;
-  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter;
-  blink::WebMediaPlayer::SurfaceLayerMode use_surface_layer_for_video =
-      GetVideoSurfaceLayerMode();
-
-  // Currently we do not support kOnDemand for MediaStreams.
-  if (use_surface_layer_for_video ==
-      blink::WebMediaPlayer::SurfaceLayerMode::kOnDemand) {
-    use_surface_layer_for_video =
-        blink::WebMediaPlayer::SurfaceLayerMode::kNever;
-  }
-
-  if (use_surface_layer_for_video ==
-      blink::WebMediaPlayer::SurfaceLayerMode::kAlways) {
-    // Run the compositor / frame submitter on its own thread.
-    video_frame_compositor_task_runner =
-        render_thread->CreateVideoFrameCompositorTaskRunner();
-    // We must use sync primitives on this thread.
-    const bool use_sync_primitives = true;
-
-    submitter = blink::WebVideoFrameSubmitter::Create(
-        base::BindRepeating(
-            &PostContextProviderToCallback,
-            RenderThreadImpl::current()->GetCompositorMainThreadTaskRunner()),
-        settings, use_sync_primitives);
-  } else {
-    video_frame_compositor_task_runner = compositor_task_runner;
-  }
+  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter =
+      CreateSubmitter(&video_frame_compositor_task_runner, settings);
 
   DCHECK(layer_tree_view);
   return new WebMediaPlayerMS(
@@ -557,13 +552,7 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
       render_thread->GetWorkerTaskRunner(), render_thread->GetGpuFactories(),
       sink_id,
       base::BindOnce(&blink::WebSurfaceLayerBridge::Create, layer_tree_view),
-      base::BindRepeating(
-          &blink::WebVideoFrameSubmitter::Create,
-          base::BindRepeating(
-              &PostContextProviderToCallback,
-              RenderThreadImpl::current()->GetCompositorMainThreadTaskRunner()),
-          settings, true),
-      use_surface_layer_for_video);
+      std::move(submitter), GetVideoSurfaceLayerMode());
 }
 
 media::RendererWebMediaPlayerDelegate*

content/renderer/media/media_factory.h

@@ -91,6 +91,13 @@ class MediaFactory {
   // interface provider is bound.
   void SetupMojo();
 
+  // Creates the VideoFrameSubmitter and its task_runner based on the current
+  // SurfaceLayerMode;
+  std::unique_ptr<blink::WebVideoFrameSubmitter> CreateSubmitter(
+      scoped_refptr<base::SingleThreadTaskRunner>*
+          video_frame_compositor_task_runner,
+      const cc::LayerTreeSettings& settings);
+
   // Creates a new WebMediaPlayer for the given |source| (either a stream or
   // URL). All pointers other than |initial_cdm| are required to be non-null.
   // The created player serves and is directed by the |client| (e.g.

content/renderer/media/stream/webmediaplayer_ms.cc

@@ -238,8 +238,7 @@ WebMediaPlayerMS::WebMediaPlayerMS(
     media::GpuVideoAcceleratorFactories* gpu_factories,
     const blink::WebString& sink_id,
     CreateSurfaceLayerBridgeCB create_bridge_callback,
-    base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
-        create_submitter_callback,
+    std::unique_ptr<blink::WebVideoFrameSubmitter> submitter,
     blink::WebMediaPlayer::SurfaceLayerMode surface_layer_mode)
     : frame_(frame),
       network_state_(WebMediaPlayer::kNetworkStateEmpty),
@@ -262,13 +261,10 @@ WebMediaPlayerMS::WebMediaPlayerMS(
       volume_multiplier_(1.0),
       should_play_upon_shown_(false),
       create_bridge_callback_(std::move(create_bridge_callback)),
-      create_submitter_callback_(create_submitter_callback),
+      submitter_(std::move(submitter)),
       surface_layer_mode_(surface_layer_mode) {
   DVLOG(1) << __func__;
   DCHECK(client);
-  // We currently do not support on demand switching to use surface layer.
-  DCHECK(surface_layer_mode_ !=
-         blink::WebMediaPlayer::SurfaceLayerMode::kOnDemand);
   DCHECK(delegate_);
   delegate_id_ = delegate_->AddObserver(this);
 
@@ -326,7 +322,7 @@ blink::WebMediaPlayer::LoadTiming WebMediaPlayerMS::Load(
 
   compositor_ = new WebMediaPlayerMSCompositor(
       compositor_task_runner_, io_task_runner_, web_stream_,
-      create_submitter_callback_, surface_layer_mode_, AsWeakPtr());
+      std::move(submitter_), surface_layer_mode_, AsWeakPtr());
 
   SetNetworkState(WebMediaPlayer::kNetworkStateLoading);
   SetReadyState(WebMediaPlayer::kReadyStateHaveNothing);
@@ -642,6 +638,9 @@ void WebMediaPlayerMS::SetVolume(double volume) {
 
 void WebMediaPlayerMS::EnterPictureInPicture(
     blink::WebMediaPlayer::PipWindowOpenedCallback callback) {
+  if (!bridge_)
+    ActivateSurfaceLayerForVideo();
+
   DCHECK(bridge_);
 
   const viz::SurfaceId& surface_id = bridge_->GetSurfaceId();
@@ -1070,27 +1069,51 @@ void WebMediaPlayerMS::OnFrameSinkDestroyed() {
   bridge_->ClearSurfaceId();
 }
 
+void WebMediaPlayerMS::ActivateSurfaceLayerForVideo() {
+  // Note that we might or might not already be in VideoLayer mode.
+  DCHECK(!bridge_);
+
+  // If we're in VideoLayer mode, then get rid of the layer.
+  if (video_layer_) {
+    client_->SetCcLayer(nullptr);
+    video_layer_ = nullptr;
+  }
+
+  bridge_ = std::move(create_bridge_callback_)
+                .Run(this, compositor_->GetUpdateSubmissionStateCallback());
+  bridge_->CreateSurfaceLayer();
+  bridge_->SetContentsOpaque(opaque_);
+
+  compositor_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(
+          &WebMediaPlayerMSCompositor::EnableSubmission, compositor_,
+          bridge_->GetSurfaceId(), video_rotation_, IsInPictureInPicture(),
+          opaque_,
+          media::BindToCurrentLoop(base::BindRepeating(
+              &WebMediaPlayerMS::OnFrameSinkDestroyed, AsWeakPtr()))));
+
+  // If the element is already in Picture-in-Picture mode, it means that it
+  // was set in this mode prior to this load, with a different
+  // WebMediaPlayerImpl. The new player needs to send its id, size and
+  // surface id to the browser process to make sure the states are properly
+  // updated.
+  // TODO(872056): the surface should be activated but for some reason, it
+  // does not. It is possible that this will no longer be needed after 872056
+  // is fixed.
+  if (client_->DisplayType() ==
+      WebMediaPlayer::DisplayType::kPictureInPicture) {
+    OnSurfaceIdUpdated(bridge_->GetSurfaceId());
+  }
+}
+
 void WebMediaPlayerMS::OnFirstFrameReceived(media::VideoRotation video_rotation,
                                             bool is_opaque) {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (surface_layer_mode_ == blink::WebMediaPlayer::SurfaceLayerMode::kAlways) {
-    DCHECK(!bridge_);
-
-    bridge_ = std::move(create_bridge_callback_)
-                  .Run(this, compositor_->GetUpdateSubmissionStateCallback());
-    bridge_->CreateSurfaceLayer();
-    bridge_->SetContentsOpaque(is_opaque);
-
-    compositor_task_runner_->PostTask(
-        FROM_HERE,
-        base::BindOnce(
-            &WebMediaPlayerMSCompositor::EnableSubmission, compositor_,
-            bridge_->GetSurfaceId(), video_rotation, IsInPictureInPicture(),
-            opaque_,
-            media::BindToCurrentLoop(base::BindRepeating(
-                &WebMediaPlayerMS::OnFrameSinkDestroyed, AsWeakPtr()))));
+    ActivateSurfaceLayerForVideo();
   }
   OnRotationChanged(video_rotation);
   OnOpacityChanged(is_opaque);

content/renderer/media/stream/webmediaplayer_ms.h

@@ -93,8 +93,7 @@ class CONTENT_EXPORT WebMediaPlayerMS
       media::GpuVideoAcceleratorFactories* gpu_factories,
       const blink::WebString& sink_id,
       CreateSurfaceLayerBridgeCB create_bridge_callback,
-      base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
-          create_submitter_callback,
+      std::unique_ptr<blink::WebVideoFrameSubmitter> submitter_,
       blink::WebMediaPlayer::SurfaceLayerMode surface_layer_mode);
 
   ~WebMediaPlayerMS() override;
@@ -254,6 +253,9 @@ class CONTENT_EXPORT WebMediaPlayerMS
 
   bool IsInPictureInPicture() const;
 
+  // Switch to SurfaceLayer, either initially or from VideoLayer.
+  void ActivateSurfaceLayerForVideo();
+
   // Need repaint due to state change.
   void RepaintInternal();
 
@@ -349,8 +351,7 @@ class CONTENT_EXPORT WebMediaPlayerMS
 
   CreateSurfaceLayerBridgeCB create_bridge_callback_;
 
-  base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
-      create_submitter_callback_;
+  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter_;
 
   // Whether the use of a surface layer instead of a video layer is enabled.
   blink::WebMediaPlayer::SurfaceLayerMode surface_layer_mode_ =

content/renderer/media/stream/webmediaplayer_ms_compositor.cc

@@ -134,8 +134,7 @@ WebMediaPlayerMSCompositor::WebMediaPlayerMSCompositor(
         video_frame_compositor_task_runner,
     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
     const blink::WebMediaStream& web_stream,
-    base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
-        create_submitter_callback,
+    std::unique_ptr<blink::WebVideoFrameSubmitter> submitter,
     blink::WebMediaPlayer::SurfaceLayerMode surface_layer_mode,
     const base::WeakPtr<WebMediaPlayerMS>& player)
     : RefCountedDeleteOnSequence<WebMediaPlayerMSCompositor>(
@@ -153,10 +152,8 @@ WebMediaPlayerMSCompositor::WebMediaPlayerMSCompositor(
       weak_ptr_factory_(this) {
   main_message_loop_ = base::MessageLoopCurrent::Get();
 
-  DCHECK(surface_layer_mode !=
-         blink::WebMediaPlayer::SurfaceLayerMode::kOnDemand);
   if (surface_layer_mode != blink::WebMediaPlayer::SurfaceLayerMode::kNever) {
-    submitter_ = create_submitter_callback.Run();
+    submitter_ = std::move(submitter);
 
     video_frame_compositor_task_runner_->PostTask(
         FROM_HERE,
@@ -220,11 +217,21 @@ void WebMediaPlayerMSCompositor::EnableSubmission(
     bool is_opaque,
     blink::WebFrameSinkDestroyedCallback frame_sink_destroyed_callback) {
   DCHECK(video_frame_compositor_task_runner_->BelongsToCurrentThread());
+
+  // If we're switching to |submitter_| from some other client, then tell it.
+  if (video_frame_provider_client_ &&
+      video_frame_provider_client_ != submitter_.get()) {
+    video_frame_provider_client_->StopUsingProvider();
+  }
+
   submitter_->SetRotation(rotation);
   submitter_->SetForceSubmit(force_submit);
   submitter_->SetIsOpaque(is_opaque);
   submitter_->EnableSubmission(id, std::move(frame_sink_destroyed_callback));
   video_frame_provider_client_ = submitter_.get();
+
+  if (!stopped_)
+    video_frame_provider_client_->StartRendering();
 }
 
 void WebMediaPlayerMSCompositor::SetForceSubmit(bool force_submit) {

content/renderer/media/stream/webmediaplayer_ms_compositor.h

@@ -67,8 +67,7 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
       scoped_refptr<base::SingleThreadTaskRunner> task_runner,
       scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
       const blink::WebMediaStream& web_stream,
-      base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
-          create_submitter_callback,
+      std::unique_ptr<blink::WebVideoFrameSubmitter> submitter,
       blink::WebMediaPlayer::SurfaceLayerMode surface_layer_mode,
       const base::WeakPtr<WebMediaPlayerMS>& player);
 

content/renderer/media/stream/webmediaplayer_ms_unittest.cc

@@ -645,11 +645,6 @@ class WebMediaPlayerMSTest
     return std::move(surface_layer_bridge_);
   }
 
-  std::unique_ptr<blink::WebVideoFrameSubmitter>
-  CreateWebMockVideoFrameSubmitter() {
-    return std::move(submitter_);
-  }
-
   base::test::ScopedTaskEnvironment task_environment_;
   MockRenderFactory* render_factory_;
   std::unique_ptr<media::MockGpuVideoAcceleratorFactories> gpu_factories_;
@@ -692,10 +687,7 @@ void WebMediaPlayerMSTest::InitializeWebMediaPlayerMS() {
       gpu_factories_.get(), blink::WebString(),
       base::BindOnce(&WebMediaPlayerMSTest::CreateMockSurfaceLayerBridge,
                      base::Unretained(this)),
-      base::BindRepeating(
-          &WebMediaPlayerMSTest::CreateWebMockVideoFrameSubmitter,
-          base::Unretained(this)),
-      surface_layer_mode);
+      std::move(submitter_), surface_layer_mode);
 }
 
 MockMediaStreamVideoRenderer* WebMediaPlayerMSTest::LoadAndGetFrameProvider(
@@ -1416,7 +1408,7 @@ TEST_P(WebMediaPlayerMSTest, PictureInPictureTriggerCallback) {
   provider->QueueFrames(timestamps);
 
   EXPECT_CALL(*submitter_ptr_, StartRendering());
-  EXPECT_CALL(*this, DisplayType());
+  EXPECT_CALL(*this, DisplayType()).Times(2);
   EXPECT_CALL(*this, DoReadyStateChanged(
                          blink::WebMediaPlayer::kReadyStateHaveMetadata));
   EXPECT_CALL(*this, DoReadyStateChanged(

third_party/blink/renderer/core/frame/picture_in_picture_controller.h

@@ -42,7 +42,6 @@ class CORE_EXPORT PictureInPictureController
     kDisabledBySystem,
     kDisabledByFeaturePolicy,
     kDisabledByAttribute,
-    kVideoLayerNotSupported,
   };
 
   // Enter Picture-in-Picture for a video element and resolve promise if any.

third_party/blink/renderer/core/html/media/html_video_element.cc

@@ -731,12 +731,6 @@ bool HTMLVideoElement::HasPictureInPictureCustomControls() const {
   return !pip_custom_controls_.empty();
 }
 
-bool HTMLVideoElement::UsesSurfaceLayer() const {
-  return GetWebMediaPlayer() &&
-         GetWebMediaPlayer()->GetVideoSurfaceLayerMode() !=
-             WebMediaPlayer::SurfaceLayerMode::kNever;
-}
-
 void HTMLVideoElement::SetIsEffectivelyFullscreen(
     blink::WebFullscreenVideoStatus status) {
   is_effectively_fullscreen_ =

third_party/blink/renderer/core/html/media/html_video_element.h

@@ -189,7 +189,6 @@ class CORE_EXPORT HTMLVideoElement final : public HTMLMediaElement,
   const std::vector<PictureInPictureControlInfo>&
   GetPictureInPictureCustomControls() const;
   bool HasPictureInPictureCustomControls() const;
-  bool UsesSurfaceLayer() const;
 
   void SetIsEffectivelyFullscreen(blink::WebFullscreenVideoStatus);
 

third_party/blink/renderer/modules/picture_in_picture/html_video_element_picture_in_picture.cc

@@ -35,10 +35,6 @@ const char kUserGestureRequired[] =
     "Must be handling a user gesture to request picture in picture.";
 const char kDisablePictureInPicturePresent[] =
     "\"disablePictureInPicture\" attribute is present.";
-const char kVideoLayerNotSupported[] =
-    "The use of VideoLayer for video Picture-in-Picture is not supported. "
-    "Please enable the enable-surfaces-for-videos flag if you wish to use this "
-    "feature.";
 }  // namespace
 
 // static
@@ -79,11 +75,6 @@ ScriptPromise HTMLVideoElementPictureInPicture::requestPictureInPicture(
           script_state,
           DOMException::Create(DOMExceptionCode::kNotSupportedError,
                                kNotAvailable));
-    case Status::kVideoLayerNotSupported:
-      return ScriptPromise::RejectWithDOMException(
-          script_state,
-          DOMException::Create(DOMExceptionCode::kNotSupportedError,
-                               kVideoLayerNotSupported));
     case Status::kEnabled:
       break;
   }

third_party/blink/renderer/modules/picture_in_picture/picture_in_picture_controller_impl.cc

@@ -79,9 +79,6 @@ PictureInPictureControllerImpl::IsElementAllowed(
   if (element.FastHasAttribute(HTMLNames::disablepictureinpictureAttr))
     return Status::kDisabledByAttribute;
 
-  if (!element.UsesSurfaceLayer())
-    return Status::kVideoLayerNotSupported;
-
   return Status::kEnabled;
 }