Gives WebMediaPlayerMSCompositor access to VideoFrameSubmitter.

This CL makes it so that WebMediaPlayerMSCompositor can make proper calls
to a VideoFrameSubmitter to use cc::Surfaces for video instead of the
video layer. With this CL, cc::Surface for Video now works end to end
with WebMediaPlayerMS.

Bug: 746182
Change-Id: I90e100ba06dc7570d6ab10d6dc3862681fe80552
Reviewed-on: https://chromium-review.googlesource.com/1090031
Commit-Queue: CJ DiMeglio <lethalantidote@chromium.org>
Reviewed-by: Emircan Uysaler <emircan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#591163}

content/renderer/media/media_factory.cc

@@ -13,6 +13,7 @@
 #include "base/task_runner_util.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "build/buildflag.h"
+#include "cc/trees/layer_tree_settings.h"
 #include "content/public/common/content_client.h"
 #include "content/public/renderer/content_renderer_client.h"
 #include "content/renderer/media/audio/audio_device_factory.h"
@@ -216,8 +217,8 @@ blink::WebMediaPlayer* MediaFactory::CreateMediaPlayer(
   blink::WebMediaStream web_stream =
       GetWebMediaStreamFromWebMediaPlayerSource(source);
   if (!web_stream.IsNull())
-    return CreateWebMediaPlayerForMediaStream(client, sink_id, security_origin,
-                                              web_frame, layer_tree_view);
+    return CreateWebMediaPlayerForMediaStream(
+        client, sink_id, security_origin, web_frame, layer_tree_view, settings);
 
   // If |source| was not a MediaStream, it must be a URL.
   // TODO(guidou): Fix this when support for other srcObject types is added.
@@ -500,7 +501,8 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
     const blink::WebString& sink_id,
     const blink::WebSecurityOrigin& security_origin,
     blink::WebLocalFrame* frame,
-    blink::WebLayerTreeView* layer_tree_view) {
+    blink::WebLayerTreeView* layer_tree_view,
+    const cc::LayerTreeSettings& settings) {
   RenderThreadImpl* const render_thread = RenderThreadImpl::current();
 
   scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner =
@@ -509,6 +511,12 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
     compositor_task_runner =
         render_frame_->GetTaskRunner(blink::TaskType::kInternalMediaRealTime);
 
+  scoped_refptr<base::SingleThreadTaskRunner>
+      media_stream_compositor_task_runner =
+          VideoSurfaceLayerEnabledForMS()
+              ? render_thread->CreateVideoFrameCompositorTaskRunner()
+              : compositor_task_runner;
+
   DCHECK(layer_tree_view);
   return new WebMediaPlayerMS(
       frame, client, GetWebMediaPlayerDelegate(),
@@ -516,10 +524,17 @@ blink::WebMediaPlayer* MediaFactory::CreateWebMediaPlayerForMediaStream(
           url::Origin(security_origin).GetURL(),
           render_frame_->GetTaskRunner(blink::TaskType::kInternalMedia)),
       CreateMediaStreamRendererFactory(), render_thread->GetIOTaskRunner(),
-      compositor_task_runner, render_thread->GetMediaThreadTaskRunner(),
+      media_stream_compositor_task_runner,
+      render_thread->GetMediaThreadTaskRunner(),
       render_thread->GetWorkerTaskRunner(), render_thread->GetGpuFactories(),
       sink_id,
       base::BindOnce(&blink::WebSurfaceLayerBridge::Create, layer_tree_view),
+      base::BindRepeating(
+          &blink::WebVideoFrameSubmitter::Create,
+          base::BindRepeating(
+              &PostContextProviderToCallback,
+              RenderThreadImpl::current()->GetCompositorMainThreadTaskRunner()),
+          settings),
       VideoSurfaceLayerEnabledForMS());
 }
 

content/renderer/media/media_factory.h

@@ -127,7 +127,8 @@ class MediaFactory {
       const blink::WebString& sink_id,
       const blink::WebSecurityOrigin& security_origin,
       blink::WebLocalFrame* frame,
-      blink::WebLayerTreeView* layer_tree_view);
+      blink::WebLayerTreeView* layer_tree_view,
+      const cc::LayerTreeSettings& settings);
 
   // Returns the media delegate for WebMediaPlayer usage.  If
   // |media_player_delegate_| is NULL, one is created.

content/renderer/media/stream/webmediaplayer_ms.cc

@@ -269,6 +269,8 @@ WebMediaPlayerMS::WebMediaPlayerMS(
     media::GpuVideoAcceleratorFactories* gpu_factories,
     const blink::WebString& sink_id,
     CreateSurfaceLayerBridgeCB create_bridge_callback,
+    base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
+        create_submitter_callback,
     bool surface_layer_for_video_enabled)
     : frame_(frame),
       network_state_(WebMediaPlayer::kNetworkStateEmpty),
@@ -291,6 +293,7 @@ WebMediaPlayerMS::WebMediaPlayerMS(
       volume_multiplier_(1.0),
       should_play_upon_shown_(false),
       create_bridge_callback_(std::move(create_bridge_callback)),
+      create_submitter_callback_(create_submitter_callback),
       surface_layer_for_video_enabled_(surface_layer_for_video_enabled) {
   DVLOG(1) << __func__;
   DCHECK(client);
@@ -349,7 +352,9 @@ blink::WebMediaPlayer::LoadTiming WebMediaPlayerMS::Load(
     web_stream_.AddObserver(this);
 
   compositor_ = new WebMediaPlayerMSCompositor(
-      compositor_task_runner_, io_task_runner_, web_stream_, AsWeakPtr());
+      compositor_task_runner_, io_task_runner_, web_stream_,
+      create_submitter_callback_, surface_layer_for_video_enabled_,
+      AsWeakPtr());
 
   SetNetworkState(WebMediaPlayer::kNetworkStateLoading);
   SetReadyState(WebMediaPlayer::kReadyStateHaveNothing);
@@ -1079,6 +1084,10 @@ bool WebMediaPlayerMS::TexImageImpl(TexImageFunctionID functionID,
   return false;
 }
 
+void WebMediaPlayerMS::OnFrameSinkDestroyed() {
+  bridge_->ClearSurfaceId();
+}
+
 void WebMediaPlayerMS::OnFirstFrameReceived(media::VideoRotation video_rotation,
                                             bool is_opaque) {
   DVLOG(1) << __func__;
@@ -1090,6 +1099,16 @@ void WebMediaPlayerMS::OnFirstFrameReceived(media::VideoRotation video_rotation,
     bridge_ = std::move(create_bridge_callback_)
                   .Run(this, compositor_->GetUpdateSubmissionStateCallback());
     bridge_->CreateSurfaceLayer();
+    bridge_->SetContentsOpaque(opaque_);
+
+    compositor_task_runner_->PostTask(
+        FROM_HERE,
+        base::BindOnce(
+            &WebMediaPlayerMSCompositor::EnableSubmission, compositor_,
+            bridge_->GetSurfaceId(), video_rotation, IsInPictureInPicture(),
+            opaque_,
+            media::BindToCurrentLoop(base::BindRepeating(
+                &WebMediaPlayerMS::OnFrameSinkDestroyed, AsWeakPtr()))));
   }
 
   SetReadyState(WebMediaPlayer::kReadyStateHaveMetadata);
@@ -1112,6 +1131,9 @@ void WebMediaPlayerMS::OnOpacityChanged(bool is_opaque) {
     DCHECK(bridge_);
 
     bridge_->SetContentsOpaque(opaque_);
+    compositor_task_runner_->PostTask(
+        FROM_HERE, base::BindOnce(&WebMediaPlayerMSCompositor::UpdateIsOpaque,
+                                  compositor_, opaque_));
   }
 }
 
@@ -1132,12 +1154,18 @@ void WebMediaPlayerMS::OnRotationChanged(media::VideoRotation video_rotation,
     get_client()->SetCcLayer(new_video_layer.get());
 
     video_layer_ = std::move(new_video_layer);
-  } else if (bridge_->GetCcLayer()) {
-    // TODO(lethalantidote): Handle rotation.
-    bridge_->SetContentsOpaque(opaque_);
+  } else {
+    compositor_task_runner_->PostTask(
+        FROM_HERE, base::BindOnce(&WebMediaPlayerMSCompositor::UpdateRotation,
+                                  compositor_, video_rotation));
   }
 }
 
+bool WebMediaPlayerMS::IsInPictureInPicture() const {
+  // TODO(apacible): Add implementation. See http://crbug/746182.
+  return false;
+}
+
 void WebMediaPlayerMS::RepaintInternal() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());

content/renderer/media/stream/webmediaplayer_ms.h

@@ -27,6 +27,7 @@ namespace blink {
 class WebLocalFrame;
 class WebMediaPlayerClient;
 class WebString;
+class WebVideoFrameSubmitter;
 }
 
 namespace media {
@@ -91,6 +92,8 @@ class CONTENT_EXPORT WebMediaPlayerMS
       media::GpuVideoAcceleratorFactories* gpu_factories,
       const blink::WebString& sink_id,
       CreateSurfaceLayerBridgeCB create_bridge_callback,
+      base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
+          create_submitter_callback,
       bool surface_layer_for_video_enabled_);
 
   ~WebMediaPlayerMS() override;
@@ -237,11 +240,18 @@ class CONTENT_EXPORT WebMediaPlayerMS
   static const gfx::Size kUseGpuMemoryBufferVideoFramesMinResolution;
 #endif  // defined(OS_WIN)
 
+  // When we lose the context_provider, we destroy the CompositorFrameSink to
+  // prevent frames from being submitted. The current surface_ids become
+  // invalid.
+  void OnFrameSinkDestroyed();
+
   void OnFirstFrameReceived(media::VideoRotation video_rotation,
                             bool is_opaque);
   void OnOpacityChanged(bool is_opaque);
   void OnRotationChanged(media::VideoRotation video_rotation, bool is_opaque);
 
+  bool IsInPictureInPicture() const;
+
   // Need repaint due to state change.
   void RepaintInternal();
 
@@ -309,6 +319,7 @@ class CONTENT_EXPORT WebMediaPlayerMS
   const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
   const scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner_;
   const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
+
   const scoped_refptr<base::TaskRunner> worker_task_runner_;
   media::GpuVideoAcceleratorFactories* gpu_factories_;
 
@@ -336,6 +347,9 @@ class CONTENT_EXPORT WebMediaPlayerMS
 
   CreateSurfaceLayerBridgeCB create_bridge_callback_;
 
+  base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
+      create_submitter_callback_;
+
   // Whether the use of a surface layer instead of a video layer is enabled.
   bool surface_layer_for_video_enabled_ = false;
 

content/renderer/media/stream/webmediaplayer_ms_compositor.h

@@ -11,7 +11,7 @@
 #include <memory>
 #include <vector>
 
-#include "base/memory/ref_counted.h"
+#include "base/memory/ref_counted_delete_on_sequence.h"
 #include "base/memory/weak_ptr.h"
 #include "base/message_loop/message_loop.h"
 #include "base/synchronization/lock.h"
@@ -20,6 +20,7 @@
 #include "cc/layers/video_frame_provider.h"
 #include "content/common/content_export.h"
 #include "media/base/media_log.h"
+#include "third_party/blink/public/platform/web_video_frame_submitter.h"
 
 namespace base {
 class SingleThreadTaskRunner;
@@ -37,6 +38,10 @@ namespace media {
 class VideoRendererAlgorithm;
 }
 
+namespace viz {
+class SurfaceId;
+}
+
 namespace content {
 class WebMediaPlayerMS;
 
@@ -51,16 +56,19 @@ class WebMediaPlayerMS;
 // frame, and submit it whenever asked by the compositor.
 class CONTENT_EXPORT WebMediaPlayerMSCompositor
     : public cc::VideoFrameProvider,
-      public base::RefCountedThreadSafe<WebMediaPlayerMSCompositor> {
+      public base::RefCountedDeleteOnSequence<WebMediaPlayerMSCompositor> {
  public:
   // This |url| represents the media stream we are rendering. |url| is used to
   // find out what web stream this WebMediaPlayerMSCompositor is playing, and
   // together with flag "--disable-rtc-smoothness-algorithm" determine whether
   // we enable algorithm or not.
   WebMediaPlayerMSCompositor(
-      scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner,
+      scoped_refptr<base::SingleThreadTaskRunner> task_runner,
       scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
       const blink::WebMediaStream& web_stream,
+      base::RepeatingCallback<std::unique_ptr<blink::WebVideoFrameSubmitter>()>
+          create_submitter_callback,
+      bool surface_layer_for_video_enabled,
       const base::WeakPtr<WebMediaPlayerMS>& player);
 
   // Can be called from any thread.
@@ -76,6 +84,24 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
   size_t total_frame_count();
   size_t dropped_frame_count();
 
+  // Signals the VideoFrameSubmitter to prepare to receive BeginFrames and
+  // submit video frames given by WebMediaPlayerMSCompositor.
+  virtual void EnableSubmission(
+      const viz::SurfaceId& id,
+      media::VideoRotation rotation,
+      bool force_submit,
+      bool is_opaque,
+      blink::WebFrameSinkDestroyedCallback frame_sink_destroyed_callback);
+
+  // Updates the rotation information for frames given to |submitter_|.
+  void UpdateRotation(media::VideoRotation rotation);
+
+  // Notifies the |submitter_| that the frames must be submitted.
+  void SetForceSubmit(bool);
+
+  // Updates the opacity information for frames given to |submitter_|.
+  void UpdateIsOpaque(bool);
+
   // VideoFrameProvider implementation.
   void SetVideoFrameProviderClient(
       cc::VideoFrameProvider::Client* client) override;
@@ -101,11 +127,19 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
   void StopUsingProvider();
 
  private:
-  friend class base::RefCountedThreadSafe<WebMediaPlayerMSCompositor>;
+  friend class base::RefCountedDeleteOnSequence<WebMediaPlayerMSCompositor>;
+  friend class base::DeleteHelper<WebMediaPlayerMSCompositor>;
   friend class WebMediaPlayerMSTest;
 
   ~WebMediaPlayerMSCompositor() override;
 
+  // Ran on the |video_frame_compositor_task_runner_| to initialize
+  // |submitter_|
+  void InitializeSubmitter();
+
+  // Signals the VideoFrameSubmitter to stop submitting frames.
+  void UpdateSubmissionState(bool);
+
   bool MapTimestampsToRenderTimeTicks(
       const std::vector<base::TimeDelta>& timestamps,
       std::vector<base::TimeTicks>* wall_clock_times);
@@ -138,7 +172,8 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
   // which is renderer main thread in this class.
   base::ThreadChecker thread_checker_;
 
-  const scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner_;
+  const scoped_refptr<base::SingleThreadTaskRunner>
+      video_frame_compositor_task_runner_;
   const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
   base::MessageLoop* main_message_loop_;
 
@@ -186,6 +221,8 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
   bool stopped_;
   bool render_started_;
 
+  std::unique_ptr<blink::WebVideoFrameSubmitter> submitter_;
+
   std::map<base::TimeDelta, base::TimeTicks> timestamps_to_clock_times_;
 
   cc::UpdateSubmissionStateCB update_submission_state_callback_;
@@ -194,6 +231,8 @@ class CONTENT_EXPORT WebMediaPlayerMSCompositor
   // |dropped_frame_count_|, and |render_started_|.
   base::Lock current_frame_lock_;
 
+  base::WeakPtrFactory<WebMediaPlayerMSCompositor> weak_ptr_factory_;
+
   DISALLOW_COPY_AND_ASSIGN(WebMediaPlayerMSCompositor);
 };
 }  // namespace content