[video-raf] Add WebMediaPlayerMS::RequestAnimationFrame()

Callbacks passed to video.requestAnimationFrame are never called when
using webRTC. This is because webRTC uses WebMediaPlayerMS instead of
WebMediaPlayerImpl, and it does not override the RequestAnimationFrame()
method. This CL implements the method and fixes the issue.

Bug: 1035028, 1012063
Change-Id: I1792dfd6f6077e6e49c39f23e607e7868611293e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1987069
Commit-Queue: Thomas Guilbert <tguilbert@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732265}

third_party/blink/public/web/modules/mediastream/webmediaplayer_ms.h

@@ -234,6 +234,8 @@ class BLINK_MODULES_EXPORT WebMediaPlayerMS
 
   void OnDisplayTypeChanged(WebMediaPlayer::DisplayType) override;
 
+  void RequestAnimationFrame() override;
+
  private:
   friend class WebMediaPlayerMSTest;
 
@@ -266,6 +268,17 @@ class BLINK_MODULES_EXPORT WebMediaPlayerMS
   void SetGpuMemoryBufferVideoForTesting(
       media::GpuMemoryBufferVideoFramePool* gpu_memory_buffer_pool);
 
+  // Returns |compositor_|'s current frame, or |current_frame_override_| if we
+  // are in the middle of a rAF callback.
+  scoped_refptr<media::VideoFrame> GetCurrentFrame() const;
+
+  // Callback used to fulfill video.requestAnimationFrame() requests.
+  void OnNewFramePresentedCallback(
+      scoped_refptr<media::VideoFrame> presented_frame,
+      base::TimeTicks presentation_time,
+      base::TimeTicks expected_presentation_time,
+      uint32_t presentation_counter);
+
   std::unique_ptr<MediaStreamInternalFrameWrapper> internal_frame_;
 
   WebMediaPlayer::NetworkState network_state_;
@@ -301,6 +314,15 @@ class BLINK_MODULES_EXPORT WebMediaPlayerMS
   scoped_refptr<WebMediaStreamAudioRenderer> audio_renderer_;  // Weak
   media::PaintCanvasVideoRenderer video_renderer_;
 
+  // Indicated whether an outstanding rAF request needs to be forwarded to
+  // |compositor_|. Set when RequestAnimationFrame() is called before Load().
+  bool pending_raf_request_ = false;
+
+  // Takes precedence over the compositor's current frame when painting or
+  // copying frames. Only set when we are in the middle of executing a
+  // video.requestAnimationFrame() callback.
+  scoped_refptr<media::VideoFrame> current_frame_override_;
+
   bool paused_;
   media::VideoTransformation video_transformation_;
 

third_party/blink/renderer/modules/mediastream/webmediaplayer_ms.cc

@@ -348,6 +348,14 @@ WebMediaPlayer::LoadTiming WebMediaPlayerMS::Load(
       compositor_task_runner_, io_task_runner_, web_stream_,
       std::move(submitter_), surface_layer_mode_, weak_this_);
 
+  // We can receive a call to RequestAnimationFrame() before |compositor_| is
+  // created. In that case, we suspend the request, and wait until now to
+  // reiniate it.
+  if (pending_raf_request_) {
+    RequestAnimationFrame();
+    pending_raf_request_ = false;
+  }
+
   SetNetworkState(WebMediaPlayer::kNetworkStateLoading);
   SetReadyState(WebMediaPlayer::kReadyStateHaveNothing);
   std::string stream_id =
@@ -731,7 +739,7 @@ WebSize WebMediaPlayerMS::NaturalSize() const {
 
 WebSize WebMediaPlayerMS::VisibleRect() const {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
-  scoped_refptr<media::VideoFrame> video_frame = compositor_->GetCurrentFrame();
+  scoped_refptr<media::VideoFrame> video_frame = GetCurrentFrame();
   if (!video_frame)
     return WebSize();
 
@@ -758,6 +766,11 @@ double WebMediaPlayerMS::Duration() const {
   return std::numeric_limits<double>::infinity();
 }
 
+scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() const {
+  return current_frame_override_ ? current_frame_override_
+                                 : compositor_->GetCurrentFrame();
+}
+
 double WebMediaPlayerMS::CurrentTime() const {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
   const base::TimeDelta current_time = compositor_->GetCurrentTime();
@@ -812,7 +825,7 @@ void WebMediaPlayerMS::Paint(cc::PaintCanvas* canvas,
   DVLOG(3) << __func__;
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
 
-  const scoped_refptr<media::VideoFrame> frame = compositor_->GetCurrentFrame();
+  const scoped_refptr<media::VideoFrame> frame = GetCurrentFrame();
 
   viz::ContextProvider* provider = nullptr;
   if (frame && frame->HasTextures()) {
@@ -980,7 +993,7 @@ bool WebMediaPlayerMS::CopyVideoTextureToPlatformTexture(
   TRACE_EVENT0("media", "copyVideoTextureToPlatformTexture");
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
 
-  scoped_refptr<media::VideoFrame> video_frame = compositor_->GetCurrentFrame();
+  scoped_refptr<media::VideoFrame> video_frame = GetCurrentFrame();
 
   if (!video_frame.get() || !video_frame->HasTextures())
     return false;
@@ -1010,7 +1023,7 @@ bool WebMediaPlayerMS::CopyVideoYUVDataToPlatformTexture(
   TRACE_EVENT0("media", "copyVideoYUVDataToPlatformTexture");
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
 
-  scoped_refptr<media::VideoFrame> video_frame = compositor_->GetCurrentFrame();
+  scoped_refptr<media::VideoFrame> video_frame = GetCurrentFrame();
 
   if (!video_frame)
     return false;
@@ -1043,8 +1056,7 @@ bool WebMediaPlayerMS::TexImageImpl(TexImageFunctionID functionID,
   TRACE_EVENT0("media", "texImageImpl");
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
 
-  const scoped_refptr<media::VideoFrame> video_frame =
-      compositor_->GetCurrentFrame();
+  const scoped_refptr<media::VideoFrame> video_frame = GetCurrentFrame();
   if (!video_frame || !video_frame->IsMappable() ||
       video_frame->HasTextures() ||
       video_frame->format() != media::PIXEL_FORMAT_Y16) {
@@ -1217,4 +1229,44 @@ void WebMediaPlayerMS::OnDisplayTypeChanged(
           display_type == WebMediaPlayer::DisplayType::kPictureInPicture));
 }
 
+void WebMediaPlayerMS::OnNewFramePresentedCallback(
+    scoped_refptr<media::VideoFrame> presented_frame,
+    base::TimeTicks presentation_time,
+    base::TimeTicks expected_presentation_time,
+    uint32_t presentation_counter) {
+  if (!main_render_task_runner_->BelongsToCurrentThread()) {
+    PostCrossThreadTask(
+        *main_render_task_runner_, FROM_HERE,
+        CrossThreadBindOnce(&WebMediaPlayerMS::OnNewFramePresentedCallback,
+                            weak_this_, presented_frame, presentation_time,
+                            expected_presentation_time, presentation_counter));
+    return;
+  }
+
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+  current_frame_override_ = std::move(presented_frame);
+  client_->OnRequestAnimationFrame(
+      presentation_time, expected_presentation_time, presentation_counter,
+      *current_frame_override_);
+  current_frame_override_.reset();
+}
+
+void WebMediaPlayerMS::RequestAnimationFrame() {
+  DCHECK(RuntimeEnabledFeatures::VideoRequestAnimationFrameEnabled());
+  DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
+
+  if (!compositor_) {
+    // Reissue the request after |compositor_| is created, in Load().
+    pending_raf_request_ = true;
+    return;
+  }
+
+  PostCrossThreadTask(
+      *compositor_task_runner_, FROM_HERE,
+      CrossThreadBindOnce(
+          &WebMediaPlayerMSCompositor::SetOnFramePresentedCallback, compositor_,
+          CrossThreadBindOnce(&WebMediaPlayerMS::OnNewFramePresentedCallback,
+                              weak_this_)));
+}
+
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.cc

@@ -506,7 +506,7 @@ void WebMediaPlayerMSCompositor::RenderUsingAlgorithm(
     return;
 
   const base::TimeDelta timestamp = frame->timestamp();
-  SetCurrentFrame(std::move(frame));
+  SetCurrentFrame(std::move(frame), deadline_min);
 
   const auto& end = timestamps_to_clock_times_.end();
   const auto& begin = timestamps_to_clock_times_.begin();
@@ -533,14 +533,15 @@ void WebMediaPlayerMSCompositor::RenderWithoutAlgorithmOnCompositor(
     base::AutoLock auto_lock(current_frame_lock_);
     if (current_frame_)
       last_render_length_ = frame->timestamp() - current_frame_->timestamp();
-    SetCurrentFrame(std::move(frame));
+    SetCurrentFrame(std::move(frame), base::nullopt);
   }
   if (video_frame_provider_client_)
     video_frame_provider_client_->DidReceiveFrame();
 }
 
 void WebMediaPlayerMSCompositor::SetCurrentFrame(
-    scoped_refptr<media::VideoFrame> frame) {
+    scoped_refptr<media::VideoFrame> frame,
+    base::Optional<base::TimeTicks> expected_presentation_time) {
   DCHECK(video_frame_compositor_task_runner_->BelongsToCurrentThread());
   current_frame_lock_.AssertAcquired();
   TRACE_EVENT_INSTANT1("media", "WebMediaPlayerMSCompositor::SetCurrentFrame",
@@ -587,19 +588,26 @@ void WebMediaPlayerMSCompositor::SetCurrentFrame(
 
   current_frame_ = std::move(frame);
 
+  base::TimeTicks now = base::TimeTicks::Now();
+
   // Complete the checks after |current_frame_| is accessible to avoid
   // deadlocks, see https://crbug.com/901744.
   PostCrossThreadTask(
       *video_frame_compositor_task_runner_, FROM_HERE,
       CrossThreadBindOnce(&WebMediaPlayerMSCompositor::CheckForFrameChanges,
                           WrapRefCounted(this), is_first_frame,
-                          has_frame_size_changed, std::move(new_rotation),
-                          std::move(new_opacity)));
+                          has_frame_size_changed, now,
+                          expected_presentation_time.value_or(now),
+                          static_cast<int>(total_frame_count_),
+                          std::move(new_rotation), std::move(new_opacity)));
 }
 
 void WebMediaPlayerMSCompositor::CheckForFrameChanges(
     bool is_first_frame,
     bool has_frame_size_changed,
+    base::TimeTicks presentation_time,
+    base::TimeTicks expected_presentation_time,
+    int frame_count,
     base::Optional<media::VideoRotation> new_frame_rotation,
     base::Optional<bool> new_frame_opacity) {
   DCHECK(video_frame_compositor_task_runner_->BelongsToCurrentThread());
@@ -609,8 +617,16 @@ void WebMediaPlayerMSCompositor::CheckForFrameChanges(
         *main_task_runner_, FROM_HERE,
         CrossThreadBindOnce(&WebMediaPlayerMS::OnFirstFrameReceived, player_,
                             *new_frame_rotation, *new_frame_opacity));
+
+    // Complete rAF requests before returning.
+    if (new_frame_presented_cb_) {
+      std::move(new_frame_presented_cb_)
+          .Run(current_frame_, presentation_time, expected_presentation_time,
+               frame_count);
+    }
     return;
   }
+
   if (new_frame_rotation.has_value()) {
     PostCrossThreadTask(
         *main_task_runner_, FROM_HERE,
@@ -632,6 +648,12 @@ void WebMediaPlayerMSCompositor::CheckForFrameChanges(
   PostCrossThreadTask(
       *main_task_runner_, FROM_HERE,
       CrossThreadBindOnce(&WebMediaPlayerMS::ResetCanvasCache, player_));
+
+  if (new_frame_presented_cb_) {
+    std::move(new_frame_presented_cb_)
+        .Run(current_frame_, presentation_time, expected_presentation_time,
+             frame_count);
+  }
 }
 
 void WebMediaPlayerMSCompositor::StartRenderingInternal() {
@@ -707,4 +729,10 @@ void WebMediaPlayerMSCompositor::SetAlgorithmEnabledForTesting(
   }
 }
 
+void WebMediaPlayerMSCompositor::SetOnFramePresentedCallback(
+    OnNewFramePresentedCB presented_cb) {
+  DCHECK(video_frame_compositor_task_runner_->BelongsToCurrentThread());
+  new_frame_presented_cb_ = std::move(presented_cb);
+}
+
 }  // namespace blink

third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.h

@@ -21,6 +21,7 @@
 #include "third_party/blink/public/platform/web_media_player.h"
 #include "third_party/blink/public/platform/web_video_frame_submitter.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
+#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h"
 #include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
 
 namespace base {
@@ -58,6 +59,12 @@ class MODULES_EXPORT WebMediaPlayerMSCompositor
       public WTF::ThreadSafeRefCounted<WebMediaPlayerMSCompositor,
                                        WebMediaPlayerMSCompositorTraits> {
  public:
+  using OnNewFramePresentedCB = CrossThreadOnceFunction<void(
+      scoped_refptr<media::VideoFrame> presented_frame,
+      base::TimeTicks presentation_time,
+      base::TimeTicks expected_presentation_time,
+      uint32_t presentation_counter)>;
+
   WebMediaPlayerMSCompositor(
       scoped_refptr<base::SingleThreadTaskRunner> task_runner,
       scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
@@ -111,6 +118,10 @@ class MODULES_EXPORT WebMediaPlayerMSCompositor
   // preparation for dtor.
   void StopUsingProvider();
 
+  // Sets a hook to be notified when a new frame is presented, to fulfill a
+  // prending video.requestAnimationFrame() request.
+  void SetOnFramePresentedCallback(OnNewFramePresentedCB presented_cb);
+
  private:
   friend class WTF::ThreadSafeRefCounted<WebMediaPlayerMSCompositor,
                                          WebMediaPlayerMSCompositorTraits>;
@@ -148,12 +159,17 @@ class MODULES_EXPORT WebMediaPlayerMSCompositor
       scoped_refptr<media::VideoFrame> frame);
 
   // Update |current_frame_| and |dropped_frame_count_|
-  void SetCurrentFrame(scoped_refptr<media::VideoFrame> frame);
+  void SetCurrentFrame(
+      scoped_refptr<media::VideoFrame> frame,
+      base::Optional<base::TimeTicks> expected_presentation_time);
   // Following the update to |current_frame_|, this will check for changes that
   // require updating video layer.
   void CheckForFrameChanges(
       bool is_first_frame,
       bool has_frame_size_changed,
+      base::TimeTicks presentation_time,
+      base::TimeTicks expected_presentation_time,
+      int frame_count,
       base::Optional<media::VideoRotation> new_frame_rotation,
       base::Optional<bool> new_frame_opacity);
 
@@ -217,6 +233,11 @@ class MODULES_EXPORT WebMediaPlayerMSCompositor
   bool stopped_;
   bool render_started_;
 
+  // Called when a new frame is enqueued, either in RenderWithoutAlgorithm() or
+  // in RenderUsingAlgorithm(). Used to fulfill video.requestAnimationFrame()
+  // requests.
+  OnNewFramePresentedCB new_frame_presented_cb_;
+
   std::unique_ptr<WebVideoFrameSubmitter> submitter_;
 
   // TODO(crbug.com/952716): Replace the use of std::map by WTF::HashMap.

third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc

@@ -39,7 +39,9 @@ using ::testing::Eq;
 using ::testing::NiceMock;
 using ::testing::Return;
 using ::testing::ReturnRef;
+using ::testing::SaveArg;
 using ::testing::StrictMock;
+using ::testing::WithArgs;
 
 namespace blink {
 
@@ -591,6 +593,25 @@ class WebMediaPlayerMSTest
   void RequestPause() override {}
   void RequestMuted(bool muted) override {}
   Features GetFeatures() override { return Features(); }
+  void OnRequestAnimationFrame(
+      base::TimeTicks presentation_time,
+      base::TimeTicks expected_presentation_time,
+      uint32_t presentation_counter,
+      const media::VideoFrame& presented_frame) override {
+    // Check if we should queue another RAF call.
+    if (chained_raf_call_count_) {
+      --chained_raf_call_count_;
+      player_->RequestAnimationFrame();
+    }
+
+    // Verify the frame size here because there is no good way of capturing
+    // a const media::VideoFrame& using mocks.
+    EXPECT_EQ(presented_frame.visible_rect().height(), kStandardHeight);
+    EXPECT_EQ(presented_frame.visible_rect().width(), kStandardWidth);
+
+    DoOnRequestAnimationFrame(presentation_time, expected_presentation_time,
+                              presentation_counter, presented_frame);
+  }
 
   // Implementation of cc::VideoFrameProvider::Client
   void StopUsingProvider() override;
@@ -627,6 +648,12 @@ class WebMediaPlayerMSTest
   MOCK_CONST_METHOD0(DisplayType, WebMediaPlayer::DisplayType());
   MOCK_CONST_METHOD0(CouldPlayIfEnoughData, bool());
 
+  MOCK_METHOD4(DoOnRequestAnimationFrame,
+               void(base::TimeTicks,
+                    base::TimeTicks,
+                    uint32_t,
+                    const media::VideoFrame&));
+
   std::unique_ptr<WebSurfaceLayerBridge> CreateMockSurfaceLayerBridge(
       WebSurfaceLayerBridgeObserver*,
       cc::UpdateSubmissionStateCB) {
@@ -647,6 +674,7 @@ class WebMediaPlayerMSTest
   NiceMock<MockSurfaceLayerBridge>* surface_layer_bridge_ptr_ = nullptr;
   NiceMock<MockWebVideoFrameSubmitter>* submitter_ptr_ = nullptr;
   bool enable_surface_layer_for_video_ = false;
+  int chained_raf_call_count_ = 0;
 
  private:
   // Main function trying to ask WebMediaPlayerMS to submit a frame for
@@ -1364,6 +1392,63 @@ TEST_P(WebMediaPlayerMSTest, HiddenPlayerTests) {
 }
 #endif
 
+TEST_P(WebMediaPlayerMSTest, RequestAnimationFrame) {
+  InitializeWebMediaPlayerMS();
+
+  MockMediaStreamVideoRenderer* provider = LoadAndGetFrameProvider(true);
+
+  const int kTestBrake = static_cast<int>(FrameType::TEST_BRAKE);
+  int tokens[] = {0,          33,  kTestBrake, 66,  100, 133, 166,
+                  kTestBrake, 200, 233,        266, 300, 333};
+  std::vector<int> timestamps(tokens, tokens + sizeof(tokens) / sizeof(int));
+  provider->QueueFrames(timestamps);
+
+  base::TimeTicks presentation_time;
+  base::TimeTicks expected_presentation_time;
+
+  // Verify a basic call to RAF.
+  player_->RequestAnimationFrame();
+  EXPECT_CALL(*this, DoOnRequestAnimationFrame(_, _, _, _))
+      .Times(1)
+      .WillOnce(DoAll(SaveArg<0>(&presentation_time),
+                      SaveArg<1>(&expected_presentation_time)));
+  message_loop_controller_.RunAndWaitForStatus(
+      media::PipelineStatus::PIPELINE_OK);
+
+  EXPECT_GT(presentation_time, base::TimeTicks());
+  EXPECT_GE(expected_presentation_time, presentation_time);
+  testing::Mock::VerifyAndClearExpectations(this);
+
+  // Make sure multiple calls to RAF only result in one call per frame to OnRAF.
+  player_->RequestAnimationFrame();
+  player_->RequestAnimationFrame();
+  player_->RequestAnimationFrame();
+
+  EXPECT_CALL(*this, DoOnRequestAnimationFrame(_, _, _, _)).Times(1);
+  message_loop_controller_.RunAndWaitForStatus(
+      media::PipelineStatus::PIPELINE_OK);
+  testing::Mock::VerifyAndClearExpectations(this);
+
+  // Make sure we can chain calls to RAF.
+  player_->RequestAnimationFrame();
+  chained_raf_call_count_ = 3;
+
+  // Verify that the presentation frame counter is monotonically increasing.
+  // NOTE: 0 is fine here as a starting value, since the counter should already
+  // be >0, because of the calls above.
+  uint32_t last_frame_counter = 0;
+  EXPECT_CALL(*this, DoOnRequestAnimationFrame(_, _, _, _))
+      .Times(chained_raf_call_count_ + 1)
+      .WillRepeatedly(WithArgs<2>([&](uint32_t frame_counter) {
+        EXPECT_GT(frame_counter, last_frame_counter);
+        last_frame_counter = frame_counter;
+      }));
+
+  message_loop_controller_.RunAndWaitForStatus(
+      media::PipelineStatus::PIPELINE_OK);
+  testing::Mock::VerifyAndClearExpectations(this);
+}
+
 INSTANTIATE_TEST_SUITE_P(All,
                          WebMediaPlayerMSTest,
                          ::testing::Combine(::testing::Bool(),