PageHandler and Viz::FrameSinkVideoCapturer

In Viz, PageHandler will no longer know that a CompositorFrame has been submitted.
This CL makes use of FrameSinkVideoCapturerImpl for screencasting, so that PageHandler
doesn't need to rely on knowing about CompositorFrame submissions.

Some information from CompositorFrameMetadata is required for screencasting. This is
sent to PageHandler by FrameSinkVideoCapturerImpl along with each frame.

Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel
Change-Id: Ie7a191944579bb7f6dda0e5384e5955764c24850
Reviewed-on: https://chromium-review.googlesource.com/969711
Commit-Queue: Jonathan Ross <jonross@chromium.org>
Reviewed-by: Xiaohan Wang <xhwang@chromium.org>
Reviewed-by: Jonathan Ross <jonross@chromium.org>
Reviewed-by: Dmitry Gozman <dgozman@chromium.org>
Reviewed-by: Yuri Wiitala <miu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#550444}

components/viz/service/frame_sinks/compositor_frame_sink_support.cc

@@ -554,11 +554,12 @@ const char* CompositorFrameSinkSupport::GetSubmitResultAsString(
 
 void CompositorFrameSinkSupport::OnAggregatedDamage(
     const LocalSurfaceId& local_surface_id,
-    const gfx::Size& frame_size_in_pixels,
+    const CompositorFrame& frame,
     const gfx::Rect& damage_rect,
     base::TimeTicks expected_display_time) const {
   DCHECK(!damage_rect.IsEmpty());
 
+  const gfx::Size& frame_size_in_pixels = frame.size_in_pixels();
   if (aggregated_damage_callback_) {
     aggregated_damage_callback_.Run(local_surface_id, frame_size_in_pixels,
                                     damage_rect, expected_display_time);
@@ -566,7 +567,7 @@ void CompositorFrameSinkSupport::OnAggregatedDamage(
 
   for (CapturableFrameSink::Client* client : capture_clients_) {
     client->OnFrameDamaged(frame_size_in_pixels, damage_rect,
-                           expected_display_time);
+                           expected_display_time, frame.metadata);
   }
 }
 

components/viz/service/frame_sinks/compositor_frame_sink_support.h

@@ -183,7 +183,7 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport
   Surface* CreateSurface(const SurfaceInfo& surface_info);
 
   void OnAggregatedDamage(const LocalSurfaceId& local_surface_id,
-                          const gfx::Size& frame_size_in_pixels,
+                          const CompositorFrame& frame,
                           const gfx::Rect& damage_rect,
                           base::TimeTicks expected_display_time) const;
 

components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h

@@ -35,9 +35,11 @@ class CapturableFrameSink {
     // |damage_rect| being the region within that has changed (never empty).
     // |expected_display_time| indicates when the content change was expected to
     // appear on the Display.
-    virtual void OnFrameDamaged(const gfx::Size& frame_size,
-                                const gfx::Rect& damage_rect,
-                                base::TimeTicks expected_display_time) = 0;
+    virtual void OnFrameDamaged(
+        const gfx::Size& frame_size,
+        const gfx::Rect& damage_rect,
+        base::TimeTicks expected_display_time,
+        const CompositorFrameMetadata& frame_metadata) = 0;
   };
 
   virtual ~CapturableFrameSink() = default;

components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.cc

@@ -46,6 +46,9 @@ constexpr media::VideoPixelFormat
 // static
 constexpr media::ColorSpace FrameSinkVideoCapturerImpl::kDefaultColorSpace;
 
+// static
+constexpr gfx::Vector2dF FrameSinkVideoCapturerImpl::kDefaultRootScrollOffset;
+
 FrameSinkVideoCapturerImpl::FrameSinkVideoCapturerImpl(
     FrameSinkVideoCapturerManager* frame_sink_manager,
     mojom::FrameSinkVideoCapturerRequest request)
@@ -59,6 +62,11 @@ FrameSinkVideoCapturerImpl::FrameSinkVideoCapturerImpl(
       capture_weak_factory_(this) {
   DCHECK(frame_sink_manager_);
 
+  // Initialize variables of |last_frame_metadata_| with default values.
+  last_frame_metadata_.device_scale_factor = kDefaultDeviceScaleFactor;
+  last_frame_metadata_.page_scale_factor = kDefaultPageScaleFactor;
+  last_frame_metadata_.root_scroll_offset = kDefaultRootScrollOffset;
+
   // Instantiate a default base::OneShotTimer instance.
   refresh_frame_retry_timer_.emplace();
 
@@ -311,19 +319,24 @@ void FrameSinkVideoCapturerImpl::RefreshSoon() {
   }
 
   MaybeCaptureFrame(VideoCaptureOracle::kRefreshRequest,
-                    gfx::Rect(oracle_.source_size()), clock_->NowTicks());
+                    gfx::Rect(oracle_.source_size()), clock_->NowTicks(),
+                    last_frame_metadata_);
 }
 
 void FrameSinkVideoCapturerImpl::OnFrameDamaged(
     const gfx::Size& frame_size,
     const gfx::Rect& damage_rect,
-    base::TimeTicks expected_display_time) {
+    base::TimeTicks expected_display_time,
+    const CompositorFrameMetadata& frame_metadata) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
   DCHECK(!frame_size.IsEmpty());
   DCHECK(!damage_rect.IsEmpty());
   DCHECK(!expected_display_time.is_null());
   DCHECK(resolved_target_);
 
+  // Cache metadata so that it can be used for refresh frames.
+  last_frame_metadata_ = frame_metadata.Clone();
+
   if (frame_size == oracle_.source_size()) {
     dirty_rect_.Union(damage_rect);
   } else {
@@ -332,13 +345,14 @@ void FrameSinkVideoCapturerImpl::OnFrameDamaged(
   }
 
   MaybeCaptureFrame(VideoCaptureOracle::kCompositorUpdate, damage_rect,
-                    expected_display_time);
+                    expected_display_time, frame_metadata);
 }
 
 void FrameSinkVideoCapturerImpl::MaybeCaptureFrame(
     VideoCaptureOracle::Event event,
     const gfx::Rect& damage_rect,
-    base::TimeTicks event_time) {
+    base::TimeTicks event_time,
+    const CompositorFrameMetadata& frame_metadata) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
   DCHECK(resolved_target_);
 
@@ -436,6 +450,14 @@ void FrameSinkVideoCapturerImpl::MaybeCaptureFrame(
   metadata->SetDouble(VideoFrameMetadata::FRAME_RATE,
                       1.0 / oracle_.min_capture_period().InSecondsF());
   metadata->SetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, event_time);
+  metadata->SetDouble(VideoFrameMetadata::DEVICE_SCALE_FACTOR,
+                      frame_metadata.device_scale_factor);
+  metadata->SetDouble(VideoFrameMetadata::PAGE_SCALE_FACTOR,
+                      frame_metadata.page_scale_factor);
+  metadata->SetDouble(VideoFrameMetadata::ROOT_SCROLL_OFFSET_X,
+                      frame_metadata.root_scroll_offset.x());
+  metadata->SetDouble(VideoFrameMetadata::ROOT_SCROLL_OFFSET_Y,
+                      frame_metadata.root_scroll_offset.y());
 
   oracle_.RecordCapture(utilization);
   const int64_t frame_number = next_capture_frame_number_++;

components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h

@@ -17,6 +17,8 @@
 #include "base/time/time.h"
 #include "base/timer/timer.h"
 #include "base/unguessable_token.h"
+#include "components/viz/common/frame_sinks/begin_frame_args.h"
+#include "components/viz/common/quads/compositor_frame_metadata.h"
 #include "components/viz/common/surfaces/frame_sink_id.h"
 #include "components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h"
 #include "components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h"
@@ -27,6 +29,7 @@
 #include "mojo/public/cpp/bindings/binding.h"
 #include "services/viz/privileged/interfaces/compositing/frame_sink_video_capture.mojom.h"
 #include "ui/gfx/geometry/rect.h"
+#include "ui/gfx/geometry/vector2d_f.h"
 
 namespace gfx {
 class Size;
@@ -108,6 +111,12 @@ class VIZ_SERVICE_EXPORT FrameSinkVideoCapturerImpl final
   static constexpr media::ColorSpace kDefaultColorSpace =
       media::COLOR_SPACE_HD_REC709;
 
+  // Default values for variables in |last_frame_metadata_|.
+  static constexpr float kDefaultDeviceScaleFactor = 1.f;
+  static constexpr float kDefaultPageScaleFactor = 1.f;
+  static constexpr gfx::Vector2dF kDefaultRootScrollOffset =
+      gfx::Vector2dF(0, 0);
+
   // The maximum number of frames in-flight in the capture pipeline, reflecting
   // the storage capacity dedicated for this purpose. Example numbers, for a
   // frame pool that is fully-allocated with 10 frames of size 1920x1080, using
@@ -156,14 +165,16 @@ class VIZ_SERVICE_EXPORT FrameSinkVideoCapturerImpl final
   // CapturableFrameSink::Client implementation:
   void OnFrameDamaged(const gfx::Size& frame_size,
                       const gfx::Rect& damage_rect,
-                      base::TimeTicks target_display_time) final;
+                      base::TimeTicks target_display_time,
+                      const CompositorFrameMetadata& frame_metadata) final;
 
   // Consults the VideoCaptureOracle to decide whether to capture a frame,
   // then ensures prerequisites are met before initiating the capture: that
   // there is a consumer present and that the pipeline is not already full.
   void MaybeCaptureFrame(media::VideoCaptureOracle::Event event,
                          const gfx::Rect& damage_rect,
-                         base::TimeTicks event_time);
+                         base::TimeTicks event_time,
+                         const CompositorFrameMetadata& frame_metadata);
 
   // Extracts the image data from the copy output |result|, populating the
   // |content_rect| region of a [possibly letterboxed] video |frame|.
@@ -233,6 +244,9 @@ class VIZ_SERVICE_EXPORT FrameSinkVideoCapturerImpl final
   // captured.
   gfx::Rect dirty_rect_;
 
+  // A cache of CompositorFrameMetadata used when a refresh frame is requested.
+  CompositorFrameMetadata last_frame_metadata_;
+
   // These are sequence counters used to ensure that the frames are being
   // delivered in the same order they are captured.
   int64_t next_capture_frame_number_ = 0;

components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl_unittest.cc

@@ -39,6 +39,29 @@ using testing::Return;
 namespace viz {
 namespace {
 
+// Returns true if |frame|'s device scale factor, page scale factor and root
+// scroll offset are equal to the expected values.
+bool CompareVarsInCompositorFrameMetadata(
+    const VideoFrame& frame,
+    float device_scale_factor,
+    float page_scale_factor,
+    const gfx::Vector2dF& root_scroll_offset) {
+  double dsf, psf, rso_x, rso_y;
+  bool valid = true;
+
+  valid &= frame.metadata()->GetDouble(
+      media::VideoFrameMetadata::DEVICE_SCALE_FACTOR, &dsf);
+  valid &= frame.metadata()->GetDouble(
+      media::VideoFrameMetadata::PAGE_SCALE_FACTOR, &psf);
+  valid &= frame.metadata()->GetDouble(
+      media::VideoFrameMetadata::ROOT_SCROLL_OFFSET_X, &rso_x);
+  valid &= frame.metadata()->GetDouble(
+      media::VideoFrameMetadata::ROOT_SCROLL_OFFSET_Y, &rso_y);
+
+  return valid && dsf == device_scale_factor && psf == page_scale_factor &&
+         gfx::Vector2dF(rso_x, rso_y) == root_scroll_offset;
+}
+
 // Dummy frame sink ID.
 constexpr FrameSinkId kFrameSinkId = FrameSinkId(1, 1);
 
@@ -335,9 +358,21 @@ class FrameSinkVideoCapturerTest : public testing::Test {
     task_runner_->FastForwardBy(GetNextVsync() - task_runner_->NowTicks());
   }
 
-  void NotifyFrameDamaged() {
+  void NotifyFrameDamaged(
+      float device_scale_factor =
+          FrameSinkVideoCapturerImpl::kDefaultDeviceScaleFactor,
+      float page_scale_factor =
+          FrameSinkVideoCapturerImpl::kDefaultPageScaleFactor,
+      gfx::Vector2dF root_scroll_offset =
+          FrameSinkVideoCapturerImpl::kDefaultRootScrollOffset) {
+    CompositorFrameMetadata metadata;
+
+    metadata.device_scale_factor = device_scale_factor;
+    metadata.page_scale_factor = page_scale_factor;
+    metadata.root_scroll_offset = root_scroll_offset;
+
     capturer_.OnFrameDamaged(kSourceSize, gfx::Rect(kSourceSize),
-                             GetNextVsync());
+                             GetNextVsync(), metadata);
   }
 
   void NotifyTargetWentAway() {
@@ -363,7 +398,7 @@ class FrameSinkVideoCapturerTest : public testing::Test {
 };
 
 // Tests that the capturer attaches to a frame sink immediately, in the case
-// where the frame sink was already known by the manger.
+// where the frame sink was already known by the manager.
 TEST_F(FrameSinkVideoCapturerTest, ResolvesTargetImmediately) {
   EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId))
       .WillRepeatedly(Return(&frame_sink_));
@@ -857,4 +892,67 @@ TEST_F(FrameSinkVideoCapturerTest, EventuallySendsARefreshFrame) {
   StopCapture();
 }
 
+// Tests that CompositorFrameMetadata variables (|device_scale_factor|,
+// |page_scale_factor| and |root_scroll_offset|) are sent along with each frame,
+// and refreshes cause variables of the cached CompositorFrameMetadata
+// (|last_frame_metadata|) to be used.
+TEST_F(FrameSinkVideoCapturerTest, CompositorFrameMetadataReachesConsumer) {
+  EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId))
+      .WillRepeatedly(Return(&frame_sink_));
+  capturer_.ChangeTarget(kFrameSinkId);
+
+  MockConsumer consumer;
+  // Initial refresh frame for starting capture, plus later refresh.
+  const int num_refresh_frames = 2;
+  const int num_update_frames = 1;
+  EXPECT_CALL(consumer, OnFrameCapturedMock(_, _, _))
+      .Times(num_refresh_frames + num_update_frames);
+  EXPECT_CALL(consumer, OnTargetLost(_)).Times(0);
+  EXPECT_CALL(consumer, OnStopped()).Times(1);
+  StartCapture(&consumer);
+
+  // With the start, an immediate refresh occurred. Simulate a copy result.
+  // Expect to see the refresh frame delivered to the consumer, along with
+  // default metadata values.
+  int cur_frame_index = 0, expected_frames_count = 1;
+  frame_sink_.SendCopyOutputResult(cur_frame_index);
+  EXPECT_EQ(expected_frames_count, consumer.num_frames_received());
+  EXPECT_TRUE(CompareVarsInCompositorFrameMetadata(
+      *(consumer.TakeFrame(cur_frame_index)),
+      FrameSinkVideoCapturerImpl::kDefaultDeviceScaleFactor,
+      FrameSinkVideoCapturerImpl::kDefaultPageScaleFactor,
+      FrameSinkVideoCapturerImpl::kDefaultRootScrollOffset));
+  consumer.SendDoneNotification(cur_frame_index);
+
+  // The metadata used to signal a frame damage and verify that it reaches the
+  // consumer.
+  const float kNewDeviceScaleFactor = 3.5;
+  const float kNewPageScaleFactor = 1.5;
+  const gfx::Vector2dF kNewRootScrollOffset = gfx::Vector2dF(100, 200);
+
+  // Notify frame damage with new metadata, and expect that the refresh frame
+  // is delivered to the consumer with this new metadata.
+  AdvanceClockToNextVsync();
+  NotifyFrameDamaged(kNewDeviceScaleFactor, kNewPageScaleFactor,
+                     kNewRootScrollOffset);
+  frame_sink_.SendCopyOutputResult(++cur_frame_index);
+  EXPECT_EQ(++expected_frames_count, consumer.num_frames_received());
+  EXPECT_TRUE(CompareVarsInCompositorFrameMetadata(
+      *(consumer.TakeFrame(cur_frame_index)), kNewDeviceScaleFactor,
+      kNewPageScaleFactor, kNewRootScrollOffset));
+  consumer.SendDoneNotification(cur_frame_index);
+
+  // Request a refresh frame. Because the refresh request was made just after
+  // the last frame capture, the refresh retry timer should be started.
+  // Expect that the refresh frame is delivered to the consumer with the same
+  // metadata from the previous frame.
+  capturer_.RequestRefreshFrame();
+  AdvanceClockForRefreshTimer();
+  EXPECT_EQ(++expected_frames_count, consumer.num_frames_received());
+  EXPECT_TRUE(CompareVarsInCompositorFrameMetadata(
+      *(consumer.TakeFrame(++cur_frame_index)), kNewDeviceScaleFactor,
+      kNewPageScaleFactor, kNewRootScrollOffset));
+  StopCapture();
+}
+
 }  // namespace viz

components/viz/service/surfaces/surface.cc

@@ -494,8 +494,7 @@ void Surface::NotifyAggregatedDamage(const gfx::Rect& damage_rect,
     return;
 
   active_frame_data_->aggregated_damage_callback.Run(
-      surface_id().local_surface_id(),
-      active_frame_data_->frame.size_in_pixels(), damage_rect,
+      surface_id().local_surface_id(), active_frame_data_->frame, damage_rect,
       expected_display_time);
 }
 

components/viz/service/surfaces/surface.h

@@ -72,7 +72,7 @@ class VIZ_SERVICE_EXPORT Surface final : public SurfaceDeadlineClient {
  public:
   using AggregatedDamageCallback =
       base::RepeatingCallback<void(const LocalSurfaceId& local_surface_id,
-                                   const gfx::Size& frame_size_in_pixels,
+                                   const CompositorFrame& frame,
                                    const gfx::Rect& damage_rect,
                                    base::TimeTicks expected_display_time)>;
   using PresentedCallback =

content/browser/devtools/devtools_video_consumer.cc

@@ -149,9 +149,8 @@ void DevToolsVideoConsumer::OnFrameCaptured(
   // Setting |frame|'s visible rect equal to |content_rect| so that only the
   // portion of the frame that contain content are used.
   frame = media::VideoFrame::WrapExternalData(
-      info->pixel_format, info->coded_size, info->visible_rect,
-      info->visible_rect.size(), static_cast<uint8_t*>(mapping.get()),
-      buffer_size, info->timestamp);
+      info->pixel_format, info->coded_size, content_rect, content_rect.size(),
+      static_cast<uint8_t*>(mapping.get()), buffer_size, info->timestamp);
   if (!frame)
     return;
   frame->AddDestructionObserver(base::BindOnce(

content/browser/devtools/protocol/page_handler.h

@@ -16,6 +16,7 @@
 #include "base/macros.h"
 #include "base/memory/weak_ptr.h"
 #include "base/time/time.h"
+#include "build/build_config.h"
 #include "components/viz/common/quads/compositor_frame_metadata.h"
 #include "content/browser/devtools/protocol/devtools_domain_handler.h"
 #include "content/browser/devtools/protocol/devtools_download_manager_delegate.h"
@@ -27,6 +28,10 @@
 #include "third_party/blink/public/platform/modules/manifest/manifest_manager.mojom.h"
 #include "url/gurl.h"
 
+#if !defined(OS_ANDROID)
+#include "content/browser/devtools/devtools_video_consumer.h"
+#endif  // !defined(OS_ANDROID)
+
 class SkBitmap;
 
 namespace base {
@@ -154,6 +159,9 @@ class PageHandler : public DevToolsDomainHandler,
   WebContentsImpl* GetWebContents();
   void NotifyScreencastVisibility(bool visible);
   void InnerSwapCompositorFrame();
+#if !defined(OS_ANDROID)
+  void OnFrameFromVideoConsumer(scoped_refptr<media::VideoFrame> frame);
+#endif  // !defined(OS_ANDROID)
   void ScreencastFrameCaptured(
       std::unique_ptr<Page::ScreencastFrameMetadata> metadata,
       const SkBitmap& bitmap);
@@ -195,6 +203,17 @@ class PageHandler : public DevToolsDomainHandler,
   int frame_counter_;
   int frames_in_flight_;
 
+#if !defined(OS_ANDROID)
+  // |video_consumer_| consumes video frames from FrameSinkVideoCapturerImpl,
+  // and provides PageHandler with these frames via OnFrameFromVideoConsumer.
+  // This is only used if Viz is enabled and if OS is not Android.
+  std::unique_ptr<DevToolsVideoConsumer> video_consumer_;
+
+  // The last surface size used to determine if frames with new sizes need
+  // to be requested. This changes due to window resizing.
+  gfx::Size last_surface_size_;
+#endif  // !defined(OS_ANDROID)
+
   RenderFrameHostImpl* host_;
   EmulationHandler* emulation_handler_;
   std::unique_ptr<Page::Frontend> frontend_;

media/base/video_frame_metadata.h

@@ -122,6 +122,16 @@ class MEDIA_EXPORT VideoFrameMetadata {
     // Whether this frame was decoded in a power efficient way.
     POWER_EFFICIENT,
 
+    // CompositorFrameMetadata variables associated with this frame. Used for
+    // remote debugging.
+    // Use Get/SetDouble() for these keys.
+    // TODO(crbug.com/832220): Use a customized dictionary value instead of
+    // using these keys directly.
+    DEVICE_SCALE_FACTOR,
+    PAGE_SCALE_FACTOR,
+    ROOT_SCROLL_OFFSET_X,
+    ROOT_SCROLL_OFFSET_Y,
+
     NUM_KEYS
   };