Use FrameSinkVideoCapturer in DevTools eyedropper tool

The eyedropper tool currently depends on being notified about arrival
of CompositorFrames from the renderer which is incompatible with the
VizDisplayCompositor feature. If viz is enabled, use the new video
capture API with is compatible with viz.

TBR=jam@chromium.org

Bug: 771336
Change-Id: I3e0edaf9cc467045a00fa05212ce79cfd58065c9
Reviewed-on: https://chromium-review.googlesource.com/949885
Commit-Queue: Saman Sami <samans@chromium.org>
Reviewed-by: John Rummell <jrummell@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: Yuri Wiitala <miu@chromium.org>
Reviewed-by: Antoine Labour <piman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#542299}

chrome/browser/DEPS

@@ -45,6 +45,7 @@ include_rules = [
   "+media/capture",
   "+media/midi",  # For midi switches
   "+media/mojo",  # For mojo media services.
+  "+media/renderers",
   "+ppapi/c",  # For various types.
   "+ppapi/host",
   "+ppapi/proxy",
@@ -74,6 +75,7 @@ include_rules = [
   "+services/ui/common",
   "+services/ui/public",
   "+services/ui/service.h",
+  "+services/viz/privileged",
   "+skia/ext",
   "+third_party/boringssl/src/include",
   "+third_party/crashpad",

chrome/browser/devtools/BUILD.gn

@@ -117,6 +117,7 @@ static_library("devtools") {
     "//base",
     "//content/public/browser",
     "//net",
+    "//services/viz/privileged/interfaces/compositing",
     "//third_party/WebKit/public:features",
     "//ui/events:dom_keycode_converter",
   ]

chrome/browser/devtools/devtools_eye_dropper.cc

@@ -6,12 +6,15 @@
 
 #include "base/bind.h"
 #include "build/build_config.h"
+#include "cc/paint/skia_paint_canvas.h"
+#include "components/viz/common/features.h"
 #include "content/public/browser/render_view_host.h"
 #include "content/public/browser/render_widget_host.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/cursor_info.h"
 #include "content/public/common/screen_info.h"
+#include "media/base/limits.h"
 #include "third_party/WebKit/public/platform/WebInputEvent.h"
 #include "third_party/WebKit/public/platform/WebMouseEvent.h"
 #include "third_party/skia/include/core/SkCanvas.h"
@@ -27,6 +30,9 @@ DevToolsEyeDropper::DevToolsEyeDropper(content::WebContents* web_contents,
       last_cursor_x_(-1),
       last_cursor_y_(-1),
       host_(nullptr),
+      video_consumer_binding_(this),
+      enable_viz_(
+          base::FeatureList::IsEnabled(features::kVizDisplayCompositor)),
       weak_factory_(this) {
   mouse_event_callback_ =
       base::Bind(&DevToolsEyeDropper::HandleMouseEvent, base::Unretained(this));
@@ -44,6 +50,29 @@ DevToolsEyeDropper::~DevToolsEyeDropper() {
 void DevToolsEyeDropper::AttachToHost(content::RenderWidgetHost* host) {
   host_ = host;
   host_->AddMouseEventCallback(mouse_event_callback_);
+
+  // TODO(crbug.com/813929): Use video capturer even if viz is not enabled.
+  if (!enable_viz_)
+    return;
+
+  // Capturing a full-page screenshot can be costly so we shouldn't do it too
+  // often. We can capture at a lower frame rate without hurting the user
+  // experience.
+  constexpr static int kMaxFrameRate = 15;
+
+  // Create and configure the video capturer.
+  video_capturer_ = host_->GetView()->CreateVideoCapturer();
+  video_capturer_->SetResolutionConstraints(
+      gfx::Size(1, 1),
+      gfx::Size(media::limits::kMaxDimension, media::limits::kMaxDimension),
+      false);
+  video_capturer_->SetMinCapturePeriod(base::TimeDelta::FromSeconds(1) /
+                                       kMaxFrameRate);
+
+  // Start video capture.
+  viz::mojom::FrameSinkVideoConsumerPtr consumer;
+  video_consumer_binding_.Bind(mojo::MakeRequest(&consumer));
+  video_capturer_->Start(std::move(consumer));
 }
 
 void DevToolsEyeDropper::DetachFromHost() {
@@ -53,6 +82,8 @@ void DevToolsEyeDropper::DetachFromHost() {
   content::CursorInfo cursor_info;
   cursor_info.type = blink::WebCursorInfo::kTypePointer;
   host_->SetCursor(cursor_info);
+  video_consumer_binding_.Close();
+  video_capturer_.reset();
   host_ = nullptr;
 }
 
@@ -85,7 +116,7 @@ void DevToolsEyeDropper::DidReceiveCompositorFrame() {
 }
 
 void DevToolsEyeDropper::UpdateFrame() {
-  if (!host_ || !host_->GetView())
+  if (enable_viz_ || !host_ || !host_->GetView())
     return;
 
   // TODO(miu): This is the wrong size. It's the size of the view on-screen, and
@@ -106,6 +137,7 @@ void DevToolsEyeDropper::ResetFrame() {
 }
 
 void DevToolsEyeDropper::FrameUpdated(const SkBitmap& bitmap) {
+  DCHECK(!enable_viz_);
   if (bitmap.drawsNothing())
     return;
   frame_ = bitmap;
@@ -271,3 +303,42 @@ void DevToolsEyeDropper::UpdateCursor() {
                                    kHotspotOffset * device_scale_factor);
   host_->SetCursor(cursor_info);
 }
+
+void DevToolsEyeDropper::OnFrameCaptured(
+    mojo::ScopedSharedBufferHandle buffer,
+    uint32_t buffer_size,
+    ::media::mojom::VideoFrameInfoPtr info,
+    const gfx::Rect& update_rect,
+    const gfx::Rect& content_rect,
+    viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr callbacks) {
+  if (!buffer.is_valid()) {
+    callbacks->Done();
+    return;
+  }
+  mojo::ScopedSharedBufferMapping mapping = buffer->Map(buffer_size);
+  if (!mapping) {
+    DLOG(ERROR) << "Shared memory mapping failed.";
+    return;
+  }
+  scoped_refptr<media::VideoFrame> frame = media::VideoFrame::WrapExternalData(
+      info->pixel_format, info->coded_size, info->visible_rect,
+      info->visible_rect.size(), static_cast<uint8_t*>(mapping.get()),
+      buffer_size, info->timestamp);
+  if (!frame)
+    return;
+  frame->AddDestructionObserver(base::BindOnce(
+      [](mojo::ScopedSharedBufferMapping mapping) {}, std::move(mapping)));
+
+  SkBitmap skbitmap;
+  skbitmap.allocN32Pixels(info->visible_rect.width(),
+                          info->visible_rect.height());
+  cc::SkiaPaintCanvas canvas(skbitmap);
+  video_renderer_.Copy(frame, &canvas, media::Context3D());
+
+  frame_ = skbitmap;
+  UpdateCursor();
+}
+
+void DevToolsEyeDropper::OnTargetLost(const viz::FrameSinkId& frame_sink_id) {}
+
+void DevToolsEyeDropper::OnStopped() {}

chrome/browser/devtools/devtools_eye_dropper.h

@@ -9,13 +9,17 @@
 #include "base/macros.h"
 #include "content/public/browser/render_widget_host.h"
 #include "content/public/browser/web_contents_observer.h"
+#include "media/renderers/paint_canvas_video_renderer.h"
+#include "mojo/public/cpp/bindings/binding.h"
+#include "services/viz/privileged/interfaces/compositing/frame_sink_video_capture.mojom.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 
 namespace blink {
 class WebMouseEvent;
 }
 
-class DevToolsEyeDropper : public content::WebContentsObserver {
+class DevToolsEyeDropper : public content::WebContentsObserver,
+                           public viz::mojom::FrameSinkVideoConsumer {
  public:
   typedef base::Callback<void(int, int, int, int)> EyeDropperCallback;
 
@@ -40,12 +44,27 @@ class DevToolsEyeDropper : public content::WebContentsObserver {
   bool HandleMouseEvent(const blink::WebMouseEvent& event);
   void UpdateCursor();
 
+  // viz::mojom::FrameSinkVideoConsumer implementation.
+  void OnFrameCaptured(
+      mojo::ScopedSharedBufferHandle buffer,
+      uint32_t buffer_size,
+      ::media::mojom::VideoFrameInfoPtr info,
+      const gfx::Rect& update_rect,
+      const gfx::Rect& content_rect,
+      viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr callbacks) override;
+  void OnTargetLost(const viz::FrameSinkId& frame_sink_id) override;
+  void OnStopped() override;
+
   EyeDropperCallback callback_;
   SkBitmap frame_;
   int last_cursor_x_;
   int last_cursor_y_;
   content::RenderWidgetHost::MouseEventCallback mouse_event_callback_;
   content::RenderWidgetHost* host_;
+  viz::mojom::FrameSinkVideoCapturerPtr video_capturer_;
+  mojo::Binding<viz::mojom::FrameSinkVideoConsumer> video_consumer_binding_;
+  const bool enable_viz_;
+  media::PaintCanvasVideoRenderer video_renderer_;
   base::WeakPtrFactory<DevToolsEyeDropper> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(DevToolsEyeDropper);

content/browser/renderer_host/render_widget_host_view_base.cc

@@ -8,6 +8,7 @@
 #include "base/logging.h"
 #include "base/unguessable_token.h"
 #include "build/build_config.h"
+#include "components/viz/host/host_frame_sink_manager.h"
 #include "components/viz/service/frame_sinks/frame_sink_manager_impl.h"
 #include "components/viz/service/surfaces/surface_hittest.h"
 #include "content/browser/accessibility/browser_accessibility_manager.h"
@@ -169,6 +170,15 @@ void RenderWidgetHostViewBase::CopyFromSurface(
   std::move(callback).Run(SkBitmap());
 }
 
+viz::mojom::FrameSinkVideoCapturerPtr
+RenderWidgetHostViewBase::CreateVideoCapturer() {
+  viz::mojom::FrameSinkVideoCapturerPtr video_capturer;
+  GetHostFrameSinkManager()->CreateVideoCapturer(
+      mojo::MakeRequest(&video_capturer));
+  video_capturer->ChangeTarget(GetFrameSinkId());
+  return video_capturer;
+}
+
 base::string16 RenderWidgetHostViewBase::GetSelectedText() {
   if (!GetTextInputManager())
     return base::string16();

content/browser/renderer_host/render_widget_host_view_base.h

@@ -119,6 +119,7 @@ class CONTENT_EXPORT RenderWidgetHostViewBase : public RenderWidgetHostView,
       const gfx::Rect& src_rect,
       const gfx::Size& output_size,
       base::OnceCallback<void(const SkBitmap&)> callback) override;
+  viz::mojom::FrameSinkVideoCapturerPtr CreateVideoCapturer() override;
   void FocusedNodeTouched(bool editable) override;
   void GetScreenInfo(ScreenInfo* screen_info) const override;
   float GetDeviceScaleFactor() const final;

content/public/browser/render_widget_host_view.h

@@ -22,10 +22,22 @@ class Rect;
 class Size;
 }
 
+namespace mojo {
+template <class T>
+class InterfacePtr;
+}
+
 namespace ui {
 class TextInputClient;
 }
 
+namespace viz {
+namespace mojom {
+class FrameSinkVideoCapturer;
+using FrameSinkVideoCapturerPtr = mojo::InterfacePtr<FrameSinkVideoCapturer>;
+}  // namespace mojom
+}  // namespace viz
+
 namespace content {
 
 class RenderWidgetHost;
@@ -167,7 +179,7 @@ class CONTENT_EXPORT RenderWidgetHostView {
   // Copies the given subset of the view's surface, optionally scales it, and
   // returns the result as a bitmap via the provided callback. This is meant for
   // one-off snapshots. For continuous video capture of the surface, please use
-  // viz::FrameSinkManager::CreateVideoCapturer() instead.
+  // CreateVideoCapturer() instead.
   //
   // |src_rect| is either the subset of the view's surface, in view coordinates,
   // or empty to indicate that all of it should be copied. This is NOT the same
@@ -190,6 +202,12 @@ class CONTENT_EXPORT RenderWidgetHostView {
       const gfx::Size& output_size,
       base::OnceCallback<void(const SkBitmap&)> callback) = 0;
 
+  // Creates a video capturer, which will allow the caller to receive a stream
+  // of media::VideoFrames captured from this view. The capturer is configured
+  // to target this view, so there is no need to call ChangeTarget() before
+  // Start(). See viz.mojom.FrameSinkVideoCapturer for documentation.
+  virtual viz::mojom::FrameSinkVideoCapturerPtr CreateVideoCapturer() = 0;
+
   // Notification that a node was touched.
   // The |editable| parameter indicates if the node is editable, for e.g.
   // an input field, etc.