Commit f2bf742a authored by Antonio Gomes's avatar Antonio Gomes Committed by Commit Bot

Onion soup content/renderer/image_capture/

In summary this CL:

- Moves content/renderer/image_capture/image_capture_frame_grabber.cc|h
  to t_p/blink/renderer/modules/imagecapture/.
- Changes t_p/blink/renderer/modules/imagecapture/image_capture.cc to
  instantiate ImageCaptureFrameGrabber directly, instead of calling out
  to RendererBlinkPlatformImpl::CreateImageCaptureFrameGrabber().
- Removes WebImageCaptureFrameGrabber altogether.
- Removes the API in [1] creates WebImageCaptureFrameGrabber, as well as
  its overrides.

Additionally, in order to create CrossThreadBind instances that pass
ScopedWebCallbacks instances, a specialized template of CrossThreadCopier
was local to [2].

[1] third_party/blink/public/platform/platform.h
[2] third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.cc

Change in [3] were needed to be extended in order to allow the use of
//media/base and //third_party/libyuv identifiers, similarly to media_capabilities
and mediastream blink modules.

[3] third_party/blink/tools/blinkpy/presubmit/audit_non_blink_usage.py

R=haraken@chromium.org
CC=​blink-reviews-blink@chromium.org

BUG=787245

Change-Id: I86f75aa6e3483db2d0b7e7243c7f9cc3239f9507
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1576468Reviewed-by: default avatarKentaro Hara <haraken@chromium.org>
Commit-Queue: Antonio Gomes <tonikitoo@igalia.com>
Cr-Commit-Position: refs/heads/master@{#653466}
parent 3caa9adf
......@@ -114,8 +114,6 @@ target(link_target_type, "renderer") {
"history_entry.h",
"history_serialization.cc",
"history_serialization.h",
"image_capture/image_capture_frame_grabber.cc",
"image_capture/image_capture_frame_grabber.h",
"image_downloader/image_downloader_base.cc",
"image_downloader/image_downloader_base.h",
"image_downloader/image_downloader_impl.cc",
......
......@@ -46,7 +46,6 @@
#include "content/renderer/dom_storage/local_storage_namespace.h"
#include "content/renderer/dom_storage/session_web_storage_namespace_impl.h"
#include "content/renderer/dom_storage/webstoragenamespace_impl.h"
#include "content/renderer/image_capture/image_capture_frame_grabber.h"
#include "content/renderer/loader/child_url_loader_factory_bundle.h"
#include "content/renderer/loader/code_cache_loader_impl.h"
#include "content/renderer/loader/resource_dispatcher.h"
......@@ -131,7 +130,6 @@ using blink::WebAudioLatencyHint;
using blink::WebBlobRegistry;
using blink::WebCanvasCaptureHandler;
using blink::WebDatabaseObserver;
using blink::WebImageCaptureFrameGrabber;
using blink::WebMediaPlayer;
using blink::WebMediaRecorderHandler;
using blink::WebMediaStream;
......@@ -821,13 +819,6 @@ void RendererBlinkPlatformImpl::CreateHTMLAudioElementCapturer(
//------------------------------------------------------------------------------
std::unique_ptr<WebImageCaptureFrameGrabber>
RendererBlinkPlatformImpl::CreateImageCaptureFrameGrabber() {
return std::make_unique<ImageCaptureFrameGrabber>();
}
//------------------------------------------------------------------------------
std::unique_ptr<webrtc::RtpCapabilities>
RendererBlinkPlatformImpl::GetRtpSenderCapabilities(
const blink::WebString& kind) {
......
......@@ -174,8 +174,6 @@ class CONTENT_EXPORT RendererBlinkPlatformImpl : public BlinkPlatformImpl {
blink::WebMediaStream* web_media_stream,
blink::WebMediaPlayer* web_media_player,
scoped_refptr<base::SingleThreadTaskRunner> task_runner) override;
std::unique_ptr<blink::WebImageCaptureFrameGrabber>
CreateImageCaptureFrameGrabber() override;
std::unique_ptr<webrtc::RtpCapabilities> GetRtpSenderCapabilities(
const blink::WebString& kind) override;
std::unique_ptr<webrtc::RtpCapabilities> GetRtpReceiverCapabilities(
......
......@@ -264,7 +264,6 @@ source_set("blink_headers") {
"platform/web_http_load_info.h",
"platform/web_icon_sizes_parser.h",
"platform/web_image.h",
"platform/web_image_capture_frame_grabber.h",
"platform/web_image_generator.h",
"platform/web_inband_text_track.h",
"platform/web_inband_text_track_client.h",
......
......@@ -110,7 +110,6 @@ class WebCrypto;
class WebDatabaseObserver;
class WebDedicatedWorker;
class WebGraphicsContext3DProvider;
class WebImageCaptureFrameGrabber;
class WebLocalFrame;
class WebMediaCapabilitiesClient;
class WebMediaPlayer;
......@@ -654,11 +653,6 @@ class BLINK_PLATFORM_EXPORT Platform {
WebMediaPlayer*,
scoped_refptr<base::SingleThreadTaskRunner>) {}
// Creates a WebImageCaptureFrameGrabber to take a snapshot of a Video Tracks.
// May return null if the functionality is not available.
virtual std::unique_ptr<WebImageCaptureFrameGrabber>
CreateImageCaptureFrameGrabber();
// Returns the most optimistic view of the capabilities of the system for
// sending or receiving media of the given kind ("audio" or "video").
virtual std::unique_ptr<webrtc::RtpCapabilities> GetRtpSenderCapabilities(
......
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_IMAGE_CAPTURE_FRAME_GRABBER_H_
#define THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_IMAGE_CAPTURE_FRAME_GRABBER_H_
#include <memory>
#include "base/single_thread_task_runner.h"
#include "third_party/blink/public/platform/web_callbacks.h"
#include "third_party/blink/public/platform/web_common.h"
#include "third_party/skia/include/core/SkRefCnt.h"
class SkImage;
namespace blink {
class WebMediaStreamTrack;
using WebImageCaptureGrabFrameCallbacks = WebCallbacks<sk_sp<SkImage>, void>;
// Platform interface of an ImageCapture class for GrabFrame() calls.
class WebImageCaptureFrameGrabber {
public:
virtual ~WebImageCaptureFrameGrabber() = default;
virtual void GrabFrame(
WebMediaStreamTrack*,
std::unique_ptr<WebImageCaptureGrabFrameCallbacks> callbacks,
scoped_refptr<base::SingleThreadTaskRunner> task_runner) = 0;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_PUBLIC_PLATFORM_WEB_IMAGE_CAPTURE_FRAME_GRABBER_H_
......@@ -8,12 +8,17 @@ blink_modules_sources("imagecapture") {
sources = [
"image_capture.cc",
"image_capture.h",
"image_capture_frame_grabber.cc",
"image_capture_frame_grabber.h",
"media_settings_range.h",
"photo_capabilities.cc",
"photo_capabilities.h",
]
deps = [
"//media",
"//media/capture/mojom:image_capture_blink",
"//skia",
"//third_party/libyuv",
]
}
include_rules = [
"+media/capture/mojom/image_capture.mojom-blink.h",
"+media/base",
"+skia/ext/platform_canvas.h",
"+third_party/libyuv",
"+third_party/skia/include/core",
]
......@@ -9,7 +9,6 @@
#include "services/service_manager/public/cpp/interface_provider.h"
#include "third_party/blink/public/platform/interface_provider.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/public/platform/web_image_capture_frame_grabber.h"
#include "third_party/blink/public/platform/web_media_stream_track.h"
#include "third_party/blink/renderer/bindings/core/v8/callback_promise_adapter.h"
#include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
......@@ -18,6 +17,7 @@
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h"
#include "third_party/blink/renderer/modules/event_target_modules.h"
#include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
#include "third_party/blink/renderer/modules/imagecapture/media_settings_range.h"
#include "third_party/blink/renderer/modules/imagecapture/photo_capabilities.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h"
......@@ -312,7 +312,7 @@ ScriptPromise ImageCapture::grabFrame(ScriptState* script_state) {
// Create |m_frameGrabber| the first time.
if (!frame_grabber_) {
frame_grabber_ = Platform::Current()->CreateImageCaptureFrameGrabber();
frame_grabber_ = std::make_unique<ImageCaptureFrameGrabber>();
}
if (!frame_grabber_) {
......
......@@ -21,10 +21,10 @@
namespace blink {
class ExceptionState;
class ImageCaptureFrameGrabber;
class MediaStreamTrack;
class PhotoCapabilities;
class ScriptPromiseResolver;
class WebImageCaptureFrameGrabber;
// TODO(mcasas): Consider adding a web test checking that this class is not
// garbage collected while it has event listeners.
......@@ -98,7 +98,7 @@ class MODULES_EXPORT ImageCapture final
void ResolveWithPhotoCapabilities(ScriptPromiseResolver*);
Member<MediaStreamTrack> stream_track_;
std::unique_ptr<WebImageCaptureFrameGrabber> frame_grabber_;
std::unique_ptr<ImageCaptureFrameGrabber> frame_grabber_;
media::mojom::blink::ImageCapturePtr service_;
Member<MediaTrackCapabilities> capabilities_;
......
......@@ -2,24 +2,23 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "content/renderer/image_capture/image_capture_frame_grabber.h"
#include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
#include "base/bind.h"
#include "cc/paint/paint_canvas.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/video_frame.h"
#include "media/base/video_types.h"
#include "media/base/video_util.h"
#include "skia/ext/platform_canvas.h"
#include "third_party/blink/public/platform/web_callbacks.h"
#include "third_party/blink/public/platform/web_media_stream_source.h"
#include "third_party/blink/public/platform/web_media_stream_track.h"
#include "third_party/blink/renderer/platform/cross_thread_functional.h"
#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h"
#include "third_party/libyuv/include/libyuv.h"
#include "third_party/skia/include/core/SkImage.h"
#include "third_party/skia/include/core/SkSurface.h"
namespace content {
using blink::WebImageCaptureGrabFrameCallbacks;
namespace blink {
namespace {
......@@ -29,22 +28,37 @@ void OnError(std::unique_ptr<WebImageCaptureGrabFrameCallbacks> callbacks) {
} // anonymous namespace
// Template specialization of [1], needed to be able to pass callbacks
// that have ScopedWebCallbacks paramaters across threads.
//
// [1] third_party/blink/renderer/platform/cross_thread_copier.h.
template <typename T>
struct CrossThreadCopier<ScopedWebCallbacks<T>>
: public CrossThreadCopierPassThrough<ScopedWebCallbacks<T>> {
STATIC_ONLY(CrossThreadCopier);
using Type = ScopedWebCallbacks<T>;
static ScopedWebCallbacks<T> Copy(ScopedWebCallbacks<T> pointer) {
return pointer;
}
};
// Ref-counted class to receive a single VideoFrame on IO thread, convert it and
// send it to |main_task_runner_|, where this class is created and destroyed.
class ImageCaptureFrameGrabber::SingleShotFrameHandler
: public base::RefCountedThreadSafe<SingleShotFrameHandler> {
: public WTF::ThreadSafeRefCounted<SingleShotFrameHandler> {
public:
SingleShotFrameHandler() : first_frame_received_(false) {}
// Receives a |frame| and converts its pixels into a SkImage via an internal
// PaintSurface and SkPixmap. Alpha channel, if any, is copied.
void OnVideoFrameOnIOThread(SkImageDeliverCB callback,
const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks current_time);
void OnVideoFrameOnIOThread(
SkImageDeliverCB callback,
scoped_refptr<base::SingleThreadTaskRunner> task_runner,
const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks current_time);
private:
friend class base::RefCountedThreadSafe<SingleShotFrameHandler>;
virtual ~SingleShotFrameHandler() {}
friend class WTF::ThreadSafeRefCounted<SingleShotFrameHandler>;
// Flag to indicate that the first frames has been processed, and subsequent
// ones can be safely discarded.
......@@ -55,6 +69,7 @@ class ImageCaptureFrameGrabber::SingleShotFrameHandler
void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
SkImageDeliverCB callback,
scoped_refptr<base::SingleThreadTaskRunner> task_runner,
const scoped_refptr<media::VideoFrame>& frame,
base::TimeTicks /* current_time */) {
DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
......@@ -73,10 +88,13 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
sk_sp<SkSurface> surface = SkSurface::MakeRaster(info);
DCHECK(surface);
auto wrapper_callback = media::BindToLoop(
std::move(task_runner), ConvertToBaseCallback(std::move(callback)));
SkPixmap pixmap;
if (!skia::GetWritablePixels(surface->getCanvas(), &pixmap)) {
DLOG(ERROR) << "Error trying to map SkSurface's pixels";
std::move(callback).Run(sk_sp<SkImage>());
std::move(wrapper_callback).Run(sk_sp<SkImage>());
return;
}
......@@ -104,25 +122,25 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
pixmap.height());
}
std::move(callback).Run(surface->makeImageSnapshot());
std::move(wrapper_callback).Run(surface->makeImageSnapshot());
}
ImageCaptureFrameGrabber::ImageCaptureFrameGrabber()
: frame_grab_in_progress_(false), weak_factory_(this) {}
ImageCaptureFrameGrabber::~ImageCaptureFrameGrabber() {
DCHECK(thread_checker_.CalledOnValidThread());
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
}
void ImageCaptureFrameGrabber::GrabFrame(
blink::WebMediaStreamTrack* track,
std::unique_ptr<blink::WebImageCaptureGrabFrameCallbacks> callbacks,
WebMediaStreamTrack* track,
std::unique_ptr<WebImageCaptureGrabFrameCallbacks> callbacks,
scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
DCHECK(thread_checker_.CalledOnValidThread());
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DCHECK(!!callbacks);
DCHECK(track && !track->IsNull() && track->GetPlatformTrack());
DCHECK_EQ(blink::WebMediaStreamSource::kTypeVideo, track->Source().GetType());
DCHECK_EQ(WebMediaStreamSource::kTypeVideo, track->Source().GetType());
if (frame_grab_in_progress_) {
// Reject grabFrame()s too close back to back.
......@@ -130,8 +148,8 @@ void ImageCaptureFrameGrabber::GrabFrame(
return;
}
auto scoped_callbacks = blink::MakeScopedWebCallbacks(
std::move(callbacks), base::BindOnce(&OnError));
auto scoped_callbacks =
MakeScopedWebCallbacks(std::move(callbacks), WTF::Bind(&OnError));
// A SingleShotFrameHandler is bound and given to the Track to guarantee that
// only one VideoFrame is converted and delivered to OnSkImage(), otherwise
......@@ -139,26 +157,24 @@ void ImageCaptureFrameGrabber::GrabFrame(
// is being processed, which might be further held up if UI is busy, see
// https://crbug.com/623042.
frame_grab_in_progress_ = true;
blink::MediaStreamVideoSink::ConnectToTrack(
MediaStreamVideoSink::ConnectToTrack(
*track,
base::BindRepeating(
ConvertToBaseCallback(CrossThreadBind(
&SingleShotFrameHandler::OnVideoFrameOnIOThread,
base::MakeRefCounted<SingleShotFrameHandler>(),
media::BindToLoop(
std::move(task_runner),
base::BindRepeating(&ImageCaptureFrameGrabber::OnSkImage,
weak_factory_.GetWeakPtr(),
base::Passed(&scoped_callbacks)))),
WTF::Passed(CrossThreadBind(
&ImageCaptureFrameGrabber::OnSkImage, weak_factory_.GetWeakPtr(),
WTF::Passed(std::move(scoped_callbacks)))),
WTF::Passed(std::move(task_runner)))),
false);
}
void ImageCaptureFrameGrabber::OnSkImage(
blink::ScopedWebCallbacks<blink::WebImageCaptureGrabFrameCallbacks>
callbacks,
ScopedWebCallbacks<WebImageCaptureGrabFrameCallbacks> callbacks,
sk_sp<SkImage> image) {
DCHECK(thread_checker_.CalledOnValidThread());
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
blink::MediaStreamVideoSink::DisconnectFromTrack();
MediaStreamVideoSink::DisconnectFromTrack();
frame_grab_in_progress_ = false;
if (image)
callbacks.PassCallbacks()->OnSuccess(image);
......@@ -166,4 +182,4 @@ void ImageCaptureFrameGrabber::OnSkImage(
callbacks.PassCallbacks()->OnError();
}
} // namespace content
} // namespace blink
......@@ -2,63 +2,64 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CONTENT_RENDERER_IMAGE_CAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
#define CONTENT_RENDERER_IMAGE_CAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
#include <memory>
#include "base/compiler_specific.h"
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "content/common/content_export.h"
#include "third_party/blink/public/platform/scoped_web_callbacks.h"
#include "third_party/blink/public/platform/web_image_capture_frame_grabber.h"
#include "third_party/blink/public/platform/web_callbacks.h"
#include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h"
#include "third_party/blink/renderer/platform/wtf/functional.h"
#include "third_party/skia/include/core/SkRefCnt.h"
class SkImage;
namespace blink {
class WebMediaStreamTrack;
}
namespace content {
// TODO(crbug.com/945851): Avoid referencing to WebCallbacks, and reference to
// CallbackPromiseAdapter directly.
using WebImageCaptureGrabFrameCallbacks = WebCallbacks<sk_sp<SkImage>, void>;
// This class grabs Video Frames from a given Media Stream Video Track, binding
// a method of an ephemeral SingleShotFrameHandler every time grabFrame() is
// called. This method receives an incoming media::VideoFrame on a background
// thread and converts it into the appropriate SkBitmap which is sent back to
// OnSkBitmap(). This class is single threaded throughout.
class CONTENT_EXPORT ImageCaptureFrameGrabber final
: public blink::WebImageCaptureFrameGrabber,
public blink::MediaStreamVideoSink {
class ImageCaptureFrameGrabber final : public MediaStreamVideoSink {
public:
using SkImageDeliverCB = base::Callback<void(sk_sp<SkImage>)>;
using SkImageDeliverCB = WTF::CrossThreadFunction<void(sk_sp<SkImage>)>;
ImageCaptureFrameGrabber();
~ImageCaptureFrameGrabber() override;
// blink::WebImageCaptureFrameGrabber implementation.
void GrabFrame(
blink::WebMediaStreamTrack* track,
std::unique_ptr<blink::WebImageCaptureGrabFrameCallbacks> callbacks,
scoped_refptr<base::SingleThreadTaskRunner> task_runner) override;
void GrabFrame(WebMediaStreamTrack* track,
std::unique_ptr<WebImageCaptureGrabFrameCallbacks> callbacks,
scoped_refptr<base::SingleThreadTaskRunner> task_runner);
private:
// Internal class to receive, convert and forward one frame.
class SingleShotFrameHandler;
void OnSkImage(blink::ScopedWebCallbacks<
blink::WebImageCaptureGrabFrameCallbacks> callbacks,
sk_sp<SkImage> image);
void OnSkImage(
ScopedWebCallbacks<WebImageCaptureGrabFrameCallbacks> callbacks,
sk_sp<SkImage> image);
// Flag to indicate that there is a frame grabbing in progress.
bool frame_grab_in_progress_;
base::ThreadChecker thread_checker_;
THREAD_CHECKER(thread_checker_);
base::WeakPtrFactory<ImageCaptureFrameGrabber> weak_factory_;
DISALLOW_COPY_AND_ASSIGN(ImageCaptureFrameGrabber);
};
} // namespace content
} // namespace blink
#endif // CONTENT_RENDERER_IMAGE_CAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_
......@@ -42,7 +42,6 @@
#include "third_party/blink/public/platform/scheduler/web_thread_scheduler.h"
#include "third_party/blink/public/platform/web_canvas_capture_handler.h"
#include "third_party/blink/public/platform/web_graphics_context_3d_provider.h"
#include "third_party/blink/public/platform/web_image_capture_frame_grabber.h"
#include "third_party/blink/public/platform/web_media_recorder_handler.h"
#include "third_party/blink/public/platform/web_media_stream_center.h"
#include "third_party/blink/public/platform/web_prerendering_support.h"
......@@ -342,11 +341,6 @@ std::unique_ptr<WebCanvasCaptureHandler> Platform::CreateCanvasCaptureHandler(
return nullptr;
}
std::unique_ptr<WebImageCaptureFrameGrabber>
Platform::CreateImageCaptureFrameGrabber() {
return nullptr;
}
std::unique_ptr<webrtc::RtpCapabilities> Platform::GetRtpSenderCapabilities(
const WebString& kind) {
return nullptr;
......
......@@ -526,6 +526,15 @@ _CONFIG = [
'display::Display',
],
},
{
'paths': [
'third_party/blink/renderer/modules/imagecapture/',
],
'allowed': [
'media::.+',
'libyuv::.+',
]
},
{
'paths': [
'third_party/blink/renderer/modules/media_capabilities/',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment