Skip to content

GitLab

T
tangled
Commit ad9ded8c authored by xianglu's avatar xianglu Committed by Commit bot
Browse files
Options

Shapedetection module: Blink side implementation 

This CL implements mojom client on blink side.
 - FaceDetector::detect() sends an image to the browser through shared_buffer;
 - FaceDetector::onDetectFace() receives a list of boundingBoxes from Chrome.
A mock mojom server is added to test the connection.

This CL also reduces the API surface by removing Detector and DetectedObject
interface, so that the hierarchy is simpler and cleaner.

BUG=646083
TEST=Pass LayoutTest detectface.html

Review-Url: https://codereview.chromium.org/2369693002
Cr-Commit-Position: refs/heads/master@{#422729}
parent 491e3bf2
Hide whitespace changes
Inline Side-by-side
Showing with 287 additions and 146 deletions
content/public/app/mojo/content_browser_manifest.json
View file @ ad9ded8c
......@@ -18,6 +18,7 @@
"blink::mojom::OffscreenCanvasCompositorFrameSinkProvider",
"blink::mojom::OffscreenCanvasSurface",
"blink::mojom::PermissionService",
"blink::mojom::ShapeDetection",
"blink::mojom::WebSocket",
"content::mojom::MemoryCoordinatorHandle",
"content::mojom::StoragePartitionService",
......
third_party/WebKit/LayoutTests/fast/shapedetection/shapedetection-creation.html
View file @ ad9ded8c
......@@ -3,22 +3,6 @@
<script src=../../resources/testharnessreport.js></script>
<script>
// This test verifies that DetectedObject can not be created
test(function() {
assert_throws(new TypeError(),
function() {
var detectedObject = new DetectedObject();
});
}, 'DetectedObject should not be instantiable.');
// This test verifies that Detector can not be created
test(function() {
assert_throws(new TypeError(),
function() {
var detector = new Detector();
});
}, 'Detector should not be instantiable.');
// This test verifies that DetectedFace can be created
test(function() {
var detectedFace = new DetectedFace();
......
third_party/WebKit/LayoutTests/http/tests/serviceworker/webexposed/global-interface-listing-service-worker-expected.txt
View file @ ad9ded8c
......@@ -122,12 +122,6 @@ interface DataView
method setUint16
method setUint32
method setUint8
interface DetectedObject
getter boundingBox
method constructor
interface Detector
method constructor
method detect
interface Event
attribute AT_TARGET
attribute BLUR
......
third_party/WebKit/LayoutTests/shapedetection/detectface.html 0 → 100644
View file @ ad9ded8c
<!DOCTYPE html>
<script src="../resources/testharness.js"></script>
<script src="../resources/testharnessreport.js"></script>
<script src="../resources/mojo-helpers.js"></script>
<script src="resources/mock-shapedetection.js"></script>
<body>
<img id='img' src='../media/content/greenbox.png'/>
</body>
<script>
// This test verifies that FaceDetector can detect(). A mock mojo
// server is implemented in mock-shapedetection.js.
async_test(function(t) {
var img = document.getElementById("img");
var theMock = null;
mockShapeDetectionReady
.then(mock => {
theMock = mock;
return new FaceDetector();
})
.catch(error => {
assert_unreached("Error creating MockShapeDetection: " + error);
})
.then(detector => {
return detector.detect(img);
})
.then(faceDetectionResult => {
const imageReceivedByMock = theMock.getFrameData();
assert_equals(imageReceivedByMock.byteLength, 2500, "imageReceivedByMock.byteLength");
const GREEN_PIXEL = 0xFF00FF00;
assert_equals(imageReceivedByMock[0], GREEN_PIXEL, "pixel must be green");
assert_equals(faceDetectionResult.length, 3, "faceDetectionResult.length");
t.done();
})
.catch(error => {
assert_unreached("Error during detect(img): " + error);
});
}, 'exercises the ShapeDetection API detect()');
</script>
third_party/WebKit/LayoutTests/shapedetection/resources/mock-shapedetection.js 0 → 100644
View file @ ad9ded8c
"use strict";
let mockShapeDetectionReady = define(
'mockShapeDetection',
['third_party/WebKit/public/platform/modules/shapedetection/shapedetection.mojom',
'mojo/public/js/bindings',
'mojo/public/js/connection',
'mojo/public/js/core',
'content/public/renderer/frame_interfaces',
], (shapeDetection, bindings, connection, mojo, interfaces) => {
class MockShapeDetection {
constructor() {
interfaces.addInterfaceOverrideForTesting(
shapeDetection.ShapeDetection.name,
pipe => this.bindToPipe(pipe));
}
bindToPipe(pipe) {
this.stub_ = connection.bindHandleToStub(pipe,
shapeDetection.ShapeDetection);
bindings.StubBindings(this.stub_).delegate = this;
}
detectFace(frame_data, width, height) {
let receivedStruct = mojo.mapBuffer(frame_data, 0, width*height*4, 0);
this.buffer_data_ = new Uint32Array(receivedStruct.buffer);
return Promise.resolve({
result: {
boundingBoxes: [
{ x : 1.0, y: 1.0, width: 100.0, height: 100.0 },
{ x : 2.0, y: 2.0, width: 200.0, height: 200.0 },
{ x : 3.0, y: 3.0, width: 300.0, height: 300.0 },
]
}
});
mojo.unmapBuffer(receivedStruct.buffer);
}
getFrameData() {
return this.buffer_data_;
}
}
return new MockShapeDetection();
});
third_party/WebKit/LayoutTests/webexposed/global-interface-listing-dedicated-worker-expected.txt
View file @ ad9ded8c
......@@ -142,14 +142,6 @@ Starting worker: resources/global-interface-listing.js
[Worker] attribute PERSISTENT
[Worker] attribute TEMPORARY
[Worker] method constructor
[Worker] interface DetectedObject
[Worker] attribute @@toStringTag
[Worker] getter boundingBox
[Worker] method constructor
[Worker] interface Detector
[Worker] attribute @@toStringTag
[Worker] method constructor
[Worker] method detect
[Worker] interface Event
[Worker] attribute @@toStringTag
[Worker] attribute AT_TARGET
......
third_party/WebKit/LayoutTests/webexposed/global-interface-listing-expected.txt
View file @ ad9ded8c
......@@ -1179,14 +1179,6 @@ interface DetectedFace
attribute @@toStringTag
getter boundingBox
method constructor
interface DetectedObject
attribute @@toStringTag
getter boundingBox
method constructor
interface Detector
attribute @@toStringTag
method constructor
method detect
interface DeviceLightEvent : Event
attribute @@toStringTag
getter value
......
third_party/WebKit/LayoutTests/webexposed/global-interface-listing-shared-worker-expected.txt
View file @ ad9ded8c
......@@ -129,14 +129,6 @@ Starting worker: resources/global-interface-listing.js
[Worker] method setUint16
[Worker] method setUint32
[Worker] method setUint8
[Worker] interface DetectedObject
[Worker] attribute @@toStringTag
[Worker] getter boundingBox
[Worker] method constructor
[Worker] interface Detector
[Worker] attribute @@toStringTag
[Worker] method constructor
[Worker] method detect
[Worker] interface Event
[Worker] attribute @@toStringTag
[Worker] attribute AT_TARGET
......
third_party/WebKit/Source/modules/modules_idl_files.gni
View file @ ad9ded8c
......@@ -238,8 +238,6 @@ modules_idl_files =
"serviceworkers/ServiceWorkerRegistration.idl",
"serviceworkers/WindowClient.idl",
"shapedetection/DetectedFace.idl",
"shapedetection/DetectedObject.idl",
"shapedetection/Detector.idl",
"shapedetection/FaceDetector.idl",
"speech/SpeechGrammar.idl",
"speech/SpeechGrammarList.idl",
......
third_party/WebKit/Source/modules/shapedetection/BUILD.gn
View file @ ad9ded8c
......@@ -8,8 +8,6 @@ blink_modules_sources("shapedetection") {
sources = [
"DetectedFace.cpp",
"DetectedFace.h",
"DetectedObject.h",
"Detector.h",
"FaceDetector.cpp",
"FaceDetector.h",
]
......
third_party/WebKit/Source/modules/shapedetection/DEPS
View file @ ad9ded8c
include_rules = [
"+bindings",
"+core",
"+heap",
"-modules",
"+modules/ModulesExport.h",
"+modules/shapedetection",
"+platform",
"+public/platform",
"+third_party/skia/include/core/SkImage.h",
"+third_party/skia/include/core/SkImageInfo.h",
"+ui/gfx/geometry",
"-web",
]
third_party/WebKit/Source/modules/shapedetection/DetectedFace.cpp
View file @ ad9ded8c
......@@ -4,10 +4,20 @@
#include "modules/shapedetection/DetectedFace.h"
#include "core/dom/DOMRect.h"
namespace blink {
DetectedFace* DetectedFace::create() {
return new DetectedFace();
}
DOMRect* DetectedFace::boundingBox() const {
return m_boundingBox.get();
}
DEFINE_TRACE(DetectedFace) {
visitor->trace(m_boundingBox);
}
} // namespace blink
third_party/WebKit/Source/modules/shapedetection/DetectedFace.h
View file @ ad9ded8c
......@@ -7,15 +7,22 @@
#include "bindings/core/v8/ScriptWrappable.h"
#include "modules/ModulesExport.h"
#include "modules/shapedetection/DetectedObject.h"
namespace blink {
class MODULES_EXPORT DetectedFace final : public DetectedObject {
class DOMRect;
class MODULES_EXPORT DetectedFace final : public GarbageCollected<DetectedFace>,
public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
static DetectedFace* create();
DOMRect* boundingBox() const;
DECLARE_TRACE();
private:
Member<DOMRect> m_boundingBox;
};
} // namespace blink
......
third_party/WebKit/Source/modules/shapedetection/DetectedFace.idl
View file @ ad9ded8c
......@@ -8,6 +8,6 @@
Constructor,
RuntimeEnabled=ShapeDetection,
] interface DetectedFace {
// TODO(xianglu): Implement missing fields. https://crbug.com/646083
// TODO(xianglu): Implement missing fields. https://crbug.com/646083
readonly attribute DOMRect boundingBox;
};
DetectedFace implements DetectedObject;
third_party/WebKit/Source/modules/shapedetection/DetectedObject.h deleted 100644 → 0
View file @ 491e3bf2
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef DetectedObject_h
#define DetectedObject_h
#include "bindings/core/v8/ScriptWrappable.h"
#include "core/dom/DOMRect.h"
#include "modules/ModulesExport.h"
namespace blink {
class MODULES_EXPORT DetectedObject : public GarbageCollected<DetectedObject>,
public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
DOMRect* boundingBox() const { return m_boundingBox.get(); }
DEFINE_INLINE_TRACE() { visitor->trace(m_boundingBox); }
private:
Member<DOMRect> m_boundingBox;
};
} // namespace blink
#endif // DetectedObject_h
third_party/WebKit/Source/modules/shapedetection/DetectedObject.idl deleted 100644 → 0
View file @ 491e3bf2
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// https://wicg.github.io/shape-detection-api/#api
[
Exposed=(Window,Worker),
RuntimeEnabled=ShapeDetection,
] interface DetectedObject {
readonly attribute DOMRect boundingBox;
};
third_party/WebKit/Source/modules/shapedetection/Detector.h deleted 100644 → 0
View file @ 491e3bf2
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef Detector_h
#define Detector_h
#include "bindings/core/v8/ScriptPromise.h"
#include "bindings/core/v8/ScriptWrappable.h"
#include "modules/ModulesExport.h"
namespace blink {
class DetectedObject;
class HTMLImageElement;
class MODULES_EXPORT Detector : public GarbageCollected<Detector>,
public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
static Detector* create();
virtual ScriptPromise detect(ScriptState*, const HTMLImageElement*) = 0;
DEFINE_INLINE_VIRTUAL_TRACE() {}
};
} // namespace blink
#endif // Detector_h
third_party/WebKit/Source/modules/shapedetection/Detector.idl deleted 100644 → 0
View file @ 491e3bf2
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// https://wicg.github.io/shape-detection-api/#api
[
Exposed=(Window,Worker),
RuntimeEnabled=ShapeDetection,
] interface Detector {
[CallWith=ScriptState] Promise<sequence<DetectedObject>> detect(HTMLImageElement htmlImageElement);
};
third_party/WebKit/Source/modules/shapedetection/FaceDetector.cpp
View file @ ad9ded8c
......@@ -4,21 +4,133 @@
#include "modules/shapedetection/FaceDetector.h"
#include "bindings/core/v8/ScriptPromiseResolver.h"
#include "core/dom/DOMException.h"
#include "core/dom/DOMRect.h"
#include "core/dom/Document.h"
#include "core/fetch/ImageResource.h"
#include "core/frame/LocalDOMWindow.h"
#include "core/frame/LocalFrame.h"
#include "core/html/HTMLImageElement.h"
#include "platform/graphics/Image.h"
#include "public/platform/InterfaceProvider.h"
#include "third_party/skia/include/core/SkImage.h"
#include "third_party/skia/include/core/SkImageInfo.h"
namespace blink {
FaceDetector* FaceDetector::create() {
return new FaceDetector();
namespace {
mojo::ScopedSharedBufferHandle getSharedBufferHandle(
const HTMLImageElement* img) {
ImageResource* const imageResource = img->cachedImage();
if (!imageResource) {
DLOG(ERROR) << "Failed to convert HTMLImageElement to ImageSource.";
return mojo::ScopedSharedBufferHandle();
}
Image* const blinkImage = imageResource->getImage();
if (!blinkImage) {
DLOG(ERROR) << "Failed to convert ImageSource to blink::Image.";
return mojo::ScopedSharedBufferHandle();
}
const sk_sp<SkImage> image = blinkImage->imageForCurrentFrame();
DCHECK_EQ(img->naturalWidth(), image->width());
DCHECK_EQ(img->naturalHeight(), image->height());
if (!image) {
DLOG(ERROR) << "Failed to convert blink::Image to sk_sp<SkImage>.";
return mojo::ScopedSharedBufferHandle();
}
const SkImageInfo skiaInfo =
SkImageInfo::MakeN32(image->width(), image->height(), image->alphaType());
const uint32_t allocationSize = skiaInfo.getSafeSize(skiaInfo.minRowBytes());
mojo::ScopedSharedBufferHandle sharedBufferHandle =
mojo::SharedBufferHandle::Create(allocationSize);
const mojo::ScopedSharedBufferMapping mappedBuffer =
sharedBufferHandle->Map(allocationSize);
DCHECK(mappedBuffer);
const SkPixmap pixmap(skiaInfo, mappedBuffer.get(), skiaInfo.minRowBytes());
if (!image->readPixels(pixmap, 0, 0)) {
DLOG(ERROR) << "Failed to read pixels from sk_sp<SkImage>.";
return mojo::ScopedSharedBufferHandle();
}
return sharedBufferHandle;
}
} // anonymous namespace
FaceDetector* FaceDetector::create(ScriptState* scriptState) {
return new FaceDetector(*scriptState->domWindow()->frame());
}
FaceDetector::FaceDetector(LocalFrame& frame) {
DCHECK(!m_service.is_bound());
DCHECK(frame.interfaceProvider());
frame.interfaceProvider()->getInterface(mojo::GetProxy(&m_service));
}
ScriptPromise FaceDetector::detect(ScriptState* scriptState,
const HTMLImageElement* image) {
const HTMLImageElement* img) {
ScriptPromiseResolver* resolver = ScriptPromiseResolver::create(scriptState);
ScriptPromise promise = resolver->promise();
resolver->reject(DOMException::create(NotSupportedError, "Not implemented"));
if (!m_service) {
resolver->reject(DOMException::create(
NotFoundError, "Face detection service unavailable."));
return promise;
}
if (!img) {
resolver->reject(DOMException::create(
SyntaxError, "The provided HTMLImageElement is empty."));
return promise;
}
// TODO(xianglu): Add security check when the spec is ready.
// https://crbug.com/646083
mojo::ScopedSharedBufferHandle sharedBufferHandle =
getSharedBufferHandle(img);
if (!sharedBufferHandle->is_valid()) {
resolver->reject(
DOMException::create(SyntaxError, "Failed to get sharedBufferHandle."));
return promise;
}
m_serviceRequests.add(resolver);
DCHECK(m_service.is_bound());
m_service->DetectFace(std::move(sharedBufferHandle), img->naturalWidth(),
img->naturalHeight(),
convertToBaseCallback(WTF::bind(
&FaceDetector::onDetectFace, wrapPersistent(this),
wrapPersistent(resolver))));
sharedBufferHandle.reset();
return promise;
}
void FaceDetector::onDetectFace(
ScriptPromiseResolver* resolver,
mojom::blink::FaceDetectionResultPtr faceDetectionResult) {
if (!m_serviceRequests.contains(resolver))
return;
HeapVector<Member<DOMRect>> detectedFaces;
for (const auto& boundingBox : faceDetectionResult->boundingBoxes) {
detectedFaces.append(DOMRect::create(boundingBox->x, boundingBox->y,
boundingBox->width,
boundingBox->height));
}
resolver->resolve(detectedFaces);
m_serviceRequests.remove(resolver);
}
DEFINE_TRACE(FaceDetector) {
visitor->trace(m_serviceRequests);
}
} // namespace blink
third_party/WebKit/Source/modules/shapedetection/FaceDetector.h
View file @ ad9ded8c
......@@ -6,18 +6,36 @@
#define FaceDetector_h
#include "bindings/core/v8/ScriptPromise.h"
#include "bindings/core/v8/ScriptPromiseResolver.h"
#include "bindings/core/v8/ScriptWrappable.h"
#include "modules/ModulesExport.h"
#include "modules/shapedetection/Detector.h"
#include "public/platform/modules/shapedetection/shapedetection.mojom-blink.h"
namespace blink {
class MODULES_EXPORT FaceDetector final : public Detector {
class Document;
class HTMLImageElement;
class LocalDOMWindow;
class LocalFrame;
class MODULES_EXPORT FaceDetector final
: public GarbageCollectedFinalized<FaceDetector>,
public ScriptWrappable {
DEFINE_WRAPPERTYPEINFO();
public:
static FaceDetector* create();
static FaceDetector* create(ScriptState*);
ScriptPromise detect(ScriptState*, const HTMLImageElement*);
DECLARE_TRACE();
private:
explicit FaceDetector(LocalFrame&);
void onDetectFace(ScriptPromiseResolver*,
mojom::blink::FaceDetectionResultPtr);
mojom::blink::ShapeDetectionPtr m_service;
HeapHashSet<Member<ScriptPromiseResolver>> m_serviceRequests;
};
} // namespace blink
......
third_party/WebKit/Source/modules/shapedetection/FaceDetector.idl
View file @ ad9ded8c
......@@ -7,7 +7,8 @@
// TODO(xianglu): Add [Constructor(optional FaceDetectorOptions faceDetectorOptions)]. https://crbug.com/646083
[
Constructor,
ConstructorCallWith=ScriptState,
RuntimeEnabled=ShapeDetection,
] interface FaceDetector {
[CallWith=ScriptState] Promise<sequence<DetectedObject>> detect(HTMLImageElement htmlImageElement);
};
FaceDetector implements Detector;
third_party/WebKit/public/BUILD.gn
View file @ ad9ded8c
......@@ -670,11 +670,13 @@ mojom("new_wrapper_types_mojo_bindings") {
"platform/modules/permissions/permission.mojom",
"platform/modules/permissions/permission_status.mojom",
"platform/modules/presentation/presentation.mojom",
"platform/modules/shapedetection/shapedetection.mojom",
"platform/modules/websockets/websocket.mojom",
"platform/referrer.mojom",
"web/window_features.mojom",
]
public_deps = [
"//ui/gfx/geometry/mojo",
"//url/mojo:url_mojom_gurl",
"//url/mojo:url_mojom_origin",
]
......
third_party/WebKit/public/platform/modules/shapedetection/OWNERS 0 → 100644
View file @ ad9ded8c
per-file *.mojom=set noparent
per-file *.mojom=file://ipc/SECURITY_OWNERS
third_party/WebKit/public/platform/modules/shapedetection/shapedetection.mojom 0 → 100644
View file @ ad9ded8c
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// https://wicg.github.io/shape-detection-api/#api
module blink.mojom;
import "ui/gfx/geometry/mojo/geometry.mojom";
// Because "//ui/gfx/geometry/mojo" is not exposed to blink, we need to declare
// a wrapper struct, so that gfx.mojom.RectF will not be directly referenced
// inside blink, and browser can still use gfx types.
struct FaceDetectionResult {
array<gfx.mojom.RectF> boundingBoxes;
};
interface ShapeDetection {
// frame_data contains tightly packed image pixels in ARGB32 format,
// row-major order.
DetectFace(handle<shared_buffer> frame_data, uint32 width, uint32 height)
=> (FaceDetectionResult result);
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment