Commit 5d2e0f71 authored by Wei Lee's avatar Wei Lee Committed by Commit Bot

Adds CameraEventObserver to listen for shutter done event when taking

photo

This CL is used to refine the timing of shutter sound and the shutter
button animation so that it could be synced to the timing when the
shutter is actually done.

Bug: 1021430
Test: Open CCA and take a picture
Change-Id: I6e5be0c9bc8aa6200917ab71165c0735a515527f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1899243Reviewed-by: default avatarKuo Jen Wei <inker@chromium.org>
Reviewed-by: default avatarShik Chen <shik@chromium.org>
Reviewed-by: default avatarRobert Sesek <rsesek@chromium.org>
Reviewed-by: default avatarRicky Liang <jcliang@chromium.org>
Commit-Queue: Wei Lee <wtlee@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715596}
parent 0478e712
......@@ -237,7 +237,8 @@ body.tablet-landscape .actions-group {
}
body:not(.streaming) #modes-group,
body.taking #modes-group {
body.taking #modes-group,
body.taking:not(.video-mode) #shutters-group {
opacity: 0.38;
}
......
......@@ -329,6 +329,42 @@ cca.mojo.DeviceOperator = class {
return isSuccess;
}
/**
* Adds observer to observe shutter event.
*
* The shutter event is defined as CAMERA3_MSG_SHUTTER in
* media/capture/video/chromeos/mojom/camera3.mojom which will be sent from
* underlying camera HAL after sensor finishes frame capturing.
*
* @param {string} deviceId The id for target camera device.
* @param {!function()} callback Callback to trigger on shutter done.
* @return {!Promise<number>} Id for the added observer.
* @throws {Error} if fails to construct device connection.
*/
async addShutterObserver(deviceId, callback) {
const observerCallbackRouter =
new cros.mojom.CameraEventObserverCallbackRouter();
observerCallbackRouter.onShutterDone.addListener(callback);
const device = await this.getDevice_(deviceId);
const {id} = await device.addCameraEventObserver(
observerCallbackRouter.$.bindNewPipeAndPassRemote());
return id;
}
/**
* Removes a shutter observer from Camera App Device.
* @param {string} deviceId The id of target camera device.
* @param {!number} observerId The id of the observer to be removed.
* @return {!Promise<boolean>} True when the observer is successfully removed.
* @throws {Error} if fails to construct device connection.
*/
async removeShutterObserver(deviceId, observerId) {
const device = await this.getDevice_(deviceId);
const {isSuccess} = await device.removeCameraEventObserver(observerId);
return isSuccess;
}
/**
* Sets reprocess option which is normally an effect to the video capture
* device before taking picture.
......
......@@ -51,21 +51,21 @@ cca.mojo.ImageCapture = function(videoTrack) {
/**
* Gets the photo capabilities with the available options/effects.
* @return {!Promise<!cca.mojo.PhotoCapabilities>} Promise for the result.
* @throws {Error} Thrown when the device operation is not supported.
* @return {!Promise<!PhotoCapabilities|cca.mojo.PhotoCapabilities>} Promise for
* the result.
*/
cca.mojo.ImageCapture.prototype.getPhotoCapabilities = async function() {
const deviceOperator = await cca.mojo.DeviceOperator.getInstance();
if (!deviceOperator) {
throw new Error('Device operation is not supported');
return this.capture_.getPhotoCapabilities();
}
const supportedEffects = [cros.mojom.Effect.NO_EFFECT];
const isPortraitModeSupported =
await deviceOperator.isPortraitModeSupported(this.deviceId_);
if (isPortraitModeSupported) {
supportedEffects.push(cros.mojom.Effect.PORTRAIT_MODE);
}
const baseCapabilities = await this.capture_.getPhotoCapabilities();
let /** !cca.mojo.PhotoCapabilities */ extendedCapabilities;
......@@ -77,29 +77,49 @@ cca.mojo.ImageCapture.prototype.getPhotoCapabilities = async function() {
* Takes single or multiple photo(s) with the specified settings and effects.
* The amount of result photo(s) depends on the specified settings and effects,
* and the first promise in the returned array will always resolve with the
* unreprocessed photo.
* unreprocessed photo. The returned array will be resolved once it received
* the shutter event.
* @param {!PhotoSettings} photoSettings Photo settings for ImageCapture's
* takePhoto().
* @param {!Array<cros.mojom.Effect>=} photoEffects Photo effects to be applied.
* @return {!Array<!Promise<!Blob>>} Array of promises for the result.
* @return {!Promise<!Array<!Promise<!Blob>>>} A promise of the array containing
* promise of each blob result.
*/
cca.mojo.ImageCapture.prototype.takePhoto = function(
photoSettings, photoEffects = []) {
cca.mojo.ImageCapture.prototype.takePhoto =
async function(photoSettings, photoEffects = []) {
/** @type {Array<!Promise<!Blob>>} */
const takes = [];
if (photoEffects) {
for (const effect of photoEffects) {
const take = (async () => {
const deviceOperator = await cca.mojo.DeviceOperator.getInstance();
if (!deviceOperator) {
throw new Error('Device operation is not supported');
}
const {data, mimeType} =
await deviceOperator.setReprocessOption(this.deviceId_, effect);
return new Blob([new Uint8Array(data)], {type: mimeType});
})();
takes.push(take);
const deviceOperator = await cca.mojo.DeviceOperator.getInstance();
if (deviceOperator === null && photoEffects.length > 0) {
throw new Error('Applying effects is not supported on this device');
}
for (const effect of photoEffects) {
const take = (async () => {
const {data, mimeType} =
await deviceOperator.setReprocessOption(this.deviceId_, effect);
return new Blob([new Uint8Array(data)], {type: mimeType});
})();
takes.push(take);
}
if (deviceOperator !== null) {
let onShutterDone;
const isShutterDone = new Promise((resolve) => {
onShutterDone = resolve;
});
const observerId =
await deviceOperator.addShutterObserver(this.deviceId_, onShutterDone);
takes.unshift(this.capture_.takePhoto(photoSettings));
await isShutterDone;
const isSuccess =
await deviceOperator.removeShutterObserver(this.deviceId_, observerId);
if (!isSuccess) {
console.error('Failed to remove shutter observer');
}
return takes;
} else {
takes.unshift(this.capture_.takePhoto(photoSettings));
return takes;
}
takes.splice(0, 0, this.capture_.takePhoto(photoSettings));
return takes;
};
......@@ -26,8 +26,7 @@ cca.sound.play = function(selector) {
var cancel;
var p = new Promise((resolve, reject) => {
var element = document.querySelector(selector);
var timeout =
setTimeout(resolve, parseInt(element.dataset.timeout || 0), 10);
var timeout = setTimeout(resolve, Number(element.dataset.timeout || 0));
cancel = () => {
clearTimeout(timeout);
reject(new Error('cancel'));
......
......@@ -83,6 +83,11 @@ cca.views.Camera = function(
const createVideoSaver = async () => resultSaver.startSaveVideo();
const playShutterEffect = () => {
cca.sound.play('#sound-shutter');
cca.util.animateOnce(this.preview_.video);
};
/**
* Modes for the camera.
* @type {cca.views.camera.Modes}
......@@ -91,7 +96,7 @@ cca.views.Camera = function(
this.modes_ = new cca.views.camera.Modes(
this.defaultMode, photoPreferrer, videoPreferrer, this.restart.bind(this),
this.doSavePhoto_.bind(this), createVideoSaver,
this.doSaveVideo_.bind(this));
this.doSaveVideo_.bind(this), playShutterEffect);
/**
* @type {?string}
......@@ -194,9 +199,6 @@ cca.views.Camera.prototype.beginTake_ = function() {
this.take_ = (async () => {
try {
await cca.views.camera.timertick.start();
if (!cca.state.get('video-mode')) {
cca.util.animateOnce(this.preview_.video);
}
await this.modes_.current.startCapture();
} catch (e) {
if (e && e.message === 'cancel') {
......
......@@ -70,6 +70,11 @@ cca.views.camera.PhotoResult;
* @return {!Promise}
*/
/**
* Callback for playing shutter effect.
* @callback PlayShutterEffect
*/
/**
* Capture modes.
* @enum {string}
......@@ -90,11 +95,12 @@ cca.views.camera.Mode = {
* @param {!DoSavePhoto} doSavePhoto
* @param {!CreateVideoSaver} createVideoSaver
* @param {!DoSaveVideo} doSaveVideo
* @param {!PlayShutterEffect} playShutterEffect
* @constructor
*/
cca.views.camera.Modes = function(
defaultMode, photoPreferrer, videoPreferrer, doSwitchMode, doSavePhoto,
createVideoSaver, doSaveVideo) {
createVideoSaver, doSaveVideo, playShutterEffect) {
/**
* @type {!DoSwitchMode}
* @private
......@@ -143,7 +149,8 @@ cca.views.camera.Modes = function(
},
'photo-mode': {
captureFactory: () => new cca.views.camera.Photo(
this.stream_, doSavePhoto, this.captureResolution_),
this.stream_, doSavePhoto, this.captureResolution_,
playShutterEffect),
isSupported: async () => true,
resolutionConfig: photoPreferrer,
v1Config: cca.views.camera.Modes.getV1Constraints.bind(this, false),
......@@ -152,7 +159,8 @@ cca.views.camera.Modes = function(
},
'square-mode': {
captureFactory: () => new cca.views.camera.Square(
this.stream_, doSavePhoto, this.captureResolution_),
this.stream_, doSavePhoto, this.captureResolution_,
playShutterEffect),
isSupported: async () => true,
resolutionConfig: photoPreferrer,
v1Config: cca.views.camera.Modes.getV1Constraints.bind(this, false),
......@@ -161,7 +169,8 @@ cca.views.camera.Modes = function(
},
'portrait-mode': {
captureFactory: () => new cca.views.camera.Portrait(
this.stream_, doSavePhoto, this.captureResolution_),
this.stream_, doSavePhoto, this.captureResolution_,
playShutterEffect),
isSupported: async (deviceId) => {
if (deviceId === null) {
return false;
......@@ -653,9 +662,11 @@ cca.views.camera.Video.prototype.captureVideo_ = async function() {
* @param {MediaStream} stream
* @param {!DoSavePhoto} doSavePhoto
* @param {?[number, number]} captureResolution
* @param {!PlayShutterEffect} playShutterEffect
* @constructor
*/
cca.views.camera.Photo = function(stream, doSavePhoto, captureResolution) {
cca.views.camera.Photo = function(
stream, doSavePhoto, captureResolution, playShutterEffect) {
cca.views.camera.ModeBase.call(this, stream, captureResolution);
/**
......@@ -667,10 +678,10 @@ cca.views.camera.Photo = function(stream, doSavePhoto, captureResolution) {
/**
* ImageCapture object to capture still photos.
* @type {?ImageCapture}
* @type {?cca.mojo.ImageCapture}
* @private
*/
this.imageCapture_ = null;
this.crosImageCapture_ = null;
/**
* The observer id for saving metadata.
......@@ -685,6 +696,13 @@ cca.views.camera.Photo = function(stream, doSavePhoto, captureResolution) {
* @private
*/
this.metadataNames_ = [];
/**
* Callback for playing shutter effect.
* @type {!PlayShutterEffect}
* @protected
*/
this.playShutterEffect_ = playShutterEffect;
};
cca.views.camera.Photo.prototype = {
......@@ -695,14 +713,9 @@ cca.views.camera.Photo.prototype = {
* @override
*/
cca.views.camera.Photo.prototype.start_ = async function() {
cca.sound.play('#sound-shutter');
if (this.imageCapture_ === null) {
try {
this.imageCapture_ = new ImageCapture(this.stream_.getVideoTracks()[0]);
} catch (e) {
cca.toast.show('error_msg_take_photo_failed');
throw e;
}
if (this.crosImageCapture_ === null) {
this.crosImageCapture_ =
new cca.mojo.ImageCapture(this.stream_.getVideoTracks()[0]);
}
const imageName = (new cca.models.Filenamer()).newImageName();
......@@ -716,7 +729,6 @@ cca.views.camera.Photo.prototype.start_ = async function() {
cca.toast.show('error_msg_take_photo_failed');
throw e;
}
await this.doSavePhoto_(result, imageName);
};
......@@ -733,15 +745,23 @@ cca.views.camera.Photo.prototype.createPhotoResult_ = async function() {
imageHeight: this.captureResolution_[1],
};
} else {
const caps = await this.imageCapture_.getPhotoCapabilities();
const caps = await this.crosImageCapture_.getPhotoCapabilities();
photoSettings = {
imageWidth: caps.imageWidth.max,
imageHeight: caps.imageHeight.max,
};
}
const blob = await this.imageCapture_.takePhoto(photoSettings);
const {width, height} = await cca.util.blobToImage(blob);
return {resolution: {width, height}, blob};
try {
const results = await this.crosImageCapture_.takePhoto(photoSettings);
this.playShutterEffect_();
const blob = await results[0];
const {width, height} = await cca.util.blobToImage(blob);
return {resolution: {width, height}, blob};
} catch (e) {
cca.toast.show('error_msg_take_photo_failed');
throw e;
}
};
/**
......@@ -820,10 +840,13 @@ cca.views.camera.Photo.prototype.removeMetadataObserver = async function() {
* @param {MediaStream} stream
* @param {!DoSavePhoto} doSavePhoto
* @param {?[number, number]} captureResolution
* @param {!PlayShutterEffect} playShutterEffect
* @constructor
*/
cca.views.camera.Square = function(stream, doSavePhoto, captureResolution) {
cca.views.camera.Photo.call(this, stream, doSavePhoto, captureResolution);
cca.views.camera.Square = function(
stream, doSavePhoto, captureResolution, playShutterEffect) {
cca.views.camera.Photo.call(
this, stream, doSavePhoto, captureResolution, playShutterEffect);
/**
* Photo saving callback from parent.
......@@ -878,17 +901,13 @@ cca.views.camera.Square.prototype.cropSquare = async function(blob) {
* @param {MediaStream} stream
* @param {!DoSavePhoto} doSavePhoto
* @param {?[number, number]} captureResolution
* @param {!PlayShutterEffect} playShutterEffect
* @constructor
*/
cca.views.camera.Portrait = function(stream, doSavePhoto, captureResolution) {
cca.views.camera.Photo.call(this, stream, doSavePhoto, captureResolution);
/**
* ImageCapture object to capture still photos.
* @type {?cca.mojo.ImageCapture}
* @private
*/
this.crosImageCapture_ = null;
cca.views.camera.Portrait = function(
stream, doSavePhoto, captureResolution, playShutterEffect) {
cca.views.camera.Photo.call(
this, stream, doSavePhoto, captureResolution, playShutterEffect);
// End of properties, seal the object.
Object.seal(this);
......@@ -902,23 +921,18 @@ cca.views.camera.Portrait.prototype = {
* @override
*/
cca.views.camera.Portrait.prototype.start_ = async function() {
cca.sound.play('#sound-shutter');
if (this.crosImageCapture_ === null) {
try {
this.crosImageCapture_ =
new cca.mojo.ImageCapture(this.stream_.getVideoTracks()[0]);
} catch (e) {
cca.toast.show('error_msg_take_photo_failed');
throw e;
}
this.crosImageCapture_ =
new cca.mojo.ImageCapture(this.stream_.getVideoTracks()[0]);
}
if (this.captureResolution_) {
var photoSettings = {
imageWidth: this.captureResolution_[0],
imageHeight: this.captureResolution_[1],
};
} else {
const caps = await this.imageCapture_.getPhotoCapabilities();
const caps = await this.crosImageCapture_.getPhotoCapabilities();
photoSettings = {
imageWidth: caps.imageWidth.max,
imageHeight: caps.imageHeight.max,
......@@ -936,8 +950,9 @@ cca.views.camera.Portrait.prototype.start_ = async function() {
}
try {
var [reference, portrait] = this.crosImageCapture_.takePhoto(
var [reference, portrait] = await this.crosImageCapture_.takePhoto(
photoSettings, [cros.mojom.Effect.PORTRAIT_MODE]);
this.playShutterEffect_();
} catch (e) {
cca.toast.show('error_msg_take_photo_failed');
throw e;
......
......@@ -67,7 +67,8 @@ CameraAppDeviceImpl::CameraAppDeviceImpl(const std::string& device_id,
camera_info_(std::move(camera_info)),
task_runner_(base::ThreadTaskRunnerHandle::Get()),
capture_intent_(cros::mojom::CaptureIntent::DEFAULT),
next_observer_id_(0),
next_metadata_observer_id_(0),
next_camera_event_observer_id_(0),
weak_ptr_factory_(
std::make_unique<base::WeakPtrFactory<CameraAppDeviceImpl>>(this)) {}
......@@ -120,12 +121,20 @@ cros::mojom::CaptureIntent CameraAppDeviceImpl::GetCaptureIntent() {
void CameraAppDeviceImpl::OnResultMetadataAvailable(
const cros::mojom::CameraMetadataPtr& metadata,
cros::mojom::StreamType streamType) {
base::AutoLock lock(observers_lock_);
base::AutoLock lock(metadata_observers_lock_);
const auto& observer_ids = stream_observer_ids_[streamType];
const auto& observer_ids = stream_metadata_observer_ids_[streamType];
for (auto& id : observer_ids) {
observers_[id]->OnMetadataAvailable(metadata.Clone());
metadata_observers_[id]->OnMetadataAvailable(metadata.Clone());
}
}
void CameraAppDeviceImpl::OnShutterDone() {
base::AutoLock lock(camera_event_observers_lock_);
for (auto& observer : camera_event_observers_) {
observer.second->OnShutterDone();
}
}
......@@ -220,12 +229,12 @@ void CameraAppDeviceImpl::AddResultMetadataObserver(
mojo::PendingRemote<cros::mojom::ResultMetadataObserver> observer,
cros::mojom::StreamType stream_type,
AddResultMetadataObserverCallback callback) {
base::AutoLock lock(observers_lock_);
base::AutoLock lock(metadata_observers_lock_);
uint32_t id = next_observer_id_++;
observers_[id] =
uint32_t id = next_metadata_observer_id_++;
metadata_observers_[id] =
mojo::Remote<cros::mojom::ResultMetadataObserver>(std::move(observer));
stream_observer_ids_[stream_type].insert(id);
stream_metadata_observer_ids_[stream_type].insert(id);
std::move(callback).Run(id);
}
......@@ -233,18 +242,38 @@ void CameraAppDeviceImpl::AddResultMetadataObserver(
void CameraAppDeviceImpl::RemoveResultMetadataObserver(
uint32_t id,
RemoveResultMetadataObserverCallback callback) {
base::AutoLock lock(observers_lock_);
base::AutoLock lock(metadata_observers_lock_);
if (observers_.erase(id) == 0) {
if (metadata_observers_.erase(id) == 0) {
std::move(callback).Run(false);
return;
}
for (auto& kv : stream_observer_ids_) {
for (auto& kv : stream_metadata_observer_ids_) {
auto& observer_ids = kv.second;
observer_ids.erase(id);
}
std::move(callback).Run(true);
}
void CameraAppDeviceImpl::AddCameraEventObserver(
mojo::PendingRemote<cros::mojom::CameraEventObserver> observer,
AddCameraEventObserverCallback callback) {
base::AutoLock lock(camera_event_observers_lock_);
uint32_t id = next_camera_event_observer_id_++;
camera_event_observers_[id] =
mojo::Remote<cros::mojom::CameraEventObserver>(std::move(observer));
std::move(callback).Run(id);
}
void CameraAppDeviceImpl::RemoveCameraEventObserver(
uint32_t id,
RemoveCameraEventObserverCallback callback) {
base::AutoLock lock(camera_event_observers_lock_);
bool is_success = camera_event_observers_.erase(id) == 1;
std::move(callback).Run(is_success);
}
} // namespace media
......@@ -79,6 +79,8 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice {
void OnResultMetadataAvailable(const cros::mojom::CameraMetadataPtr& metadata,
const cros::mojom::StreamType stream_type);
void OnShutterDone();
void SetReprocessResult(SetReprocessOptionCallback callback,
const int32_t status,
media::mojom::BlobPtr blob);
......@@ -105,6 +107,14 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice {
uint32_t id,
RemoveResultMetadataObserverCallback callback) override;
void AddCameraEventObserver(
mojo::PendingRemote<cros::mojom::CameraEventObserver> observer,
AddCameraEventObserverCallback callback) override;
void RemoveCameraEventObserver(
uint32_t id,
RemoveCameraEventObserverCallback callback) override;
private:
std::string device_id_;
......@@ -126,15 +136,22 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice {
cros::mojom::CaptureIntent capture_intent_;
base::Lock observers_lock_;
base::Lock metadata_observers_lock_;
uint32_t next_observer_id_;
base::Lock camera_event_observers_lock_;
uint32_t next_metadata_observer_id_;
uint32_t next_camera_event_observer_id_;
base::flat_map<uint32_t, mojo::Remote<cros::mojom::ResultMetadataObserver>>
observers_;
metadata_observers_;
base::flat_map<cros::mojom::StreamType, base::flat_set<uint32_t>>
stream_observer_ids_;
stream_metadata_observer_ids_;
base::flat_map<uint32_t, mojo::Remote<cros::mojom::CameraEventObserver>>
camera_event_observers_;
std::unique_ptr<base::WeakPtrFactory<CameraAppDeviceImpl>> weak_ptr_factory_;
......
......@@ -109,6 +109,15 @@ interface CameraAppDevice {
// AddResultMetadataObserver.
// If the ResultMetadataObserver is found, |is_success| returns true.
RemoveResultMetadataObserver(uint32 id) => (bool is_success);
// Adds an observer for camera events and returns the observer |id|. The
// |observer| is the remote of the observer to be added.
AddCameraEventObserver(pending_remote<CameraEventObserver> observer)
=> (uint32 id);
// Removes the camera events observer according to given observer |id|. Sets
// |is_success| to true if remove successfully, false otherwise.
RemoveCameraEventObserver(uint32 id) => (bool is_success);
};
// Interface for camera device to send camera metadata to Chrome Camera App.
......@@ -117,3 +126,9 @@ interface ResultMetadataObserver {
// is produced.
OnMetadataAvailable(CameraMetadata camera_metadata);
};
// Interface for observing camera events such as shutter done.
interface CameraEventObserver {
// Triggered when the shutter is done for a still capture image.
OnShutterDone();
};
......@@ -274,7 +274,7 @@ void RequestManager::PrepareCaptureRequest() {
// 3. Preview + Capture (BlobOutput)
std::set<StreamType> stream_types;
cros::mojom::CameraMetadataPtr settings;
TakePhotoCallback callback = base::DoNothing();
TakePhotoCallback callback = base::NullCallback();
base::Optional<uint64_t> input_buffer_id;
cros::mojom::Effect reprocess_effect = cros::mojom::Effect::NO_EFFECT;
......@@ -657,6 +657,10 @@ void RequestManager::Notify(cros::mojom::Camera3NotifyMsgPtr message) {
first_frame_shutter_time_ = reference_time;
}
pending_result.timestamp = reference_time - first_frame_shutter_time_;
if (camera_app_device_ && pending_result.still_capture_callback) {
camera_app_device_->OnShutterDone();
}
TrySubmitPendingBuffers(frame_number);
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment