Commit f9eaa531 authored by Réda Housni Alaoui's avatar Réda Housni Alaoui Committed by Commit Bot

Win video capture: use IMFCaptureEngine for Media Foundation

- Full rewrite of the MediaFoundation implementation video part to use
IMFCaptureEngine
- Implementation of takePhoto, setPhotoOptions and getPhotoCapabilities
- takePhoto triggers a still image capture with the highest available
resolution without stopping the video stream thanks to IMFCaptureEngine

TEST=adapted video_capture_device_unittest.cc and
webrtc_image_capture_browsertest.cc; launch Chrome with
--force-mediafoundation on Win8+ and capture video using
e.g. https://webrtc.github.io/samples/src/content/getusermedia/gum/

R=mcasas@chromium.org

Bug: 730068
Change-Id: Ib8e7f475d8120a63dd08c7b215c1eaf2c6f3d800
Reviewed-on: https://chromium-review.googlesource.com/734042
Commit-Queue: Christian Fremerey <chfremer@chromium.org>
Reviewed-by: default avatarMiguel Casas <mcasas@chromium.org>
Reviewed-by: default avatarChristian Fremerey <chfremer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521435}
parent e7e22942
......@@ -667,6 +667,7 @@ Raveendra Karu <r.karu@samsung.com>
Ravi Nanjundappa <nravi.n@samsung.com>
Ravi Phaneendra Kasibhatla <r.kasibhatla@samsung.com>
Ravi Phaneendra Kasibhatla <ravi.kasibhatla@motorola.com>
Réda Housni Alaoui <alaoui.rda@gmail.com>
Refael Ackermann <refack@gmail.com>
Renata Hodovan <rhodovan.u-szeged@partner.samsung.com>
Rene Bolldorf <rb@radix.io>
......@@ -913,6 +914,7 @@ BlackBerry Limited <*@blackberry.com>
Canonical Limited <*@canonical.com>
Code Aurora Forum <*@codeaurora.org>
Comodo CA Limited
Cosium <*@cosium.com>
Endless Mobile, Inc. <*@endlessm.com>
Facebook, Inc. <*@fb.com>
Facebook, Inc. <*@oculus.com>
......
......@@ -46,13 +46,26 @@ namespace {
static const char kImageCaptureHtmlFile[] = "/media/image_capture_test.html";
enum class Camera {
FAKE,
DEFAULT,
#if defined(OS_WIN)
// Media Foundation is only available in Windows versions >= 7, below that the
// following flag has no effect
WIN_MEDIA_FOUNDATION
#endif
};
// TODO(mcasas): enable real-camera tests by disabling the Fake Device for
// platforms where the ImageCaptureCode is landed, https://crbug.com/656810
static struct TargetCamera {
bool use_fake;
} const kTargetCameras[] = {{true},
Camera camera;
} const kTargetCameras[] = {{Camera::FAKE},
#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_ANDROID)
{false}
{Camera::DEFAULT},
#endif
#if defined(OS_WIN)
{Camera::WIN_MEDIA_FOUNDATION}
#endif
};
......@@ -145,11 +158,23 @@ class WebRtcImageCaptureSucceedsBrowserTest
void SetUpCommandLine(base::CommandLine* command_line) override {
WebRtcImageCaptureBrowserTestBase::SetUpCommandLine(command_line);
if (std::get<0>(GetParam()).use_fake) {
base::CommandLine::ForCurrentProcess()->AppendSwitch(
switches::kUseFakeDeviceForMediaStream);
ASSERT_TRUE(base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kUseFakeDeviceForMediaStream));
switch (std::get<0>(GetParam()).camera) {
case Camera::FAKE:
base::CommandLine::ForCurrentProcess()->AppendSwitch(
switches::kUseFakeDeviceForMediaStream);
ASSERT_TRUE(base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kUseFakeDeviceForMediaStream));
break;
#if defined(OS_WIN)
case Camera::WIN_MEDIA_FOUNDATION:
base::CommandLine::ForCurrentProcess()->AppendSwitch(
switches::kForceMediaFoundationVideoCapture);
ASSERT_TRUE(base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kForceMediaFoundationVideoCapture));
break;
#endif
default:
break;
}
}
......@@ -157,7 +182,7 @@ class WebRtcImageCaptureSucceedsBrowserTest
// TODO(chfremer): Enable test cases using the video capture service with
// real cameras as soon as root cause for https://crbug.com/733582 is
// understood and resolved.
if ((!std::get<0>(GetParam()).use_fake) &&
if ((std::get<0>(GetParam()).camera != Camera::FAKE) &&
(std::get<1>(GetParam()).use_video_capture_service)) {
LOG(INFO) << "Skipping this test case";
return true;
......
......@@ -6,7 +6,7 @@
<body>
<script type="text/javascript" src="webrtc_test_utilities.js"></script>
<script>
const WIDTH = 320;
const WIDTH = 640;
/** @const */ var CONSTRAINTS = { width: { max : WIDTH } };
// Returns a Promise resolved with |object| after a delay of |delayInMs|.
......
......@@ -28,9 +28,11 @@
#include "testing/gtest/include/gtest/gtest.h"
#if defined(OS_WIN)
#include <mfcaptureengine.h>
#include "base/win/scoped_com_initializer.h"
#include "base/win/windows_version.h" // For fine-grained suppression.
#include "media/capture/video/win/video_capture_device_factory_win.h"
#include "media/capture/video/win/video_capture_device_mf_win.h"
#endif
#if defined(OS_MACOSX)
......@@ -88,9 +90,16 @@
#define MAYBE_GetPhotoState DISABLED_GetPhotoState
#endif
// Wrap the TEST_P macro into another one to allow to preprocess |test_name|
// macros. Needed until https://github.com/google/googletest/issues/389 is
// fixed.
#define WRAPPED_TEST_P(test_case_name, test_name) \
TEST_P(test_case_name, test_name)
using ::testing::_;
using ::testing::Invoke;
using ::testing::SaveArg;
using ::testing::Return;
namespace media {
namespace {
......@@ -116,6 +125,35 @@ static bool IsDeviceUsableForTesting(
};
#endif
enum VideoCaptureImplementationTweak {
NONE,
#if defined(OS_WIN)
WIN_MEDIA_FOUNDATION
#endif
};
#if defined(OS_WIN)
class MockMFPhotoCallback final : public IMFCaptureEngineOnSampleCallback {
public:
~MockMFPhotoCallback() {}
MOCK_METHOD2(DoQueryInterface, HRESULT(REFIID, void**));
MOCK_METHOD0(DoAddRef, ULONG(void));
MOCK_METHOD0(DoRelease, ULONG(void));
MOCK_METHOD1(DoOnSample, HRESULT(IMFSample*));
STDMETHOD(QueryInterface)(REFIID riid, void** object) override {
return DoQueryInterface(riid, object);
}
STDMETHOD_(ULONG, AddRef)() override { return DoAddRef(); }
STDMETHOD_(ULONG, Release)() override { return DoRelease(); }
STDMETHOD(OnSample)(IMFSample* sample) override { return DoOnSample(sample); }
};
#endif
class MockVideoCaptureClient : public VideoCaptureDevice::Client {
public:
MOCK_METHOD0(DoReserveOutputBuffer, void(void));
......@@ -261,7 +299,19 @@ testing::Environment* const mojo_test_env =
} // namespace
class VideoCaptureDeviceTest : public testing::TestWithParam<gfx::Size> {
class VideoCaptureDeviceTest
: public testing::TestWithParam<
std::tuple<gfx::Size, VideoCaptureImplementationTweak>> {
public:
#if defined(OS_WIN)
scoped_refptr<IMFCaptureEngineOnSampleCallback> CreateMockPhotoCallback(
MockMFPhotoCallback* mock_photo_callback,
VideoCaptureDevice::TakePhotoCallback callback,
VideoCaptureFormat format) {
return scoped_refptr<IMFCaptureEngineOnSampleCallback>(mock_photo_callback);
}
#endif
protected:
typedef VideoCaptureDevice::Client Client;
......@@ -294,6 +344,10 @@ class VideoCaptureDeviceTest : public testing::TestWithParam<gfx::Size> {
static_cast<VideoCaptureDeviceFactoryAndroid*>(
video_capture_device_factory_.get())
->ConfigureForTesting();
#elif defined(OS_WIN)
static_cast<VideoCaptureDeviceFactoryWin*>(
video_capture_device_factory_.get())
->set_use_media_foundation_for_testing(UseWinMediaFoundation());
#endif
EXPECT_CALL(*video_capture_client_, DoReserveOutputBuffer()).Times(0);
EXPECT_CALL(*video_capture_client_, DoOnIncomingCapturedBuffer()).Times(0);
......@@ -301,6 +355,12 @@ class VideoCaptureDeviceTest : public testing::TestWithParam<gfx::Size> {
.Times(0);
}
#if defined(OS_WIN)
bool UseWinMediaFoundation() {
return std::get<1>(GetParam()) == WIN_MEDIA_FOUNDATION;
}
#endif
void ResetWithNewClient() {
video_capture_client_.reset(new MockVideoCaptureClient(base::Bind(
&VideoCaptureDeviceTest::OnFrameCaptured, base::Unretained(this))));
......@@ -407,7 +467,7 @@ class VideoCaptureDeviceTest : public testing::TestWithParam<gfx::Size> {
#define MAYBE_OpenInvalidDevice OpenInvalidDevice
#endif
// Tries to allocate an invalid device and verifies it doesn't work.
TEST_F(VideoCaptureDeviceTest, MAYBE_OpenInvalidDevice) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, MAYBE_OpenInvalidDevice) {
VideoCaptureDeviceDescriptor invalid_descriptor;
invalid_descriptor.device_id = "jibberish";
invalid_descriptor.display_name = "jibberish";
......@@ -439,12 +499,12 @@ TEST_F(VideoCaptureDeviceTest, MAYBE_OpenInvalidDevice) {
}
// Allocates the first enumerated device, and expects a frame.
TEST_P(VideoCaptureDeviceTest, CaptureWithSize) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, CaptureWithSize) {
const auto descriptor = FindUsableDeviceDescriptor();
if (!descriptor)
return;
const gfx::Size& size = GetParam();
const gfx::Size& size = std::get<0>(GetParam());
if (!IsCaptureSizeSupported(*descriptor, size))
return;
const int width = size.width();
......@@ -474,14 +534,22 @@ TEST_P(VideoCaptureDeviceTest, CaptureWithSize) {
}
const gfx::Size kCaptureSizes[] = {gfx::Size(640, 480), gfx::Size(1280, 720)};
const VideoCaptureImplementationTweak kCaptureImplementationTweaks[] = {
NONE,
#if defined(OS_WIN)
WIN_MEDIA_FOUNDATION
#endif
};
INSTANTIATE_TEST_CASE_P(VideoCaptureDeviceTests,
VideoCaptureDeviceTest,
testing::ValuesIn(kCaptureSizes));
INSTANTIATE_TEST_CASE_P(
VideoCaptureDeviceTests,
VideoCaptureDeviceTest,
testing::Combine(testing::ValuesIn(kCaptureSizes),
testing::ValuesIn(kCaptureImplementationTweaks)));
// Allocates a device with an uncommon resolution and verifies frames are
// captured in a close, much more typical one.
TEST_F(VideoCaptureDeviceTest, MAYBE_AllocateBadSize) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, MAYBE_AllocateBadSize) {
const auto descriptor = FindUsableDeviceDescriptor();
if (!descriptor)
return;
......@@ -508,7 +576,7 @@ TEST_F(VideoCaptureDeviceTest, MAYBE_AllocateBadSize) {
}
// Cause hangs on Windows, Linux. Fails Android. https://crbug.com/417824
TEST_F(VideoCaptureDeviceTest, DISABLED_ReAllocateCamera) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, DISABLED_ReAllocateCamera) {
const auto descriptor = FindUsableDeviceDescriptor();
if (!descriptor)
return;
......@@ -552,7 +620,7 @@ TEST_F(VideoCaptureDeviceTest, DISABLED_ReAllocateCamera) {
}
// Starts the camera in 720p to try and capture MJPEG format.
TEST_F(VideoCaptureDeviceTest, MAYBE_CaptureMjpeg) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, MAYBE_CaptureMjpeg) {
std::unique_ptr<VideoCaptureDeviceDescriptor> device_descriptor =
GetFirstDeviceDescriptorSupportingPixelFormat(PIXEL_FORMAT_MJPEG);
if (!device_descriptor) {
......@@ -589,7 +657,7 @@ TEST_F(VideoCaptureDeviceTest, MAYBE_CaptureMjpeg) {
device->StopAndDeAllocate();
}
TEST_F(VideoCaptureDeviceTest, NoCameraSupportsPixelFormatMax) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, NoCameraSupportsPixelFormatMax) {
// Use PIXEL_FORMAT_MAX to iterate all device names for testing
// GetDeviceSupportedFormats().
std::unique_ptr<VideoCaptureDeviceDescriptor> device_descriptor =
......@@ -601,7 +669,7 @@ TEST_F(VideoCaptureDeviceTest, NoCameraSupportsPixelFormatMax) {
// Starts the camera and verifies that a photo can be taken. The correctness of
// the photo is enforced by MockImageCaptureClient.
TEST_F(VideoCaptureDeviceTest, MAYBE_TakePhoto) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, MAYBE_TakePhoto) {
const auto descriptor = FindUsableDeviceDescriptor();
if (!descriptor)
return;
......@@ -650,7 +718,7 @@ TEST_F(VideoCaptureDeviceTest, MAYBE_TakePhoto) {
}
// Starts the camera and verifies that the photo capabilities can be retrieved.
TEST_F(VideoCaptureDeviceTest, MAYBE_GetPhotoState) {
WRAPPED_TEST_P(VideoCaptureDeviceTest, MAYBE_GetPhotoState) {
const auto descriptor = FindUsableDeviceDescriptor();
if (!descriptor)
return;
......@@ -701,4 +769,55 @@ TEST_F(VideoCaptureDeviceTest, MAYBE_GetPhotoState) {
device->StopAndDeAllocate();
}
#if defined(OS_WIN)
// Verifies that the photo callback is correctly released by MediaFoundation
WRAPPED_TEST_P(VideoCaptureDeviceTest, CheckPhotoCallbackRelease) {
if (!UseWinMediaFoundation())
return;
std::unique_ptr<VideoCaptureDeviceDescriptor> descriptor =
GetFirstDeviceDescriptorSupportingPixelFormat(PIXEL_FORMAT_MJPEG);
if (!descriptor) {
DVLOG(1) << "No usable media foundation device descriptor. Exiting test.";
return;
}
MockMFPhotoCallback* callback = new MockMFPhotoCallback();
EXPECT_CALL(*callback, DoQueryInterface(_, _)).WillRepeatedly(Return(S_OK));
EXPECT_CALL(*callback, DoAddRef()).WillOnce(Return(1U));
EXPECT_CALL(*callback, DoRelease()).WillOnce(Return(1U));
EXPECT_CALL(*callback, DoOnSample(_)).WillOnce(Return(S_OK));
EXPECT_CALL(*video_capture_client_, OnError(_, _)).Times(0);
EXPECT_CALL(*video_capture_client_, OnStarted());
std::unique_ptr<VideoCaptureDevice> device(
video_capture_device_factory_->CreateDevice(*descriptor));
ASSERT_TRUE(device);
static_cast<VideoCaptureDeviceMFWin*>(device.get())
->set_create_mf_photo_callback_for_testing(
base::Bind(&VideoCaptureDeviceTest::CreateMockPhotoCallback,
base::Unretained(this), callback));
VideoCaptureParams capture_params;
capture_params.requested_format.frame_size.SetSize(320, 240);
capture_params.requested_format.frame_rate = 30;
capture_params.requested_format.pixel_format = PIXEL_FORMAT_MJPEG;
device->AllocateAndStart(capture_params, std::move(video_capture_client_));
VideoCaptureDevice::TakePhotoCallback scoped_callback = base::BindOnce(
&MockImageCaptureClient::DoOnPhotoTaken, image_capture_client_);
base::RunLoop run_loop;
base::Closure quit_closure = BindToCurrentLoop(run_loop.QuitClosure());
EXPECT_CALL(*image_capture_client_.get(), OnCorrectPhotoTaken())
.WillOnce(RunClosure(quit_closure));
device->TakePhoto(std::move(scoped_callback));
run_loop.Run();
device->StopAndDeAllocate();
}
#endif
}; // namespace media
......@@ -69,10 +69,9 @@ static bool IsDeviceBlacklistedForQueryingDetailedFrameRates(
static bool LoadMediaFoundationDlls() {
static const wchar_t* const kMfDLLs[] = {
L"%WINDIR%\\system32\\mf.dll",
L"%WINDIR%\\system32\\mfplat.dll",
L"%WINDIR%\\system32\\mf.dll", L"%WINDIR%\\system32\\mfplat.dll",
L"%WINDIR%\\system32\\mfreadwrite.dll",
};
L"%WINDIR%\\system32\\MFCaptureEngine.dll"};
for (const wchar_t* kMfDLL : kMfDLLs) {
wchar_t path[MAX_PATH] = {0};
......@@ -86,8 +85,13 @@ static bool LoadMediaFoundationDlls() {
static bool PrepareVideoCaptureAttributesMediaFoundation(
IMFAttributes** attributes,
int count) {
if (!InitializeMediaFoundation())
// Once https://bugs.chromium.org/p/chromium/issues/detail?id=791615 is fixed,
// we must make sure that this method succeeds in capture_unittests context
// when MediaFoundation is enabled.
if (!VideoCaptureDeviceFactoryWin::PlatformSupportsMediaFoundation() ||
!InitializeMediaFoundation()) {
return false;
}
if (FAILED(MFCreateAttributes(attributes, count)))
return false;
......@@ -286,8 +290,9 @@ static void GetDeviceSupportedFormatsMediaFoundation(
DWORD stream_index = 0;
ComPtr<IMFMediaType> type;
while (SUCCEEDED(reader->GetNativeMediaType(kFirstVideoStream, stream_index,
type.GetAddressOf()))) {
while (SUCCEEDED(hr = reader->GetNativeMediaType(
static_cast<DWORD>(MF_SOURCE_READER_FIRST_VIDEO_STREAM),
stream_index, type.GetAddressOf()))) {
UINT32 width, height;
hr = MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &width, &height);
if (FAILED(hr)) {
......
......@@ -30,10 +30,12 @@ class CAPTURE_EXPORT VideoCaptureDeviceFactoryWin
const VideoCaptureDeviceDescriptor& device_descriptor,
VideoCaptureFormats* supported_formats) override;
void set_use_media_foundation_for_testing(bool use) {
use_media_foundation_ = use;
}
private:
// Media Foundation is available in Win7 and later, use it if explicitly
// forced via flag, else use DirectShow.
const bool use_media_foundation_;
bool use_media_foundation_;
DISALLOW_COPY_AND_ASSIGN(VideoCaptureDeviceFactoryWin);
};
......
......@@ -3,12 +3,13 @@
// found in the LICENSE file.
// Windows specific implementation of VideoCaptureDevice.
// DirectShow is used for capturing. DirectShow provide its own threads
// for capturing.
// MediaFoundation is used for capturing. MediaFoundation provides its own
// threads for capturing.
#ifndef MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_MF_WIN_H_
#define MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_MF_WIN_H_
#include <mfcaptureengine.h>
#include <mfidl.h>
#include <mfreadwrite.h>
#include <stdint.h>
......@@ -16,9 +17,9 @@
#include <vector>
#include "base/callback_forward.h"
#include "base/macros.h"
#include "base/sequence_checker.h"
#include "base/synchronization/lock.h"
#include "media/capture/capture_export.h"
#include "media/capture/video/video_capture_device.h"
......@@ -30,10 +31,12 @@ class Location;
namespace media {
class MFReaderCallback;
class MFVideoCallback;
const DWORD kFirstVideoStream =
static_cast<DWORD>(MF_SOURCE_READER_FIRST_VIDEO_STREAM);
const DWORD kPreferredVideoPreviewStream = static_cast<DWORD>(
MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW);
const DWORD kPreferredPhotoStream =
static_cast<DWORD>(MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO);
class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice {
public:
......@@ -51,6 +54,10 @@ class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice {
const VideoCaptureParams& params,
std::unique_ptr<VideoCaptureDevice::Client> client) override;
void StopAndDeAllocate() override;
void TakePhoto(TakePhotoCallback callback) override;
void GetPhotoState(GetPhotoStateCallback callback) override;
void SetPhotoOptions(mojom::PhotoSettingsPtr settings,
SetPhotoOptionsCallback callback) override;
// Captured new video data.
void OnIncomingCapturedData(const uint8_t* data,
......@@ -58,19 +65,33 @@ class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice {
int rotation,
base::TimeTicks reference_time,
base::TimeDelta timestamp);
void OnEvent(IMFMediaEvent* media_event);
using CreateMFPhotoCallbackCB =
base::Callback<scoped_refptr<IMFCaptureEngineOnSampleCallback>(
VideoCaptureDevice::TakePhotoCallback callback,
VideoCaptureFormat format)>;
void set_create_mf_photo_callback_for_testing(CreateMFPhotoCallbackCB cb) {
create_mf_photo_callback_ = cb;
}
private:
void OnError(const base::Location& from_here, HRESULT hr);
VideoCaptureDeviceDescriptor descriptor_;
Microsoft::WRL::ComPtr<IMFActivate> device_;
scoped_refptr<MFReaderCallback> callback_;
CreateMFPhotoCallbackCB create_mf_photo_callback_;
scoped_refptr<MFVideoCallback> video_callback_;
// Guards the below variables from concurrent access between methods running
// on |sequence_checker_| and calls to OnIncomingCapturedData() and OnEvent()
// made by MediaFoundation on threads outside of our control.
base::Lock lock_;
base::Lock lock_; // Used to guard the below variables.
std::unique_ptr<VideoCaptureDevice::Client> client_;
Microsoft::WRL::ComPtr<IMFSourceReader> reader_;
VideoCaptureFormat capture_format_;
bool capture_;
Microsoft::WRL::ComPtr<IMFCaptureEngine> engine_;
VideoCaptureFormat capture_video_format_;
bool is_started_;
SEQUENCE_CHECKER(sequence_checker_);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment