Commit 2839be2c authored by Xiangjun Zhang's avatar Xiangjun Zhang Committed by Commit Bot

Mirroring service: Add video capture client implementation.

Add the video capture client implementation for the Mirroring service.
It requests to start video capturing through the
media::mojom::VideoCaptureHost interface, and receive the captured
video frames through media::mojom::VideoCaptureObserver interface.

Bug: 734672
Change-Id: I8007d9c8cbca1639eee8c5659f0b34a30c85e375
Reviewed-on: https://chromium-review.googlesource.com/954056Reviewed-by: default avatarYuri Wiitala <miu@chromium.org>
Commit-Queue: Xiangjun Zhang <xjz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#543908}
parent 332bb11a
......@@ -2704,7 +2704,7 @@ test("unit_tests") {
"//components/content_settings/core/test:test_support",
"//components/data_reduction_proxy/core/browser:test_support",
"//components/data_use_measurement/core",
"//components/mirroring/browser:unittests",
"//components/mirroring:mirroring_tests",
"//components/nacl/common:buildflags",
"//components/ntp_snippets:test_support",
"//components/optimization_guide",
......
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import("//testing/test.gni")
group("mirroring_tests") {
testonly = true
public_deps = [
"browser:unittests",
"service:unittests",
]
}
test("mirroring_unittests") {
deps = [
":mirroring_tests",
"//media/test:run_all_unittests",
]
}
......@@ -63,9 +63,7 @@ SingleClientVideoCaptureHost::SingleClientVideoCaptureHost(
SingleClientVideoCaptureHost::~SingleClientVideoCaptureHost() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// If started, Stop() must be called before destruction.
DCHECK(!observer_);
Stop(0);
}
void SingleClientVideoCaptureHost::Start(
......@@ -104,6 +102,9 @@ void SingleClientVideoCaptureHost::Stop(int32_t device_id) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
if (!observer_)
return;
// Returns all the buffers.
for (const auto& entry : buffer_context_map_) {
OnFinishedConsumingBuffer(
......
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import("//testing/test.gni")
source_set("service") {
sources = [
"video_capture_client.cc",
"video_capture_client.h",
]
public_deps = [
"//base",
]
deps = [
"//media/capture/mojom:video_capture",
"//media/mojo/common:common",
"//mojo/public/cpp/bindings",
]
}
source_set("unittests") {
testonly = true
sources = [
"video_capture_client_unittest.cc",
]
deps = [
":service",
"//base",
"//base/test:test_support",
"//media/capture/mojom:video_capture",
"//mojo/public/cpp/bindings",
"//testing/gmock",
"//testing/gtest",
]
}
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/mirroring/service/video_capture_client.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/video_frame.h"
namespace mirroring {
namespace {
// Required by mojom::VideoCaptureHost interface. Can be any number.
constexpr int kDeviceId = 0;
} // namespace
VideoCaptureClient::VideoCaptureClient(media::mojom::VideoCaptureHostPtr host)
: video_capture_host_(std::move(host)),
binding_(this),
weak_factory_(this) {
DCHECK(video_capture_host_);
}
VideoCaptureClient::~VideoCaptureClient() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
Stop();
}
void VideoCaptureClient::Start(FrameDeliverCallback deliver_callback,
base::OnceClosure error_callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
DCHECK(!deliver_callback.is_null());
frame_deliver_callback_ = std::move(deliver_callback);
error_callback_ = std::move(error_callback);
media::mojom::VideoCaptureObserverPtr observer;
binding_.Bind(mojo::MakeRequest(&observer));
video_capture_host_->Start(kDeviceId, 0, media::VideoCaptureParams(),
std::move(observer));
}
void VideoCaptureClient::Stop() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
video_capture_host_->Stop(kDeviceId);
}
void VideoCaptureClient::Pause() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
if (frame_deliver_callback_.is_null())
return;
frame_deliver_callback_.Reset();
video_capture_host_->Pause(kDeviceId);
}
void VideoCaptureClient::Resume(FrameDeliverCallback deliver_callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK(!deliver_callback.is_null());
if (!frame_deliver_callback_.is_null()) {
return;
}
frame_deliver_callback_ = std::move(deliver_callback);
video_capture_host_->Resume(kDeviceId, 0, media::VideoCaptureParams());
}
void VideoCaptureClient::RequestRefreshFrame() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (frame_deliver_callback_.is_null())
return;
video_capture_host_->RequestRefreshFrame(kDeviceId);
}
void VideoCaptureClient::OnStateChanged(media::mojom::VideoCaptureState state) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__ << " state: " << state;
switch (state) {
case media::mojom::VideoCaptureState::STARTED:
RequestRefreshFrame();
break;
case media::mojom::VideoCaptureState::FAILED:
if (!error_callback_.is_null())
std::move(error_callback_).Run();
break;
case media::mojom::VideoCaptureState::PAUSED:
case media::mojom::VideoCaptureState::RESUMED:
break;
case media::mojom::VideoCaptureState::STOPPED:
case media::mojom::VideoCaptureState::ENDED:
client_buffers_.clear();
mapped_buffers_.clear();
weak_factory_.InvalidateWeakPtrs();
error_callback_.Reset();
frame_deliver_callback_.Reset();
binding_.Close();
break;
}
}
void VideoCaptureClient::OnBufferCreated(
int32_t buffer_id,
mojo::ScopedSharedBufferHandle handle) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(handle.is_valid());
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
const auto insert_result =
client_buffers_.emplace(std::make_pair(buffer_id, std::move(handle)));
DCHECK(insert_result.second);
}
void VideoCaptureClient::OnBufferReady(int32_t buffer_id,
media::mojom::VideoFrameInfoPtr info) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
bool consume_buffer = !frame_deliver_callback_.is_null();
if ((info->pixel_format != media::PIXEL_FORMAT_I420 &&
info->pixel_format != media::PIXEL_FORMAT_Y16) ||
info->storage_type != media::VideoPixelStorage::CPU) {
consume_buffer = false;
LOG(DFATAL) << "Wrong pixel format or storage, got pixel format:"
<< VideoPixelFormatToString(info->pixel_format)
<< ", storage:" << static_cast<int>(info->storage_type);
}
if (!consume_buffer) {
video_capture_host_->ReleaseBuffer(kDeviceId, buffer_id, -1.0);
return;
}
base::TimeTicks reference_time;
media::VideoFrameMetadata frame_metadata;
frame_metadata.MergeInternalValuesFrom(*info->metadata);
const bool success = frame_metadata.GetTimeTicks(
media::VideoFrameMetadata::REFERENCE_TIME, &reference_time);
DCHECK(success);
if (first_frame_ref_time_.is_null())
first_frame_ref_time_ = reference_time;
// If the timestamp is not prepared, we use reference time to make a rough
// estimate. e.g. ThreadSafeCaptureOracle::DidCaptureFrame().
// TODO(crbug/618407): Fix upstream capturers to always set timestamp and
// reference time.
if (info->timestamp.is_zero())
info->timestamp = reference_time - first_frame_ref_time_;
// Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
TRACE_EVENT_SCOPE_THREAD, "timestamp",
(reference_time - base::TimeTicks()).InMicroseconds(),
"time_delta", info->timestamp.InMicroseconds());
const auto& buffer_iter = client_buffers_.find(buffer_id);
DCHECK(buffer_iter != client_buffers_.end());
auto mapping_iter = mapped_buffers_.find(buffer_id);
const size_t buffer_size =
media::VideoFrame::AllocationSize(info->pixel_format, info->coded_size);
if (mapping_iter != mapped_buffers_.end() &&
buffer_size > mapping_iter->second.second) {
// Unmaps shared memory for too-small region.
mapped_buffers_.erase(mapping_iter);
mapping_iter = mapped_buffers_.end();
}
if (mapping_iter == mapped_buffers_.end()) {
const auto insert_result = mapped_buffers_.emplace(std::make_pair(
buffer_id,
MappingAndSize(buffer_iter->second->Map(buffer_size), buffer_size)));
DCHECK(insert_result.second);
mapping_iter = insert_result.first;
if (!mapping_iter->second.first) {
VLOG(1) << __func__ << ": Mapping Error";
mapped_buffers_.erase(mapping_iter);
video_capture_host_->ReleaseBuffer(kDeviceId, buffer_id, -1.0);
return;
}
}
DCHECK(mapping_iter != mapped_buffers_.end());
const auto& buffer = mapping_iter->second;
scoped_refptr<media::VideoFrame> frame = media::VideoFrame::WrapExternalData(
info->pixel_format, info->coded_size, info->visible_rect,
info->visible_rect.size(), reinterpret_cast<uint8_t*>(buffer.first.get()),
buffer.second, info->timestamp);
if (!frame) {
video_capture_host_->ReleaseBuffer(kDeviceId, buffer_id, -1.0);
return;
}
BufferFinishedCallback buffer_finished_callback = media::BindToCurrentLoop(
base::BindOnce(&VideoCaptureClient::OnClientBufferFinished,
weak_factory_.GetWeakPtr(), buffer_id));
frame->AddDestructionObserver(
base::BindOnce(&VideoCaptureClient::DidFinishConsumingFrame,
frame->metadata(), std::move(buffer_finished_callback)));
frame->metadata()->MergeInternalValuesFrom(*info->metadata);
frame_deliver_callback_.Run(frame, reference_time);
}
void VideoCaptureClient::OnBufferDestroyed(int32_t buffer_id) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
const auto& buffer_iter = client_buffers_.find(buffer_id);
if (buffer_iter != client_buffers_.end())
client_buffers_.erase(buffer_iter);
const auto& mapping_iter = mapped_buffers_.find(buffer_id);
if (mapping_iter != mapped_buffers_.end())
mapped_buffers_.erase(mapping_iter);
}
void VideoCaptureClient::OnClientBufferFinished(
int buffer_id,
double consumer_resource_utilization) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": buffer_id=" << buffer_id;
// Buffer was already destroyed.
if (client_buffers_.find(buffer_id) == client_buffers_.end()) {
DCHECK(mapped_buffers_.find(buffer_id) == mapped_buffers_.end());
return;
}
video_capture_host_->ReleaseBuffer(kDeviceId, buffer_id,
consumer_resource_utilization);
}
// static
void VideoCaptureClient::DidFinishConsumingFrame(
const media::VideoFrameMetadata* metadata,
BufferFinishedCallback callback) {
// Note: This function may be called on any thread by the VideoFrame
// destructor. |metadata| is still valid for read-access at this point.
double consumer_resource_utilization = -1.0;
if (!metadata->GetDouble(media::VideoFrameMetadata::RESOURCE_UTILIZATION,
&consumer_resource_utilization)) {
consumer_resource_utilization = -1.0;
}
DCHECK(!callback.is_null());
std::move(callback).Run(consumer_resource_utilization);
}
} // namespace mirroring
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef COMPONENTS_MIRRORING_SERVICE_VIDEO_CAPTURE_CLIENT_H_
#define COMPONENTS_MIRRORING_SERVICE_VIDEO_CAPTURE_CLIENT_H_
#include "base/callback.h"
#include "base/containers/flat_map.h"
#include "base/memory/weak_ptr.h"
#include "base/sequence_checker.h"
#include "base/time/time.h"
#include "media/capture/mojom/video_capture.mojom.h"
#include "mojo/public/cpp/bindings/binding.h"
#include "mojo/public/cpp/system/buffer.h"
namespace media {
class VideoFrame;
} // namespace media
namespace mirroring {
// On Start(), this class connects to |host| through the
// media::mojom::VideoCaptureHost interface and requests to launch a video
// capture device. After the device is started, the captured video frames are
// received through the media::mojom::VideoCaptureObserver interface.
class VideoCaptureClient : public media::mojom::VideoCaptureObserver {
public:
explicit VideoCaptureClient(media::mojom::VideoCaptureHostPtr host);
~VideoCaptureClient() override;
using FrameDeliverCallback =
base::RepeatingCallback<void(scoped_refptr<media::VideoFrame> video_frame,
base::TimeTicks estimated_capture_time)>;
void Start(FrameDeliverCallback deliver_callback,
base::OnceClosure error_callback);
void Stop();
// Will stop delivering frames on this call.
void Pause();
void Resume(FrameDeliverCallback deliver_callback);
// Requests to receive a refreshed captured video frame. Do nothing if the
// capturing device is not started or the capturing is paused.
void RequestRefreshFrame();
// media::mojom::VideoCaptureObserver implementations.
void OnStateChanged(media::mojom::VideoCaptureState state) override;
void OnBufferCreated(int32_t buffer_id,
mojo::ScopedSharedBufferHandle handle) override;
void OnBufferReady(int32_t buffer_id,
media::mojom::VideoFrameInfoPtr info) override;
void OnBufferDestroyed(int32_t buffer_id) override;
private:
using BufferFinishedCallback =
base::OnceCallback<void(double consumer_resource_utilization)>;
// Called by the VideoFrame destructor.
static void DidFinishConsumingFrame(const media::VideoFrameMetadata* metadata,
BufferFinishedCallback callback);
// Reports the utilization and returns the buffer.
void OnClientBufferFinished(int buffer_id,
double consumer_resource_utilization);
const media::mojom::VideoCaptureHostPtr video_capture_host_;
// Called when capturing failed to start.
base::OnceClosure error_callback_;
mojo::Binding<media::mojom::VideoCaptureObserver> binding_;
using ClientBufferMap =
base::flat_map<int32_t, mojo::ScopedSharedBufferHandle>;
// Stores the buffer handler on OnBufferCreated(). |buffer_id| is the key.
ClientBufferMap client_buffers_;
using MappingAndSize = std::pair<mojo::ScopedSharedBufferMapping, uint32_t>;
using MappingMap = base::flat_map<int32_t, MappingAndSize>;
// Stores the mapped buffers and their size. Each buffer is added the first
// time the mapping is done or a larger size is requested.
// |buffer_id| is the key to this map.
MappingMap mapped_buffers_;
// The reference time for the first frame. Used to calculate the timestamp of
// the captured frame if not provided in the frame info.
base::TimeTicks first_frame_ref_time_;
// The callback to deliver the received frame.
FrameDeliverCallback frame_deliver_callback_;
SEQUENCE_CHECKER(sequence_checker_);
base::WeakPtrFactory<VideoCaptureClient> weak_factory_;
DISALLOW_COPY_AND_ASSIGN(VideoCaptureClient);
};
} // namespace mirroring
#endif // COMPONENTS_MIRRORING_SERVICE_VIDEO_CAPTURE_CLIENT_H_
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/mirroring/service/video_capture_client.h"
#include "base/run_loop.h"
#include "base/test/mock_callback.h"
#include "base/test/scoped_task_environment.h"
#include "mojo/public/cpp/bindings/binding.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
using ::testing::InvokeWithoutArgs;
using ::testing::_;
namespace mirroring {
namespace {
class FakeVideoCaptureHost final : public media::mojom::VideoCaptureHost {
public:
explicit FakeVideoCaptureHost(media::mojom::VideoCaptureHostRequest request)
: binding_(this, std::move(request)) {}
~FakeVideoCaptureHost() override {}
// mojom::VideoCaptureHost implementations
MOCK_METHOD1(RequestRefreshFrame, void(int32_t));
MOCK_METHOD3(ReleaseBuffer, void(int32_t, int32_t, double));
MOCK_METHOD1(Pause, void(int32_t));
MOCK_METHOD3(Resume,
void(int32_t, int32_t, const media::VideoCaptureParams&));
MOCK_METHOD0(OnStopped, void());
void Start(int32_t device_id,
int32_t session_id,
const media::VideoCaptureParams& params,
media::mojom::VideoCaptureObserverPtr observer) override {
client_ = std::move(observer);
client_->OnStateChanged(media::mojom::VideoCaptureState::STARTED);
}
void Stop(int32_t device_id) override {
client_->OnStateChanged(media::mojom::VideoCaptureState::ENDED);
client_.reset();
OnStopped();
}
void GetDeviceSupportedFormats(
int32_t device_id,
int32_t session_id,
GetDeviceSupportedFormatsCallback callback) override {}
void GetDeviceFormatsInUse(int32_t device_id,
int32_t session_id,
GetDeviceFormatsInUseCallback callback) override {}
private:
mojo::Binding<media::mojom::VideoCaptureHost> binding_;
media::mojom::VideoCaptureObserverPtr client_;
DISALLOW_COPY_AND_ASSIGN(FakeVideoCaptureHost);
};
media::mojom::VideoFrameInfoPtr GetVideoFrameInfo(const gfx::Size& size) {
media::VideoFrameMetadata metadata;
metadata.SetDouble(media::VideoFrameMetadata::FRAME_RATE, 30);
metadata.SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME,
base::TimeTicks());
return media::mojom::VideoFrameInfo::New(
base::TimeDelta(), metadata.CopyInternalValues(),
media::PIXEL_FORMAT_I420, media::VideoPixelStorage::CPU, size,
gfx::Rect(size));
}
} // namespace
class VideoCaptureClientTest : public ::testing::Test {
public:
VideoCaptureClientTest() {
media::mojom::VideoCaptureHostPtr host;
host_impl_ =
std::make_unique<FakeVideoCaptureHost>(mojo::MakeRequest(&host));
client_ = std::make_unique<VideoCaptureClient>(std::move(host));
}
~VideoCaptureClientTest() override {
if (client_) {
base::RunLoop run_loop;
EXPECT_CALL(*host_impl_, OnStopped())
.WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
client_->Stop();
run_loop.Run();
}
scoped_task_environment_.RunUntilIdle();
}
MOCK_METHOD1(OnFrameReceived, void(const gfx::Size&));
void OnFrameReady(scoped_refptr<media::VideoFrame> video_frame,
base::TimeTicks estimated_capture_time) {
video_frame->metadata()->SetDouble(
media::VideoFrameMetadata::RESOURCE_UTILIZATION, 0.6);
OnFrameReceived(video_frame->coded_size());
}
protected:
base::test::ScopedTaskEnvironment scoped_task_environment_;
std::unique_ptr<FakeVideoCaptureHost> host_impl_;
std::unique_ptr<VideoCaptureClient> client_;
};
TEST_F(VideoCaptureClientTest, Basic) {
base::MockCallback<base::OnceClosure> error_cb;
EXPECT_CALL(error_cb, Run()).Times(0);
{
base::RunLoop run_loop;
// Expect to call RequestRefreshFrame() after capturing started.
EXPECT_CALL(*host_impl_, RequestRefreshFrame(_))
.WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
client_->Start(base::BindRepeating(&VideoCaptureClientTest::OnFrameReady,
base::Unretained(this)),
error_cb.Get());
run_loop.Run();
}
scoped_task_environment_.RunUntilIdle();
client_->OnBufferCreated(0, mojo::SharedBufferHandle::Create(100000));
scoped_task_environment_.RunUntilIdle();
{
base::RunLoop run_loop;
// Expect to receive one frame.
EXPECT_CALL(*this, OnFrameReceived(gfx::Size(128, 64))).Times(1);
// Expect to return the buffer after the frame is consumed.
EXPECT_CALL(*host_impl_, ReleaseBuffer(_, 0, 0.6))
.WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
client_->OnBufferReady(0, GetVideoFrameInfo(gfx::Size(128, 64)));
run_loop.Run();
}
scoped_task_environment_.RunUntilIdle();
// Received a smaller size video frame in the same buffer.
{
base::RunLoop run_loop;
// Expect to receive one frame.
EXPECT_CALL(*this, OnFrameReceived(gfx::Size(64, 32))).Times(1);
// Expect to return the buffer after the frame is consumed.
EXPECT_CALL(*host_impl_, ReleaseBuffer(_, 0, 0.6))
.WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
client_->OnBufferReady(0, GetVideoFrameInfo(gfx::Size(64, 32)));
run_loop.Run();
}
scoped_task_environment_.RunUntilIdle();
// Received a larger size video frame in the same buffer.
{
base::RunLoop run_loop;
// Expect to receive one frame.
EXPECT_CALL(*this, OnFrameReceived(gfx::Size(320, 180))).Times(1);
// Expect to return the buffer after the frame is consumed.
EXPECT_CALL(*host_impl_, ReleaseBuffer(_, 0, 0.6))
.WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit));
client_->OnBufferReady(0, GetVideoFrameInfo(gfx::Size(320, 180)));
run_loop.Run();
}
scoped_task_environment_.RunUntilIdle();
}
} // namespace mirroring
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment