Skip to content

GitLab

T
tangled
Commit cbacab9d authored by Xing Liu's avatar Xing Liu Committed by Commit Bot
Browse files
Options

Media Thumbnail Parser: Use empty implementation when ffmpeg is disabled. 

VideoFrameExtractor should not be built when media_use_ffmpeg build
flag is false.

This CL splits ThumbnailMediaParser to an interface and use empty
implementation based on media_use_ffmpeg.

There are other build errors, this CL only fixes the one in
VideoFrameExtractor use case.

Bug: 1066404
Change-Id: Ib856899b4d465a817634b468866d3fb62d7c5053
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2131036Reviewed-by: default avatarDale Curtis <dalecurtis@chromium.org>
Reviewed-by: default avatarDavid Trainor <dtrainor@chromium.org>
Commit-Queue: Xing Liu <xingliu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#757195}
parent f5417f2d
Hide whitespace changes
Inline Side-by-side
Showing with 526 additions and 413 deletions
chrome/browser/thumbnail/generator/BUILD.gn
View file @ cbacab9d
......@@ -2,6 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import("//media/media_options.gni")
if (is_android) {
import("//build/config/android/config.gni")
import("//build/config/android/rules.gni")
......@@ -42,6 +44,18 @@ source_set("generator") {
"android/thumbnail_media_parser.h",
]
if (media_use_ffmpeg) {
sources += [
"android/thumbnail_media_parser_impl.cc",
"android/thumbnail_media_parser_impl.h",
]
} else {
sources += [
"android/noop_thumbnail_media_parser.cc",
"android/noop_thumbnail_media_parser.h",
]
}
deps += [
":jni_headers",
"//third_party/android_opengl/etc1",
......
chrome/browser/thumbnail/generator/android/noop_thumbnail_media_parser.cc 0 → 100644
View file @ cbacab9d
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/thumbnail/generator/android/noop_thumbnail_media_parser.h"
#include "third_party/skia/include/core/SkBitmap.h"
void NoopThumbnailMediaParser::Start(ParseCompleteCB parse_complete_cb) {
DCHECK(parse_complete_cb);
std::move(parse_complete_cb)
.Run(false /*success*/, chrome::mojom::MediaMetadata::New(), SkBitmap());
}
chrome/browser/thumbnail/generator/android/noop_thumbnail_media_parser.h 0 → 100644
View file @ cbacab9d
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_NOOP_THUMBNAIL_MEDIA_PARSER_H_
#define CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_NOOP_THUMBNAIL_MEDIA_PARSER_H_
#include "chrome/browser/thumbnail/generator/android/thumbnail_media_parser.h"
// Empty implementation used when ENABLE_FFMPEG build flag is false.
class NoopThumbnailMediaParser : public ThumbnailMediaParser {
public:
NoopThumbnailMediaParser() = default;
NoopThumbnailMediaParser(const NoopThumbnailMediaParser&) = delete;
NoopThumbnailMediaParser& operator=(const NoopThumbnailMediaParser&) = delete;
~NoopThumbnailMediaParser() override = default;
private:
// ThumbnailMediaParser implementation.
void Start(ParseCompleteCB parse_complete_cb) override;
};
#endif // CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_NOOP_THUMBNAIL_MEDIA_PARSER_H_
chrome/browser/thumbnail/generator/android/thumbnail_generator.cc
View file @ cbacab9d
......@@ -106,7 +106,7 @@ void ThumbnailGenerator::RetrieveThumbnail(
// Retrieve video thumbnail.
if (base::StartsWith(mime_type, "video/",
base::CompareCase::INSENSITIVE_ASCII)) {
auto parser = std::make_unique<ThumbnailMediaParser>(mime_type, file_path);
auto parser = ThumbnailMediaParser::Create(mime_type, file_path);
parser->Start(base::BindOnce(&ThumbnailGenerator::OnVideoThumbnailRetrieved,
weak_factory_.GetWeakPtr(),
std::move(java_callback), icon_size,
......
chrome/browser/thumbnail/generator/android/thumbnail_media_parser.cc
View file @ cbacab9d
// Copyright 2019 The Chromium Authors. All rights reserved.
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/thumbnail/generator/android/thumbnail_media_parser.h"
#include "base/bind.h"
#include "base/files/file.h"
#include "base/files/file_util.h"
#include "base/numerics/safe_conversions.h"
#include "base/task/post_task.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "base/task_runner_util.h"
#include "cc/paint/skia_paint_canvas.h"
#include "chrome/browser/thumbnail/generator/android/local_media_data_source_factory.h"
#include "content/public/browser/android/gpu_video_accelerator_factories_provider.h"
#include "content/public/browser/media_service.h"
#include "media/base/overlay_info.h"
#include "media/base/video_thumbnail_decoder.h"
#include "media/mojo/clients/mojo_video_decoder.h"
#include "media/mojo/mojom/media_service.mojom.h"
#include "media/mojo/services/media_interface_provider.h"
#include "media/renderers/paint_canvas_video_renderer.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "mojo/public/cpp/bindings/pending_remote.h"
#include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
#include "third_party/skia/include/core/SkBitmap.h"
namespace {
// The maximum duration to parse media file.
const base::TimeDelta kTimeOut = base::TimeDelta::FromSeconds(8);
// Returns if the mime type is video or audio.
bool IsSupportedMediaMimeType(const std::string& mime_type) {
return base::StartsWith(mime_type, "audio/",
base::CompareCase::INSENSITIVE_ASCII) ||
base::StartsWith(mime_type, "video/",
base::CompareCase::INSENSITIVE_ASCII);
}
void OnRequestOverlayInfo(bool decoder_requires_restart_for_overlay,
media::ProvideOverlayInfoCB overlay_info_cb) {
// No android overlay associated with video thumbnail.
if (overlay_info_cb)
std::move(overlay_info_cb).Run(media::OverlayInfo());
}
int64_t GetFileSize(const base::FilePath& file_path) {
int64_t size = 0;
if (!base::GetFileSize(file_path, &size))
return -1;
return size;
}
} // namespace
ThumbnailMediaParser::ThumbnailMediaParser(const std::string& mime_type,
const base::FilePath& file_path)
: mime_type_(mime_type),
file_path_(file_path),
file_task_runner_(
base::ThreadPool::CreateSingleThreadTaskRunner({base::MayBlock()})),
decode_done_(false) {}
ThumbnailMediaParser::~ThumbnailMediaParser() = default;
void ThumbnailMediaParser::Start(ParseCompleteCB parse_complete_cb) {
RecordMediaParserEvent(MediaParserEvent::kInitialize);
parse_complete_cb_ = std::move(parse_complete_cb);
timer_.Start(
FROM_HERE, kTimeOut,
base::BindOnce(&ThumbnailMediaParser::OnError, weak_factory_.GetWeakPtr(),
MediaParserEvent::kTimeout));
// Only process media mime types.
if (!IsSupportedMediaMimeType(mime_type_)) {
OnError(MediaParserEvent::kUnsupportedMimeType);
return;
}
// Get the size of the file if needed.
base::PostTaskAndReplyWithResult(
file_task_runner_.get(), FROM_HERE,
base::BindOnce(&GetFileSize, file_path_),
base::BindOnce(&ThumbnailMediaParser::OnReadFileSize,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParser::OnReadFileSize(int64_t file_size) {
if (file_size < 0) {
OnError(MediaParserEvent::kReadFileError);
return;
}
size_ = file_size;
RetrieveMediaParser();
}
void ThumbnailMediaParser::OnMediaParserCreated() {
auto media_source_factory = std::make_unique<LocalMediaDataSourceFactory>(
file_path_, file_task_runner_);
mojo::PendingRemote<chrome::mojom::MediaDataSource> source;
media_data_source_ = media_source_factory->CreateMediaDataSource(
source.InitWithNewPipeAndPassReceiver(),
base::BindRepeating(&ThumbnailMediaParser::OnMediaDataReady,
weak_factory_.GetWeakPtr()));
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataStart);
media_parser()->ParseMediaMetadata(
mime_type_, size_, false /* get_attached_images */, std::move(source),
base::BindOnce(&ThumbnailMediaParser::OnMediaMetadataParsed,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParser::OnConnectionError() {
OnError(MediaParserEvent::kUtilityConnectionError);
}
void ThumbnailMediaParser::OnMediaMetadataParsed(
bool parse_success,
chrome::mojom::MediaMetadataPtr metadata,
const std::vector<metadata::AttachedImage>& attached_images) {
if (!parse_success) {
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataFailed);
OnError(MediaParserEvent::kMetadataFailed);
return;
}
metadata_ = std::move(metadata);
DCHECK(metadata_);
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataComplete);
// For audio file, we only need metadata and poster.
if (base::StartsWith(mime_type_, "audio/",
base::CompareCase::INSENSITIVE_ASCII)) {
NotifyComplete(SkBitmap());
return;
}
DCHECK(base::StartsWith(mime_type_, "video/",
base::CompareCase::INSENSITIVE_ASCII));
// Start to retrieve video thumbnail.
base::ThreadTaskRunnerHandle::Get()->PostTask(
FROM_HERE,
base::BindOnce(&ThumbnailMediaParser::RetrieveEncodedVideoFrame,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParser::RetrieveEncodedVideoFrame() {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoThumbnailStart);
media_data_source_.reset();
auto media_source_factory = std::make_unique<LocalMediaDataSourceFactory>(
file_path_, file_task_runner_);
mojo::PendingRemote<chrome::mojom::MediaDataSource> source;
media_data_source_ = media_source_factory->CreateMediaDataSource(
source.InitWithNewPipeAndPassReceiver(),
base::BindRepeating(&ThumbnailMediaParser::OnMediaDataReady,
weak_factory_.GetWeakPtr()));
media_parser()->ExtractVideoFrame(
mime_type_, base::saturated_cast<uint32_t>(size_), std::move(source),
base::BindOnce(&ThumbnailMediaParser::OnVideoFrameRetrieved,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParser::OnVideoFrameRetrieved(
bool success,
chrome::mojom::VideoFrameDataPtr video_frame_data,
const base::Optional<media::VideoDecoderConfig>& config) {
if (!success) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoFrameExtractionFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
video_frame_data_ = std::move(video_frame_data);
DCHECK(config.has_value());
config_ = config.value();
// For vp8, vp9 codec, we directly do software decoding in utility process.
// Render now.
if (video_frame_data_->which() ==
chrome::mojom::VideoFrameData::Tag::DECODED_FRAME) {
decode_done_ = true;
RenderVideoFrame(std::move(video_frame_data_->get_decoded_frame()));
return;
}
// For other codec, the encoded frame is retrieved in utility process, send
// the data to GPU process to do hardware decoding.
if (video_frame_data_->get_encoded_data().empty()) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoFrameExtractionFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
// Starts to decode with MojoVideoDecoder.
content::CreateGpuVideoAcceleratorFactories(base::BindRepeating(
&ThumbnailMediaParser::OnGpuVideoAcceleratorFactoriesReady,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParser::OnGpuVideoAcceleratorFactoriesReady(
std::unique_ptr<media::GpuVideoAcceleratorFactories> factories) {
gpu_factories_ = std::move(factories);
DecodeVideoFrame();
}
void ThumbnailMediaParser::DecodeVideoFrame() {
mojo::PendingRemote<media::mojom::VideoDecoder> video_decoder_remote;
GetMediaInterfaceFactory()->CreateVideoDecoder(
video_decoder_remote.InitWithNewPipeAndPassReceiver());
// Build and config the decoder.
DCHECK(gpu_factories_);
auto mojo_decoder = std::make_unique<media::MojoVideoDecoder>(
base::ThreadTaskRunnerHandle::Get(), gpu_factories_.get(), this,
std::move(video_decoder_remote),
media::VideoDecoderImplementation::kDefault,
base::BindRepeating(&OnRequestOverlayInfo), gfx::ColorSpace());
decoder_ = std::make_unique<media::VideoThumbnailDecoder>(
std::move(mojo_decoder), config_,
std::move(video_frame_data_->get_encoded_data()));
decoder_->Start(base::BindOnce(&ThumbnailMediaParser::OnVideoFrameDecoded,
weak_factory_.GetWeakPtr()));
video_frame_data_.reset();
}
void ThumbnailMediaParser::OnVideoFrameDecoded(
scoped_refptr<media::VideoFrame> frame) {
if (!frame) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoDecodeFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
DCHECK(frame->HasTextures());
decode_done_ = true;
RenderVideoFrame(std::move(frame));
}
void ThumbnailMediaParser::RenderVideoFrame(
scoped_refptr<media::VideoFrame> video_frame) {
auto* context_provider =
gpu_factories_ ? gpu_factories_->GetMediaContextProvider() : nullptr;
media::PaintCanvasVideoRenderer renderer;
SkBitmap bitmap;
bitmap.allocN32Pixels(video_frame->visible_rect().width(),
video_frame->visible_rect().height());
// Draw the video frame to |bitmap|.
cc::SkiaPaintCanvas canvas(bitmap);
renderer.Copy(video_frame, &canvas, context_provider);
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoThumbnailComplete);
NotifyComplete(std::move(bitmap));
}
media::mojom::InterfaceFactory*
ThumbnailMediaParser::GetMediaInterfaceFactory() {
if (!media_interface_factory_) {
mojo::PendingRemote<service_manager::mojom::InterfaceProvider> interfaces;
media_interface_provider_ = std::make_unique<media::MediaInterfaceProvider>(
interfaces.InitWithNewPipeAndPassReceiver());
content::GetMediaService().CreateInterfaceFactory(
media_interface_factory_.BindNewPipeAndPassReceiver(),
std::move(interfaces));
media_interface_factory_.set_disconnect_handler(
base::BindOnce(&ThumbnailMediaParser::OnDecoderConnectionError,
base::Unretained(this)));
}
return media_interface_factory_.get();
}
void ThumbnailMediaParser::OnDecoderConnectionError() {
OnError(MediaParserEvent::kGpuConnectionError);
}
void ThumbnailMediaParser::OnMediaDataReady(
chrome::mojom::MediaDataSource::ReadCallback callback,
std::unique_ptr<std::string> data) {
// TODO(xingliu): Change media_parser.mojom to move the data instead of copy.
if (media_parser())
std::move(callback).Run(std::vector<uint8_t>(data->begin(), data->end()));
}
void ThumbnailMediaParser::NotifyComplete(SkBitmap bitmap) {
DCHECK(metadata_);
DCHECK(parse_complete_cb_);
RecordMediaParserEvent(MediaParserEvent::kSuccess);
std::move(parse_complete_cb_)
.Run(true, std::move(metadata_), std::move(bitmap));
}
void ThumbnailMediaParser::OnError(MediaParserEvent event) {
DCHECK(parse_complete_cb_);
RecordMediaParserEvent(MediaParserEvent::kFailure);
RecordMediaParserEvent(event);
std::move(parse_complete_cb_)
.Run(false, chrome::mojom::MediaMetadata::New(), SkBitmap());
#include "media/media_buildflags.h"
#if BUILDFLAG(ENABLE_FFMPEG)
#include "chrome/browser/thumbnail/generator/android/thumbnail_media_parser_impl.h"
#else
#include "chrome/browser/thumbnail/generator/android/noop_thumbnail_media_parser.h"
#endif
std::unique_ptr<ThumbnailMediaParser> ThumbnailMediaParser::Create(
const std::string& mime_type,
const base::FilePath& file_path) {
#if BUILDFLAG(ENABLE_FFMPEG)
return std::make_unique<ThumbnailMediaParserImpl>(mime_type, file_path);
#else
return std::make_unique<NoopThumbnailMediaParser>();
#endif
}
chrome/browser/thumbnail/generator/android/thumbnail_media_parser.h
View file @ cbacab9d
// Copyright 2019 The Chromium Authors. All rights reserved.
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
......@@ -7,132 +7,39 @@
#include <memory>
#include <string>
#include <vector>
#include "base/callback.h"
#include "base/files/file_path.h"
#include "base/macros.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/sequenced_task_runner.h"
#include "base/timer/timer.h"
#include "chrome/browser/thumbnail/generator/android/stats.h"
#include "chrome/common/media_galleries/metadata_types.h"
#include "chrome/services/media_gallery_util/public/cpp/media_parser_provider.h"
#include "chrome/services/media_gallery_util/public/mojom/media_parser.mojom-forward.h"
#include "media/base/media_log.h"
#include "media/mojo/mojom/interface_factory.mojom.h"
#include "mojo/public/cpp/bindings/remote.h"
namespace media {
class GpuVideoAcceleratorFactories;
class MediaInterfaceProvider;
class MojoVideoDecoder;
class VideoDecoderConfig;
class VideoThumbnailDecoder;
} // namespace media
class SkBitmap;
// Parse local media files, including media metadata and thumbnails.
// Metadata is always parsed in utility process for both audio and video files.
//
// For video file, the thumbnail will be the a video key frame. The frame
// extraction always happens in utility process. The decoding may happen in
// utility or GPU process based on video codec.
class ThumbnailMediaParser : public MediaParserProvider,
public media::MediaLog {
class ThumbnailMediaParser {
public:
using ParseCompleteCB =
base::OnceCallback<void(bool success,
chrome::mojom::MediaMetadataPtr media_metadata,
SkBitmap bitmap)>;
ThumbnailMediaParser(const std::string& mime_type,
const base::FilePath& file_path);
~ThumbnailMediaParser() override;
// Creates the parser, may return an empty implementation when ffmpeg is
// disabled.
static std::unique_ptr<ThumbnailMediaParser> Create(
const std::string& mime_type,
const base::FilePath& file_path);
ThumbnailMediaParser() = default;
ThumbnailMediaParser(const ThumbnailMediaParser&) = delete;
ThumbnailMediaParser& operator=(const ThumbnailMediaParser&) = delete;
virtual ~ThumbnailMediaParser() = default;
// Parse media metadata and thumbnail in a local file. All file IO will run on
// |file_task_runner|. The metadata is parsed in an utility process safely.
// The thumbnail is retrieved from GPU process or utility process based on
// different codec.
void Start(ParseCompleteCB parse_complete_cb);
private:
void OnReadFileSize(int64_t file_size);
// MediaParserProvider implementation:
void OnMediaParserCreated() override;
void OnConnectionError() override;
// Called after media metadata are parsed.
void OnMediaMetadataParsed(
bool parse_success,
chrome::mojom::MediaMetadataPtr metadata,
const std::vector<metadata::AttachedImage>& attached_images);
// Retrieves an encoded video frame.
void RetrieveEncodedVideoFrame();
void OnVideoFrameRetrieved(
bool success,
chrome::mojom::VideoFrameDataPtr video_frame_data,
const base::Optional<media::VideoDecoderConfig>& config);
// Decodes the video frame.
void OnGpuVideoAcceleratorFactoriesReady(
std::unique_ptr<media::GpuVideoAcceleratorFactories>);
void DecodeVideoFrame();
void OnVideoFrameDecoded(scoped_refptr<media::VideoFrame> decoded_frame);
// Renders the video frame to bitmap.
void RenderVideoFrame(scoped_refptr<media::VideoFrame> video_frame);
media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
void OnDecoderConnectionError();
// Overlays media data source read operation. Gradually read data from media
// file.
void OnMediaDataReady(chrome::mojom::MediaDataSource::ReadCallback callback,
std::unique_ptr<std::string> data);
void NotifyComplete(SkBitmap bitmap);
void OnError(MediaParserEvent event);
int64_t size_;
std::string mime_type_;
base::FilePath file_path_;
ParseCompleteCB parse_complete_cb_;
chrome::mojom::MediaMetadataPtr metadata_;
// Used to read media files chunks to feed to IPC channel.
std::unique_ptr<chrome::mojom::MediaDataSource> media_data_source_;
// The task runner to do blocking disk IO.
scoped_refptr<base::SequencedTaskRunner> file_task_runner_;
// A timer to prevent unresponsive Android decoder during video file parsing,
// and the parser will fail gracefully after timeout.
base::OneShotTimer timer_;
// Cached video frame data, which contains either encoded frame or decoded
// video frame. Encoded frame is extracted with ffmpeg, the data can be large
// for high resolution video.
chrome::mojom::VideoFrameDataPtr video_frame_data_;
// Objects used to decode the video into media::VideoFrame with
// MojoVideoDecoder.
media::VideoDecoderConfig config_;
std::unique_ptr<media::VideoThumbnailDecoder> decoder_;
mojo::Remote<media::mojom::InterfaceFactory> media_interface_factory_;
std::unique_ptr<media::MediaInterfaceProvider> media_interface_provider_;
std::unique_ptr<media::GpuVideoAcceleratorFactories> gpu_factories_;
bool decode_done_;
base::WeakPtrFactory<ThumbnailMediaParser> weak_factory_{this};
DISALLOW_COPY_AND_ASSIGN(ThumbnailMediaParser);
virtual void Start(ParseCompleteCB parse_complete_cb) = 0;
};
#endif // CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_THUMBNAIL_MEDIA_PARSER_H_
chrome/browser/thumbnail/generator/android/thumbnail_media_parser_bridge.cc
View file @ cbacab9d
......@@ -64,7 +64,7 @@ ThumbnailMediaParserBridge::ThumbnailMediaParserBridge(
const std::string& mime_type,
const base::FilePath& file_path,
ThumbnailMediaParser::ParseCompleteCB parse_complete_cb)
: parser_(std::make_unique<ThumbnailMediaParser>(mime_type, file_path)),
: parser_(ThumbnailMediaParser::Create(mime_type, file_path)),
parse_complete_cb_(std::move(parse_complete_cb)) {}
ThumbnailMediaParserBridge::~ThumbnailMediaParserBridge() = default;
......
chrome/browser/thumbnail/generator/android/thumbnail_media_parser_impl.cc 0 → 100644
View file @ cbacab9d
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/thumbnail/generator/android/thumbnail_media_parser_impl.h"
#include "base/bind.h"
#include "base/files/file.h"
#include "base/files/file_util.h"
#include "base/numerics/safe_conversions.h"
#include "base/task/post_task.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "base/task_runner_util.h"
#include "cc/paint/skia_paint_canvas.h"
#include "chrome/browser/thumbnail/generator/android/local_media_data_source_factory.h"
#include "content/public/browser/android/gpu_video_accelerator_factories_provider.h"
#include "content/public/browser/media_service.h"
#include "media/base/overlay_info.h"
#include "media/base/video_thumbnail_decoder.h"
#include "media/mojo/clients/mojo_video_decoder.h"
#include "media/mojo/mojom/media_service.mojom.h"
#include "media/mojo/services/media_interface_provider.h"
#include "media/renderers/paint_canvas_video_renderer.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "mojo/public/cpp/bindings/pending_remote.h"
#include "services/viz/public/cpp/gpu/context_provider_command_buffer.h"
#include "third_party/skia/include/core/SkBitmap.h"
namespace {
// The maximum duration to parse media file.
const base::TimeDelta kTimeOut = base::TimeDelta::FromSeconds(8);
// Returns if the mime type is video or audio.
bool IsSupportedMediaMimeType(const std::string& mime_type) {
return base::StartsWith(mime_type, "audio/",
base::CompareCase::INSENSITIVE_ASCII) ||
base::StartsWith(mime_type, "video/",
base::CompareCase::INSENSITIVE_ASCII);
}
void OnRequestOverlayInfo(bool decoder_requires_restart_for_overlay,
media::ProvideOverlayInfoCB overlay_info_cb) {
// No android overlay associated with video thumbnail.
if (overlay_info_cb)
std::move(overlay_info_cb).Run(media::OverlayInfo());
}
int64_t GetFileSize(const base::FilePath& file_path) {
int64_t size = 0;
if (!base::GetFileSize(file_path, &size))
return -1;
return size;
}
} // namespace
ThumbnailMediaParserImpl::ThumbnailMediaParserImpl(
const std::string& mime_type,
const base::FilePath& file_path)
: mime_type_(mime_type),
file_path_(file_path),
file_task_runner_(
base::ThreadPool::CreateSingleThreadTaskRunner({base::MayBlock()})),
decode_done_(false) {}
ThumbnailMediaParserImpl::~ThumbnailMediaParserImpl() = default;
void ThumbnailMediaParserImpl::Start(ParseCompleteCB parse_complete_cb) {
RecordMediaParserEvent(MediaParserEvent::kInitialize);
parse_complete_cb_ = std::move(parse_complete_cb);
timer_.Start(
FROM_HERE, kTimeOut,
base::BindOnce(&ThumbnailMediaParserImpl::OnError,
weak_factory_.GetWeakPtr(), MediaParserEvent::kTimeout));
// Only process media mime types.
if (!IsSupportedMediaMimeType(mime_type_)) {
OnError(MediaParserEvent::kUnsupportedMimeType);
return;
}
// Get the size of the file if needed.
base::PostTaskAndReplyWithResult(
file_task_runner_.get(), FROM_HERE,
base::BindOnce(&GetFileSize, file_path_),
base::BindOnce(&ThumbnailMediaParserImpl::OnReadFileSize,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParserImpl::OnReadFileSize(int64_t file_size) {
if (file_size < 0) {
OnError(MediaParserEvent::kReadFileError);
return;
}
size_ = file_size;
RetrieveMediaParser();
}
void ThumbnailMediaParserImpl::OnMediaParserCreated() {
auto media_source_factory = std::make_unique<LocalMediaDataSourceFactory>(
file_path_, file_task_runner_);
mojo::PendingRemote<chrome::mojom::MediaDataSource> source;
media_data_source_ = media_source_factory->CreateMediaDataSource(
source.InitWithNewPipeAndPassReceiver(),
base::BindRepeating(&ThumbnailMediaParserImpl::OnMediaDataReady,
weak_factory_.GetWeakPtr()));
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataStart);
media_parser()->ParseMediaMetadata(
mime_type_, size_, false /* get_attached_images */, std::move(source),
base::BindOnce(&ThumbnailMediaParserImpl::OnMediaMetadataParsed,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParserImpl::OnConnectionError() {
OnError(MediaParserEvent::kUtilityConnectionError);
}
void ThumbnailMediaParserImpl::OnMediaMetadataParsed(
bool parse_success,
chrome::mojom::MediaMetadataPtr metadata,
const std::vector<metadata::AttachedImage>& attached_images) {
if (!parse_success) {
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataFailed);
OnError(MediaParserEvent::kMetadataFailed);
return;
}
metadata_ = std::move(metadata);
DCHECK(metadata_);
RecordMediaMetadataEvent(MediaMetadataEvent::kMetadataComplete);
// For audio file, we only need metadata and poster.
if (base::StartsWith(mime_type_, "audio/",
base::CompareCase::INSENSITIVE_ASCII)) {
NotifyComplete(SkBitmap());
return;
}
DCHECK(base::StartsWith(mime_type_, "video/",
base::CompareCase::INSENSITIVE_ASCII));
// Start to retrieve video thumbnail.
base::ThreadTaskRunnerHandle::Get()->PostTask(
FROM_HERE,
base::BindOnce(&ThumbnailMediaParserImpl::RetrieveEncodedVideoFrame,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParserImpl::RetrieveEncodedVideoFrame() {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoThumbnailStart);
media_data_source_.reset();
auto media_source_factory = std::make_unique<LocalMediaDataSourceFactory>(
file_path_, file_task_runner_);
mojo::PendingRemote<chrome::mojom::MediaDataSource> source;
media_data_source_ = media_source_factory->CreateMediaDataSource(
source.InitWithNewPipeAndPassReceiver(),
base::BindRepeating(&ThumbnailMediaParserImpl::OnMediaDataReady,
weak_factory_.GetWeakPtr()));
media_parser()->ExtractVideoFrame(
mime_type_, base::saturated_cast<uint32_t>(size_), std::move(source),
base::BindOnce(&ThumbnailMediaParserImpl::OnVideoFrameRetrieved,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParserImpl::OnVideoFrameRetrieved(
bool success,
chrome::mojom::VideoFrameDataPtr video_frame_data,
const base::Optional<media::VideoDecoderConfig>& config) {
if (!success) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoFrameExtractionFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
video_frame_data_ = std::move(video_frame_data);
DCHECK(config.has_value());
config_ = config.value();
// For vp8, vp9 codec, we directly do software decoding in utility process.
// Render now.
if (video_frame_data_->which() ==
chrome::mojom::VideoFrameData::Tag::DECODED_FRAME) {
decode_done_ = true;
RenderVideoFrame(std::move(video_frame_data_->get_decoded_frame()));
return;
}
// For other codec, the encoded frame is retrieved in utility process, send
// the data to GPU process to do hardware decoding.
if (video_frame_data_->get_encoded_data().empty()) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoFrameExtractionFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
// Starts to decode with MojoVideoDecoder.
content::CreateGpuVideoAcceleratorFactories(base::BindRepeating(
&ThumbnailMediaParserImpl::OnGpuVideoAcceleratorFactoriesReady,
weak_factory_.GetWeakPtr()));
}
void ThumbnailMediaParserImpl::OnGpuVideoAcceleratorFactoriesReady(
std::unique_ptr<media::GpuVideoAcceleratorFactories> factories) {
gpu_factories_ = std::move(factories);
DecodeVideoFrame();
}
void ThumbnailMediaParserImpl::DecodeVideoFrame() {
mojo::PendingRemote<media::mojom::VideoDecoder> video_decoder_remote;
GetMediaInterfaceFactory()->CreateVideoDecoder(
video_decoder_remote.InitWithNewPipeAndPassReceiver());
// Build and config the decoder.
DCHECK(gpu_factories_);
auto mojo_decoder = std::make_unique<media::MojoVideoDecoder>(
base::ThreadTaskRunnerHandle::Get(), gpu_factories_.get(), this,
std::move(video_decoder_remote),
media::VideoDecoderImplementation::kDefault,
base::BindRepeating(&OnRequestOverlayInfo), gfx::ColorSpace());
decoder_ = std::make_unique<media::VideoThumbnailDecoder>(
std::move(mojo_decoder), config_,
std::move(video_frame_data_->get_encoded_data()));
decoder_->Start(base::BindOnce(&ThumbnailMediaParserImpl::OnVideoFrameDecoded,
weak_factory_.GetWeakPtr()));
video_frame_data_.reset();
}
void ThumbnailMediaParserImpl::OnVideoFrameDecoded(
scoped_refptr<media::VideoFrame> frame) {
if (!frame) {
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoDecodeFailed);
OnError(MediaParserEvent::kVideoThumbnailFailed);
return;
}
DCHECK(frame->HasTextures());
decode_done_ = true;
RenderVideoFrame(std::move(frame));
}
void ThumbnailMediaParserImpl::RenderVideoFrame(
scoped_refptr<media::VideoFrame> video_frame) {
auto* context_provider =
gpu_factories_ ? gpu_factories_->GetMediaContextProvider() : nullptr;
media::PaintCanvasVideoRenderer renderer;
SkBitmap bitmap;
bitmap.allocN32Pixels(video_frame->visible_rect().width(),
video_frame->visible_rect().height());
// Draw the video frame to |bitmap|.
cc::SkiaPaintCanvas canvas(bitmap);
renderer.Copy(video_frame, &canvas, context_provider);
RecordVideoThumbnailEvent(VideoThumbnailEvent::kVideoThumbnailComplete);
NotifyComplete(std::move(bitmap));
}
media::mojom::InterfaceFactory*
ThumbnailMediaParserImpl::GetMediaInterfaceFactory() {
if (!media_interface_factory_) {
mojo::PendingRemote<service_manager::mojom::InterfaceProvider> interfaces;
media_interface_provider_ = std::make_unique<media::MediaInterfaceProvider>(
interfaces.InitWithNewPipeAndPassReceiver());
content::GetMediaService().CreateInterfaceFactory(
media_interface_factory_.BindNewPipeAndPassReceiver(),
std::move(interfaces));
media_interface_factory_.set_disconnect_handler(
base::BindOnce(&ThumbnailMediaParserImpl::OnDecoderConnectionError,
base::Unretained(this)));
}
return media_interface_factory_.get();
}
void ThumbnailMediaParserImpl::OnDecoderConnectionError() {
OnError(MediaParserEvent::kGpuConnectionError);
}
void ThumbnailMediaParserImpl::OnMediaDataReady(
chrome::mojom::MediaDataSource::ReadCallback callback,
std::unique_ptr<std::string> data) {
// TODO(xingliu): Change media_parser.mojom to move the data instead of copy.
if (media_parser())
std::move(callback).Run(std::vector<uint8_t>(data->begin(), data->end()));
}
void ThumbnailMediaParserImpl::NotifyComplete(SkBitmap bitmap) {
DCHECK(metadata_);
DCHECK(parse_complete_cb_);
RecordMediaParserEvent(MediaParserEvent::kSuccess);
std::move(parse_complete_cb_)
.Run(true, std::move(metadata_), std::move(bitmap));
}
void ThumbnailMediaParserImpl::OnError(MediaParserEvent event) {
DCHECK(parse_complete_cb_);
RecordMediaParserEvent(MediaParserEvent::kFailure);
RecordMediaParserEvent(event);
std::move(parse_complete_cb_)
.Run(false, chrome::mojom::MediaMetadata::New(), SkBitmap());
}
chrome/browser/thumbnail/generator/android/thumbnail_media_parser_impl.h 0 → 100644
View file @ cbacab9d
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_THUMBNAIL_MEDIA_PARSER_IMPL_H_
#define CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_THUMBNAIL_MEDIA_PARSER_IMPL_H_
#include <memory>
#include <string>
#include <vector>
#include "base/callback.h"
#include "base/files/file_path.h"
#include "base/macros.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/sequenced_task_runner.h"
#include "base/timer/timer.h"
#include "chrome/browser/thumbnail/generator/android/stats.h"
#include "chrome/browser/thumbnail/generator/android/thumbnail_media_parser.h"
#include "chrome/common/media_galleries/metadata_types.h"
#include "media/base/media_log.h"
#include "media/mojo/mojom/interface_factory.mojom.h"
#include "mojo/public/cpp/bindings/remote.h"
namespace media {
class GpuVideoAcceleratorFactories;
class MediaInterfaceProvider;
class MojoVideoDecoder;
class VideoDecoderConfig;
class VideoThumbnailDecoder;
} // namespace media
class SkBitmap;
// Parse media file in remote process using ffmpeg to extract video frame.
// The decoding may happen in utility or GPU process based on video codec.
// For video file, the thumbnail will be the a video key frame.
class ThumbnailMediaParserImpl : public ThumbnailMediaParser,
public MediaParserProvider,
public media::MediaLog {
public:
ThumbnailMediaParserImpl(const std::string& mime_type,
const base::FilePath& file_path);
~ThumbnailMediaParserImpl() override;
// ThumbnailMediaParser implementation.
void Start(ParseCompleteCB parse_complete_cb) override;
private:
void OnReadFileSize(int64_t file_size);
// MediaParserProvider implementation:
void OnMediaParserCreated() override;
void OnConnectionError() override;
// Called after media metadata are parsed.
void OnMediaMetadataParsed(
bool parse_success,
chrome::mojom::MediaMetadataPtr metadata,
const std::vector<metadata::AttachedImage>& attached_images);
// Retrieves an encoded video frame.
void RetrieveEncodedVideoFrame();
void OnVideoFrameRetrieved(
bool success,
chrome::mojom::VideoFrameDataPtr video_frame_data,
const base::Optional<media::VideoDecoderConfig>& config);
// Decodes the video frame.
void OnGpuVideoAcceleratorFactoriesReady(
std::unique_ptr<media::GpuVideoAcceleratorFactories>);
void DecodeVideoFrame();
void OnVideoFrameDecoded(scoped_refptr<media::VideoFrame> decoded_frame);
// Renders the video frame to bitmap.
void RenderVideoFrame(scoped_refptr<media::VideoFrame> video_frame);
media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
void OnDecoderConnectionError();
// Overlays media data source read operation. Gradually read data from media
// file.
void OnMediaDataReady(chrome::mojom::MediaDataSource::ReadCallback callback,
std::unique_ptr<std::string> data);
void NotifyComplete(SkBitmap bitmap);
void OnError(MediaParserEvent event);
int64_t size_;
std::string mime_type_;
base::FilePath file_path_;
ParseCompleteCB parse_complete_cb_;
chrome::mojom::MediaMetadataPtr metadata_;
// Used to read media files chunks to feed to IPC channel.
std::unique_ptr<chrome::mojom::MediaDataSource> media_data_source_;
// The task runner to do blocking disk IO.
scoped_refptr<base::SequencedTaskRunner> file_task_runner_;
// A timer to prevent unresponsive Android decoder during video file parsing,
// and the parser will fail gracefully after timeout.
base::OneShotTimer timer_;
// Cached video frame data, which contains either encoded frame or decoded
// video frame. Encoded frame is extracted with ffmpeg, the data can be large
// for high resolution video.
chrome::mojom::VideoFrameDataPtr video_frame_data_;
// Objects used to decode the video into media::VideoFrame with
// MojoVideoDecoder.
media::VideoDecoderConfig config_;
std::unique_ptr<media::VideoThumbnailDecoder> decoder_;
mojo::Remote<media::mojom::InterfaceFactory> media_interface_factory_;
std::unique_ptr<media::MediaInterfaceProvider> media_interface_provider_;
std::unique_ptr<media::GpuVideoAcceleratorFactories> gpu_factories_;
bool decode_done_;
base::WeakPtrFactory<ThumbnailMediaParserImpl> weak_factory_{this};
DISALLOW_COPY_AND_ASSIGN(ThumbnailMediaParserImpl);
};
#endif // CHROME_BROWSER_THUMBNAIL_GENERATOR_ANDROID_THUMBNAIL_MEDIA_PARSER_IMPL_H_
media/filters/BUILD.gn
View file @ cbacab9d
......@@ -187,6 +187,13 @@ jumbo_source_set("filters") {
"ffmpeg_video_decoder.h",
]
}
if (is_android) {
sources += [
"android/video_frame_extractor.cc",
"android/video_frame_extractor.h",
]
}
}
if (is_android) {
......@@ -199,8 +206,6 @@ jumbo_source_set("filters") {
sources += [
"android/media_codec_audio_decoder.cc",
"android/media_codec_audio_decoder.h",
"android/video_frame_extractor.cc",
"android/video_frame_extractor.h",
]
deps += [ "//media/base/android" ]
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment